5228db97cc
Model: promotion/qwen3-8b-aaai27-p3-ronpo-alpha050_anchor0035_lr7p5e8-s42 Source: Original Platform
4904 lines
186 KiB
JSON
4904 lines
186 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2860548271752086,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.2888891100883484,
|
|
"drift/rejected_abs": 0.17491090297698975,
|
|
"epoch": 0.0015891934843067143,
|
|
"grad_norm": 0.90625,
|
|
"learning_rate": 3.3333333333333334e-09,
|
|
"logits/chosen": -2.22796893119812,
|
|
"logits/rejected": -2.1537139415740967,
|
|
"logps/chosen": -0.25039365887641907,
|
|
"logps/rejected": -0.2549845278263092,
|
|
"loss": 0.2590796649456024,
|
|
"loss/core": 0.24525848031044006,
|
|
"loss/preference_sft": 0.2586572766304016,
|
|
"loss/reference_anchor": 0.3690255582332611,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8888912200927734,
|
|
"rewards/margins": 1.140271782875061,
|
|
"rewards/rejected": 1.7486193180084229,
|
|
"ronpo/logit": 0.11402717977762222,
|
|
"ronpo/residual": 0.12274861335754395,
|
|
"ronpo/residual_abs": 0.22723272442817688,
|
|
"ronpo/target": -0.008721430785953999,
|
|
"ronpo/target_abs": 0.06909500062465668,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25118276476860046,
|
|
"drift/rejected_abs": 0.1736268401145935,
|
|
"epoch": 0.0031783869686134287,
|
|
"grad_norm": 5.21875,
|
|
"learning_rate": 7.5e-09,
|
|
"logits/chosen": -1.9810645580291748,
|
|
"logits/rejected": -1.9216926097869873,
|
|
"logps/chosen": -0.2643340528011322,
|
|
"logps/rejected": -0.2714827060699463,
|
|
"loss": 0.15867051482200623,
|
|
"loss/core": 0.14627718925476074,
|
|
"loss/preference_sft": 0.2700865864753723,
|
|
"loss/reference_anchor": 0.3270857632160187,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.511749267578125,
|
|
"rewards/margins": 0.7762027978897095,
|
|
"rewards/rejected": 1.735546350479126,
|
|
"ronpo/logit": 0.0776202604174614,
|
|
"ronpo/residual": 0.0606955885887146,
|
|
"ronpo/residual_abs": 0.17922377586364746,
|
|
"ronpo/target": 0.016924677416682243,
|
|
"ronpo/target_abs": 0.08231900632381439,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14721375703811646,
|
|
"drift/rejected_abs": 0.0799868255853653,
|
|
"epoch": 0.004767580452920143,
|
|
"grad_norm": 7.5,
|
|
"learning_rate": 1.1666666666666667e-08,
|
|
"logits/chosen": -2.291767120361328,
|
|
"logits/rejected": -2.2457759380340576,
|
|
"logps/chosen": -0.2763332426548004,
|
|
"logps/rejected": -0.28252172470092773,
|
|
"loss": 0.049369972199201584,
|
|
"loss/core": 0.04628817364573479,
|
|
"loss/preference_sft": 0.2778027057647705,
|
|
"loss/reference_anchor": 0.060271114110946655,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4715213775634766,
|
|
"rewards/margins": 0.6733793020248413,
|
|
"rewards/rejected": 0.7981420755386353,
|
|
"ronpo/logit": 0.06733792275190353,
|
|
"ronpo/residual": 0.09835626929998398,
|
|
"ronpo/residual_abs": 0.1334967315196991,
|
|
"ronpo/target": -0.031018350273370743,
|
|
"ronpo/target_abs": 0.076286181807518,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14123335480690002,
|
|
"drift/rejected_abs": 0.0699237510561943,
|
|
"epoch": 0.006356773937226857,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 1.5833333333333333e-08,
|
|
"logits/chosen": -2.2912278175354004,
|
|
"logits/rejected": -2.2170040607452393,
|
|
"logps/chosen": -0.27552682161331177,
|
|
"logps/rejected": -0.27909940481185913,
|
|
"loss": 0.02950289286673069,
|
|
"loss/core": 0.02670697309076786,
|
|
"loss/preference_sft": 0.27772125601768494,
|
|
"loss/reference_anchor": 0.05211133509874344,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.4118921756744385,
|
|
"rewards/margins": 0.7152848839759827,
|
|
"rewards/rejected": 0.6966072916984558,
|
|
"ronpo/logit": 0.07152849435806274,
|
|
"ronpo/residual": 0.06994546204805374,
|
|
"ronpo/residual_abs": 0.10728304088115692,
|
|
"ronpo/target": 0.001583026722073555,
|
|
"ronpo/target_abs": 0.0675927996635437,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23433640599250793,
|
|
"drift/rejected_abs": 0.07729706168174744,
|
|
"epoch": 0.007945967421533572,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 2e-08,
|
|
"logits/chosen": -2.353553295135498,
|
|
"logits/rejected": -2.392822504043579,
|
|
"logps/chosen": -0.23983506858348846,
|
|
"logps/rejected": -0.24817518889904022,
|
|
"loss": 0.30295810103416443,
|
|
"loss/core": 0.29502326250076294,
|
|
"loss/preference_sft": 0.2504972517490387,
|
|
"loss/reference_anchor": 0.20166020095348358,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.342874050140381,
|
|
"rewards/margins": 1.5709974765777588,
|
|
"rewards/rejected": 0.7718764543533325,
|
|
"ronpo/logit": 0.15709975361824036,
|
|
"ronpo/residual": 0.1602698713541031,
|
|
"ronpo/residual_abs": 0.20184986293315887,
|
|
"ronpo/target": -0.0031701307743787766,
|
|
"ronpo/target_abs": 0.07585608959197998,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20710475742816925,
|
|
"drift/rejected_abs": 0.0870819017291069,
|
|
"epoch": 0.009535160905840286,
|
|
"grad_norm": 316.0,
|
|
"learning_rate": 2.4166666666666668e-08,
|
|
"logits/chosen": -2.2282848358154297,
|
|
"logits/rejected": -2.124019145965576,
|
|
"logps/chosen": -0.28481242060661316,
|
|
"logps/rejected": -0.28323936462402344,
|
|
"loss": 0.1839178502559662,
|
|
"loss/core": 0.17509128153324127,
|
|
"loss/preference_sft": 0.2906222939491272,
|
|
"loss/reference_anchor": 0.22312648594379425,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0701568126678467,
|
|
"rewards/margins": 1.1993378400802612,
|
|
"rewards/rejected": 0.870819091796875,
|
|
"ronpo/logit": 0.11993379890918732,
|
|
"ronpo/residual": 0.10416283458471298,
|
|
"ronpo/residual_abs": 0.17567749321460724,
|
|
"ronpo/target": 0.015770956873893738,
|
|
"ronpo/target_abs": 0.0844925194978714,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20917043089866638,
|
|
"drift/rejected_abs": 0.13052448630332947,
|
|
"epoch": 0.011124354390147001,
|
|
"grad_norm": 2.3125,
|
|
"learning_rate": 2.8333333333333332e-08,
|
|
"logits/chosen": -2.122420310974121,
|
|
"logits/rejected": -2.0597376823425293,
|
|
"logps/chosen": -0.2706679403781891,
|
|
"logps/rejected": -0.28155890107154846,
|
|
"loss": 0.04921208694577217,
|
|
"loss/core": 0.04241333529353142,
|
|
"loss/preference_sft": 0.2808118462562561,
|
|
"loss/reference_anchor": 0.16616889834403992,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.091449737548828,
|
|
"rewards/margins": 0.7882283926010132,
|
|
"rewards/rejected": 1.303221344947815,
|
|
"ronpo/logit": 0.07882285118103027,
|
|
"ronpo/residual": 0.06731168925762177,
|
|
"ronpo/residual_abs": 0.12287186086177826,
|
|
"ronpo/target": 0.011511145159602165,
|
|
"ronpo/target_abs": 0.07380068302154541,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13368353247642517,
|
|
"drift/rejected_abs": 0.10889682918787003,
|
|
"epoch": 0.012713547874453715,
|
|
"grad_norm": 430.0,
|
|
"learning_rate": 3.25e-08,
|
|
"logits/chosen": -2.080500364303589,
|
|
"logits/rejected": -2.039989471435547,
|
|
"logps/chosen": -0.31930357217788696,
|
|
"logps/rejected": -0.32469144463539124,
|
|
"loss": 0.22682690620422363,
|
|
"loss/core": 0.220375657081604,
|
|
"loss/preference_sft": 0.32273247838020325,
|
|
"loss/reference_anchor": 0.15204796195030212,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.335686445236206,
|
|
"rewards/margins": 0.24776522815227509,
|
|
"rewards/rejected": 1.0879215002059937,
|
|
"ronpo/logit": 0.024776523932814598,
|
|
"ronpo/residual": 0.01710623875260353,
|
|
"ronpo/residual_abs": 0.15854685008525848,
|
|
"ronpo/target": 0.007670276798307896,
|
|
"ronpo/target_abs": 0.06935645639896393,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19959691166877747,
|
|
"drift/rejected_abs": 0.10330881178379059,
|
|
"epoch": 0.014302741358760428,
|
|
"grad_norm": 3.90625,
|
|
"learning_rate": 3.6666666666666664e-08,
|
|
"logits/chosen": -2.1476857662200928,
|
|
"logits/rejected": -2.0756335258483887,
|
|
"logps/chosen": -0.30488723516464233,
|
|
"logps/rejected": -0.29676884412765503,
|
|
"loss": 0.08849599957466125,
|
|
"loss/core": 0.08234541118144989,
|
|
"loss/preference_sft": 0.3082302510738373,
|
|
"loss/reference_anchor": 0.14490821957588196,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9952192306518555,
|
|
"rewards/margins": 0.9640136957168579,
|
|
"rewards/rejected": 1.0312055349349976,
|
|
"ronpo/logit": 0.09640135616064072,
|
|
"ronpo/residual": 0.09527552872896194,
|
|
"ronpo/residual_abs": 0.14935743808746338,
|
|
"ronpo/target": 0.0011258311569690704,
|
|
"ronpo/target_abs": 0.0794779434800148,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20762768387794495,
|
|
"drift/rejected_abs": 0.17166680097579956,
|
|
"epoch": 0.015891934843067144,
|
|
"grad_norm": 8.5625,
|
|
"learning_rate": 4.083333333333333e-08,
|
|
"logits/chosen": -2.2317585945129395,
|
|
"logits/rejected": -2.281120538711548,
|
|
"logps/chosen": -0.38824811577796936,
|
|
"logps/rejected": -0.37904253602027893,
|
|
"loss": 0.48683279752731323,
|
|
"loss/core": 0.4693141579627991,
|
|
"loss/preference_sft": 0.3856825530529022,
|
|
"loss/reference_anchor": 0.4619651436805725,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.075303554534912,
|
|
"rewards/margins": 1.4245798587799072,
|
|
"rewards/rejected": 0.6507236361503601,
|
|
"ronpo/logit": 0.1424579918384552,
|
|
"ronpo/residual": 0.1403721421957016,
|
|
"ronpo/residual_abs": 0.23278239369392395,
|
|
"ronpo/target": 0.002085856394842267,
|
|
"ronpo/target_abs": 0.07827635854482651,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2422821968793869,
|
|
"drift/rejected_abs": 0.11313442885875702,
|
|
"epoch": 0.01748112832737386,
|
|
"grad_norm": 1.4140625,
|
|
"learning_rate": 4.5e-08,
|
|
"logits/chosen": -2.13415265083313,
|
|
"logits/rejected": -2.1069016456604004,
|
|
"logps/chosen": -0.3025519549846649,
|
|
"logps/rejected": -0.29403483867645264,
|
|
"loss": 0.2851434350013733,
|
|
"loss/core": 0.27573922276496887,
|
|
"loss/preference_sft": 0.29142332077026367,
|
|
"loss/reference_anchor": 0.2395510971546173,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.42053484916687,
|
|
"rewards/margins": 1.2919930219650269,
|
|
"rewards/rejected": 1.1285418272018433,
|
|
"ronpo/logit": 0.1291993260383606,
|
|
"ronpo/residual": 0.12877634167671204,
|
|
"ronpo/residual_abs": 0.18722160160541534,
|
|
"ronpo/target": 0.00042299646884202957,
|
|
"ronpo/target_abs": 0.0751943588256836,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2859105169773102,
|
|
"drift/rejected_abs": 0.07585777342319489,
|
|
"epoch": 0.01907032181168057,
|
|
"grad_norm": 6.75,
|
|
"learning_rate": 4.9166666666666664e-08,
|
|
"logits/chosen": -2.022763252258301,
|
|
"logits/rejected": -2.0409018993377686,
|
|
"logps/chosen": -0.2585347890853882,
|
|
"logps/rejected": -0.26091212034225464,
|
|
"loss": 0.6952535510063171,
|
|
"loss/core": 0.6805825233459473,
|
|
"loss/preference_sft": 0.2590915560722351,
|
|
"loss/reference_anchor": 0.3932636082172394,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 2.8586783409118652,
|
|
"rewards/margins": 2.1142590045928955,
|
|
"rewards/rejected": 0.7444195747375488,
|
|
"ronpo/logit": 0.21142590045928955,
|
|
"ronpo/residual": 0.22805149853229523,
|
|
"ronpo/residual_abs": 0.26630499958992004,
|
|
"ronpo/target": -0.01662559062242508,
|
|
"ronpo/target_abs": 0.07312311232089996,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23467068374156952,
|
|
"drift/rejected_abs": 0.08460647612810135,
|
|
"epoch": 0.020659515295987287,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 5.3333333333333334e-08,
|
|
"logits/chosen": -2.391279458999634,
|
|
"logits/rejected": -2.461313247680664,
|
|
"logps/chosen": -0.2505510151386261,
|
|
"logps/rejected": -0.24410393834114075,
|
|
"loss": 0.23585395514965057,
|
|
"loss/core": 0.22895380854606628,
|
|
"loss/preference_sft": 0.25251513719558716,
|
|
"loss/reference_anchor": 0.1718951165676117,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3467066287994385,
|
|
"rewards/margins": 1.5009348392486572,
|
|
"rewards/rejected": 0.8457720875740051,
|
|
"ronpo/logit": 0.15009348094463348,
|
|
"ronpo/residual": 0.12950626015663147,
|
|
"ronpo/residual_abs": 0.1809195727109909,
|
|
"ronpo/target": 0.02058723010122776,
|
|
"ronpo/target_abs": 0.07401974499225616,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2189807891845703,
|
|
"drift/rejected_abs": 0.11199836432933807,
|
|
"epoch": 0.022248708780294002,
|
|
"grad_norm": 3.640625,
|
|
"learning_rate": 5.75e-08,
|
|
"logits/chosen": -2.13312029838562,
|
|
"logits/rejected": -2.1325230598449707,
|
|
"logps/chosen": -0.2632366716861725,
|
|
"logps/rejected": -0.26798808574676514,
|
|
"loss": 0.07575295865535736,
|
|
"loss/core": 0.07079951465129852,
|
|
"loss/preference_sft": 0.27200424671173096,
|
|
"loss/reference_anchor": 0.11432687193155289,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.18943190574646,
|
|
"rewards/margins": 1.069993495941162,
|
|
"rewards/rejected": 1.1194385290145874,
|
|
"ronpo/logit": 0.10699935257434845,
|
|
"ronpo/residual": 0.09644700586795807,
|
|
"ronpo/residual_abs": 0.1400878131389618,
|
|
"ronpo/target": 0.010552345775067806,
|
|
"ronpo/target_abs": 0.07036933302879333,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2812674641609192,
|
|
"drift/rejected_abs": 0.14779499173164368,
|
|
"epoch": 0.023837902264600714,
|
|
"grad_norm": 6.65625,
|
|
"learning_rate": 6.166666666666666e-08,
|
|
"logits/chosen": -2.1063144207000732,
|
|
"logits/rejected": -2.0400238037109375,
|
|
"logps/chosen": -0.2795808017253876,
|
|
"logps/rejected": -0.27232033014297485,
|
|
"loss": 0.6679071187973022,
|
|
"loss/core": 0.6522617340087891,
|
|
"loss/preference_sft": 0.27452582120895386,
|
|
"loss/reference_anchor": 0.4195597171783447,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.8126060962677,
|
|
"rewards/margins": 1.365581750869751,
|
|
"rewards/rejected": 1.4470244646072388,
|
|
"ronpo/logit": 0.1365581750869751,
|
|
"ronpo/residual": 0.14009013772010803,
|
|
"ronpo/residual_abs": 0.30546045303344727,
|
|
"ronpo/target": -0.003531957510858774,
|
|
"ronpo/target_abs": 0.0828089714050293,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32651928067207336,
|
|
"drift/rejected_abs": 0.1381281465291977,
|
|
"epoch": 0.02542709574890743,
|
|
"grad_norm": 41.0,
|
|
"learning_rate": 6.583333333333333e-08,
|
|
"logits/chosen": -2.0517067909240723,
|
|
"logits/rejected": -2.019787311553955,
|
|
"logps/chosen": -0.270771861076355,
|
|
"logps/rejected": -0.27388402819633484,
|
|
"loss": 0.371652215719223,
|
|
"loss/core": 0.35672515630722046,
|
|
"loss/preference_sft": 0.27029484510421753,
|
|
"loss/reference_anchor": 0.3994576334953308,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.265192747116089,
|
|
"rewards/margins": 1.8877226114273071,
|
|
"rewards/rejected": 1.3774702548980713,
|
|
"ronpo/logit": 0.18877224624156952,
|
|
"ronpo/residual": 0.18959882855415344,
|
|
"ronpo/residual_abs": 0.23960533738136292,
|
|
"ronpo/target": -0.0008265804499387741,
|
|
"ronpo/target_abs": 0.08044739812612534,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.229955792427063,
|
|
"drift/rejected_abs": 0.07845465838909149,
|
|
"epoch": 0.027016289233214145,
|
|
"grad_norm": 1000.0,
|
|
"learning_rate": 6.999999999999999e-08,
|
|
"logits/chosen": -2.366101026535034,
|
|
"logits/rejected": -2.30869722366333,
|
|
"logps/chosen": -0.29234808683395386,
|
|
"logps/rejected": -0.26663321256637573,
|
|
"loss": 0.579170286655426,
|
|
"loss/core": 0.5666659474372864,
|
|
"loss/preference_sft": 0.2720109522342682,
|
|
"loss/reference_anchor": 0.3300662636756897,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2987306118011475,
|
|
"rewards/margins": 1.5194584131240845,
|
|
"rewards/rejected": 0.779272198677063,
|
|
"ronpo/logit": 0.15194585919380188,
|
|
"ronpo/residual": 0.1387050896883011,
|
|
"ronpo/residual_abs": 0.20297744870185852,
|
|
"ronpo/target": 0.013240769505500793,
|
|
"ronpo/target_abs": 0.07829339802265167,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1896975040435791,
|
|
"drift/rejected_abs": 0.16012468934059143,
|
|
"epoch": 0.028605482717520857,
|
|
"grad_norm": 1.1875,
|
|
"learning_rate": 7.416666666666666e-08,
|
|
"logits/chosen": -2.2618637084960938,
|
|
"logits/rejected": -2.2726664543151855,
|
|
"logps/chosen": -0.2831419110298157,
|
|
"logps/rejected": -0.2784458100795746,
|
|
"loss": 0.26859554648399353,
|
|
"loss/core": 0.26028066873550415,
|
|
"loss/preference_sft": 0.2823284864425659,
|
|
"loss/reference_anchor": 0.20933446288108826,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8969752788543701,
|
|
"rewards/margins": 0.29952192306518555,
|
|
"rewards/rejected": 1.5974533557891846,
|
|
"ronpo/logit": 0.029952192679047585,
|
|
"ronpo/residual": 0.023403458297252655,
|
|
"ronpo/residual_abs": 0.17928209900856018,
|
|
"ronpo/target": 0.006548731122165918,
|
|
"ronpo/target_abs": 0.08282124251127243,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22641654312610626,
|
|
"drift/rejected_abs": 0.10155995935201645,
|
|
"epoch": 0.030194676201827572,
|
|
"grad_norm": 3.8125,
|
|
"learning_rate": 7.499548723985053e-08,
|
|
"logits/chosen": -2.3383660316467285,
|
|
"logits/rejected": -2.2575364112854004,
|
|
"logps/chosen": -0.2721652388572693,
|
|
"logps/rejected": -0.270842969417572,
|
|
"loss": 0.3992326259613037,
|
|
"loss/core": 0.39081233739852905,
|
|
"loss/preference_sft": 0.2746056616306305,
|
|
"loss/reference_anchor": 0.21311792731285095,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2641654014587402,
|
|
"rewards/margins": 1.2714884281158447,
|
|
"rewards/rejected": 0.9926770925521851,
|
|
"ronpo/logit": 0.12714883685112,
|
|
"ronpo/residual": 0.14186282455921173,
|
|
"ronpo/residual_abs": 0.22912614047527313,
|
|
"ronpo/target": -0.014713977463543415,
|
|
"ronpo/target_abs": 0.08614644408226013,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.253028929233551,
|
|
"drift/rejected_abs": 0.10068502277135849,
|
|
"epoch": 0.03178386968613429,
|
|
"grad_norm": 10.0,
|
|
"learning_rate": 7.497715601321609e-08,
|
|
"logits/chosen": -1.9719501733779907,
|
|
"logits/rejected": -2.0437729358673096,
|
|
"logps/chosen": -0.2982824444770813,
|
|
"logps/rejected": -0.28864842653274536,
|
|
"loss": 0.24105651676654816,
|
|
"loss/core": 0.23307080566883087,
|
|
"loss/preference_sft": 0.28916245698928833,
|
|
"loss/reference_anchor": 0.19924673438072205,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.5282044410705566,
|
|
"rewards/margins": 1.5234105587005615,
|
|
"rewards/rejected": 1.0047937631607056,
|
|
"ronpo/logit": 0.1523410826921463,
|
|
"ronpo/residual": 0.1546805202960968,
|
|
"ronpo/residual_abs": 0.19531986117362976,
|
|
"ronpo/target": -0.0023394643794745207,
|
|
"ronpo/target_abs": 0.07932108640670776,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3908888101577759,
|
|
"drift/rejected_abs": 0.12839046120643616,
|
|
"epoch": 0.033373063170441,
|
|
"grad_norm": 95.0,
|
|
"learning_rate": 7.494473116090843e-08,
|
|
"logits/chosen": -1.92897629737854,
|
|
"logits/rejected": -1.9361374378204346,
|
|
"logps/chosen": -0.2795645296573639,
|
|
"logps/rejected": -0.28167638182640076,
|
|
"loss": 0.6834398508071899,
|
|
"loss/core": 0.6608570218086243,
|
|
"loss/preference_sft": 0.2787085473537445,
|
|
"loss/reference_anchor": 0.617350697517395,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.908715009689331,
|
|
"rewards/margins": 2.635611057281494,
|
|
"rewards/rejected": 1.273103952407837,
|
|
"ronpo/logit": 0.26356109976768494,
|
|
"ronpo/residual": 0.27055853605270386,
|
|
"ronpo/residual_abs": 0.3101016879081726,
|
|
"ronpo/target": -0.00699741393327713,
|
|
"ronpo/target_abs": 0.08270318806171417,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2439744919538498,
|
|
"drift/rejected_abs": 0.12347511947154999,
|
|
"epoch": 0.03496225665474772,
|
|
"grad_norm": 8.0,
|
|
"learning_rate": 7.489822487658843e-08,
|
|
"logits/chosen": -2.243443250656128,
|
|
"logits/rejected": -2.2027456760406494,
|
|
"logps/chosen": -0.29370397329330444,
|
|
"logps/rejected": -0.28651857376098633,
|
|
"loss": 0.12330013513565063,
|
|
"loss/core": 0.11514942348003387,
|
|
"loss/preference_sft": 0.29073047637939453,
|
|
"loss/reference_anchor": 0.20380404591560364,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.438559055328369,
|
|
"rewards/margins": 1.2159253358840942,
|
|
"rewards/rejected": 1.2226336002349854,
|
|
"ronpo/logit": 0.12159254401922226,
|
|
"ronpo/residual": 0.12451876699924469,
|
|
"ronpo/residual_abs": 0.18701641261577606,
|
|
"ronpo/target": -0.002926218556240201,
|
|
"ronpo/target_abs": 0.08683927357196808,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27595922350883484,
|
|
"drift/rejected_abs": 0.1378968507051468,
|
|
"epoch": 0.03655145013905443,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 7.483765464936766e-08,
|
|
"logits/chosen": -1.8842188119888306,
|
|
"logits/rejected": -1.7752964496612549,
|
|
"logps/chosen": -0.2784324586391449,
|
|
"logps/rejected": -0.2820046842098236,
|
|
"loss": 0.1508234441280365,
|
|
"loss/core": 0.14237406849861145,
|
|
"loss/preference_sft": 0.2840380072593689,
|
|
"loss/reference_anchor": 0.21300740540027618,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.757049560546875,
|
|
"rewards/margins": 1.3863294124603271,
|
|
"rewards/rejected": 1.3707202672958374,
|
|
"ronpo/logit": 0.13863296806812286,
|
|
"ronpo/residual": 0.14248371124267578,
|
|
"ronpo/residual_abs": 0.18661606311798096,
|
|
"ronpo/target": -0.003850756911560893,
|
|
"ronpo/target_abs": 0.07908544689416885,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25356635451316833,
|
|
"drift/rejected_abs": 0.11751619726419449,
|
|
"epoch": 0.03814064362336114,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 7.476304325723143e-08,
|
|
"logits/chosen": -2.017167568206787,
|
|
"logits/rejected": -2.0651519298553467,
|
|
"logps/chosen": -0.3016659915447235,
|
|
"logps/rejected": -0.2997520864009857,
|
|
"loss": 0.2998911142349243,
|
|
"loss/core": 0.28765398263931274,
|
|
"loss/preference_sft": 0.3130350410938263,
|
|
"loss/reference_anchor": 0.31832972168922424,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.534654140472412,
|
|
"rewards/margins": 1.3620463609695435,
|
|
"rewards/rejected": 1.1726077795028687,
|
|
"ronpo/logit": 0.13620463013648987,
|
|
"ronpo/residual": 0.13591532409191132,
|
|
"ronpo/residual_abs": 0.20360629260540009,
|
|
"ronpo/target": 0.0002892996999435127,
|
|
"ronpo/target_abs": 0.07245339453220367,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1270354688167572,
|
|
"drift/rejected_abs": 0.07065033167600632,
|
|
"epoch": 0.03972983710766786,
|
|
"grad_norm": 1.3359375,
|
|
"learning_rate": 7.467441875847293e-08,
|
|
"logits/chosen": -2.2720069885253906,
|
|
"logits/rejected": -2.192866802215576,
|
|
"logps/chosen": -0.2744554877281189,
|
|
"logps/rejected": -0.28156858682632446,
|
|
"loss": 0.04993171617388725,
|
|
"loss/core": 0.04652109742164612,
|
|
"loss/preference_sft": 0.2717655599117279,
|
|
"loss/reference_anchor": 0.07026959955692291,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2697632312774658,
|
|
"rewards/margins": 0.5650413036346436,
|
|
"rewards/rejected": 0.7047218680381775,
|
|
"ronpo/logit": 0.05650413781404495,
|
|
"ronpo/residual": 0.06540760397911072,
|
|
"ronpo/residual_abs": 0.11058205366134644,
|
|
"ronpo/target": -0.008903468027710915,
|
|
"ronpo/target_abs": 0.07374493777751923,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3694007992744446,
|
|
"drift/rejected_abs": 0.15201464295387268,
|
|
"epoch": 0.04131903059197457,
|
|
"grad_norm": 16.875,
|
|
"learning_rate": 7.457181448114169e-08,
|
|
"logits/chosen": -2.0490753650665283,
|
|
"logits/rejected": -1.9781653881072998,
|
|
"logps/chosen": -0.280802845954895,
|
|
"logps/rejected": -0.30160802602767944,
|
|
"loss": 0.4936361312866211,
|
|
"loss/core": 0.4678853154182434,
|
|
"loss/preference_sft": 0.2995707094669342,
|
|
"loss/reference_anchor": 0.7057813405990601,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.6940078735351562,
|
|
"rewards/margins": 2.1743321418762207,
|
|
"rewards/rejected": 1.5196759700775146,
|
|
"ronpo/logit": 0.21743318438529968,
|
|
"ronpo/residual": 0.2153353989124298,
|
|
"ronpo/residual_abs": 0.2628825604915619,
|
|
"ronpo/target": 0.0020978045649826527,
|
|
"ronpo/target_abs": 0.06583398580551147,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22835186123847961,
|
|
"drift/rejected_abs": 0.17439039051532745,
|
|
"epoch": 0.04290822407628129,
|
|
"grad_norm": 90.0,
|
|
"learning_rate": 7.445526901051018e-08,
|
|
"logits/chosen": -2.157517910003662,
|
|
"logits/rejected": -2.1436400413513184,
|
|
"logps/chosen": -0.2549034357070923,
|
|
"logps/rejected": -0.27489176392555237,
|
|
"loss": 0.35979098081588745,
|
|
"loss/core": 0.3489346206188202,
|
|
"loss/preference_sft": 0.26387128233909607,
|
|
"loss/reference_anchor": 0.28379374742507935,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2820842266082764,
|
|
"rewards/margins": 0.5401636958122253,
|
|
"rewards/rejected": 1.7419202327728271,
|
|
"ronpo/logit": 0.054016370326280594,
|
|
"ronpo/residual": 0.05434161424636841,
|
|
"ronpo/residual_abs": 0.2039295881986618,
|
|
"ronpo/target": -0.0003252413007430732,
|
|
"ronpo/target_abs": 0.06969951838254929,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2118876725435257,
|
|
"drift/rejected_abs": 0.1539703756570816,
|
|
"epoch": 0.044497417560588004,
|
|
"grad_norm": 38.75,
|
|
"learning_rate": 7.432482617456355e-08,
|
|
"logits/chosen": -2.0652270317077637,
|
|
"logits/rejected": -2.0345349311828613,
|
|
"logps/chosen": -0.28724318742752075,
|
|
"logps/rejected": -0.28770583868026733,
|
|
"loss": 0.2550918161869049,
|
|
"loss/core": 0.2464626580476761,
|
|
"loss/preference_sft": 0.28313058614730835,
|
|
"loss/reference_anchor": 0.2182338684797287,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.117487668991089,
|
|
"rewards/margins": 0.5793629884719849,
|
|
"rewards/rejected": 1.538124918937683,
|
|
"ronpo/logit": 0.057936299592256546,
|
|
"ronpo/residual": 0.04774229973554611,
|
|
"ronpo/residual_abs": 0.19685055315494537,
|
|
"ronpo/target": 0.010194000788033009,
|
|
"ronpo/target_abs": 0.0766717940568924,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3104633092880249,
|
|
"drift/rejected_abs": 0.13752777874469757,
|
|
"epoch": 0.04608661104489471,
|
|
"grad_norm": 213.0,
|
|
"learning_rate": 7.41805350275177e-08,
|
|
"logits/chosen": -2.1027872562408447,
|
|
"logits/rejected": -2.1394100189208984,
|
|
"logps/chosen": -0.2751466631889343,
|
|
"logps/rejected": -0.2653554081916809,
|
|
"loss": 0.20231513679027557,
|
|
"loss/core": 0.18823571503162384,
|
|
"loss/preference_sft": 0.2743663191795349,
|
|
"loss/reference_anchor": 0.37483280897140503,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.104318380355835,
|
|
"rewards/margins": 1.7324186563491821,
|
|
"rewards/rejected": 1.3718997240066528,
|
|
"ronpo/logit": 0.17324186861515045,
|
|
"ronpo/residual": 0.1822124421596527,
|
|
"ronpo/residual_abs": 0.214836984872818,
|
|
"ronpo/target": -0.008970573544502258,
|
|
"ronpo/target_abs": 0.07618303596973419,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.40220266580581665,
|
|
"drift/rejected_abs": 0.11431588977575302,
|
|
"epoch": 0.04767580452920143,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 7.402244983137196e-08,
|
|
"logits/chosen": -2.0275425910949707,
|
|
"logits/rejected": -2.13136625289917,
|
|
"logps/chosen": -0.29549291729927063,
|
|
"logps/rejected": -0.2743282914161682,
|
|
"loss": 0.994286835193634,
|
|
"loss/core": 0.9691765904426575,
|
|
"loss/preference_sft": 0.2976817190647125,
|
|
"loss/reference_anchor": 0.6876654624938965,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.022026062011719,
|
|
"rewards/margins": 2.8812575340270996,
|
|
"rewards/rejected": 1.1407687664031982,
|
|
"ronpo/logit": 0.28812575340270996,
|
|
"ronpo/residual": 0.2670125663280487,
|
|
"ronpo/residual_abs": 0.32439857721328735,
|
|
"ronpo/target": 0.021113192662596703,
|
|
"ronpo/target_abs": 0.08010004460811615,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.311504065990448,
|
|
"drift/rejected_abs": 0.10349141061306,
|
|
"epoch": 0.04926499801350814,
|
|
"grad_norm": 4.875,
|
|
"learning_rate": 7.385063003550334e-08,
|
|
"logits/chosen": -2.169419765472412,
|
|
"logits/rejected": -2.1742537021636963,
|
|
"logps/chosen": -0.2659375071525574,
|
|
"logps/rejected": -0.26293236017227173,
|
|
"loss": 0.6533135175704956,
|
|
"loss/core": 0.6341246366500854,
|
|
"loss/preference_sft": 0.2658475339412689,
|
|
"loss/reference_anchor": 0.5216662287712097,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1136386394500732,
|
|
"rewards/margins": 2.081437826156616,
|
|
"rewards/rejected": 1.0322009325027466,
|
|
"ronpo/logit": 0.20814375579357147,
|
|
"ronpo/residual": 0.20334522426128387,
|
|
"ronpo/residual_abs": 0.2493644505739212,
|
|
"ronpo/target": 0.004798541776835918,
|
|
"ronpo/target_abs": 0.07737194001674652,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20044144988059998,
|
|
"drift/rejected_abs": 0.06772055476903915,
|
|
"epoch": 0.05085419149781486,
|
|
"grad_norm": 844.0,
|
|
"learning_rate": 7.36651402543102e-08,
|
|
"logits/chosen": -2.206714153289795,
|
|
"logits/rejected": -2.2234230041503906,
|
|
"logps/chosen": -0.3195026218891144,
|
|
"logps/rejected": -0.27392783761024475,
|
|
"loss": 0.5297176241874695,
|
|
"loss/core": 0.517520546913147,
|
|
"loss/preference_sft": 0.3230019211769104,
|
|
"loss/reference_anchor": 0.3161885142326355,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0040805339813232,
|
|
"rewards/margins": 1.3281657695770264,
|
|
"rewards/rejected": 0.6759150624275208,
|
|
"ronpo/logit": 0.13281656801700592,
|
|
"ronpo/residual": 0.12692351639270782,
|
|
"ronpo/residual_abs": 0.18193206191062927,
|
|
"ronpo/target": 0.00589305954053998,
|
|
"ronpo/target_abs": 0.0840735211968422,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1306372582912445,
|
|
"drift/rejected_abs": 0.0512150414288044,
|
|
"epoch": 0.052443384982121574,
|
|
"grad_norm": 4.46875,
|
|
"learning_rate": 7.346605024291327e-08,
|
|
"logits/chosen": -2.128614902496338,
|
|
"logits/rejected": -2.0060853958129883,
|
|
"logps/chosen": -0.2849295735359192,
|
|
"logps/rejected": -0.2936997413635254,
|
|
"loss": 0.11120221763849258,
|
|
"loss/core": 0.10780216753482819,
|
|
"loss/preference_sft": 0.28842681646347046,
|
|
"loss/reference_anchor": 0.06830208003520966,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.3055182695388794,
|
|
"rewards/margins": 0.7997378706932068,
|
|
"rewards/rejected": 0.5057803392410278,
|
|
"ronpo/logit": 0.07997378706932068,
|
|
"ronpo/residual": 0.07414297759532928,
|
|
"ronpo/residual_abs": 0.13741916418075562,
|
|
"ronpo/target": 0.005830812267959118,
|
|
"ronpo/target_abs": 0.07218202203512192,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31605249643325806,
|
|
"drift/rejected_abs": 0.16812530159950256,
|
|
"epoch": 0.05403257846642829,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 7.325343487092362e-08,
|
|
"logits/chosen": -2.0245869159698486,
|
|
"logits/rejected": -2.0181970596313477,
|
|
"logps/chosen": -0.28985458612442017,
|
|
"logps/rejected": -0.2747141420841217,
|
|
"loss": 0.4409272074699402,
|
|
"loss/core": 0.42512696981430054,
|
|
"loss/preference_sft": 0.2895664572715759,
|
|
"loss/reference_anchor": 0.42247915267944336,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.159921407699585,
|
|
"rewards/margins": 1.4804481267929077,
|
|
"rewards/rejected": 1.6794731616973877,
|
|
"ronpo/logit": 0.14804483950138092,
|
|
"ronpo/residual": 0.14191299676895142,
|
|
"ronpo/residual_abs": 0.25526586174964905,
|
|
"ronpo/target": 0.006131820380687714,
|
|
"ronpo/target_abs": 0.07981385290622711,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19817472994327545,
|
|
"drift/rejected_abs": 0.11151386797428131,
|
|
"epoch": 0.055621771950735005,
|
|
"grad_norm": 26.375,
|
|
"learning_rate": 7.302737409428734e-08,
|
|
"logits/chosen": -2.2624623775482178,
|
|
"logits/rejected": -2.1980812549591064,
|
|
"logps/chosen": -0.29418444633483887,
|
|
"logps/rejected": -0.30473509430885315,
|
|
"loss": 0.05296384170651436,
|
|
"loss/core": 0.0463070385158062,
|
|
"loss/preference_sft": 0.2961450517177582,
|
|
"loss/reference_anchor": 0.1605798900127411,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9817469120025635,
|
|
"rewards/margins": 0.8666085004806519,
|
|
"rewards/rejected": 1.1151385307312012,
|
|
"ronpo/logit": 0.08666086941957474,
|
|
"ronpo/residual": 0.09623171389102936,
|
|
"ronpo/residual_abs": 0.13510657846927643,
|
|
"ronpo/target": -0.009570842608809471,
|
|
"ronpo/target_abs": 0.08157099038362503,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.47317036986351013,
|
|
"drift/rejected_abs": 0.22457854449748993,
|
|
"epoch": 0.057210965435041714,
|
|
"grad_norm": 30.125,
|
|
"learning_rate": 7.278795292521733e-08,
|
|
"logits/chosen": -2.0715465545654297,
|
|
"logits/rejected": -2.0496222972869873,
|
|
"logps/chosen": -0.28974977135658264,
|
|
"logps/rejected": -0.28343138098716736,
|
|
"loss": 0.9135662913322449,
|
|
"loss/core": 0.8731704950332642,
|
|
"loss/preference_sft": 0.2855917811393738,
|
|
"loss/reference_anchor": 1.1256051063537598,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.731703758239746,
|
|
"rewards/margins": 2.486184597015381,
|
|
"rewards/rejected": 2.2455201148986816,
|
|
"ronpo/logit": 0.24861843883991241,
|
|
"ronpo/residual": 0.26165086030960083,
|
|
"ronpo/residual_abs": 0.29524511098861694,
|
|
"ronpo/target": -0.013032438233494759,
|
|
"ronpo/target_abs": 0.0726519376039505,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12218070030212402,
|
|
"drift/rejected_abs": 0.06443633139133453,
|
|
"epoch": 0.05880015891934843,
|
|
"grad_norm": 4.59375,
|
|
"learning_rate": 7.253526140022365e-08,
|
|
"logits/chosen": -2.228654384613037,
|
|
"logits/rejected": -2.33091139793396,
|
|
"logps/chosen": -0.29517388343811035,
|
|
"logps/rejected": -0.2922295331954956,
|
|
"loss": 0.027977192774415016,
|
|
"loss/core": 0.025672132149338722,
|
|
"loss/preference_sft": 0.2922584116458893,
|
|
"loss/reference_anchor": 0.03663304075598717,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.2214815616607666,
|
|
"rewards/margins": 0.5821638107299805,
|
|
"rewards/rejected": 0.6393176317214966,
|
|
"ronpo/logit": 0.05821638181805611,
|
|
"ronpo/residual": 0.06779725104570389,
|
|
"ronpo/residual_abs": 0.1186339408159256,
|
|
"ronpo/target": -0.009580868296325207,
|
|
"ronpo/target_abs": 0.08872409909963608,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12466661632061005,
|
|
"drift/rejected_abs": 0.05561278015375137,
|
|
"epoch": 0.060389352403655144,
|
|
"grad_norm": 0.8671875,
|
|
"learning_rate": 7.226939454625452e-08,
|
|
"logits/chosen": -2.314230442047119,
|
|
"logits/rejected": -2.4117889404296875,
|
|
"logps/chosen": -0.3054582476615906,
|
|
"logps/rejected": -0.3027587831020355,
|
|
"loss": 0.05369839817285538,
|
|
"loss/core": 0.050449974834918976,
|
|
"loss/preference_sft": 0.3041912019252777,
|
|
"loss/reference_anchor": 0.0623929388821125,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.245711088180542,
|
|
"rewards/margins": 0.6905202865600586,
|
|
"rewards/rejected": 0.5551908612251282,
|
|
"ronpo/logit": 0.06905203312635422,
|
|
"ronpo/residual": 0.062486421316862106,
|
|
"ronpo/residual_abs": 0.13185548782348633,
|
|
"ronpo/target": 0.00656561553478241,
|
|
"ronpo/target_abs": 0.08973196148872375,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21235451102256775,
|
|
"drift/rejected_abs": 0.11808476597070694,
|
|
"epoch": 0.06197854588796186,
|
|
"grad_norm": 6.40625,
|
|
"learning_rate": 7.199045234496058e-08,
|
|
"logits/chosen": -2.054386615753174,
|
|
"logits/rejected": -2.0310075283050537,
|
|
"logps/chosen": -0.2783626914024353,
|
|
"logps/rejected": -0.27244699001312256,
|
|
"loss": 0.05899541452527046,
|
|
"loss/core": 0.05395280569791794,
|
|
"loss/preference_sft": 0.27698707580566406,
|
|
"loss/reference_anchor": 0.11637584865093231,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.1214401721954346,
|
|
"rewards/margins": 0.9426848292350769,
|
|
"rewards/rejected": 1.1787551641464233,
|
|
"ronpo/logit": 0.09426848590373993,
|
|
"ronpo/residual": 0.09450878202915192,
|
|
"ronpo/residual_abs": 0.1405004858970642,
|
|
"ronpo/target": -0.00024029240012168884,
|
|
"ronpo/target_abs": 0.07883597910404205,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2827044129371643,
|
|
"drift/rejected_abs": 0.08958513289690018,
|
|
"epoch": 0.06356773937226858,
|
|
"grad_norm": 8.875,
|
|
"learning_rate": 7.169853969509582e-08,
|
|
"logits/chosen": -2.036879777908325,
|
|
"logits/rejected": -2.1572928428649902,
|
|
"logps/chosen": -0.28990262746810913,
|
|
"logps/rejected": -0.2788967192173004,
|
|
"loss": 0.4873984456062317,
|
|
"loss/core": 0.4752472937107086,
|
|
"loss/preference_sft": 0.28781306743621826,
|
|
"loss/reference_anchor": 0.3183947503566742,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8262548446655273,
|
|
"rewards/margins": 1.9317960739135742,
|
|
"rewards/rejected": 0.8944588899612427,
|
|
"ronpo/logit": 0.19317962229251862,
|
|
"ronpo/residual": 0.19387605786323547,
|
|
"ronpo/residual_abs": 0.2424454241991043,
|
|
"ronpo/target": -0.000696455710567534,
|
|
"ronpo/target_abs": 0.0863131508231163,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2943757474422455,
|
|
"drift/rejected_abs": 0.10825987160205841,
|
|
"epoch": 0.06515693285657528,
|
|
"grad_norm": 101.5,
|
|
"learning_rate": 7.139376637306952e-08,
|
|
"logits/chosen": -2.000290632247925,
|
|
"logits/rejected": -2.0813472270965576,
|
|
"logps/chosen": -0.2815471589565277,
|
|
"logps/rejected": -0.2869817614555359,
|
|
"loss": 0.7546241879463196,
|
|
"loss/core": 0.7346261739730835,
|
|
"loss/preference_sft": 0.287017822265625,
|
|
"loss/reference_anchor": 0.5426746606826782,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.9414560794830322,
|
|
"rewards/margins": 1.859922170639038,
|
|
"rewards/rejected": 1.0815337896347046,
|
|
"ronpo/logit": 0.18599221110343933,
|
|
"ronpo/residual": 0.21781902015209198,
|
|
"ronpo/residual_abs": 0.2372894585132599,
|
|
"ronpo/target": -0.031826816499233246,
|
|
"ronpo/target_abs": 0.06753235310316086,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21875524520874023,
|
|
"drift/rejected_abs": 0.058099307119846344,
|
|
"epoch": 0.066746126340882,
|
|
"grad_norm": 3.40625,
|
|
"learning_rate": 7.107624699166376e-08,
|
|
"logits/chosen": -2.2508599758148193,
|
|
"logits/rejected": -2.1692118644714355,
|
|
"logps/chosen": -0.3082744777202606,
|
|
"logps/rejected": -0.31492576003074646,
|
|
"loss": 1.0409514904022217,
|
|
"loss/core": 1.020789384841919,
|
|
"loss/preference_sft": 0.3110334277153015,
|
|
"loss/reference_anchor": 0.5449555516242981,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.1875526905059814,
|
|
"rewards/margins": 1.6086879968643188,
|
|
"rewards/rejected": 0.5788646340370178,
|
|
"ronpo/logit": 0.16086876392364502,
|
|
"ronpo/residual": 0.16442081332206726,
|
|
"ronpo/residual_abs": 0.21720726788043976,
|
|
"ronpo/target": -0.003552014008164406,
|
|
"ronpo/target_abs": 0.07966691255569458,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32707658410072327,
|
|
"drift/rejected_abs": 0.164951890707016,
|
|
"epoch": 0.06833531982518871,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 7.074610095693229e-08,
|
|
"logits/chosen": -2.1667943000793457,
|
|
"logits/rejected": -2.154695510864258,
|
|
"logps/chosen": -0.26537930965423584,
|
|
"logps/rejected": -0.2718598246574402,
|
|
"loss": 0.29404348134994507,
|
|
"loss/core": 0.27320724725723267,
|
|
"loss/preference_sft": 0.2671200931072235,
|
|
"loss/reference_anchor": 0.5686091780662537,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.270766019821167,
|
|
"rewards/margins": 1.6218563318252563,
|
|
"rewards/rejected": 1.648909330368042,
|
|
"ronpo/logit": 0.1621856391429901,
|
|
"ronpo/residual": 0.16278550028800964,
|
|
"ronpo/residual_abs": 0.21330192685127258,
|
|
"ronpo/target": -0.0005998473498038948,
|
|
"ronpo/target_abs": 0.07392500340938568,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11985316127538681,
|
|
"drift/rejected_abs": 0.08097579330205917,
|
|
"epoch": 0.06992451330949544,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 7.040345242329681e-08,
|
|
"logits/chosen": -2.214625120162964,
|
|
"logits/rejected": -2.082509756088257,
|
|
"logps/chosen": -0.30458956956863403,
|
|
"logps/rejected": -0.2999945878982544,
|
|
"loss": 0.06141216307878494,
|
|
"loss/core": 0.05872996523976326,
|
|
"loss/preference_sft": 0.302223265171051,
|
|
"loss/reference_anchor": 0.046411849558353424,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.1941266059875488,
|
|
"rewards/margins": 0.3879542350769043,
|
|
"rewards/rejected": 0.806172251701355,
|
|
"ronpo/logit": 0.03879542648792267,
|
|
"ronpo/residual": 0.04558854550123215,
|
|
"ronpo/residual_abs": 0.13173437118530273,
|
|
"ronpo/target": -0.006793119013309479,
|
|
"ronpo/target_abs": 0.07642895728349686,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23159165680408478,
|
|
"drift/rejected_abs": 0.10313665866851807,
|
|
"epoch": 0.07151370679380215,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 7.004843024685753e-08,
|
|
"logits/chosen": -2.178675651550293,
|
|
"logits/rejected": -2.195631504058838,
|
|
"logps/chosen": -0.2871968150138855,
|
|
"logps/rejected": -0.2853408753871918,
|
|
"loss": 0.20537880063056946,
|
|
"loss/core": 0.19829311966896057,
|
|
"loss/preference_sft": 0.28042349219322205,
|
|
"loss/reference_anchor": 0.1744067519903183,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.3159165382385254,
|
|
"rewards/margins": 1.2847979068756104,
|
|
"rewards/rejected": 1.031118631362915,
|
|
"ronpo/logit": 0.12847980856895447,
|
|
"ronpo/residual": 0.11979977041482925,
|
|
"ronpo/residual_abs": 0.16873392462730408,
|
|
"ronpo/target": 0.008680026978254318,
|
|
"ronpo/target_abs": 0.06639182567596436,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26562970876693726,
|
|
"drift/rejected_abs": 0.0964142382144928,
|
|
"epoch": 0.07310290027810885,
|
|
"grad_norm": 2.578125,
|
|
"learning_rate": 6.968116793693563e-08,
|
|
"logits/chosen": -2.240947723388672,
|
|
"logits/rejected": -2.258277654647827,
|
|
"logps/chosen": -0.2617165446281433,
|
|
"logps/rejected": -0.26082921028137207,
|
|
"loss": 0.40380316972732544,
|
|
"loss/core": 0.3928808569908142,
|
|
"loss/preference_sft": 0.2601296007633209,
|
|
"loss/reference_anchor": 0.2860526442527771,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.656296730041504,
|
|
"rewards/margins": 1.6946455240249634,
|
|
"rewards/rejected": 0.9616514444351196,
|
|
"ronpo/logit": 0.16946455836296082,
|
|
"ronpo/residual": 0.16977901756763458,
|
|
"ronpo/residual_abs": 0.21348360180854797,
|
|
"ronpo/target": -0.0003144521324429661,
|
|
"ronpo/target_abs": 0.07685980945825577,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.46096473932266235,
|
|
"drift/rejected_abs": 0.2360442876815796,
|
|
"epoch": 0.07469209376241558,
|
|
"grad_norm": 63.25,
|
|
"learning_rate": 6.930180360586597e-08,
|
|
"logits/chosen": -2.1169967651367188,
|
|
"logits/rejected": -2.118549108505249,
|
|
"logps/chosen": -0.28103750944137573,
|
|
"logps/rejected": -0.28609365224838257,
|
|
"loss": 1.180558443069458,
|
|
"loss/core": 1.1377079486846924,
|
|
"loss/preference_sft": 0.28971725702285767,
|
|
"loss/reference_anchor": 1.1953245401382446,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.6081223487854,
|
|
"rewards/margins": 2.326866626739502,
|
|
"rewards/rejected": 2.2812554836273193,
|
|
"ronpo/logit": 0.23268663883209229,
|
|
"ronpo/residual": 0.2368365228176117,
|
|
"ronpo/residual_abs": 0.38683050870895386,
|
|
"ronpo/target": -0.0041498770006000996,
|
|
"ronpo/target_abs": 0.06436969339847565,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29032057523727417,
|
|
"drift/rejected_abs": 0.16610702872276306,
|
|
"epoch": 0.07628128724672228,
|
|
"grad_norm": 169.0,
|
|
"learning_rate": 6.89104799170586e-08,
|
|
"logits/chosen": -2.1400163173675537,
|
|
"logits/rejected": -2.216646671295166,
|
|
"logps/chosen": -0.2673208713531494,
|
|
"logps/rejected": -0.2859800159931183,
|
|
"loss": 0.206389382481575,
|
|
"loss/core": 0.18720188736915588,
|
|
"loss/preference_sft": 0.2810494303703308,
|
|
"loss/reference_anchor": 0.520110011100769,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9032058715820312,
|
|
"rewards/margins": 1.2437211275100708,
|
|
"rewards/rejected": 1.6594845056533813,
|
|
"ronpo/logit": 0.12437212467193604,
|
|
"ronpo/residual": 0.10429469496011734,
|
|
"ronpo/residual_abs": 0.1717267632484436,
|
|
"ronpo/target": 0.020077422261238098,
|
|
"ronpo/target_abs": 0.07924573123455048,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18758592009544373,
|
|
"drift/rejected_abs": 0.0732673853635788,
|
|
"epoch": 0.07787048073102901,
|
|
"grad_norm": 2.25,
|
|
"learning_rate": 6.85073440313489e-08,
|
|
"logits/chosen": -2.178632974624634,
|
|
"logits/rejected": -2.1970858573913574,
|
|
"logps/chosen": -0.29963111877441406,
|
|
"logps/rejected": -0.3051955997943878,
|
|
"loss": 0.2845226526260376,
|
|
"loss/core": 0.277618944644928,
|
|
"loss/preference_sft": 0.3048124611377716,
|
|
"loss/reference_anchor": 0.1667684018611908,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8743228912353516,
|
|
"rewards/margins": 1.169947624206543,
|
|
"rewards/rejected": 0.7043755054473877,
|
|
"ronpo/logit": 0.11699476093053818,
|
|
"ronpo/residual": 0.12534533441066742,
|
|
"ronpo/residual_abs": 0.17828436195850372,
|
|
"ronpo/target": -0.008350581862032413,
|
|
"ronpo/target_abs": 0.08381186425685883,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36205315589904785,
|
|
"drift/rejected_abs": 0.0857582613825798,
|
|
"epoch": 0.07945967421533572,
|
|
"grad_norm": 7.6875,
|
|
"learning_rate": 6.809254755165645e-08,
|
|
"logits/chosen": -2.110821485519409,
|
|
"logits/rejected": -2.0873301029205322,
|
|
"logps/chosen": -0.30966055393218994,
|
|
"logps/rejected": -0.3097647726535797,
|
|
"loss": 1.2090619802474976,
|
|
"loss/core": 1.1840286254882812,
|
|
"loss/preference_sft": 0.3095891773700714,
|
|
"loss/reference_anchor": 0.6842803955078125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.619492769241333,
|
|
"rewards/margins": 2.763693332672119,
|
|
"rewards/rejected": 0.8557994961738586,
|
|
"ronpo/logit": 0.2763693332672119,
|
|
"ronpo/residual": 0.2691985070705414,
|
|
"ronpo/residual_abs": 0.32348698377609253,
|
|
"ronpo/target": 0.007170822471380234,
|
|
"ronpo/target_abs": 0.07173763960599899,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.44590863585472107,
|
|
"drift/rejected_abs": 0.1995648592710495,
|
|
"epoch": 0.08104886769964244,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 6.766624646597332e-08,
|
|
"logits/chosen": -2.1040661334991455,
|
|
"logits/rejected": -2.106905937194824,
|
|
"logps/chosen": -0.26315388083457947,
|
|
"logps/rejected": -0.2701566815376282,
|
|
"loss": 0.7402136325836182,
|
|
"loss/core": 0.7059299349784851,
|
|
"loss/preference_sft": 0.27266380190849304,
|
|
"loss/reference_anchor": 0.9522676467895508,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 4.455777168273926,
|
|
"rewards/margins": 2.532688617706299,
|
|
"rewards/rejected": 1.9230890274047852,
|
|
"ronpo/logit": 0.25326889753341675,
|
|
"ronpo/residual": 0.2603941559791565,
|
|
"ronpo/residual_abs": 0.30658119916915894,
|
|
"ronpo/target": -0.007125276140868664,
|
|
"ronpo/target_abs": 0.08376964181661606,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33060067892074585,
|
|
"drift/rejected_abs": 0.0853637307882309,
|
|
"epoch": 0.08263806118394915,
|
|
"grad_norm": 880.0,
|
|
"learning_rate": 6.722860108870342e-08,
|
|
"logits/chosen": -2.1947333812713623,
|
|
"logits/rejected": -2.1691174507141113,
|
|
"logps/chosen": -0.280915766954422,
|
|
"logps/rejected": -0.28027015924453735,
|
|
"loss": 0.8541151881217957,
|
|
"loss/core": 0.8351130485534668,
|
|
"loss/preference_sft": 0.2944938540458679,
|
|
"loss/reference_anchor": 0.5134744644165039,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.304910182952881,
|
|
"rewards/margins": 2.455112934112549,
|
|
"rewards/rejected": 0.849797248840332,
|
|
"ronpo/logit": 0.24551133811473846,
|
|
"ronpo/residual": 0.2461731731891632,
|
|
"ronpo/residual_abs": 0.2894217371940613,
|
|
"ronpo/target": -0.000661846250295639,
|
|
"ronpo/target_abs": 0.07702169567346573,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18190790712833405,
|
|
"drift/rejected_abs": 0.08150045573711395,
|
|
"epoch": 0.08422725466825585,
|
|
"grad_norm": 16.0,
|
|
"learning_rate": 6.677977600037472e-08,
|
|
"logits/chosen": -2.1558258533477783,
|
|
"logits/rejected": -2.1550559997558594,
|
|
"logps/chosen": -0.2850190997123718,
|
|
"logps/rejected": -0.29064029455184937,
|
|
"loss": 0.0755200982093811,
|
|
"loss/core": 0.07134844362735748,
|
|
"loss/preference_sft": 0.28953614830970764,
|
|
"loss/reference_anchor": 0.0902366042137146,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8185592889785767,
|
|
"rewards/margins": 1.0096482038497925,
|
|
"rewards/rejected": 0.8089107275009155,
|
|
"ronpo/logit": 0.10096482932567596,
|
|
"ronpo/residual": 0.08930448442697525,
|
|
"ronpo/residual_abs": 0.14850127696990967,
|
|
"ronpo/target": 0.011660362593829632,
|
|
"ronpo/target_abs": 0.07749392092227936,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29220855236053467,
|
|
"drift/rejected_abs": 0.16033557057380676,
|
|
"epoch": 0.08581644815256258,
|
|
"grad_norm": 2.328125,
|
|
"learning_rate": 6.631993998574737e-08,
|
|
"logits/chosen": -2.2243332862854004,
|
|
"logits/rejected": -2.183389663696289,
|
|
"logps/chosen": -0.2799707055091858,
|
|
"logps/rejected": -0.28995639085769653,
|
|
"loss": 0.3449793756008148,
|
|
"loss/core": 0.3193924129009247,
|
|
"loss/preference_sft": 0.28027012944221497,
|
|
"loss/reference_anchor": 0.7030280828475952,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.920807361602783,
|
|
"rewards/margins": 1.320322036743164,
|
|
"rewards/rejected": 1.60048508644104,
|
|
"ronpo/logit": 0.13203221559524536,
|
|
"ronpo/residual": 0.11397619545459747,
|
|
"ronpo/residual_abs": 0.1874753087759018,
|
|
"ronpo/target": 0.01805601641535759,
|
|
"ronpo/target_abs": 0.089533731341362,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1789226233959198,
|
|
"drift/rejected_abs": 0.10433357954025269,
|
|
"epoch": 0.08740564163686929,
|
|
"grad_norm": 7.40625,
|
|
"learning_rate": 6.584926597034054e-08,
|
|
"logits/chosen": -2.2748570442199707,
|
|
"logits/rejected": -2.3492331504821777,
|
|
"logps/chosen": -0.27904680371284485,
|
|
"logps/rejected": -0.2677331864833832,
|
|
"loss": 0.04550812020897865,
|
|
"loss/core": 0.04103340208530426,
|
|
"loss/preference_sft": 0.27703091502189636,
|
|
"loss/reference_anchor": 0.10014583170413971,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.7892261743545532,
|
|
"rewards/margins": 0.7504833936691284,
|
|
"rewards/rejected": 1.0387427806854248,
|
|
"ronpo/logit": 0.07504834234714508,
|
|
"ronpo/residual": 0.06096462160348892,
|
|
"ronpo/residual_abs": 0.12806007266044617,
|
|
"ronpo/target": 0.014083716087043285,
|
|
"ronpo/target_abs": 0.07628627121448517,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1692146509885788,
|
|
"drift/rejected_abs": 0.11026661098003387,
|
|
"epoch": 0.08899483512117601,
|
|
"grad_norm": 50.0,
|
|
"learning_rate": 6.536793095540228e-08,
|
|
"logits/chosen": -2.18296480178833,
|
|
"logits/rejected": -2.1851444244384766,
|
|
"logps/chosen": -0.2847527265548706,
|
|
"logps/rejected": -0.29109668731689453,
|
|
"loss": 0.08967806398868561,
|
|
"loss/core": 0.08531448990106583,
|
|
"loss/preference_sft": 0.28928548097610474,
|
|
"loss/reference_anchor": 0.09574474394321442,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6921466588974,
|
|
"rewards/margins": 0.5903476476669312,
|
|
"rewards/rejected": 1.1017990112304688,
|
|
"ronpo/logit": 0.05903477221727371,
|
|
"ronpo/residual": 0.04740891978144646,
|
|
"ronpo/residual_abs": 0.1459716111421585,
|
|
"ronpo/target": 0.011625855229794979,
|
|
"ronpo/target_abs": 0.067486971616745,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4365122318267822,
|
|
"drift/rejected_abs": 0.14117935299873352,
|
|
"epoch": 0.09058402860548272,
|
|
"grad_norm": 9.875,
|
|
"learning_rate": 6.487611595134655e-08,
|
|
"logits/chosen": -2.2247390747070312,
|
|
"logits/rejected": -2.196688175201416,
|
|
"logps/chosen": -0.28154322504997253,
|
|
"logps/rejected": -0.2904406487941742,
|
|
"loss": 1.4670677185058594,
|
|
"loss/core": 1.4230749607086182,
|
|
"loss/preference_sft": 0.2900314927101135,
|
|
"loss/reference_anchor": 1.2279331684112549,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.364717960357666,
|
|
"rewards/margins": 2.953922986984253,
|
|
"rewards/rejected": 1.4107954502105713,
|
|
"ronpo/logit": 0.29539233446121216,
|
|
"ronpo/residual": 0.3029133677482605,
|
|
"ronpo/residual_abs": 0.355401873588562,
|
|
"ronpo/target": -0.0075210509821772575,
|
|
"ronpo/target_abs": 0.07652939856052399,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.299263060092926,
|
|
"drift/rejected_abs": 0.13971708714962006,
|
|
"epoch": 0.09217322208978942,
|
|
"grad_norm": 84.0,
|
|
"learning_rate": 6.43740059096826e-08,
|
|
"logits/chosen": -2.013880968093872,
|
|
"logits/rejected": -2.087587594985962,
|
|
"logps/chosen": -0.2723047435283661,
|
|
"logps/rejected": -0.28252094984054565,
|
|
"loss": 0.28749948740005493,
|
|
"loss/core": 0.2771393358707428,
|
|
"loss/preference_sft": 0.27838370203971863,
|
|
"loss/reference_anchor": 0.2681671977043152,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.991696357727051,
|
|
"rewards/margins": 1.5951488018035889,
|
|
"rewards/rejected": 1.396547555923462,
|
|
"ronpo/logit": 0.1595148742198944,
|
|
"ronpo/residual": 0.15534573793411255,
|
|
"ronpo/residual_abs": 0.23441383242607117,
|
|
"ronpo/target": 0.004169141408056021,
|
|
"ronpo/target_abs": 0.08591829240322113,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19509822130203247,
|
|
"drift/rejected_abs": 0.07152483612298965,
|
|
"epoch": 0.09376241557409615,
|
|
"grad_norm": 192.0,
|
|
"learning_rate": 6.386178965346232e-08,
|
|
"logits/chosen": -2.1909687519073486,
|
|
"logits/rejected": -2.1839189529418945,
|
|
"logps/chosen": -0.30111661553382874,
|
|
"logps/rejected": -0.3057703375816345,
|
|
"loss": 0.2420537918806076,
|
|
"loss/core": 0.23458893597126007,
|
|
"loss/preference_sft": 0.29602617025375366,
|
|
"loss/reference_anchor": 0.18367911875247955,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9500179290771484,
|
|
"rewards/margins": 1.2395662069320679,
|
|
"rewards/rejected": 0.7104517817497253,
|
|
"ronpo/logit": 0.12395663559436798,
|
|
"ronpo/residual": 0.12118228524923325,
|
|
"ronpo/residual_abs": 0.16567404568195343,
|
|
"ronpo/target": 0.0027743394020944834,
|
|
"ronpo/target_abs": 0.06994090974330902,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1367836445569992,
|
|
"drift/rejected_abs": 0.08203138411045074,
|
|
"epoch": 0.09535160905840286,
|
|
"grad_norm": 5.65625,
|
|
"learning_rate": 6.333965980627154e-08,
|
|
"logits/chosen": -2.4043996334075928,
|
|
"logits/rejected": -2.4086849689483643,
|
|
"logps/chosen": -0.29549726843833923,
|
|
"logps/rejected": -0.2862420082092285,
|
|
"loss": 0.030557457357645035,
|
|
"loss/core": 0.027756040915846825,
|
|
"loss/preference_sft": 0.29091423749923706,
|
|
"loss/reference_anchor": 0.05094895511865616,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.3668571710586548,
|
|
"rewards/margins": 0.5511847734451294,
|
|
"rewards/rejected": 0.8156723976135254,
|
|
"ronpo/logit": 0.05511848255991936,
|
|
"ronpo/residual": 0.05358175188302994,
|
|
"ronpo/residual_abs": 0.10816502571105957,
|
|
"ronpo/target": 0.001536733703687787,
|
|
"ronpo/target_abs": 0.07785020023584366,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22175483405590057,
|
|
"drift/rejected_abs": 0.10230524837970734,
|
|
"epoch": 0.09694080254270958,
|
|
"grad_norm": 2.90625,
|
|
"learning_rate": 6.280781271979229e-08,
|
|
"logits/chosen": -2.0780434608459473,
|
|
"logits/rejected": -2.1060025691986084,
|
|
"logps/chosen": -0.30538249015808105,
|
|
"logps/rejected": -0.29317212104797363,
|
|
"loss": 0.18119879066944122,
|
|
"loss/core": 0.17126640677452087,
|
|
"loss/preference_sft": 0.3047506511211395,
|
|
"loss/reference_anchor": 0.25330713391304016,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2160160541534424,
|
|
"rewards/margins": 1.1955595016479492,
|
|
"rewards/rejected": 1.0204565525054932,
|
|
"ronpo/logit": 0.11955595016479492,
|
|
"ronpo/residual": 0.12216754257678986,
|
|
"ronpo/residual_abs": 0.17102210223674774,
|
|
"ronpo/target": -0.0026116042863577604,
|
|
"ronpo/target_abs": 0.07656402885913849,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.229892298579216,
|
|
"drift/rejected_abs": 0.15156972408294678,
|
|
"epoch": 0.09852999602701629,
|
|
"grad_norm": 226.0,
|
|
"learning_rate": 6.226644839996278e-08,
|
|
"logits/chosen": -2.0961735248565674,
|
|
"logits/rejected": -1.9890098571777344,
|
|
"logps/chosen": -0.30494624376296997,
|
|
"logps/rejected": -0.29889440536499023,
|
|
"loss": 0.08074851334095001,
|
|
"loss/core": 0.06823646277189255,
|
|
"loss/preference_sft": 0.3070228397846222,
|
|
"loss/reference_anchor": 0.32678505778312683,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2979280948638916,
|
|
"rewards/margins": 0.7833226323127747,
|
|
"rewards/rejected": 1.5146055221557617,
|
|
"ronpo/logit": 0.07833226025104523,
|
|
"ronpo/residual": 0.0831359401345253,
|
|
"ronpo/residual_abs": 0.14295317232608795,
|
|
"ronpo/target": -0.0048036882653832436,
|
|
"ronpo/target_abs": 0.0696224644780159,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21884290874004364,
|
|
"drift/rejected_abs": 0.1715756207704544,
|
|
"epoch": 0.10011918951132301,
|
|
"grad_norm": 1.96875,
|
|
"learning_rate": 6.171577043176358e-08,
|
|
"logits/chosen": -2.206495761871338,
|
|
"logits/rejected": -2.2399115562438965,
|
|
"logps/chosen": -0.2883833348751068,
|
|
"logps/rejected": -0.2834603190422058,
|
|
"loss": 0.11746679246425629,
|
|
"loss/core": 0.08842774480581284,
|
|
"loss/preference_sft": 0.2884564995765686,
|
|
"loss/reference_anchor": 0.8008415102958679,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1873815059661865,
|
|
"rewards/margins": 0.4734341502189636,
|
|
"rewards/rejected": 1.7139472961425781,
|
|
"ronpo/logit": 0.047343410551548004,
|
|
"ronpo/residual": 0.03924637287855148,
|
|
"ronpo/residual_abs": 0.13970692455768585,
|
|
"ronpo/target": 0.008097042329609394,
|
|
"ronpo/target_abs": 0.07850007712841034,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30623868107795715,
|
|
"drift/rejected_abs": 0.15175890922546387,
|
|
"epoch": 0.10170838299562972,
|
|
"grad_norm": 5.15625,
|
|
"learning_rate": 6.115598590265742e-08,
|
|
"logits/chosen": -2.048631191253662,
|
|
"logits/rejected": -2.0478978157043457,
|
|
"logps/chosen": -0.2675008773803711,
|
|
"logps/rejected": -0.26748839020729065,
|
|
"loss": 0.41966015100479126,
|
|
"loss/core": 0.3942663371562958,
|
|
"loss/preference_sft": 0.26846352219581604,
|
|
"loss/reference_anchor": 0.6986923217773438,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.06076717376709,
|
|
"rewards/margins": 1.5459158420562744,
|
|
"rewards/rejected": 1.5148515701293945,
|
|
"ronpo/logit": 0.15459157526493073,
|
|
"ronpo/residual": 0.15290841460227966,
|
|
"ronpo/residual_abs": 0.209668830037117,
|
|
"ronpo/target": 0.0016831799875944853,
|
|
"ronpo/target_abs": 0.0813584178686142,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2687004804611206,
|
|
"drift/rejected_abs": 0.06457342952489853,
|
|
"epoch": 0.10329757647993643,
|
|
"grad_norm": 22.5,
|
|
"learning_rate": 6.058730532471218e-08,
|
|
"logits/chosen": -2.212750196456909,
|
|
"logits/rejected": -2.2377688884735107,
|
|
"logps/chosen": -0.3067273497581482,
|
|
"logps/rejected": -0.2894396185874939,
|
|
"loss": 1.0871208906173706,
|
|
"loss/core": 1.0651031732559204,
|
|
"loss/preference_sft": 0.3019077479839325,
|
|
"loss/reference_anchor": 0.5988877415657043,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.686971426010132,
|
|
"rewards/margins": 2.045879602432251,
|
|
"rewards/rejected": 0.6410917043685913,
|
|
"ronpo/logit": 0.20458796620368958,
|
|
"ronpo/residual": 0.2081504613161087,
|
|
"ronpo/residual_abs": 0.24282391369342804,
|
|
"ronpo/target": -0.003562526311725378,
|
|
"ronpo/target_abs": 0.07360132783651352,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25193846225738525,
|
|
"drift/rejected_abs": 0.1257767379283905,
|
|
"epoch": 0.10488676996424315,
|
|
"grad_norm": 15.5,
|
|
"learning_rate": 6.000994255543592e-08,
|
|
"logits/chosen": -2.3243191242218018,
|
|
"logits/rejected": -2.323913812637329,
|
|
"logps/chosen": -0.265578955411911,
|
|
"logps/rejected": -0.26127713918685913,
|
|
"loss": 0.5734900832176208,
|
|
"loss/core": 0.5604211091995239,
|
|
"loss/preference_sft": 0.27206915616989136,
|
|
"loss/reference_anchor": 0.34619513154029846,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.5193843841552734,
|
|
"rewards/margins": 1.2678606510162354,
|
|
"rewards/rejected": 1.2515236139297485,
|
|
"ronpo/logit": 0.12678608298301697,
|
|
"ronpo/residual": 0.14241120219230652,
|
|
"ronpo/residual_abs": 0.2712676525115967,
|
|
"ronpo/target": -0.015625139698386192,
|
|
"ronpo/target_abs": 0.09194107353687286,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4483170509338379,
|
|
"drift/rejected_abs": 0.2124291956424713,
|
|
"epoch": 0.10647596344854986,
|
|
"grad_norm": 13.0625,
|
|
"learning_rate": 5.942411471735374e-08,
|
|
"logits/chosen": -1.9917030334472656,
|
|
"logits/rejected": -1.9397776126861572,
|
|
"logps/chosen": -0.29203617572784424,
|
|
"logps/rejected": -0.2925509214401245,
|
|
"loss": 0.7571202516555786,
|
|
"loss/core": 0.7059234380722046,
|
|
"loss/preference_sft": 0.29065781831741333,
|
|
"loss/reference_anchor": 1.4337010383605957,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.481464385986328,
|
|
"rewards/margins": 2.3606128692626953,
|
|
"rewards/rejected": 2.120851516723633,
|
|
"ronpo/logit": 0.23606130480766296,
|
|
"ronpo/residual": 0.25377950072288513,
|
|
"ronpo/residual_abs": 0.3121553659439087,
|
|
"ronpo/target": -0.017718184739351273,
|
|
"ronpo/target_abs": 0.07344368100166321,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.45905232429504395,
|
|
"drift/rejected_abs": 0.16429868340492249,
|
|
"epoch": 0.10806515693285658,
|
|
"grad_norm": 82.0,
|
|
"learning_rate": 5.883004211635694e-08,
|
|
"logits/chosen": -1.8839290142059326,
|
|
"logits/rejected": -1.9421613216400146,
|
|
"logps/chosen": -0.28718405961990356,
|
|
"logps/rejected": -0.2739580273628235,
|
|
"loss": 0.9680835008621216,
|
|
"loss/core": 0.93462735414505,
|
|
"loss/preference_sft": 0.2868141829967499,
|
|
"loss/reference_anchor": 0.927208423614502,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.588636875152588,
|
|
"rewards/margins": 2.951829195022583,
|
|
"rewards/rejected": 1.6368080377578735,
|
|
"ronpo/logit": 0.29518288373947144,
|
|
"ronpo/residual": 0.2900192439556122,
|
|
"ronpo/residual_abs": 0.33183783292770386,
|
|
"ronpo/target": 0.005163692403584719,
|
|
"ronpo/target_abs": 0.06677814573049545,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2457018792629242,
|
|
"drift/rejected_abs": 0.12155888229608536,
|
|
"epoch": 0.10965435041716329,
|
|
"grad_norm": 63.25,
|
|
"learning_rate": 5.8227948158854975e-08,
|
|
"logits/chosen": -2.178396463394165,
|
|
"logits/rejected": -2.251084566116333,
|
|
"logps/chosen": -0.27320748567581177,
|
|
"logps/rejected": -0.27947288751602173,
|
|
"loss": 0.1250918060541153,
|
|
"loss/core": 0.1128639206290245,
|
|
"loss/preference_sft": 0.28871798515319824,
|
|
"loss/reference_anchor": 0.3204962909221649,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4553937911987305,
|
|
"rewards/margins": 1.2922033071517944,
|
|
"rewards/rejected": 1.1631903648376465,
|
|
"ronpo/logit": 0.12922033667564392,
|
|
"ronpo/residual": 0.1374775469303131,
|
|
"ronpo/residual_abs": 0.1687041074037552,
|
|
"ronpo/target": -0.008257212117314339,
|
|
"ronpo/target_abs": 0.06108090281486511,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41426244378089905,
|
|
"drift/rejected_abs": 0.1653062105178833,
|
|
"epoch": 0.11124354390147001,
|
|
"grad_norm": 1.578125,
|
|
"learning_rate": 5.761805926776142e-08,
|
|
"logits/chosen": -2.268873691558838,
|
|
"logits/rejected": -2.296691417694092,
|
|
"logps/chosen": -0.23340287804603577,
|
|
"logps/rejected": -0.24630527198314667,
|
|
"loss": 0.9605229496955872,
|
|
"loss/core": 0.9214657545089722,
|
|
"loss/preference_sft": 0.233280748128891,
|
|
"loss/reference_anchor": 1.0925902128219604,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.142511367797852,
|
|
"rewards/margins": 2.4894490242004395,
|
|
"rewards/rejected": 1.6530622243881226,
|
|
"ronpo/logit": 0.24894490838050842,
|
|
"ronpo/residual": 0.24280676245689392,
|
|
"ronpo/residual_abs": 0.2877592444419861,
|
|
"ronpo/target": 0.006138127762824297,
|
|
"ronpo/target_abs": 0.06590522080659866,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28039026260375977,
|
|
"drift/rejected_abs": 0.10265352576971054,
|
|
"epoch": 0.11283273738577672,
|
|
"grad_norm": 18.25,
|
|
"learning_rate": 5.700060479734557e-08,
|
|
"logits/chosen": -2.1270534992218018,
|
|
"logits/rejected": -2.02483868598938,
|
|
"logps/chosen": -0.28407421708106995,
|
|
"logps/rejected": -0.27686986327171326,
|
|
"loss": 0.40096959471702576,
|
|
"loss/core": 0.3891247808933258,
|
|
"loss/preference_sft": 0.2740500569343567,
|
|
"loss/reference_anchor": 0.3110194802284241,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8039023876190186,
|
|
"rewards/margins": 1.780269980430603,
|
|
"rewards/rejected": 1.0236326456069946,
|
|
"ronpo/logit": 0.17802700400352478,
|
|
"ronpo/residual": 0.16591885685920715,
|
|
"ronpo/residual_abs": 0.22742481529712677,
|
|
"ronpo/target": 0.012108135037124157,
|
|
"ronpo/target_abs": 0.081517793238163,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2177037000656128,
|
|
"drift/rejected_abs": 0.0887526124715805,
|
|
"epoch": 0.11442193087008343,
|
|
"grad_norm": 11.0,
|
|
"learning_rate": 5.6375816946981745e-08,
|
|
"logits/chosen": -2.2079660892486572,
|
|
"logits/rejected": -2.2729287147521973,
|
|
"logps/chosen": -0.26719698309898376,
|
|
"logps/rejected": -0.25824791193008423,
|
|
"loss": 0.08846821635961533,
|
|
"loss/core": 0.08424760401248932,
|
|
"loss/preference_sft": 0.26704221963882446,
|
|
"loss/reference_anchor": 0.09388500452041626,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1766204833984375,
|
|
"rewards/margins": 1.2892426252365112,
|
|
"rewards/rejected": 0.8873778581619263,
|
|
"ronpo/logit": 0.12892426550388336,
|
|
"ronpo/residual": 0.13182158768177032,
|
|
"ronpo/residual_abs": 0.17560000717639923,
|
|
"ronpo/target": -0.002897311467677355,
|
|
"ronpo/target_abs": 0.07039253413677216,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2076272964477539,
|
|
"drift/rejected_abs": 0.13079994916915894,
|
|
"epoch": 0.11601112435439015,
|
|
"grad_norm": 2.203125,
|
|
"learning_rate": 5.574393067382849e-08,
|
|
"logits/chosen": -2.094231128692627,
|
|
"logits/rejected": -2.132718324661255,
|
|
"logps/chosen": -0.28609710931777954,
|
|
"logps/rejected": -0.3247900903224945,
|
|
"loss": 0.05453226715326309,
|
|
"loss/core": 0.040341801941394806,
|
|
"loss/preference_sft": 0.29053995013237,
|
|
"loss/reference_anchor": 0.37638789415359497,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0743491649627686,
|
|
"rewards/margins": 0.7902168035507202,
|
|
"rewards/rejected": 1.284132480621338,
|
|
"ronpo/logit": 0.07902167737483978,
|
|
"ronpo/residual": 0.0732402577996254,
|
|
"ronpo/residual_abs": 0.12813961505889893,
|
|
"ronpo/target": 0.005781428888440132,
|
|
"ronpo/target_abs": 0.08649028837680817,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31887754797935486,
|
|
"drift/rejected_abs": 0.2154131680727005,
|
|
"epoch": 0.11760031783869686,
|
|
"grad_norm": 13.0625,
|
|
"learning_rate": 5.51051836044709e-08,
|
|
"logits/chosen": -2.1323890686035156,
|
|
"logits/rejected": -2.1665499210357666,
|
|
"logps/chosen": -0.2781253159046173,
|
|
"logps/rejected": -0.28154605627059937,
|
|
"loss": 0.14984655380249023,
|
|
"loss/core": 0.10850384086370468,
|
|
"loss/preference_sft": 0.28171804547309875,
|
|
"loss/reference_anchor": 1.1530487537384033,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1884636878967285,
|
|
"rewards/margins": 1.0351537466049194,
|
|
"rewards/rejected": 2.1533100605010986,
|
|
"ronpo/logit": 0.1035153716802597,
|
|
"ronpo/residual": 0.09588007628917694,
|
|
"ronpo/residual_abs": 0.16027481853961945,
|
|
"ronpo/target": 0.007635296322405338,
|
|
"ronpo/target_abs": 0.0714506059885025,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21330508589744568,
|
|
"drift/rejected_abs": 0.09820286929607391,
|
|
"epoch": 0.11918951132300358,
|
|
"grad_norm": 19.25,
|
|
"learning_rate": 5.445981594555891e-08,
|
|
"logits/chosen": -2.2786364555358887,
|
|
"logits/rejected": -2.3025929927825928,
|
|
"logps/chosen": -0.29398036003112793,
|
|
"logps/rejected": -0.2887040376663208,
|
|
"loss": 0.14517758786678314,
|
|
"loss/core": 0.1390768587589264,
|
|
"loss/preference_sft": 0.2918010950088501,
|
|
"loss/reference_anchor": 0.14512670040130615,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.132155179977417,
|
|
"rewards/margins": 1.1507148742675781,
|
|
"rewards/rejected": 0.9814404249191284,
|
|
"ronpo/logit": 0.11507149785757065,
|
|
"ronpo/residual": 0.12740284204483032,
|
|
"ronpo/residual_abs": 0.17654451727867126,
|
|
"ronpo/target": -0.01233135536313057,
|
|
"ronpo/target_abs": 0.08002184331417084,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2749318480491638,
|
|
"drift/rejected_abs": 0.10027341544628143,
|
|
"epoch": 0.12077870480731029,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 5.3808070393475435e-08,
|
|
"logits/chosen": -2.1572823524475098,
|
|
"logits/rejected": -2.1735129356384277,
|
|
"logps/chosen": -0.2892719805240631,
|
|
"logps/rejected": -0.28280091285705566,
|
|
"loss": 0.5006850361824036,
|
|
"loss/core": 0.4873785972595215,
|
|
"loss/preference_sft": 0.28779616951942444,
|
|
"loss/reference_anchor": 0.35140353441238403,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7480342388153076,
|
|
"rewards/margins": 1.75465989112854,
|
|
"rewards/rejected": 0.9933745265007019,
|
|
"ronpo/logit": 0.17546597123146057,
|
|
"ronpo/residual": 0.16056114435195923,
|
|
"ronpo/residual_abs": 0.2221776247024536,
|
|
"ronpo/target": 0.014904849231243134,
|
|
"ronpo/target_abs": 0.07894332706928253,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2522042691707611,
|
|
"drift/rejected_abs": 0.1486947387456894,
|
|
"epoch": 0.12236789829161701,
|
|
"grad_norm": 62.75,
|
|
"learning_rate": 5.315019204306817e-08,
|
|
"logits/chosen": -2.2786965370178223,
|
|
"logits/rejected": -2.2488064765930176,
|
|
"logps/chosen": -0.27131175994873047,
|
|
"logps/rejected": -0.26389139890670776,
|
|
"loss": 0.17779920995235443,
|
|
"loss/core": 0.16905324161052704,
|
|
"loss/preference_sft": 0.26512569189071655,
|
|
"loss/reference_anchor": 0.22337183356285095,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.521192789077759,
|
|
"rewards/margins": 1.0361357927322388,
|
|
"rewards/rejected": 1.4850571155548096,
|
|
"ronpo/logit": 0.10361359268426895,
|
|
"ronpo/residual": 0.0915413573384285,
|
|
"ronpo/residual_abs": 0.15346641838550568,
|
|
"ronpo/target": 0.012072229757905006,
|
|
"ronpo/target_abs": 0.08567623794078827,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2105921506881714,
|
|
"drift/rejected_abs": 0.12278739362955093,
|
|
"epoch": 0.12395709177592372,
|
|
"grad_norm": 3.0625,
|
|
"learning_rate": 5.248642829547946e-08,
|
|
"logits/chosen": -2.0767416954040527,
|
|
"logits/rejected": -2.117583751678467,
|
|
"logps/chosen": -0.3050832748413086,
|
|
"logps/rejected": -0.2946804165840149,
|
|
"loss": 0.08686129003763199,
|
|
"loss/core": 0.07733297348022461,
|
|
"loss/preference_sft": 0.2956315875053406,
|
|
"loss/reference_anchor": 0.2426745593547821,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.1027276515960693,
|
|
"rewards/margins": 0.8792668581008911,
|
|
"rewards/rejected": 1.2234607934951782,
|
|
"ronpo/logit": 0.08792667090892792,
|
|
"ronpo/residual": 0.10677643120288849,
|
|
"ronpo/residual_abs": 0.141727477312088,
|
|
"ronpo/target": -0.018849752843379974,
|
|
"ronpo/target_abs": 0.07699684798717499,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2143975794315338,
|
|
"drift/rejected_abs": 0.07005457580089569,
|
|
"epoch": 0.12554628526023043,
|
|
"grad_norm": 2.703125,
|
|
"learning_rate": 5.181702876510881e-08,
|
|
"logits/chosen": -2.1568331718444824,
|
|
"logits/rejected": -2.2212538719177246,
|
|
"logps/chosen": -0.3092118203639984,
|
|
"logps/rejected": -0.2957898676395416,
|
|
"loss": 0.3654123842716217,
|
|
"loss/core": 0.3564660847187042,
|
|
"loss/preference_sft": 0.30225998163223267,
|
|
"loss/reference_anchor": 0.2253817319869995,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.142920732498169,
|
|
"rewards/margins": 1.4426730871200562,
|
|
"rewards/rejected": 0.700247585773468,
|
|
"ronpo/logit": 0.14426730573177338,
|
|
"ronpo/residual": 0.14315283298492432,
|
|
"ronpo/residual_abs": 0.18677830696105957,
|
|
"ronpo/target": 0.0011144764721393585,
|
|
"ronpo/target_abs": 0.07045630365610123,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19173476099967957,
|
|
"drift/rejected_abs": 0.08401096612215042,
|
|
"epoch": 0.12713547874453715,
|
|
"grad_norm": 9.1875,
|
|
"learning_rate": 5.114224518574318e-08,
|
|
"logits/chosen": -2.2294397354125977,
|
|
"logits/rejected": -2.318349599838257,
|
|
"logps/chosen": -0.25690674781799316,
|
|
"logps/rejected": -0.26110321283340454,
|
|
"loss": 0.07373060286045074,
|
|
"loss/core": 0.06859154254198074,
|
|
"loss/preference_sft": 0.2571546137332916,
|
|
"loss/reference_anchor": 0.12111492455005646,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.916455864906311,
|
|
"rewards/margins": 1.0823928117752075,
|
|
"rewards/rejected": 0.8340628743171692,
|
|
"ronpo/logit": 0.10823929309844971,
|
|
"ronpo/residual": 0.10606695711612701,
|
|
"ronpo/residual_abs": 0.15262378752231598,
|
|
"ronpo/target": 0.0021723308600485325,
|
|
"ronpo/target_abs": 0.07407379150390625,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17076550424098969,
|
|
"drift/rejected_abs": 0.1465177685022354,
|
|
"epoch": 0.12872467222884387,
|
|
"grad_norm": 1.5859375,
|
|
"learning_rate": 5.046233131589012e-08,
|
|
"logits/chosen": -2.2917768955230713,
|
|
"logits/rejected": -2.300046443939209,
|
|
"logps/chosen": -0.29497990012168884,
|
|
"logps/rejected": -0.2882741093635559,
|
|
"loss": 0.28003552556037903,
|
|
"loss/core": 0.27095627784729004,
|
|
"loss/preference_sft": 0.28535428643226624,
|
|
"loss/reference_anchor": 0.2308715283870697,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7055221796035767,
|
|
"rewards/margins": 0.24134886264801025,
|
|
"rewards/rejected": 1.464173436164856,
|
|
"ronpo/logit": 0.024134885519742966,
|
|
"ronpo/residual": 0.025067275390028954,
|
|
"ronpo/residual_abs": 0.1801082193851471,
|
|
"ronpo/target": -0.0009323973208665848,
|
|
"ronpo/target_abs": 0.07486466318368912,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18705379962921143,
|
|
"drift/rejected_abs": 0.05877493694424629,
|
|
"epoch": 0.13031386571315057,
|
|
"grad_norm": 0.94921875,
|
|
"learning_rate": 4.977754284334966e-08,
|
|
"logits/chosen": -2.2271358966827393,
|
|
"logits/rejected": -2.1403744220733643,
|
|
"logps/chosen": -0.28943097591400146,
|
|
"logps/rejected": -0.28723686933517456,
|
|
"loss": 0.34275034070014954,
|
|
"loss/core": 0.33544498682022095,
|
|
"loss/preference_sft": 0.2885206937789917,
|
|
"loss/reference_anchor": 0.1798722892999649,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8687069416046143,
|
|
"rewards/margins": 1.3043181896209717,
|
|
"rewards/rejected": 0.5643887519836426,
|
|
"ronpo/logit": 0.13043180108070374,
|
|
"ronpo/residual": 0.1327742636203766,
|
|
"ronpo/residual_abs": 0.1725582629442215,
|
|
"ronpo/target": -0.0023424329701811075,
|
|
"ronpo/target_abs": 0.07028114050626755,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12771040201187134,
|
|
"drift/rejected_abs": 0.05937645584344864,
|
|
"epoch": 0.1319030591974573,
|
|
"grad_norm": 18.75,
|
|
"learning_rate": 4.9088137289060525e-08,
|
|
"logits/chosen": -2.2670741081237793,
|
|
"logits/rejected": -2.297637701034546,
|
|
"logps/chosen": -0.2853187322616577,
|
|
"logps/rejected": -0.27392154932022095,
|
|
"loss": 0.0417989157140255,
|
|
"loss/core": 0.03925047814846039,
|
|
"loss/preference_sft": 0.27748462557792664,
|
|
"loss/reference_anchor": 0.04506412148475647,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.277104139328003,
|
|
"rewards/margins": 0.6841433644294739,
|
|
"rewards/rejected": 0.5929605960845947,
|
|
"ronpo/logit": 0.06841433793306351,
|
|
"ronpo/residual": 0.06507981568574905,
|
|
"ronpo/residual_abs": 0.1218712329864502,
|
|
"ronpo/target": 0.003334520850330591,
|
|
"ronpo/target_abs": 0.07405976951122284,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3893112242221832,
|
|
"drift/rejected_abs": 0.1252056360244751,
|
|
"epoch": 0.133492252681764,
|
|
"grad_norm": 1.9921875,
|
|
"learning_rate": 4.8394373910257054e-08,
|
|
"logits/chosen": -2.1123807430267334,
|
|
"logits/rejected": -2.231128692626953,
|
|
"logps/chosen": -0.2718810737133026,
|
|
"logps/rejected": -0.272949755191803,
|
|
"loss": 0.8291887044906616,
|
|
"loss/core": 0.8047215342521667,
|
|
"loss/preference_sft": 0.27599674463272095,
|
|
"loss/reference_anchor": 0.6714580059051514,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.8931126594543457,
|
|
"rewards/margins": 2.735173463821411,
|
|
"rewards/rejected": 1.1579387187957764,
|
|
"ronpo/logit": 0.27351734042167664,
|
|
"ronpo/residual": 0.26163390278816223,
|
|
"ronpo/residual_abs": 0.302009254693985,
|
|
"ronpo/target": 0.011883469298481941,
|
|
"ronpo/target_abs": 0.07862479239702225,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.42474645376205444,
|
|
"drift/rejected_abs": 0.13992993533611298,
|
|
"epoch": 0.1350814461660707,
|
|
"grad_norm": 304.0,
|
|
"learning_rate": 4.769651360297318e-08,
|
|
"logits/chosen": -2.1638529300689697,
|
|
"logits/rejected": -2.1579020023345947,
|
|
"logps/chosen": -0.29796314239501953,
|
|
"logps/rejected": -0.29541391134262085,
|
|
"loss": 1.2007313966751099,
|
|
"loss/core": 1.1638450622558594,
|
|
"loss/preference_sft": 0.29007643461227417,
|
|
"loss/reference_anchor": 1.0248842239379883,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.246431350708008,
|
|
"rewards/margins": 2.849513530731201,
|
|
"rewards/rejected": 1.3969179391860962,
|
|
"ronpo/logit": 0.284951388835907,
|
|
"ronpo/residual": 0.2653922438621521,
|
|
"ronpo/residual_abs": 0.3415285050868988,
|
|
"ronpo/target": 0.019559111446142197,
|
|
"ronpo/target_abs": 0.09040865302085876,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18750756978988647,
|
|
"drift/rejected_abs": 0.09092601388692856,
|
|
"epoch": 0.13667063965037743,
|
|
"grad_norm": 11.375,
|
|
"learning_rate": 4.699481880393007e-08,
|
|
"logits/chosen": -2.430100679397583,
|
|
"logits/rejected": -2.416260242462158,
|
|
"logps/chosen": -0.27946940064430237,
|
|
"logps/rejected": -0.2753836512565613,
|
|
"loss": 0.09714354574680328,
|
|
"loss/core": 0.09125364571809769,
|
|
"loss/preference_sft": 0.273917555809021,
|
|
"loss/reference_anchor": 0.1408909112215042,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8739410638809204,
|
|
"rewards/margins": 0.9661051630973816,
|
|
"rewards/rejected": 0.9078359603881836,
|
|
"ronpo/logit": 0.09661051630973816,
|
|
"ronpo/residual": 0.090634785592556,
|
|
"ronpo/residual_abs": 0.14002564549446106,
|
|
"ronpo/target": 0.005975720938295126,
|
|
"ronpo/target_abs": 0.06694401055574417,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1731262505054474,
|
|
"drift/rejected_abs": 0.0927744135260582,
|
|
"epoch": 0.13825983313468415,
|
|
"grad_norm": 11.875,
|
|
"learning_rate": 4.628955339184442e-08,
|
|
"logits/chosen": -2.182917356491089,
|
|
"logits/rejected": -2.1770148277282715,
|
|
"logps/chosen": -0.27976903319358826,
|
|
"logps/rejected": -0.281031996011734,
|
|
"loss": 0.04321626201272011,
|
|
"loss/core": 0.03948911279439926,
|
|
"loss/preference_sft": 0.27680492401123047,
|
|
"loss/reference_anchor": 0.07880944013595581,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7302322387695312,
|
|
"rewards/margins": 0.80370032787323,
|
|
"rewards/rejected": 0.9265319108963013,
|
|
"ronpo/logit": 0.08037003129720688,
|
|
"ronpo/residual": 0.06914183497428894,
|
|
"ronpo/residual_abs": 0.1280720978975296,
|
|
"ronpo/target": 0.011228211224079132,
|
|
"ronpo/target_abs": 0.06739668548107147,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31899359822273254,
|
|
"drift/rejected_abs": 0.1331823766231537,
|
|
"epoch": 0.13984902661899087,
|
|
"grad_norm": 3.984375,
|
|
"learning_rate": 4.55809825881945e-08,
|
|
"logits/chosen": -2.2753958702087402,
|
|
"logits/rejected": -2.300555467605591,
|
|
"logps/chosen": -0.2945733666419983,
|
|
"logps/rejected": -0.29113486409187317,
|
|
"loss": 0.7341649532318115,
|
|
"loss/core": 0.7019937634468079,
|
|
"loss/preference_sft": 0.29861825704574585,
|
|
"loss/reference_anchor": 0.8893128633499146,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1885926723480225,
|
|
"rewards/margins": 1.86493718624115,
|
|
"rewards/rejected": 1.323655366897583,
|
|
"ronpo/logit": 0.18649372458457947,
|
|
"ronpo/residual": 0.19063273072242737,
|
|
"ronpo/residual_abs": 0.24889390170574188,
|
|
"ronpo/target": -0.004139023832976818,
|
|
"ronpo/target_abs": 0.08812376856803894,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19011065363883972,
|
|
"drift/rejected_abs": 0.16861161589622498,
|
|
"epoch": 0.14143822010329757,
|
|
"grad_norm": 286.0,
|
|
"learning_rate": 4.486937285748113e-08,
|
|
"logits/chosen": -2.2313015460968018,
|
|
"logits/rejected": -2.1554415225982666,
|
|
"logps/chosen": -0.280648797750473,
|
|
"logps/rejected": -0.27201026678085327,
|
|
"loss": 0.22713658213615417,
|
|
"loss/core": 0.21747353672981262,
|
|
"loss/preference_sft": 0.2815045714378357,
|
|
"loss/reference_anchor": 0.24793705344200134,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9011064767837524,
|
|
"rewards/margins": 0.21516799926757812,
|
|
"rewards/rejected": 1.6859385967254639,
|
|
"ronpo/logit": 0.021516799926757812,
|
|
"ronpo/residual": 0.026197385042905807,
|
|
"ronpo/residual_abs": 0.18634097278118134,
|
|
"ronpo/target": -0.0046805874444544315,
|
|
"ronpo/target_abs": 0.0748194083571434,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3207959234714508,
|
|
"drift/rejected_abs": 0.1021362692117691,
|
|
"epoch": 0.1430274135876043,
|
|
"grad_norm": 31.25,
|
|
"learning_rate": 4.415499180702132e-08,
|
|
"logits/chosen": -2.263270854949951,
|
|
"logits/rejected": -2.3271467685699463,
|
|
"logps/chosen": -0.2672632336616516,
|
|
"logps/rejected": -0.2835923731327057,
|
|
"loss": 0.7797986268997192,
|
|
"loss/core": 0.7587357759475708,
|
|
"loss/preference_sft": 0.2777632474899292,
|
|
"loss/reference_anchor": 0.5740165114402771,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2072951793670654,
|
|
"rewards/margins": 2.1891021728515625,
|
|
"rewards/rejected": 1.018193006515503,
|
|
"ronpo/logit": 0.21891024708747864,
|
|
"ronpo/residual": 0.228356271982193,
|
|
"ronpo/residual_abs": 0.27203264832496643,
|
|
"ronpo/target": -0.00944601558148861,
|
|
"ronpo/target_abs": 0.06918533146381378,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2078581303358078,
|
|
"drift/rejected_abs": 0.09944683313369751,
|
|
"epoch": 0.144616607071911,
|
|
"grad_norm": 9.875,
|
|
"learning_rate": 4.3438108086312104e-08,
|
|
"logits/chosen": -2.315324306488037,
|
|
"logits/rejected": -2.3483715057373047,
|
|
"logps/chosen": -0.2843065857887268,
|
|
"logps/rejected": -0.2822841703891754,
|
|
"loss": 0.1076749935746193,
|
|
"loss/core": 0.10172615200281143,
|
|
"loss/preference_sft": 0.2944115698337555,
|
|
"loss/reference_anchor": 0.14052584767341614,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0785813331604004,
|
|
"rewards/margins": 1.0842633247375488,
|
|
"rewards/rejected": 0.994317889213562,
|
|
"ronpo/logit": 0.10842633247375488,
|
|
"ronpo/residual": 0.10938017070293427,
|
|
"ronpo/residual_abs": 0.16080966591835022,
|
|
"ronpo/target": -0.0009538428857922554,
|
|
"ronpo/target_abs": 0.07275767624378204,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37791651487350464,
|
|
"drift/rejected_abs": 0.16409462690353394,
|
|
"epoch": 0.1462058005562177,
|
|
"grad_norm": 115.5,
|
|
"learning_rate": 4.271899128600245e-08,
|
|
"logits/chosen": -1.805584192276001,
|
|
"logits/rejected": -1.6658241748809814,
|
|
"logps/chosen": -0.2807144224643707,
|
|
"logps/rejected": -0.30006617307662964,
|
|
"loss": 0.5550448298454285,
|
|
"loss/core": 0.5333111882209778,
|
|
"loss/preference_sft": 0.29056912660598755,
|
|
"loss/reference_anchor": 0.5919061303138733,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.7732338905334473,
|
|
"rewards/margins": 2.1390435695648193,
|
|
"rewards/rejected": 1.634190320968628,
|
|
"ronpo/logit": 0.21390438079833984,
|
|
"ronpo/residual": 0.2101598083972931,
|
|
"ronpo/residual_abs": 0.26126962900161743,
|
|
"ronpo/target": 0.0037445551715791225,
|
|
"ronpo/target_abs": 0.07518764585256577,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3444373905658722,
|
|
"drift/rejected_abs": 0.23399516940116882,
|
|
"epoch": 0.14779499404052443,
|
|
"grad_norm": 318.0,
|
|
"learning_rate": 4.199791183651124e-08,
|
|
"logits/chosen": -2.0957283973693848,
|
|
"logits/rejected": -2.116079092025757,
|
|
"logps/chosen": -0.2696365416049957,
|
|
"logps/rejected": -0.2735532522201538,
|
|
"loss": 0.8012321591377258,
|
|
"loss/core": 0.7794309854507446,
|
|
"loss/preference_sft": 0.277563214302063,
|
|
"loss/reference_anchor": 0.5951352119445801,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4441075325012207,
|
|
"rewards/margins": 1.1061354875564575,
|
|
"rewards/rejected": 2.3379721641540527,
|
|
"ronpo/logit": 0.11061354726552963,
|
|
"ronpo/residual": 0.099038265645504,
|
|
"ronpo/residual_abs": 0.29626208543777466,
|
|
"ronpo/target": 0.011575284413993359,
|
|
"ronpo/target_abs": 0.06793732196092606,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29984086751937866,
|
|
"drift/rejected_abs": 0.11699891090393066,
|
|
"epoch": 0.14938418752483115,
|
|
"grad_norm": 2.640625,
|
|
"learning_rate": 4.127514090632942e-08,
|
|
"logits/chosen": -2.075214147567749,
|
|
"logits/rejected": -2.0434682369232178,
|
|
"logps/chosen": -0.254089891910553,
|
|
"logps/rejected": -0.2596934139728546,
|
|
"loss": 0.7762528657913208,
|
|
"loss/core": 0.7569972276687622,
|
|
"loss/preference_sft": 0.2593012750148773,
|
|
"loss/reference_anchor": 0.5242301821708679,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.996891975402832,
|
|
"rewards/margins": 1.8279199600219727,
|
|
"rewards/rejected": 1.1689717769622803,
|
|
"ronpo/logit": 0.18279200792312622,
|
|
"ronpo/residual": 0.1851007044315338,
|
|
"ronpo/residual_abs": 0.25966882705688477,
|
|
"ronpo/target": -0.0023087128065526485,
|
|
"ronpo/target_abs": 0.07931704819202423,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25055521726608276,
|
|
"drift/rejected_abs": 0.09638573229312897,
|
|
"epoch": 0.15097338100913787,
|
|
"grad_norm": 8.3125,
|
|
"learning_rate": 4.055095030004469e-08,
|
|
"logits/chosen": -2.195185661315918,
|
|
"logits/rejected": -2.1315507888793945,
|
|
"logps/chosen": -0.30518704652786255,
|
|
"logps/rejected": -0.29946842789649963,
|
|
"loss": 0.2174142301082611,
|
|
"loss/core": 0.21045875549316406,
|
|
"loss/preference_sft": 0.3060879111289978,
|
|
"loss/reference_anchor": 0.16811935603618622,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5054726600646973,
|
|
"rewards/margins": 1.542541265487671,
|
|
"rewards/rejected": 0.9629314541816711,
|
|
"ronpo/logit": 0.15425412356853485,
|
|
"ronpo/residual": 0.15975168347358704,
|
|
"ronpo/residual_abs": 0.21089980006217957,
|
|
"ronpo/target": -0.0054975529201328754,
|
|
"ronpo/target_abs": 0.07147940248250961,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22197389602661133,
|
|
"drift/rejected_abs": 0.1093837171792984,
|
|
"epoch": 0.15256257449344457,
|
|
"grad_norm": 1.5390625,
|
|
"learning_rate": 3.982561235612682e-08,
|
|
"logits/chosen": -2.1202902793884277,
|
|
"logits/rejected": -2.142862319946289,
|
|
"logps/chosen": -0.2731131911277771,
|
|
"logps/rejected": -0.2682819962501526,
|
|
"loss": 0.08932836353778839,
|
|
"loss/core": 0.08391264826059341,
|
|
"loss/preference_sft": 0.2691730856895447,
|
|
"loss/reference_anchor": 0.12781760096549988,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2168147563934326,
|
|
"rewards/margins": 1.1324942111968994,
|
|
"rewards/rejected": 1.0843205451965332,
|
|
"ronpo/logit": 0.11324940621852875,
|
|
"ronpo/residual": 0.13424549996852875,
|
|
"ronpo/residual_abs": 0.1686258614063263,
|
|
"ronpo/target": -0.020996104925870895,
|
|
"ronpo/target_abs": 0.07291201502084732,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29734545946121216,
|
|
"drift/rejected_abs": 0.1604335755109787,
|
|
"epoch": 0.1541517679777513,
|
|
"grad_norm": 6.125,
|
|
"learning_rate": 3.90993998445124e-08,
|
|
"logits/chosen": -1.9032106399536133,
|
|
"logits/rejected": -2.025702953338623,
|
|
"logps/chosen": -0.28302717208862305,
|
|
"logps/rejected": -0.28014808893203735,
|
|
"loss": 0.12766507267951965,
|
|
"loss/core": 0.1155250072479248,
|
|
"loss/preference_sft": 0.28657975792884827,
|
|
"loss/reference_anchor": 0.3182009756565094,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.972499370574951,
|
|
"rewards/margins": 1.370253086090088,
|
|
"rewards/rejected": 1.6022460460662842,
|
|
"ronpo/logit": 0.13702531158924103,
|
|
"ronpo/residual": 0.1244589239358902,
|
|
"ronpo/residual_abs": 0.1808883100748062,
|
|
"ronpo/target": 0.012566402554512024,
|
|
"ronpo/target_abs": 0.08073397725820541,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36853092908859253,
|
|
"drift/rejected_abs": 0.15083953738212585,
|
|
"epoch": 0.15574096146205801,
|
|
"grad_norm": 320.0,
|
|
"learning_rate": 3.837258586402711e-08,
|
|
"logits/chosen": -2.1028637886047363,
|
|
"logits/rejected": -2.0833709239959717,
|
|
"logps/chosen": -0.2817224860191345,
|
|
"logps/rejected": -0.2832174301147461,
|
|
"loss": 0.6819409132003784,
|
|
"loss/core": 0.6572893857955933,
|
|
"loss/preference_sft": 0.2822495102882385,
|
|
"loss/reference_anchor": 0.6761033535003662,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.6840882301330566,
|
|
"rewards/margins": 2.1773343086242676,
|
|
"rewards/rejected": 1.50675368309021,
|
|
"ronpo/logit": 0.2177334576845169,
|
|
"ronpo/residual": 0.2175024449825287,
|
|
"ronpo/residual_abs": 0.26726433634757996,
|
|
"ronpo/target": 0.00023099928512237966,
|
|
"ronpo/target_abs": 0.07836781442165375,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22081363201141357,
|
|
"drift/rejected_abs": 0.18465587496757507,
|
|
"epoch": 0.1573301549463647,
|
|
"grad_norm": 378.0,
|
|
"learning_rate": 3.7645443739684514e-08,
|
|
"logits/chosen": -2.126253843307495,
|
|
"logits/rejected": -2.0972957611083984,
|
|
"logps/chosen": -0.2862113416194916,
|
|
"logps/rejected": -0.2739894986152649,
|
|
"loss": 1.195561170578003,
|
|
"loss/core": 1.1711348295211792,
|
|
"loss/preference_sft": 0.2842695713043213,
|
|
"loss/reference_anchor": 0.6694715619087219,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2055563926696777,
|
|
"rewards/margins": 0.3634433150291443,
|
|
"rewards/rejected": 1.8421132564544678,
|
|
"ronpo/logit": 0.03634430468082428,
|
|
"ronpo/residual": 0.03679324686527252,
|
|
"ronpo/residual_abs": 0.3129720687866211,
|
|
"ronpo/target": -0.00044894107850268483,
|
|
"ronpo/target_abs": 0.0857948586344719,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2433653324842453,
|
|
"drift/rejected_abs": 0.14672759175300598,
|
|
"epoch": 0.15891934843067143,
|
|
"grad_norm": 3.578125,
|
|
"learning_rate": 3.6918246919899624e-08,
|
|
"logits/chosen": -2.231562376022339,
|
|
"logits/rejected": -2.089289903640747,
|
|
"logps/chosen": -0.2816031575202942,
|
|
"logps/rejected": -0.2773313522338867,
|
|
"loss": 0.23030047118663788,
|
|
"loss/core": 0.21950602531433105,
|
|
"loss/preference_sft": 0.27701348066329956,
|
|
"loss/reference_anchor": 0.28071126341819763,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4336533546447754,
|
|
"rewards/margins": 0.9874534606933594,
|
|
"rewards/rejected": 1.446199893951416,
|
|
"ronpo/logit": 0.09874534606933594,
|
|
"ronpo/residual": 0.10226426273584366,
|
|
"ronpo/residual_abs": 0.20653577148914337,
|
|
"ronpo/target": -0.0035189155023545027,
|
|
"ronpo/target_abs": 0.07701299339532852,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16075539588928223,
|
|
"drift/rejected_abs": 0.1068691611289978,
|
|
"epoch": 0.16050854191497815,
|
|
"grad_norm": 4.40625,
|
|
"learning_rate": 3.619126887365622e-08,
|
|
"logits/chosen": -2.2844841480255127,
|
|
"logits/rejected": -2.2047648429870605,
|
|
"logps/chosen": -0.2954888343811035,
|
|
"logps/rejected": -0.280442476272583,
|
|
"loss": 0.028839338570833206,
|
|
"loss/core": 0.025353241711854935,
|
|
"loss/preference_sft": 0.29189762473106384,
|
|
"loss/reference_anchor": 0.0704130083322525,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6047475337982178,
|
|
"rewards/margins": 0.536501944065094,
|
|
"rewards/rejected": 1.0682456493377686,
|
|
"ronpo/logit": 0.05365020036697388,
|
|
"ronpo/residual": 0.03312177211046219,
|
|
"ronpo/residual_abs": 0.1087198257446289,
|
|
"ronpo/target": 0.020528428256511688,
|
|
"ronpo/target_abs": 0.0795794278383255,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3148565888404846,
|
|
"drift/rejected_abs": 0.16070185601711273,
|
|
"epoch": 0.16209773539928488,
|
|
"grad_norm": 2.609375,
|
|
"learning_rate": 3.5464782987666235e-08,
|
|
"logits/chosen": -1.940596342086792,
|
|
"logits/rejected": -1.9633086919784546,
|
|
"logps/chosen": -0.26210013031959534,
|
|
"logps/rejected": -0.26328200101852417,
|
|
"loss": 0.426157146692276,
|
|
"loss/core": 0.40358486771583557,
|
|
"loss/preference_sft": 0.263058602809906,
|
|
"loss/reference_anchor": 0.6186167597770691,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1481029987335205,
|
|
"rewards/margins": 1.5438060760498047,
|
|
"rewards/rejected": 1.6042969226837158,
|
|
"ronpo/logit": 0.15438061952590942,
|
|
"ronpo/residual": 0.15045881271362305,
|
|
"ronpo/residual_abs": 0.1978466957807541,
|
|
"ronpo/target": 0.00392179936170578,
|
|
"ronpo/target_abs": 0.06807820498943329,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2575669288635254,
|
|
"drift/rejected_abs": 0.15305769443511963,
|
|
"epoch": 0.16368692888359157,
|
|
"grad_norm": 5.75,
|
|
"learning_rate": 3.473906246356032e-08,
|
|
"logits/chosen": -2.141854763031006,
|
|
"logits/rejected": -2.2106332778930664,
|
|
"logps/chosen": -0.2897399663925171,
|
|
"logps/rejected": -0.27617400884628296,
|
|
"loss": 0.3367363512516022,
|
|
"loss/core": 0.32625168561935425,
|
|
"loss/preference_sft": 0.28051096200942993,
|
|
"loss/reference_anchor": 0.2715080976486206,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.574777126312256,
|
|
"rewards/margins": 1.0462206602096558,
|
|
"rewards/rejected": 1.528556227684021,
|
|
"ronpo/logit": 0.10462206602096558,
|
|
"ronpo/residual": 0.09638858586549759,
|
|
"ronpo/residual_abs": 0.214156836271286,
|
|
"ronpo/target": 0.008233482949435711,
|
|
"ronpo/target_abs": 0.07465226948261261,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21658673882484436,
|
|
"drift/rejected_abs": 0.10446375608444214,
|
|
"epoch": 0.1652761223678983,
|
|
"grad_norm": 2.0625,
|
|
"learning_rate": 3.401438021514779e-08,
|
|
"logits/chosen": -2.1490402221679688,
|
|
"logits/rejected": -2.1608524322509766,
|
|
"logps/chosen": -0.29304903745651245,
|
|
"logps/rejected": -0.3020794689655304,
|
|
"loss": 0.2509724795818329,
|
|
"loss/core": 0.23218867182731628,
|
|
"loss/preference_sft": 0.2949676513671875,
|
|
"loss/reference_anchor": 0.5071809887886047,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.165867328643799,
|
|
"rewards/margins": 1.1303069591522217,
|
|
"rewards/rejected": 1.0355603694915771,
|
|
"ronpo/logit": 0.11303069442510605,
|
|
"ronpo/residual": 0.10117007791996002,
|
|
"ronpo/residual_abs": 0.1607782542705536,
|
|
"ronpo/target": 0.011860623024404049,
|
|
"ronpo/target_abs": 0.08329831808805466,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2207900583744049,
|
|
"drift/rejected_abs": 0.06476399302482605,
|
|
"epoch": 0.16686531585220502,
|
|
"grad_norm": 7.625,
|
|
"learning_rate": 3.329100876578495e-08,
|
|
"logits/chosen": -1.9039376974105835,
|
|
"logits/rejected": -2.0480172634124756,
|
|
"logps/chosen": -0.3111041188240051,
|
|
"logps/rejected": -0.3204375207424164,
|
|
"loss": 0.27312102913856506,
|
|
"loss/core": 0.2654520869255066,
|
|
"loss/preference_sft": 0.32211512327194214,
|
|
"loss/reference_anchor": 0.18690235912799835,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2061574459075928,
|
|
"rewards/margins": 1.5601660013198853,
|
|
"rewards/rejected": 0.6459915637969971,
|
|
"ronpo/logit": 0.15601658821105957,
|
|
"ronpo/residual": 0.16684645414352417,
|
|
"ronpo/residual_abs": 0.20652273297309875,
|
|
"ronpo/target": -0.010829855687916279,
|
|
"ronpo/target_abs": 0.08058717846870422,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1540854275226593,
|
|
"drift/rejected_abs": 0.06486465036869049,
|
|
"epoch": 0.1684545093365117,
|
|
"grad_norm": 1.3046875,
|
|
"learning_rate": 3.256922014589015e-08,
|
|
"logits/chosen": -2.324816942214966,
|
|
"logits/rejected": -2.319702625274658,
|
|
"logps/chosen": -0.26595860719680786,
|
|
"logps/rejected": -0.26108628511428833,
|
|
"loss": 0.06081946939229965,
|
|
"loss/core": 0.05783188343048096,
|
|
"loss/preference_sft": 0.26668626070022583,
|
|
"loss/reference_anchor": 0.05869119241833687,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 1.5408542156219482,
|
|
"rewards/margins": 0.8943354487419128,
|
|
"rewards/rejected": 0.6465188264846802,
|
|
"ronpo/logit": 0.08943354338407516,
|
|
"ronpo/residual": 0.09258811175823212,
|
|
"ronpo/residual_abs": 0.13264910876750946,
|
|
"ronpo/target": -0.003154561622068286,
|
|
"ronpo/target_abs": 0.0733359232544899,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3481886386871338,
|
|
"drift/rejected_abs": 0.14496779441833496,
|
|
"epoch": 0.17004370282081843,
|
|
"grad_norm": 2.40625,
|
|
"learning_rate": 3.184928579064438e-08,
|
|
"logits/chosen": -2.240485429763794,
|
|
"logits/rejected": -2.2552919387817383,
|
|
"logps/chosen": -0.3145575523376465,
|
|
"logps/rejected": -0.3206997513771057,
|
|
"loss": 1.516591191291809,
|
|
"loss/core": 1.477942705154419,
|
|
"loss/preference_sft": 0.31602850556373596,
|
|
"loss/reference_anchor": 1.0726332664489746,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.4811675548553467,
|
|
"rewards/margins": 2.195894241333008,
|
|
"rewards/rejected": 1.2852733135223389,
|
|
"ronpo/logit": 0.21958939731121063,
|
|
"ronpo/residual": 0.23421013355255127,
|
|
"ronpo/residual_abs": 0.340684711933136,
|
|
"ronpo/target": -0.014620688743889332,
|
|
"ronpo/target_abs": 0.08300719410181046,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2462986409664154,
|
|
"drift/rejected_abs": 0.10847475379705429,
|
|
"epoch": 0.17163289630512515,
|
|
"grad_norm": 3.234375,
|
|
"learning_rate": 3.1131476437915473e-08,
|
|
"logits/chosen": -2.202176570892334,
|
|
"logits/rejected": -2.2374606132507324,
|
|
"logps/chosen": -0.27796033024787903,
|
|
"logps/rejected": -0.2893964350223541,
|
|
"loss": 0.22307059168815613,
|
|
"loss/core": 0.20895227789878845,
|
|
"loss/preference_sft": 0.28652340173721313,
|
|
"loss/reference_anchor": 0.3747285306453705,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.462252140045166,
|
|
"rewards/margins": 1.377599835395813,
|
|
"rewards/rejected": 1.084652304649353,
|
|
"ronpo/logit": 0.1377600133419037,
|
|
"ronpo/residual": 0.13737379014492035,
|
|
"ronpo/residual_abs": 0.19108590483665466,
|
|
"ronpo/target": 0.0003861982258968055,
|
|
"ronpo/target_abs": 0.06714820116758347,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.39844200015068054,
|
|
"drift/rejected_abs": 0.2459792196750641,
|
|
"epoch": 0.17322208978943188,
|
|
"grad_norm": 5.1875,
|
|
"learning_rate": 3.0416062026444745e-08,
|
|
"logits/chosen": -1.9908326864242554,
|
|
"logits/rejected": -2.015047550201416,
|
|
"logps/chosen": -0.26075753569602966,
|
|
"logps/rejected": -0.27910637855529785,
|
|
"loss": 0.45975977182388306,
|
|
"loss/core": 0.42998823523521423,
|
|
"loss/preference_sft": 0.2777344584465027,
|
|
"loss/reference_anchor": 0.8228422999382019,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.9832959175109863,
|
|
"rewards/margins": 1.527392029762268,
|
|
"rewards/rejected": 2.455904006958008,
|
|
"ronpo/logit": 0.15273919701576233,
|
|
"ronpo/residual": 0.1590009331703186,
|
|
"ronpo/residual_abs": 0.25908511877059937,
|
|
"ronpo/target": -0.006261733826249838,
|
|
"ronpo/target_abs": 0.08037153631448746,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2750501334667206,
|
|
"drift/rejected_abs": 0.13786940276622772,
|
|
"epoch": 0.17481128327373857,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 2.970331159433403e-08,
|
|
"logits/chosen": -2.1441848278045654,
|
|
"logits/rejected": -2.100531578063965,
|
|
"logps/chosen": -0.25560611486434937,
|
|
"logps/rejected": -0.26517319679260254,
|
|
"loss": 0.3065684139728546,
|
|
"loss/core": 0.2939642667770386,
|
|
"loss/preference_sft": 0.2637214660644531,
|
|
"loss/reference_anchor": 0.3337470293045044,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.750082492828369,
|
|
"rewards/margins": 1.3717514276504517,
|
|
"rewards/rejected": 1.378330945968628,
|
|
"ronpo/logit": 0.13717512786388397,
|
|
"ronpo/residual": 0.16072766482830048,
|
|
"ronpo/residual_abs": 0.20835892856121063,
|
|
"ronpo/target": -0.023552536964416504,
|
|
"ronpo/target_abs": 0.08319397270679474,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12603361904621124,
|
|
"drift/rejected_abs": 0.05538574978709221,
|
|
"epoch": 0.1764004767580453,
|
|
"grad_norm": 0.87109375,
|
|
"learning_rate": 2.8993493177871495e-08,
|
|
"logits/chosen": -2.215672016143799,
|
|
"logits/rejected": -2.2594668865203857,
|
|
"logps/chosen": -0.31073468923568726,
|
|
"logps/rejected": -0.30866894125938416,
|
|
"loss": 0.06362307071685791,
|
|
"loss/core": 0.05991923063993454,
|
|
"loss/preference_sft": 0.3092317581176758,
|
|
"loss/reference_anchor": 0.07490097731351852,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.2597297430038452,
|
|
"rewards/margins": 0.7066539525985718,
|
|
"rewards/rejected": 0.5530757308006287,
|
|
"ronpo/logit": 0.0706654042005539,
|
|
"ronpo/residual": 0.05782570317387581,
|
|
"ronpo/residual_abs": 0.1277032196521759,
|
|
"ronpo/target": 0.012839692644774914,
|
|
"ronpo/target_abs": 0.08341912925243378,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26530271768569946,
|
|
"drift/rejected_abs": 0.12235991656780243,
|
|
"epoch": 0.17798967024235202,
|
|
"grad_norm": 17.25,
|
|
"learning_rate": 2.8286873710734132e-08,
|
|
"logits/chosen": -2.085667133331299,
|
|
"logits/rejected": -2.0912811756134033,
|
|
"logps/chosen": -0.27223190665245056,
|
|
"logps/rejected": -0.2789652347564697,
|
|
"loss": 0.25534918904304504,
|
|
"loss/core": 0.24563512206077576,
|
|
"loss/preference_sft": 0.2783053517341614,
|
|
"loss/reference_anchor": 0.24971409142017365,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.650958299636841,
|
|
"rewards/margins": 1.4280980825424194,
|
|
"rewards/rejected": 1.2228600978851318,
|
|
"ronpo/logit": 0.14280982315540314,
|
|
"ronpo/residual": 0.14116224646568298,
|
|
"ronpo/residual_abs": 0.1777869462966919,
|
|
"ronpo/target": 0.0016475621378049254,
|
|
"ronpo/target_abs": 0.06457264721393585,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2612389326095581,
|
|
"drift/rejected_abs": 0.11688123643398285,
|
|
"epoch": 0.1795788637266587,
|
|
"grad_norm": 30.75,
|
|
"learning_rate": 2.7583718923604928e-08,
|
|
"logits/chosen": -2.042419910430908,
|
|
"logits/rejected": -2.0682761669158936,
|
|
"logps/chosen": -0.33960863947868347,
|
|
"logps/rejected": -0.3336796164512634,
|
|
"loss": 0.359559029340744,
|
|
"loss/core": 0.34349602460861206,
|
|
"loss/preference_sft": 0.34128159284591675,
|
|
"loss/reference_anchor": 0.4248151183128357,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6094517707824707,
|
|
"rewards/margins": 1.4438995122909546,
|
|
"rewards/rejected": 1.1655519008636475,
|
|
"ronpo/logit": 0.14438995718955994,
|
|
"ronpo/residual": 0.13987357914447784,
|
|
"ronpo/residual_abs": 0.19552505016326904,
|
|
"ronpo/target": 0.004516408313065767,
|
|
"ronpo/target_abs": 0.07334297150373459,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4135201871395111,
|
|
"drift/rejected_abs": 0.09967617690563202,
|
|
"epoch": 0.18116805721096543,
|
|
"grad_norm": 7.78125,
|
|
"learning_rate": 2.688429324424236e-08,
|
|
"logits/chosen": -2.1476290225982666,
|
|
"logits/rejected": -2.1181139945983887,
|
|
"logps/chosen": -0.27395740151405334,
|
|
"logps/rejected": -0.2648341655731201,
|
|
"loss": 1.3773685693740845,
|
|
"loss/core": 1.3479124307632446,
|
|
"loss/preference_sft": 0.27103132009506226,
|
|
"loss/reference_anchor": 0.8145006895065308,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.134523868560791,
|
|
"rewards/margins": 3.1398844718933105,
|
|
"rewards/rejected": 0.99463951587677,
|
|
"ronpo/logit": 0.31398844718933105,
|
|
"ronpo/residual": 0.30398809909820557,
|
|
"ronpo/residual_abs": 0.3727145493030548,
|
|
"ronpo/target": 0.010000362992286682,
|
|
"ronpo/target_abs": 0.09326197952032089,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19183699786663055,
|
|
"drift/rejected_abs": 0.09668353945016861,
|
|
"epoch": 0.18275725069527216,
|
|
"grad_norm": 3.484375,
|
|
"learning_rate": 2.6188859698040042e-08,
|
|
"logits/chosen": -2.0081419944763184,
|
|
"logits/rejected": -2.0951578617095947,
|
|
"logps/chosen": -0.2836478650569916,
|
|
"logps/rejected": -0.27917319536209106,
|
|
"loss": 0.0594443753361702,
|
|
"loss/core": 0.05546221882104874,
|
|
"loss/preference_sft": 0.2888682782649994,
|
|
"loss/reference_anchor": 0.08488909900188446,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9143584966659546,
|
|
"rewards/margins": 0.9508224725723267,
|
|
"rewards/rejected": 0.9635359048843384,
|
|
"ronpo/logit": 0.09508226811885834,
|
|
"ronpo/residual": 0.10423971712589264,
|
|
"ronpo/residual_abs": 0.1455569565296173,
|
|
"ronpo/target": -0.009157449007034302,
|
|
"ronpo/target_abs": 0.07435912638902664,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3637532591819763,
|
|
"drift/rejected_abs": 0.16664664447307587,
|
|
"epoch": 0.18434644417957885,
|
|
"grad_norm": 1.4453125,
|
|
"learning_rate": 2.5497679809113563e-08,
|
|
"logits/chosen": -2.001953125,
|
|
"logits/rejected": -2.0324103832244873,
|
|
"logps/chosen": -0.2893816828727722,
|
|
"logps/rejected": -0.2681158483028412,
|
|
"loss": 0.548211395740509,
|
|
"loss/core": 0.5249797105789185,
|
|
"loss/preference_sft": 0.28467053174972534,
|
|
"loss/reference_anchor": 0.6352949738502502,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6368331909179688,
|
|
"rewards/margins": 1.9708592891693115,
|
|
"rewards/rejected": 1.6659743785858154,
|
|
"ronpo/logit": 0.1970859318971634,
|
|
"ronpo/residual": 0.1857091784477234,
|
|
"ronpo/residual_abs": 0.2529819905757904,
|
|
"ronpo/target": 0.011376758106052876,
|
|
"ronpo/target_abs": 0.06672044098377228,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23147885501384735,
|
|
"drift/rejected_abs": 0.09522049129009247,
|
|
"epoch": 0.18593563766388557,
|
|
"grad_norm": 1.8203125,
|
|
"learning_rate": 2.481101350195196e-08,
|
|
"logits/chosen": -1.9583909511566162,
|
|
"logits/rejected": -1.982860803604126,
|
|
"logps/chosen": -0.28692710399627686,
|
|
"logps/rejected": -0.2864198684692383,
|
|
"loss": 0.15366187691688538,
|
|
"loss/core": 0.14675399661064148,
|
|
"loss/preference_sft": 0.2956783175468445,
|
|
"loss/reference_anchor": 0.16780035197734833,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.313133955001831,
|
|
"rewards/margins": 1.362217903137207,
|
|
"rewards/rejected": 0.9509159922599792,
|
|
"ronpo/logit": 0.13622179627418518,
|
|
"ronpo/residual": 0.13147194683551788,
|
|
"ronpo/residual_abs": 0.17332948744297028,
|
|
"ronpo/target": 0.004749844316393137,
|
|
"ronpo/target_abs": 0.06973059475421906,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.259533554315567,
|
|
"drift/rejected_abs": 0.12255913019180298,
|
|
"epoch": 0.1875248311481923,
|
|
"grad_norm": 1.3671875,
|
|
"learning_rate": 2.4129119003670752e-08,
|
|
"logits/chosen": -2.1177847385406494,
|
|
"logits/rejected": -2.0852158069610596,
|
|
"logps/chosen": -0.3071213364601135,
|
|
"logps/rejected": -0.31466570496559143,
|
|
"loss": 0.545588493347168,
|
|
"loss/core": 0.5284724831581116,
|
|
"loss/preference_sft": 0.315735787153244,
|
|
"loss/reference_anchor": 0.4574582576751709,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.592897891998291,
|
|
"rewards/margins": 1.3681689500808716,
|
|
"rewards/rejected": 1.2247288227081299,
|
|
"ronpo/logit": 0.13681688904762268,
|
|
"ronpo/residual": 0.14628082513809204,
|
|
"ronpo/residual_abs": 0.19750835001468658,
|
|
"ronpo/target": -0.009463920257985592,
|
|
"ronpo/target_abs": 0.07016940414905548,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23209957778453827,
|
|
"drift/rejected_abs": 0.11176522821187973,
|
|
"epoch": 0.18911402463249902,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 2.3452252746903295e-08,
|
|
"logits/chosen": -1.974440336227417,
|
|
"logits/rejected": -1.9321467876434326,
|
|
"logps/chosen": -0.28932034969329834,
|
|
"logps/rejected": -0.29376018047332764,
|
|
"loss": 0.1980138123035431,
|
|
"loss/core": 0.19030292332172394,
|
|
"loss/preference_sft": 0.29327264428138733,
|
|
"loss/reference_anchor": 0.19098426401615143,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3200583457946777,
|
|
"rewards/margins": 1.2056306600570679,
|
|
"rewards/rejected": 1.1144278049468994,
|
|
"ronpo/logit": 0.12056305259466171,
|
|
"ronpo/residual": 0.10401593148708344,
|
|
"ronpo/residual_abs": 0.1900263875722885,
|
|
"ronpo/target": 0.016547124832868576,
|
|
"ronpo/target_abs": 0.08112061023712158,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2035134732723236,
|
|
"drift/rejected_abs": 0.1420602947473526,
|
|
"epoch": 0.1907032181168057,
|
|
"grad_norm": 147.0,
|
|
"learning_rate": 2.278066927336698e-08,
|
|
"logits/chosen": -2.131030797958374,
|
|
"logits/rejected": -2.104788303375244,
|
|
"logps/chosen": -0.29952484369277954,
|
|
"logps/rejected": -0.320384681224823,
|
|
"loss": 0.45714807510375977,
|
|
"loss/core": 0.4465022087097168,
|
|
"loss/preference_sft": 0.3080054819583893,
|
|
"loss/reference_anchor": 0.2733660936355591,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0339159965515137,
|
|
"rewards/margins": 0.6155134439468384,
|
|
"rewards/rejected": 1.4184025526046753,
|
|
"ronpo/logit": 0.06155133247375488,
|
|
"ronpo/residual": 0.06355226784944534,
|
|
"ronpo/residual_abs": 0.23413407802581787,
|
|
"ronpo/target": -0.0020009521394968033,
|
|
"ronpo/target_abs": 0.06787214428186417,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34611085057258606,
|
|
"drift/rejected_abs": 0.15783950686454773,
|
|
"epoch": 0.19229241160111243,
|
|
"grad_norm": 1.2890625,
|
|
"learning_rate": 2.2114621138140448e-08,
|
|
"logits/chosen": -2.026627540588379,
|
|
"logits/rejected": -1.951611876487732,
|
|
"logps/chosen": -0.283507376909256,
|
|
"logps/rejected": -0.27926164865493774,
|
|
"loss": 0.4765016436576843,
|
|
"loss/core": 0.45531606674194336,
|
|
"loss/preference_sft": 0.2730947434902191,
|
|
"loss/reference_anchor": 0.5779938101768494,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.452455520629883,
|
|
"rewards/margins": 1.884347915649414,
|
|
"rewards/rejected": 1.5681076049804688,
|
|
"ronpo/logit": 0.18843480944633484,
|
|
"ronpo/residual": 0.18703269958496094,
|
|
"ronpo/residual_abs": 0.2455645054578781,
|
|
"ronpo/target": 0.0014021346578374505,
|
|
"ronpo/target_abs": 0.0817280188202858,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3367292284965515,
|
|
"drift/rejected_abs": 0.152937650680542,
|
|
"epoch": 0.19388160508541916,
|
|
"grad_norm": 95.5,
|
|
"learning_rate": 2.145435881468797e-08,
|
|
"logits/chosen": -2.018470525741577,
|
|
"logits/rejected": -1.968064308166504,
|
|
"logps/chosen": -0.3080677092075348,
|
|
"logps/rejected": -0.29313474893569946,
|
|
"loss": 0.3543425500392914,
|
|
"loss/core": 0.3362095355987549,
|
|
"loss/preference_sft": 0.31406843662261963,
|
|
"loss/reference_anchor": 0.4866800904273987,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.365661144256592,
|
|
"rewards/margins": 1.8380382061004639,
|
|
"rewards/rejected": 1.527622938156128,
|
|
"ronpo/logit": 0.18380382657051086,
|
|
"ronpo/residual": 0.18959875404834747,
|
|
"ronpo/residual_abs": 0.23765261471271515,
|
|
"ronpo/target": -0.005794919561594725,
|
|
"ronpo/target_abs": 0.08199281990528107,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25560295581817627,
|
|
"drift/rejected_abs": 0.11357559263706207,
|
|
"epoch": 0.19547079856972585,
|
|
"grad_norm": 125.0,
|
|
"learning_rate": 2.0800130600666648e-08,
|
|
"logits/chosen": -2.1990818977355957,
|
|
"logits/rejected": -2.159752368927002,
|
|
"logps/chosen": -0.28720757365226746,
|
|
"logps/rejected": -0.28762027621269226,
|
|
"loss": 0.293748140335083,
|
|
"loss/core": 0.28089436888694763,
|
|
"loss/preference_sft": 0.293092280626297,
|
|
"loss/reference_anchor": 0.33794093132019043,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.5560295581817627,
|
|
"rewards/margins": 1.4323694705963135,
|
|
"rewards/rejected": 1.1236598491668701,
|
|
"ronpo/logit": 0.1432369500398636,
|
|
"ronpo/residual": 0.14470703899860382,
|
|
"ronpo/residual_abs": 0.18086595833301544,
|
|
"ronpo/target": -0.0014700808096677065,
|
|
"ronpo/target_abs": 0.06537780165672302,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22184021770954132,
|
|
"drift/rejected_abs": 0.1088438630104065,
|
|
"epoch": 0.19705999205403257,
|
|
"grad_norm": 13.875,
|
|
"learning_rate": 2.0152182524551813e-08,
|
|
"logits/chosen": -2.0863685607910156,
|
|
"logits/rejected": -2.0217607021331787,
|
|
"logps/chosen": -0.29196107387542725,
|
|
"logps/rejected": -0.2876456379890442,
|
|
"loss": 0.10439170897006989,
|
|
"loss/core": 0.0983448475599289,
|
|
"loss/preference_sft": 0.2921106219291687,
|
|
"loss/reference_anchor": 0.1435559093952179,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2182328701019287,
|
|
"rewards/margins": 1.1298673152923584,
|
|
"rewards/rejected": 1.0883656740188599,
|
|
"ronpo/logit": 0.1129867285490036,
|
|
"ronpo/residual": 0.11576826870441437,
|
|
"ronpo/residual_abs": 0.16257163882255554,
|
|
"ronpo/target": -0.0027815259527415037,
|
|
"ronpo/target_abs": 0.06601886451244354,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24640531837940216,
|
|
"drift/rejected_abs": 0.09290675818920135,
|
|
"epoch": 0.1986491855383393,
|
|
"grad_norm": 45.25,
|
|
"learning_rate": 1.951075825311578e-08,
|
|
"logits/chosen": -2.0369296073913574,
|
|
"logits/rejected": -2.0407700538635254,
|
|
"logps/chosen": -0.2959621846675873,
|
|
"logps/rejected": -0.29841041564941406,
|
|
"loss": 0.3487691283226013,
|
|
"loss/core": 0.33975881338119507,
|
|
"loss/preference_sft": 0.2982815206050873,
|
|
"loss/reference_anchor": 0.22760829329490662,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4635446071624756,
|
|
"rewards/margins": 1.5408732891082764,
|
|
"rewards/rejected": 0.9226716160774231,
|
|
"ronpo/logit": 0.15408733487129211,
|
|
"ronpo/residual": 0.17024736106395721,
|
|
"ronpo/residual_abs": 0.2152116596698761,
|
|
"ronpo/target": -0.016160037368535995,
|
|
"ronpo/target_abs": 0.07987652719020844,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23432044684886932,
|
|
"drift/rejected_abs": 0.08939666301012039,
|
|
"epoch": 0.20023837902264602,
|
|
"grad_norm": 2.875,
|
|
"learning_rate": 1.887609899979465e-08,
|
|
"logits/chosen": -2.3256447315216064,
|
|
"logits/rejected": -2.416283130645752,
|
|
"logps/chosen": -0.2670096158981323,
|
|
"logps/rejected": -0.2636256814002991,
|
|
"loss": 0.3040315806865692,
|
|
"loss/core": 0.29226741194725037,
|
|
"loss/preference_sft": 0.2660647928714752,
|
|
"loss/reference_anchor": 0.30951163172721863,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.3428587913513184,
|
|
"rewards/margins": 1.4511168003082275,
|
|
"rewards/rejected": 0.891741931438446,
|
|
"ronpo/logit": 0.14511170983314514,
|
|
"ronpo/residual": 0.1341066062450409,
|
|
"ronpo/residual_abs": 0.19294190406799316,
|
|
"ronpo/target": 0.011005090549588203,
|
|
"ronpo/target_abs": 0.06993237882852554,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2895382344722748,
|
|
"drift/rejected_abs": 0.10728726536035538,
|
|
"epoch": 0.2018275725069527,
|
|
"grad_norm": 16.375,
|
|
"learning_rate": 1.824844343397797e-08,
|
|
"logits/chosen": -2.1119954586029053,
|
|
"logits/rejected": -2.150533676147461,
|
|
"logps/chosen": -0.2655460238456726,
|
|
"logps/rejected": -0.2733895182609558,
|
|
"loss": 0.38659775257110596,
|
|
"loss/core": 0.37493616342544556,
|
|
"loss/preference_sft": 0.2679237723350525,
|
|
"loss/reference_anchor": 0.30639657378196716,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.8941664695739746,
|
|
"rewards/margins": 1.8319135904312134,
|
|
"rewards/rejected": 1.0622529983520508,
|
|
"ronpo/logit": 0.18319137394428253,
|
|
"ronpo/residual": 0.1691356599330902,
|
|
"ronpo/residual_abs": 0.2294747531414032,
|
|
"ronpo/target": 0.014055708423256874,
|
|
"ronpo/target_abs": 0.07849180698394775,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32430991530418396,
|
|
"drift/rejected_abs": 0.09809484332799911,
|
|
"epoch": 0.20341676599125944,
|
|
"grad_norm": 1.703125,
|
|
"learning_rate": 1.762802759125482e-08,
|
|
"logits/chosen": -2.017073154449463,
|
|
"logits/rejected": -2.0082268714904785,
|
|
"logps/chosen": -0.31356340646743774,
|
|
"logps/rejected": -0.31727951765060425,
|
|
"loss": 0.7281572818756104,
|
|
"loss/core": 0.707539439201355,
|
|
"loss/preference_sft": 0.3098931312561035,
|
|
"loss/reference_anchor": 0.5580906867980957,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.2425026893615723,
|
|
"rewards/margins": 2.297203779220581,
|
|
"rewards/rejected": 0.9452990293502808,
|
|
"ronpo/logit": 0.22972038388252258,
|
|
"ronpo/residual": 0.2376583367586136,
|
|
"ronpo/residual_abs": 0.27416878938674927,
|
|
"ronpo/target": -0.007937933318316936,
|
|
"ronpo/target_abs": 0.07765815407037735,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26535215973854065,
|
|
"drift/rejected_abs": 0.11467768251895905,
|
|
"epoch": 0.20500595947556616,
|
|
"grad_norm": 7.9375,
|
|
"learning_rate": 1.7015084784650487e-08,
|
|
"logits/chosen": -2.123807430267334,
|
|
"logits/rejected": -2.129242181777954,
|
|
"logps/chosen": -0.3087415099143982,
|
|
"logps/rejected": -0.2945675551891327,
|
|
"loss": 0.3146616816520691,
|
|
"loss/core": 0.30256468057632446,
|
|
"loss/preference_sft": 0.3151400685310364,
|
|
"loss/reference_anchor": 0.31411463022232056,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.6520421504974365,
|
|
"rewards/margins": 1.5085344314575195,
|
|
"rewards/rejected": 1.143507957458496,
|
|
"ronpo/logit": 0.15085344016551971,
|
|
"ronpo/residual": 0.14317695796489716,
|
|
"ronpo/residual_abs": 0.20519272983074188,
|
|
"ronpo/target": 0.007676492445170879,
|
|
"ronpo/target_abs": 0.08047398179769516,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15140828490257263,
|
|
"drift/rejected_abs": 0.09236256033182144,
|
|
"epoch": 0.20659515295987285,
|
|
"grad_norm": 1.84375,
|
|
"learning_rate": 1.6409845516887003e-08,
|
|
"logits/chosen": -2.207329273223877,
|
|
"logits/rejected": -2.135587215423584,
|
|
"logps/chosen": -0.3012005686759949,
|
|
"logps/rejected": -0.29473382234573364,
|
|
"loss": 0.0722493976354599,
|
|
"loss/core": 0.06875967979431152,
|
|
"loss/preference_sft": 0.3042813539505005,
|
|
"loss/reference_anchor": 0.06927807629108429,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5135061740875244,
|
|
"rewards/margins": 0.6719859838485718,
|
|
"rewards/rejected": 0.8415201902389526,
|
|
"ronpo/logit": 0.06719859689474106,
|
|
"ronpo/residual": 0.0728544145822525,
|
|
"ronpo/residual_abs": 0.12853005528450012,
|
|
"ronpo/target": -0.005655817687511444,
|
|
"ronpo/target_abs": 0.07488958537578583,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20458777248859406,
|
|
"drift/rejected_abs": 0.06509005278348923,
|
|
"epoch": 0.20818434644417957,
|
|
"grad_norm": 172.0,
|
|
"learning_rate": 1.5812537393700506e-08,
|
|
"logits/chosen": -2.2815442085266113,
|
|
"logits/rejected": -2.287656784057617,
|
|
"logps/chosen": -0.28355464339256287,
|
|
"logps/rejected": -0.29816484451293945,
|
|
"loss": 0.39056888222694397,
|
|
"loss/core": 0.3819330930709839,
|
|
"loss/preference_sft": 0.2903548777103424,
|
|
"loss/reference_anchor": 0.2177002727985382,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0455703735351562,
|
|
"rewards/margins": 1.397626519203186,
|
|
"rewards/rejected": 0.6479436755180359,
|
|
"ronpo/logit": 0.13976263999938965,
|
|
"ronpo/residual": 0.16489243507385254,
|
|
"ronpo/residual_abs": 0.20165614783763885,
|
|
"ronpo/target": -0.025129785761237144,
|
|
"ronpo/target_abs": 0.08682605624198914,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17608028650283813,
|
|
"drift/rejected_abs": 0.07316999137401581,
|
|
"epoch": 0.2097735399284863,
|
|
"grad_norm": 8.0,
|
|
"learning_rate": 1.522338503824803e-08,
|
|
"logits/chosen": -2.2558116912841797,
|
|
"logits/rejected": -2.2970988750457764,
|
|
"logps/chosen": -0.29492825269699097,
|
|
"logps/rejected": -0.29021143913269043,
|
|
"loss": 0.12977449595928192,
|
|
"loss/core": 0.1249893456697464,
|
|
"loss/preference_sft": 0.29489555954933167,
|
|
"loss/reference_anchor": 0.10722937434911728,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.7590672969818115,
|
|
"rewards/margins": 1.0307245254516602,
|
|
"rewards/rejected": 0.7283426523208618,
|
|
"ronpo/logit": 0.10307244956493378,
|
|
"ronpo/residual": 0.1016714796423912,
|
|
"ronpo/residual_abs": 0.14400455355644226,
|
|
"ronpo/target": 0.0014009776059538126,
|
|
"ronpo/target_abs": 0.06940744817256927,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2352174073457718,
|
|
"drift/rejected_abs": 0.13656437397003174,
|
|
"epoch": 0.21136273341279302,
|
|
"grad_norm": 0.79296875,
|
|
"learning_rate": 1.464261000663589e-08,
|
|
"logits/chosen": -2.079434871673584,
|
|
"logits/rejected": -2.024263858795166,
|
|
"logps/chosen": -0.2820441424846649,
|
|
"logps/rejected": -0.2857016921043396,
|
|
"loss": 0.23443233966827393,
|
|
"loss/core": 0.2236245572566986,
|
|
"loss/preference_sft": 0.2865526080131531,
|
|
"loss/reference_anchor": 0.28013843297958374,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.349414825439453,
|
|
"rewards/margins": 0.9838077425956726,
|
|
"rewards/rejected": 1.3656069040298462,
|
|
"ronpo/logit": 0.09838077425956726,
|
|
"ronpo/residual": 0.11156570911407471,
|
|
"ronpo/residual_abs": 0.17216257750988007,
|
|
"ronpo/target": -0.013184946961700916,
|
|
"ronpo/target_abs": 0.07473040372133255,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20989990234375,
|
|
"drift/rejected_abs": 0.07849745452404022,
|
|
"epoch": 0.2129519268970997,
|
|
"grad_norm": 5.40625,
|
|
"learning_rate": 1.40704307046015e-08,
|
|
"logits/chosen": -2.347775459289551,
|
|
"logits/rejected": -2.3573763370513916,
|
|
"logps/chosen": -0.2676052153110504,
|
|
"logps/rejected": -0.27207422256469727,
|
|
"loss": 0.21361777186393738,
|
|
"loss/core": 0.20699867606163025,
|
|
"loss/preference_sft": 0.26641157269477844,
|
|
"loss/reference_anchor": 0.16247610747814178,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0982730388641357,
|
|
"rewards/margins": 1.3146512508392334,
|
|
"rewards/rejected": 0.7836218476295471,
|
|
"ronpo/logit": 0.1314651221036911,
|
|
"ronpo/residual": 0.16295532882213593,
|
|
"ronpo/residual_abs": 0.18369440734386444,
|
|
"ronpo/target": -0.03149022161960602,
|
|
"ronpo/target_abs": 0.07089079916477203,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4036272466182709,
|
|
"drift/rejected_abs": 0.08043356984853745,
|
|
"epoch": 0.21454112038140644,
|
|
"grad_norm": 2.21875,
|
|
"learning_rate": 1.3507062305379941e-08,
|
|
"logits/chosen": -1.9893615245819092,
|
|
"logits/rejected": -2.1025524139404297,
|
|
"logps/chosen": -0.2764974534511566,
|
|
"logps/rejected": -0.27073296904563904,
|
|
"loss": 1.4822903871536255,
|
|
"loss/core": 1.4529348611831665,
|
|
"loss/preference_sft": 0.27298080921173096,
|
|
"loss/reference_anchor": 0.8114289045333862,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.034860134124756,
|
|
"rewards/margins": 3.2362799644470215,
|
|
"rewards/rejected": 0.7985801100730896,
|
|
"ronpo/logit": 0.3236280679702759,
|
|
"ronpo/residual": 0.3155183792114258,
|
|
"ronpo/residual_abs": 0.37980663776397705,
|
|
"ronpo/target": 0.008109642192721367,
|
|
"ronpo/target_abs": 0.08955520391464233,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19523149728775024,
|
|
"drift/rejected_abs": 0.11687342822551727,
|
|
"epoch": 0.21613031386571316,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 1.2952716668786109e-08,
|
|
"logits/chosen": -1.9566532373428345,
|
|
"logits/rejected": -1.9517863988876343,
|
|
"logps/chosen": -0.28758880496025085,
|
|
"logps/rejected": -0.28461915254592896,
|
|
"loss": 0.066526398062706,
|
|
"loss/core": 0.05917876213788986,
|
|
"loss/preference_sft": 0.28697067499160767,
|
|
"loss/reference_anchor": 0.18123537302017212,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.950860619544983,
|
|
"rewards/margins": 0.7826768159866333,
|
|
"rewards/rejected": 1.1681835651397705,
|
|
"ronpo/logit": 0.07826767861843109,
|
|
"ronpo/residual": 0.07248474657535553,
|
|
"ronpo/residual_abs": 0.13983985781669617,
|
|
"ronpo/target": 0.0057829380966722965,
|
|
"ronpo/target_abs": 0.08430159091949463,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18238039314746857,
|
|
"drift/rejected_abs": 0.09428268671035767,
|
|
"epoch": 0.21771950735001985,
|
|
"grad_norm": 3.3125,
|
|
"learning_rate": 1.2407602261542828e-08,
|
|
"logits/chosen": -1.9722627401351929,
|
|
"logits/rejected": -2.0617997646331787,
|
|
"logps/chosen": -0.2924751937389374,
|
|
"logps/rejected": -0.29996949434280396,
|
|
"loss": 0.054906345903873444,
|
|
"loss/core": 0.0511222667992115,
|
|
"loss/preference_sft": 0.30433911085128784,
|
|
"loss/reference_anchor": 0.07768283784389496,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8226171731948853,
|
|
"rewards/margins": 0.8817318677902222,
|
|
"rewards/rejected": 0.9408853650093079,
|
|
"ronpo/logit": 0.08817318081855774,
|
|
"ronpo/residual": 0.08928640186786652,
|
|
"ronpo/residual_abs": 0.133590966463089,
|
|
"ronpo/target": -0.0011132096406072378,
|
|
"ronpo/target_abs": 0.07680922746658325,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.45133644342422485,
|
|
"drift/rejected_abs": 0.18786491453647614,
|
|
"epoch": 0.21930870083432658,
|
|
"grad_norm": 434.0,
|
|
"learning_rate": 1.1871924078885106e-08,
|
|
"logits/chosen": -1.929914116859436,
|
|
"logits/rejected": -1.9743080139160156,
|
|
"logps/chosen": -0.2774960696697235,
|
|
"logps/rejected": -0.26925212144851685,
|
|
"loss": 0.8445002436637878,
|
|
"loss/core": 0.8047998547554016,
|
|
"loss/preference_sft": 0.2817479968070984,
|
|
"loss/reference_anchor": 1.1061266660690308,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.513057708740234,
|
|
"rewards/margins": 2.6355862617492676,
|
|
"rewards/rejected": 1.8774709701538086,
|
|
"ronpo/logit": 0.26355868577957153,
|
|
"ronpo/residual": 0.26055908203125,
|
|
"ronpo/residual_abs": 0.3207409679889679,
|
|
"ronpo/target": 0.002999568125233054,
|
|
"ronpo/target_abs": 0.0797749012708664,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1836492121219635,
|
|
"drift/rejected_abs": 0.10975848138332367,
|
|
"epoch": 0.2208978943186333,
|
|
"grad_norm": 12.375,
|
|
"learning_rate": 1.1345883567469826e-08,
|
|
"logits/chosen": -2.1611971855163574,
|
|
"logits/rejected": -2.147216558456421,
|
|
"logps/chosen": -0.2980535626411438,
|
|
"logps/rejected": -0.30087488889694214,
|
|
"loss": 0.11188419908285141,
|
|
"loss/core": 0.10657135397195816,
|
|
"loss/preference_sft": 0.2982610762119293,
|
|
"loss/reference_anchor": 0.1219698041677475,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8356151580810547,
|
|
"rewards/margins": 0.7388324737548828,
|
|
"rewards/rejected": 1.0967828035354614,
|
|
"ronpo/logit": 0.07388325035572052,
|
|
"ronpo/residual": 0.06399385631084442,
|
|
"ronpo/residual_abs": 0.16593526303768158,
|
|
"ronpo/target": 0.00988937821239233,
|
|
"ronpo/target_abs": 0.07012668997049332,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2231002300977707,
|
|
"drift/rejected_abs": 0.13982199132442474,
|
|
"epoch": 0.22248708780294002,
|
|
"grad_norm": 6.5625,
|
|
"learning_rate": 1.0829678549619859e-08,
|
|
"logits/chosen": -2.218724489212036,
|
|
"logits/rejected": -2.1778321266174316,
|
|
"logps/chosen": -0.32158035039901733,
|
|
"logps/rejected": -0.3115960955619812,
|
|
"loss": 0.43686145544052124,
|
|
"loss/core": 0.42180243134498596,
|
|
"loss/preference_sft": 0.31540364027023315,
|
|
"loss/reference_anchor": 0.39871746301651,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2305080890655518,
|
|
"rewards/margins": 0.8341946601867676,
|
|
"rewards/rejected": 1.3963135480880737,
|
|
"ronpo/logit": 0.08341946452856064,
|
|
"ronpo/residual": 0.08280835300683975,
|
|
"ronpo/residual_abs": 0.19750329852104187,
|
|
"ronpo/target": 0.0006111121037974954,
|
|
"ronpo/target_abs": 0.0668053850531578,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1843855381011963,
|
|
"drift/rejected_abs": 0.08389295637607574,
|
|
"epoch": 0.22407628128724671,
|
|
"grad_norm": 3.328125,
|
|
"learning_rate": 1.0323503148931167e-08,
|
|
"logits/chosen": -2.261746644973755,
|
|
"logits/rejected": -2.3672595024108887,
|
|
"logps/chosen": -0.26710596680641174,
|
|
"logps/rejected": -0.27074798941612244,
|
|
"loss": 0.08147259801626205,
|
|
"loss/core": 0.07696676254272461,
|
|
"loss/preference_sft": 0.27835264801979065,
|
|
"loss/reference_anchor": 0.10090293735265732,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8432375192642212,
|
|
"rewards/margins": 1.0078269243240356,
|
|
"rewards/rejected": 0.8354104161262512,
|
|
"ronpo/logit": 0.10078269243240356,
|
|
"ronpo/residual": 0.10459145158529282,
|
|
"ronpo/residual_abs": 0.15122047066688538,
|
|
"ronpo/target": -0.0038087614811956882,
|
|
"ronpo/target_abs": 0.08357212692499161,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2887698709964752,
|
|
"drift/rejected_abs": 0.10348562896251678,
|
|
"epoch": 0.22566547477155344,
|
|
"grad_norm": 107.5,
|
|
"learning_rate": 9.827547717270884e-09,
|
|
"logits/chosen": -2.164956569671631,
|
|
"logits/rejected": -2.177293300628662,
|
|
"logps/chosen": -0.29969245195388794,
|
|
"logps/rejected": -0.29687872529029846,
|
|
"loss": 0.6983797550201416,
|
|
"loss/core": 0.6833909749984741,
|
|
"loss/preference_sft": 0.2986063063144684,
|
|
"loss/reference_anchor": 0.3983899652957916,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.8871381282806396,
|
|
"rewards/margins": 1.9205951690673828,
|
|
"rewards/rejected": 0.9665428400039673,
|
|
"ronpo/logit": 0.19205954670906067,
|
|
"ronpo/residual": 0.18855328857898712,
|
|
"ronpo/residual_abs": 0.2506033778190613,
|
|
"ronpo/target": 0.003506250213831663,
|
|
"ronpo/target_abs": 0.06901577860116959,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1665627658367157,
|
|
"drift/rejected_abs": 0.0922744944691658,
|
|
"epoch": 0.22725466825586016,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 9.34199876319378e-09,
|
|
"logits/chosen": -2.0484366416931152,
|
|
"logits/rejected": -1.9690176248550415,
|
|
"logps/chosen": -0.3025513291358948,
|
|
"logps/rejected": -0.30018630623817444,
|
|
"loss": 0.07143449038267136,
|
|
"loss/core": 0.06748920679092407,
|
|
"loss/preference_sft": 0.3050025403499603,
|
|
"loss/reference_anchor": 0.08222213387489319,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6645787954330444,
|
|
"rewards/margins": 0.7441354990005493,
|
|
"rewards/rejected": 0.9204433560371399,
|
|
"ronpo/logit": 0.07441355288028717,
|
|
"ronpo/residual": 0.06392879039049149,
|
|
"ronpo/residual_abs": 0.13627013564109802,
|
|
"ronpo/target": 0.010484759695827961,
|
|
"ronpo/target_abs": 0.0739731639623642,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3448890745639801,
|
|
"drift/rejected_abs": 0.14354726672172546,
|
|
"epoch": 0.22884386174016685,
|
|
"grad_norm": 482.0,
|
|
"learning_rate": 8.867038881803992e-09,
|
|
"logits/chosen": -1.9485433101654053,
|
|
"logits/rejected": -2.0286898612976074,
|
|
"logps/chosen": -0.2920309901237488,
|
|
"logps/rejected": -0.2847922146320343,
|
|
"loss": 0.5865887403488159,
|
|
"loss/core": 0.5658643841743469,
|
|
"loss/preference_sft": 0.2796561121940613,
|
|
"loss/reference_anchor": 0.5641579031944275,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.4488911628723145,
|
|
"rewards/margins": 2.0442280769348145,
|
|
"rewards/rejected": 1.4046627283096313,
|
|
"ronpo/logit": 0.20442280173301697,
|
|
"ronpo/residual": 0.18873873353004456,
|
|
"ronpo/residual_abs": 0.24582239985466003,
|
|
"ronpo/target": 0.015684079378843307,
|
|
"ronpo/target_abs": 0.07453683018684387,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2959468960762024,
|
|
"drift/rejected_abs": 0.06901375949382782,
|
|
"epoch": 0.23043305522447358,
|
|
"grad_norm": 4.0625,
|
|
"learning_rate": 8.402846686088465e-09,
|
|
"logits/chosen": -2.08479642868042,
|
|
"logits/rejected": -2.0257925987243652,
|
|
"logps/chosen": -0.29016560316085815,
|
|
"logps/rejected": -0.2911625802516937,
|
|
"loss": 1.0842726230621338,
|
|
"loss/core": 1.0630663633346558,
|
|
"loss/preference_sft": 0.2913043797016144,
|
|
"loss/reference_anchor": 0.5767573118209839,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.959277629852295,
|
|
"rewards/margins": 2.2727534770965576,
|
|
"rewards/rejected": 0.6865240931510925,
|
|
"ronpo/logit": 0.22727534174919128,
|
|
"ronpo/residual": 0.20963330566883087,
|
|
"ronpo/residual_abs": 0.27996546030044556,
|
|
"ronpo/target": 0.017642047256231308,
|
|
"ronpo/target_abs": 0.08470751345157623,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31818053126335144,
|
|
"drift/rejected_abs": 0.09311111271381378,
|
|
"epoch": 0.2320222487087803,
|
|
"grad_norm": 2656.0,
|
|
"learning_rate": 7.949596739747928e-09,
|
|
"logits/chosen": -2.1799933910369873,
|
|
"logits/rejected": -2.214536190032959,
|
|
"logps/chosen": -0.2960435748100281,
|
|
"logps/rejected": -0.2987004816532135,
|
|
"loss": 1.0148961544036865,
|
|
"loss/core": 0.9916189908981323,
|
|
"loss/preference_sft": 0.30723002552986145,
|
|
"loss/reference_anchor": 0.6343376040458679,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.18082332611084,
|
|
"rewards/margins": 2.2561564445495605,
|
|
"rewards/rejected": 0.9246670007705688,
|
|
"ronpo/logit": 0.22561565041542053,
|
|
"ronpo/residual": 0.211147278547287,
|
|
"ronpo/residual_abs": 0.27076631784439087,
|
|
"ronpo/target": 0.014468356966972351,
|
|
"ronpo/target_abs": 0.07857581973075867,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41159167885780334,
|
|
"drift/rejected_abs": 0.15270794928073883,
|
|
"epoch": 0.23361144219308702,
|
|
"grad_norm": 9.9375,
|
|
"learning_rate": 7.507459491550653e-09,
|
|
"logits/chosen": -1.872210144996643,
|
|
"logits/rejected": -1.9234001636505127,
|
|
"logps/chosen": -0.31087052822113037,
|
|
"logps/rejected": -0.28266483545303345,
|
|
"loss": 1.28376042842865,
|
|
"loss/core": 1.2525687217712402,
|
|
"loss/preference_sft": 0.30879443883895874,
|
|
"loss/reference_anchor": 0.8603096008300781,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.114415645599365,
|
|
"rewards/margins": 2.59476900100708,
|
|
"rewards/rejected": 1.5196462869644165,
|
|
"ronpo/logit": 0.259476900100708,
|
|
"ronpo/residual": 0.2532740831375122,
|
|
"ronpo/residual_abs": 0.36283624172210693,
|
|
"ronpo/target": 0.006202817894518375,
|
|
"ronpo/target_abs": 0.09357239305973053,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.40805894136428833,
|
|
"drift/rejected_abs": 0.17579728364944458,
|
|
"epoch": 0.23520063567739372,
|
|
"grad_norm": 6.1875,
|
|
"learning_rate": 7.076601211233599e-09,
|
|
"logits/chosen": -2.061128854751587,
|
|
"logits/rejected": -2.0528006553649902,
|
|
"logps/chosen": -0.27439576387405396,
|
|
"logps/rejected": -0.2714352607727051,
|
|
"loss": 0.8380556106567383,
|
|
"loss/core": 0.8007208704948425,
|
|
"loss/preference_sft": 0.2815498411655426,
|
|
"loss/reference_anchor": 1.0385510921478271,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.079874515533447,
|
|
"rewards/margins": 2.3227553367614746,
|
|
"rewards/rejected": 1.7571195363998413,
|
|
"ronpo/logit": 0.23227553069591522,
|
|
"ronpo/residual": 0.21789321303367615,
|
|
"ronpo/residual_abs": 0.29552918672561646,
|
|
"ronpo/target": 0.01438230276107788,
|
|
"ronpo/target_abs": 0.08076776564121246,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2793598175048828,
|
|
"drift/rejected_abs": 0.1420426368713379,
|
|
"epoch": 0.23678982916170044,
|
|
"grad_norm": 4.1875,
|
|
"learning_rate": 6.6571839269751625e-09,
|
|
"logits/chosen": -1.9656693935394287,
|
|
"logits/rejected": -1.9407997131347656,
|
|
"logps/chosen": -0.310263454914093,
|
|
"logps/rejected": -0.2949405312538147,
|
|
"loss": 0.4082837998867035,
|
|
"loss/core": 0.39522331953048706,
|
|
"loss/preference_sft": 0.302360862493515,
|
|
"loss/reference_anchor": 0.34292274713516235,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7910044193267822,
|
|
"rewards/margins": 1.4027537107467651,
|
|
"rewards/rejected": 1.3882505893707275,
|
|
"ronpo/logit": 0.14027537405490875,
|
|
"ronpo/residual": 0.14826272428035736,
|
|
"ronpo/residual_abs": 0.2111869603395462,
|
|
"ronpo/target": -0.007987345568835735,
|
|
"ronpo/target_abs": 0.06692041456699371,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4917718470096588,
|
|
"drift/rejected_abs": 0.09017399698495865,
|
|
"epoch": 0.23837902264600716,
|
|
"grad_norm": 25.875,
|
|
"learning_rate": 6.249365364463068e-09,
|
|
"logits/chosen": -1.8745949268341064,
|
|
"logits/rejected": -2.0258686542510986,
|
|
"logps/chosen": -0.32771915197372437,
|
|
"logps/rejected": -0.3020108938217163,
|
|
"loss": 1.669013261795044,
|
|
"loss/core": 1.633209228515625,
|
|
"loss/preference_sft": 0.3086247444152832,
|
|
"loss/reference_anchor": 0.9921077489852905,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.917025566101074,
|
|
"rewards/margins": 4.016125679016113,
|
|
"rewards/rejected": 0.9008998870849609,
|
|
"ronpo/logit": 0.4016125798225403,
|
|
"ronpo/residual": 0.39909854531288147,
|
|
"ronpo/residual_abs": 0.4497567117214203,
|
|
"ronpo/target": 0.002514071064069867,
|
|
"ronpo/target_abs": 0.08284027129411697,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5578740835189819,
|
|
"drift/rejected_abs": 0.25196394324302673,
|
|
"epoch": 0.23996821613031386,
|
|
"grad_norm": 176.0,
|
|
"learning_rate": 5.853298887580098e-09,
|
|
"logits/chosen": -1.7803856134414673,
|
|
"logits/rejected": -1.7086734771728516,
|
|
"logps/chosen": -0.26622751355171204,
|
|
"logps/rejected": -0.2793108820915222,
|
|
"loss": 1.8586667776107788,
|
|
"loss/core": 1.813750982284546,
|
|
"loss/preference_sft": 0.258129745721817,
|
|
"loss/reference_anchor": 1.257493019104004,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 5.57759952545166,
|
|
"rewards/margins": 3.06099009513855,
|
|
"rewards/rejected": 2.5166096687316895,
|
|
"ronpo/logit": 0.3060990273952484,
|
|
"ronpo/residual": 0.29286080598831177,
|
|
"ronpo/residual_abs": 0.49150973558425903,
|
|
"ronpo/target": 0.013238211162388325,
|
|
"ronpo/target_abs": 0.07374797761440277,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2570843994617462,
|
|
"drift/rejected_abs": 0.10265134274959564,
|
|
"epoch": 0.24155740961462058,
|
|
"grad_norm": 1.515625,
|
|
"learning_rate": 5.4691334407302935e-09,
|
|
"logits/chosen": -2.2019875049591064,
|
|
"logits/rejected": -2.2530570030212402,
|
|
"logps/chosen": -0.26051491498947144,
|
|
"logps/rejected": -0.2609294652938843,
|
|
"loss": 0.4085308611392975,
|
|
"loss/core": 0.3968563675880432,
|
|
"loss/preference_sft": 0.27111831307411194,
|
|
"loss/reference_anchor": 0.3064435124397278,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5702319145202637,
|
|
"rewards/margins": 1.548225998878479,
|
|
"rewards/rejected": 1.0220061540603638,
|
|
"ronpo/logit": 0.15482258796691895,
|
|
"ronpo/residual": 0.14783695340156555,
|
|
"ronpo/residual_abs": 0.20230190455913544,
|
|
"ronpo/target": 0.0069856420159339905,
|
|
"ronpo/target_abs": 0.0757329910993576,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16770626604557037,
|
|
"drift/rejected_abs": 0.09157443791627884,
|
|
"epoch": 0.2431466030989273,
|
|
"grad_norm": 1.203125,
|
|
"learning_rate": 5.097013492826982e-09,
|
|
"logits/chosen": -2.3989851474761963,
|
|
"logits/rejected": -2.3377420902252197,
|
|
"logps/chosen": -0.2859249711036682,
|
|
"logps/rejected": -0.28967946767807007,
|
|
"loss": 0.04619941860437393,
|
|
"loss/core": 0.041453104466199875,
|
|
"loss/preference_sft": 0.28654763102531433,
|
|
"loss/reference_anchor": 0.10695407539606094,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6764370203018188,
|
|
"rewards/margins": 0.7609391212463379,
|
|
"rewards/rejected": 0.9154976606369019,
|
|
"ronpo/logit": 0.07609390467405319,
|
|
"ronpo/residual": 0.09639062732458115,
|
|
"ronpo/residual_abs": 0.12980125844478607,
|
|
"ronpo/target": -0.020296704024076462,
|
|
"ronpo/target_abs": 0.06380324065685272,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33088406920433044,
|
|
"drift/rejected_abs": 0.10067299753427505,
|
|
"epoch": 0.24473579658323402,
|
|
"grad_norm": 49.5,
|
|
"learning_rate": 4.737078982963998e-09,
|
|
"logits/chosen": -1.9620716571807861,
|
|
"logits/rejected": -2.093608856201172,
|
|
"logps/chosen": -0.2551668882369995,
|
|
"logps/rejected": -0.27200862765312195,
|
|
"loss": 0.6154618263244629,
|
|
"loss/core": 0.6015649437904358,
|
|
"loss/preference_sft": 0.26770323514938354,
|
|
"loss/reference_anchor": 0.3702848255634308,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.3076298236846924,
|
|
"rewards/margins": 2.3778507709503174,
|
|
"rewards/rejected": 0.929779052734375,
|
|
"ronpo/logit": 0.23778507113456726,
|
|
"ronpo/residual": 0.2486831247806549,
|
|
"ronpo/residual_abs": 0.29293423891067505,
|
|
"ronpo/target": -0.010898038744926453,
|
|
"ronpo/target_abs": 0.08667393028736115,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34422239661216736,
|
|
"drift/rejected_abs": 0.14332148432731628,
|
|
"epoch": 0.24632499006754072,
|
|
"grad_norm": 1.125,
|
|
"learning_rate": 4.389465267790238e-09,
|
|
"logits/chosen": -1.8292220830917358,
|
|
"logits/rejected": -1.8172327280044556,
|
|
"logps/chosen": -0.2949088513851166,
|
|
"logps/rejected": -0.2962006628513336,
|
|
"loss": 0.5463058948516846,
|
|
"loss/core": 0.5306318402290344,
|
|
"loss/preference_sft": 0.29769080877304077,
|
|
"loss/reference_anchor": 0.41806188225746155,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.440873384475708,
|
|
"rewards/margins": 2.0447137355804443,
|
|
"rewards/rejected": 1.3961597681045532,
|
|
"ronpo/logit": 0.2044713944196701,
|
|
"ronpo/residual": 0.21325626969337463,
|
|
"ronpo/residual_abs": 0.2596508264541626,
|
|
"ronpo/target": -0.008784900419414043,
|
|
"ronpo/target_abs": 0.08347609639167786,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3502070903778076,
|
|
"drift/rejected_abs": 0.15574514865875244,
|
|
"epoch": 0.24791418355184744,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 4.05430307060764e-09,
|
|
"logits/chosen": -2.158703565597534,
|
|
"logits/rejected": -2.1818182468414307,
|
|
"logps/chosen": -0.2700011134147644,
|
|
"logps/rejected": -0.2787485718727112,
|
|
"loss": 0.3507295846939087,
|
|
"loss/core": 0.33437737822532654,
|
|
"loss/preference_sft": 0.27247267961502075,
|
|
"loss/reference_anchor": 0.43995794653892517,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.501782178878784,
|
|
"rewards/margins": 1.9450876712799072,
|
|
"rewards/rejected": 1.556694507598877,
|
|
"ronpo/logit": 0.19450877606868744,
|
|
"ronpo/residual": 0.18948352336883545,
|
|
"ronpo/residual_abs": 0.27817171812057495,
|
|
"ronpo/target": 0.0050252461805939674,
|
|
"ronpo/target_abs": 0.0843028575181961,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.47840428352355957,
|
|
"drift/rejected_abs": 0.1084831953048706,
|
|
"epoch": 0.24950337703615416,
|
|
"grad_norm": 4.3125,
|
|
"learning_rate": 3.731718432211417e-09,
|
|
"logits/chosen": -2.1047401428222656,
|
|
"logits/rejected": -2.21636962890625,
|
|
"logps/chosen": -0.2704011797904968,
|
|
"logps/rejected": -0.26196351647377014,
|
|
"loss": 1.810654878616333,
|
|
"loss/core": 1.7768656015396118,
|
|
"loss/preference_sft": 0.26972612738609314,
|
|
"loss/reference_anchor": 0.9384282827377319,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.778249263763428,
|
|
"rewards/margins": 3.708155870437622,
|
|
"rewards/rejected": 1.0700935125350952,
|
|
"ronpo/logit": 0.37081560492515564,
|
|
"ronpo/residual": 0.3735385537147522,
|
|
"ronpo/residual_abs": 0.4183095097541809,
|
|
"ronpo/target": -0.0027229245752096176,
|
|
"ronpo/target_abs": 0.08420609682798386,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3598479926586151,
|
|
"drift/rejected_abs": 0.12107064574956894,
|
|
"epoch": 0.25109257052046086,
|
|
"grad_norm": 191.0,
|
|
"learning_rate": 3.4218326634913406e-09,
|
|
"logits/chosen": -2.0067882537841797,
|
|
"logits/rejected": -2.0267016887664795,
|
|
"logps/chosen": -0.31197962164878845,
|
|
"logps/rejected": -0.29012903571128845,
|
|
"loss": 0.4827456474304199,
|
|
"loss/core": 0.46548813581466675,
|
|
"loss/preference_sft": 0.307406485080719,
|
|
"loss/reference_anchor": 0.46233057975769043,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.597403049468994,
|
|
"rewards/margins": 2.416919231414795,
|
|
"rewards/rejected": 1.1804835796356201,
|
|
"ronpo/logit": 0.2416919767856598,
|
|
"ronpo/residual": 0.2469751387834549,
|
|
"ronpo/residual_abs": 0.3075826168060303,
|
|
"ronpo/target": -0.005283153150230646,
|
|
"ronpo/target_abs": 0.07869095355272293,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24024005234241486,
|
|
"drift/rejected_abs": 0.11211339384317398,
|
|
"epoch": 0.2526817640047676,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 3.1247622998116148e-09,
|
|
"logits/chosen": -2.133969783782959,
|
|
"logits/rejected": -2.1671390533447266,
|
|
"logps/chosen": -0.25962501764297485,
|
|
"logps/rejected": -0.25368732213974,
|
|
"loss": 0.12422094494104385,
|
|
"loss/core": 0.11874745786190033,
|
|
"loss/preference_sft": 0.2524915635585785,
|
|
"loss/reference_anchor": 0.13113582134246826,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.401646375656128,
|
|
"rewards/margins": 1.2813524007797241,
|
|
"rewards/rejected": 1.1202940940856934,
|
|
"ronpo/logit": 0.12813523411750793,
|
|
"ronpo/residual": 0.1344965100288391,
|
|
"ronpo/residual_abs": 0.19210845232009888,
|
|
"ronpo/target": -0.006361273117363453,
|
|
"ronpo/target_abs": 0.07786816358566284,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2830193042755127,
|
|
"drift/rejected_abs": 0.15909279882907867,
|
|
"epoch": 0.2542709574890743,
|
|
"grad_norm": 16.75,
|
|
"learning_rate": 2.840619057186745e-09,
|
|
"logits/chosen": -2.169642448425293,
|
|
"logits/rejected": -2.2404894828796387,
|
|
"logps/chosen": -0.2703879475593567,
|
|
"logps/rejected": -0.2701282203197479,
|
|
"loss": 0.2124510258436203,
|
|
"loss/core": 0.1838868260383606,
|
|
"loss/preference_sft": 0.2592373192310333,
|
|
"loss/reference_anchor": 0.7901961803436279,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8296594619750977,
|
|
"rewards/margins": 1.2395825386047363,
|
|
"rewards/rejected": 1.5900771617889404,
|
|
"ronpo/logit": 0.12395823001861572,
|
|
"ronpo/residual": 0.1275547742843628,
|
|
"ronpo/residual_abs": 0.1707366406917572,
|
|
"ronpo/target": -0.003596528898924589,
|
|
"ronpo/target_abs": 0.06492110341787338,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3190115988254547,
|
|
"drift/rejected_abs": 0.13805243372917175,
|
|
"epoch": 0.255860150973381,
|
|
"grad_norm": 217.0,
|
|
"learning_rate": 2.5695097902696582e-09,
|
|
"logits/chosen": -2.2752726078033447,
|
|
"logits/rejected": -2.237405300140381,
|
|
"logps/chosen": -0.30355507135391235,
|
|
"logps/rejected": -0.3101564049720764,
|
|
"loss": 0.8964834213256836,
|
|
"loss/core": 0.8716424703598022,
|
|
"loss/preference_sft": 0.2993467450141907,
|
|
"loss/reference_anchor": 0.6798084378242493,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1900362968444824,
|
|
"rewards/margins": 1.8883402347564697,
|
|
"rewards/rejected": 1.3016960620880127,
|
|
"ronpo/logit": 0.1888340413570404,
|
|
"ronpo/residual": 0.1919974535703659,
|
|
"ronpo/residual_abs": 0.29645854234695435,
|
|
"ronpo/target": -0.0031634189654141665,
|
|
"ronpo/target_abs": 0.07282833755016327,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21134933829307556,
|
|
"drift/rejected_abs": 0.11000971496105194,
|
|
"epoch": 0.25744934445768775,
|
|
"grad_norm": 7.40625,
|
|
"learning_rate": 2.311536452168106e-09,
|
|
"logits/chosen": -2.0637238025665283,
|
|
"logits/rejected": -2.114058494567871,
|
|
"logps/chosen": -0.28332144021987915,
|
|
"logps/rejected": -0.28769245743751526,
|
|
"loss": 0.04527657851576805,
|
|
"loss/core": 0.04074571654200554,
|
|
"loss/preference_sft": 0.2843453884124756,
|
|
"loss/reference_anchor": 0.10101868957281113,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1122398376464844,
|
|
"rewards/margins": 1.0128422975540161,
|
|
"rewards/rejected": 1.0993976593017578,
|
|
"ronpo/logit": 0.10128422826528549,
|
|
"ronpo/residual": 0.07714606821537018,
|
|
"ronpo/residual_abs": 0.1348331868648529,
|
|
"ronpo/target": 0.02413817122578621,
|
|
"ronpo/target_abs": 0.07104022800922394,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2593992352485657,
|
|
"drift/rejected_abs": 0.16390137374401093,
|
|
"epoch": 0.25903853794199444,
|
|
"grad_norm": 12.125,
|
|
"learning_rate": 2.0667960561041974e-09,
|
|
"logits/chosen": -2.107893228530884,
|
|
"logits/rejected": -2.0309271812438965,
|
|
"logps/chosen": -0.2672129273414612,
|
|
"logps/rejected": -0.2773855924606323,
|
|
"loss": 0.2365700900554657,
|
|
"loss/core": 0.22861385345458984,
|
|
"loss/preference_sft": 0.2656956911087036,
|
|
"loss/reference_anchor": 0.20075194537639618,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.593992233276367,
|
|
"rewards/margins": 0.9726476669311523,
|
|
"rewards/rejected": 1.6213443279266357,
|
|
"ronpo/logit": 0.09726477414369583,
|
|
"ronpo/residual": 0.08035314828157425,
|
|
"ronpo/residual_abs": 0.22539310157299042,
|
|
"ronpo/target": 0.016911622136831284,
|
|
"ronpo/target_abs": 0.08380316197872162,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1997155249118805,
|
|
"drift/rejected_abs": 0.1135120838880539,
|
|
"epoch": 0.26062773142630113,
|
|
"grad_norm": 1.828125,
|
|
"learning_rate": 1.8353806389317424e-09,
|
|
"logits/chosen": -2.0197744369506836,
|
|
"logits/rejected": -1.9759705066680908,
|
|
"logps/chosen": -0.2759111225605011,
|
|
"logps/rejected": -0.28906577825546265,
|
|
"loss": 0.06284098327159882,
|
|
"loss/core": 0.056352388113737106,
|
|
"loss/preference_sft": 0.27423539757728577,
|
|
"loss/reference_anchor": 0.15796485543251038,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.993607521057129,
|
|
"rewards/margins": 0.8597732782363892,
|
|
"rewards/rejected": 1.1338340044021606,
|
|
"ronpo/logit": 0.08597733825445175,
|
|
"ronpo/residual": 0.08307936042547226,
|
|
"ronpo/residual_abs": 0.14249305427074432,
|
|
"ronpo/target": 0.0028979722410440445,
|
|
"ronpo/target_abs": 0.07825162261724472,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2867492735385895,
|
|
"drift/rejected_abs": 0.18776382505893707,
|
|
"epoch": 0.2622169249106079,
|
|
"grad_norm": 67.5,
|
|
"learning_rate": 1.6173772265249478e-09,
|
|
"logits/chosen": -2.2137134075164795,
|
|
"logits/rejected": -2.161581516265869,
|
|
"logps/chosen": -0.2998419404029846,
|
|
"logps/rejected": -0.3022862374782562,
|
|
"loss": 0.3233335018157959,
|
|
"loss/core": 0.29298847913742065,
|
|
"loss/preference_sft": 0.3009437024593353,
|
|
"loss/reference_anchor": 0.8369055986404419,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8670103549957275,
|
|
"rewards/margins": 0.9923250079154968,
|
|
"rewards/rejected": 1.8746850490570068,
|
|
"ronpo/logit": 0.0992325022816658,
|
|
"ronpo/residual": 0.09889347851276398,
|
|
"ronpo/residual_abs": 0.20365075767040253,
|
|
"ronpo/target": 0.00033902880386449397,
|
|
"ronpo/target_abs": 0.07141275703907013,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18498516082763672,
|
|
"drift/rejected_abs": 0.11326082795858383,
|
|
"epoch": 0.2638061183949146,
|
|
"grad_norm": 35.5,
|
|
"learning_rate": 1.4128678010515385e-09,
|
|
"logits/chosen": -2.2336018085479736,
|
|
"logits/rejected": -2.234981060028076,
|
|
"logps/chosen": -0.2774164378643036,
|
|
"logps/rejected": -0.2730526328086853,
|
|
"loss": 0.13492383062839508,
|
|
"loss/core": 0.12916125357151031,
|
|
"loss/preference_sft": 0.27852898836135864,
|
|
"loss/reference_anchor": 0.13679227232933044,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8494479656219482,
|
|
"rewards/margins": 0.7175174951553345,
|
|
"rewards/rejected": 1.1319307088851929,
|
|
"ronpo/logit": 0.07175175100564957,
|
|
"ronpo/residual": 0.05915927141904831,
|
|
"ronpo/residual_abs": 0.17424622178077698,
|
|
"ronpo/target": 0.012592477723956108,
|
|
"ronpo/target_abs": 0.0779508426785469,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4061504006385803,
|
|
"drift/rejected_abs": 0.16050119698047638,
|
|
"epoch": 0.2653953118792213,
|
|
"grad_norm": 17.25,
|
|
"learning_rate": 1.2219292701426076e-09,
|
|
"logits/chosen": -2.190483808517456,
|
|
"logits/rejected": -2.155860424041748,
|
|
"logps/chosen": -0.24286535382270813,
|
|
"logps/rejected": -0.24661651253700256,
|
|
"loss": 1.0600229501724243,
|
|
"loss/core": 1.0351521968841553,
|
|
"loss/preference_sft": 0.24154765903949738,
|
|
"loss/reference_anchor": 0.686432957649231,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 4.060586452484131,
|
|
"rewards/margins": 2.4555745124816895,
|
|
"rewards/rejected": 1.6050119400024414,
|
|
"ronpo/logit": 0.24555747210979462,
|
|
"ronpo/residual": 0.233054518699646,
|
|
"ronpo/residual_abs": 0.3451952636241913,
|
|
"ronpo/target": 0.012502968311309814,
|
|
"ronpo/target_abs": 0.07277142256498337,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28720027208328247,
|
|
"drift/rejected_abs": 0.13702043890953064,
|
|
"epoch": 0.266984505363528,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 1.0446334379708085e-09,
|
|
"logits/chosen": -2.3651621341705322,
|
|
"logits/rejected": -2.4263358116149902,
|
|
"logps/chosen": -0.26630455255508423,
|
|
"logps/rejected": -0.2645034193992615,
|
|
"loss": 0.3639865517616272,
|
|
"loss/core": 0.34910276532173157,
|
|
"loss/preference_sft": 0.2732158899307251,
|
|
"loss/reference_anchor": 0.39793118834495544,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.8720028400421143,
|
|
"rewards/margins": 1.5028787851333618,
|
|
"rewards/rejected": 1.369124174118042,
|
|
"ronpo/logit": 0.15028786659240723,
|
|
"ronpo/residual": 0.15997345745563507,
|
|
"ronpo/residual_abs": 0.20985810458660126,
|
|
"ronpo/target": -0.009685601107776165,
|
|
"ronpo/target_abs": 0.08229205757379532,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29841169714927673,
|
|
"drift/rejected_abs": 0.18531523644924164,
|
|
"epoch": 0.2685736988478347,
|
|
"grad_norm": 6.65625,
|
|
"learning_rate": 8.810469782477639e-10,
|
|
"logits/chosen": -2.1420059204101562,
|
|
"logits/rejected": -2.135344982147217,
|
|
"logps/chosen": -0.24753496050834656,
|
|
"logps/rejected": -0.23413236439228058,
|
|
"loss": 0.0860786885023117,
|
|
"loss/core": 0.07448365539312363,
|
|
"loss/preference_sft": 0.24143639206886292,
|
|
"loss/reference_anchor": 0.30714288353919983,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.9827346801757812,
|
|
"rewards/margins": 1.1305488348007202,
|
|
"rewards/rejected": 1.8521859645843506,
|
|
"ronpo/logit": 0.11305488646030426,
|
|
"ronpo/residual": 0.10625892877578735,
|
|
"ronpo/residual_abs": 0.16595976054668427,
|
|
"ronpo/target": 0.006795956287533045,
|
|
"ronpo/target_abs": 0.08061005175113678,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41207045316696167,
|
|
"drift/rejected_abs": 0.1471131145954132,
|
|
"epoch": 0.2701628923321414,
|
|
"grad_norm": 82.5,
|
|
"learning_rate": 7.31231409150776e-10,
|
|
"logits/chosen": -1.953844428062439,
|
|
"logits/rejected": -1.8874391317367554,
|
|
"logps/chosen": -0.2713695466518402,
|
|
"logps/rejected": -0.2831560969352722,
|
|
"loss": 0.9032095670700073,
|
|
"loss/core": 0.8764148950576782,
|
|
"loss/preference_sft": 0.2667026221752167,
|
|
"loss/reference_anchor": 0.7388928532600403,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.118669509887695,
|
|
"rewards/margins": 2.6498663425445557,
|
|
"rewards/rejected": 1.4688036441802979,
|
|
"ronpo/logit": 0.26498663425445557,
|
|
"ronpo/residual": 0.2674812972545624,
|
|
"ronpo/residual_abs": 0.32319194078445435,
|
|
"ronpo/target": -0.0024946490302681923,
|
|
"ronpo/target_abs": 0.07483317703008652,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20735082030296326,
|
|
"drift/rejected_abs": 0.07366625219583511,
|
|
"epoch": 0.27175208581644816,
|
|
"grad_norm": 2.21875,
|
|
"learning_rate": 5.952430701883781e-10,
|
|
"logits/chosen": -2.3780345916748047,
|
|
"logits/rejected": -2.365642547607422,
|
|
"logps/chosen": -0.27689987421035767,
|
|
"logps/rejected": -0.2746134400367737,
|
|
"loss": 0.4126233160495758,
|
|
"loss/core": 0.40223345160484314,
|
|
"loss/preference_sft": 0.27305346727371216,
|
|
"loss/reference_anchor": 0.2695460915565491,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.073309898376465,
|
|
"rewards/margins": 1.336647391319275,
|
|
"rewards/rejected": 0.7366625070571899,
|
|
"ronpo/logit": 0.13366474211215973,
|
|
"ronpo/residual": 0.1408972442150116,
|
|
"ronpo/residual_abs": 0.18301846086978912,
|
|
"ronpo/target": -0.0072325123474001884,
|
|
"ronpo/target_abs": 0.07463272660970688,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1881171017885208,
|
|
"drift/rejected_abs": 0.10389295965433121,
|
|
"epoch": 0.27334127930075486,
|
|
"grad_norm": 24.875,
|
|
"learning_rate": 4.731331010133288e-10,
|
|
"logits/chosen": -2.2028229236602783,
|
|
"logits/rejected": -2.12843656539917,
|
|
"logps/chosen": -0.2806995213031769,
|
|
"logps/rejected": -0.2797695994377136,
|
|
"loss": 0.1201997771859169,
|
|
"loss/core": 0.11440330743789673,
|
|
"loss/preference_sft": 0.28371673822402954,
|
|
"loss/reference_anchor": 0.13724195957183838,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8811709880828857,
|
|
"rewards/margins": 0.8511055111885071,
|
|
"rewards/rejected": 1.0300655364990234,
|
|
"ronpo/logit": 0.08511054515838623,
|
|
"ronpo/residual": 0.07745916396379471,
|
|
"ronpo/residual_abs": 0.1381129026412964,
|
|
"ronpo/target": 0.007651382125914097,
|
|
"ronpo/target_abs": 0.06799162924289703,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24889037013053894,
|
|
"drift/rejected_abs": 0.12972140312194824,
|
|
"epoch": 0.2749304727850616,
|
|
"grad_norm": 0.75,
|
|
"learning_rate": 3.649474221911114e-10,
|
|
"logits/chosen": -2.090186595916748,
|
|
"logits/rejected": -2.0353407859802246,
|
|
"logps/chosen": -0.27941226959228516,
|
|
"logps/rejected": -0.28891220688819885,
|
|
"loss": 0.11919602006673813,
|
|
"loss/core": 0.11207473278045654,
|
|
"loss/preference_sft": 0.2893761694431305,
|
|
"loss/reference_anchor": 0.1745276153087616,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.488442897796631,
|
|
"rewards/margins": 1.1927058696746826,
|
|
"rewards/rejected": 1.2957369089126587,
|
|
"ronpo/logit": 0.11927058547735214,
|
|
"ronpo/residual": 0.11880163848400116,
|
|
"ronpo/residual_abs": 0.18198655545711517,
|
|
"ronpo/target": 0.0004689468187279999,
|
|
"ronpo/target_abs": 0.07780329138040543,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.46323925256729126,
|
|
"drift/rejected_abs": 0.21974530816078186,
|
|
"epoch": 0.2765196662693683,
|
|
"grad_norm": 532.0,
|
|
"learning_rate": 2.7072671793106074e-10,
|
|
"logits/chosen": -2.0998735427856445,
|
|
"logits/rejected": -2.087890625,
|
|
"logps/chosen": -0.26934343576431274,
|
|
"logps/rejected": -0.27645033597946167,
|
|
"loss": 0.7379623651504517,
|
|
"loss/core": 0.7018780708312988,
|
|
"loss/preference_sft": 0.2766755223274231,
|
|
"loss/reference_anchor": 1.0033100843429565,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.631134986877441,
|
|
"rewards/margins": 2.4353508949279785,
|
|
"rewards/rejected": 2.195784091949463,
|
|
"ronpo/logit": 0.24353507161140442,
|
|
"ronpo/residual": 0.2610352635383606,
|
|
"ronpo/residual_abs": 0.31035056710243225,
|
|
"ronpo/target": -0.017500165849924088,
|
|
"ronpo/target_abs": 0.0813521146774292,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3059462904930115,
|
|
"drift/rejected_abs": 0.15434163808822632,
|
|
"epoch": 0.278108859753675,
|
|
"grad_norm": 5.0,
|
|
"learning_rate": 1.9050642078670547e-10,
|
|
"logits/chosen": -1.9450798034667969,
|
|
"logits/rejected": -1.9134384393692017,
|
|
"logps/chosen": -0.2710108757019043,
|
|
"logps/rejected": -0.2715742588043213,
|
|
"loss": 0.22777685523033142,
|
|
"loss/core": 0.21646137535572052,
|
|
"loss/preference_sft": 0.27517765760421753,
|
|
"loss/reference_anchor": 0.2957829236984253,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0580132007598877,
|
|
"rewards/margins": 1.5153281688690186,
|
|
"rewards/rejected": 1.5426852703094482,
|
|
"ronpo/logit": 0.15153281390666962,
|
|
"ronpo/residual": 0.1544264703989029,
|
|
"ronpo/residual_abs": 0.21350125968456268,
|
|
"ronpo/target": -0.002893655328080058,
|
|
"ronpo/target_abs": 0.08072160184383392,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24425086379051208,
|
|
"drift/rejected_abs": 0.0907929316163063,
|
|
"epoch": 0.27969805323798175,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 1.2431669833102288e-10,
|
|
"logits/chosen": -2.1156883239746094,
|
|
"logits/rejected": -2.1574745178222656,
|
|
"logps/chosen": -0.27831846475601196,
|
|
"logps/rejected": -0.29058295488357544,
|
|
"loss": 0.42100924253463745,
|
|
"loss/core": 0.41138726472854614,
|
|
"loss/preference_sft": 0.2927462160587311,
|
|
"loss/reference_anchor": 0.2456396371126175,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4425082206726074,
|
|
"rewards/margins": 1.5455248355865479,
|
|
"rewards/rejected": 0.8969835042953491,
|
|
"ronpo/logit": 0.15455248951911926,
|
|
"ronpo/residual": 0.1457471400499344,
|
|
"ronpo/residual_abs": 0.20269238948822021,
|
|
"ronpo/target": 0.008805358782410622,
|
|
"ronpo/target_abs": 0.08658818900585175,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26528865098953247,
|
|
"drift/rejected_abs": 0.1037355437874794,
|
|
"epoch": 0.28128724672228844,
|
|
"grad_norm": 5.625,
|
|
"learning_rate": 7.218244181167594e-11,
|
|
"logits/chosen": -2.1619465351104736,
|
|
"logits/rejected": -2.223400592803955,
|
|
"logps/chosen": -0.29743996262550354,
|
|
"logps/rejected": -0.2958257794380188,
|
|
"loss": 0.3847818076610565,
|
|
"loss/core": 0.36538928747177124,
|
|
"loss/preference_sft": 0.2988830506801605,
|
|
"loss/reference_anchor": 0.5241828560829163,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.6526153087615967,
|
|
"rewards/margins": 1.6183156967163086,
|
|
"rewards/rejected": 1.034299612045288,
|
|
"ronpo/logit": 0.1618315875530243,
|
|
"ronpo/residual": 0.1641121506690979,
|
|
"ronpo/residual_abs": 0.20270681381225586,
|
|
"ronpo/target": -0.0022805710323154926,
|
|
"ronpo/target_abs": 0.07088293880224228,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2995665669441223,
|
|
"drift/rejected_abs": 0.12155304849147797,
|
|
"epoch": 0.28287644020659514,
|
|
"grad_norm": 3.765625,
|
|
"learning_rate": 3.4123256790405397e-11,
|
|
"logits/chosen": -2.307978868484497,
|
|
"logits/rejected": -2.2648723125457764,
|
|
"logps/chosen": -0.2815799415111542,
|
|
"logps/rejected": -0.2809089720249176,
|
|
"loss": 0.4028624892234802,
|
|
"loss/core": 0.3849177956581116,
|
|
"loss/preference_sft": 0.286651074886322,
|
|
"loss/reference_anchor": 0.48404186964035034,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.99299693107605,
|
|
"rewards/margins": 1.7814449071884155,
|
|
"rewards/rejected": 1.2115522623062134,
|
|
"ronpo/logit": 0.17814448475837708,
|
|
"ronpo/residual": 0.18465931713581085,
|
|
"ronpo/residual_abs": 0.22240400314331055,
|
|
"ronpo/target": -0.006514822132885456,
|
|
"ronpo/target_abs": 0.07123463600873947,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30803316831588745,
|
|
"drift/rejected_abs": 0.12433400005102158,
|
|
"epoch": 0.2844656336909019,
|
|
"grad_norm": 11.5,
|
|
"learning_rate": 1.0153455770185725e-11,
|
|
"logits/chosen": -2.0798468589782715,
|
|
"logits/rejected": -2.068073272705078,
|
|
"logps/chosen": -0.2840914726257324,
|
|
"logps/rejected": -0.2947894334793091,
|
|
"loss": 0.3525415360927582,
|
|
"loss/core": 0.33794763684272766,
|
|
"loss/preference_sft": 0.29777130484580994,
|
|
"loss/reference_anchor": 0.38719049096107483,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0781116485595703,
|
|
"rewards/margins": 1.9379209280014038,
|
|
"rewards/rejected": 1.1401903629302979,
|
|
"ronpo/logit": 0.19379207491874695,
|
|
"ronpo/residual": 0.1745019257068634,
|
|
"ronpo/residual_abs": 0.2425132542848587,
|
|
"ronpo/target": 0.019290190190076828,
|
|
"ronpo/target_abs": 0.08941071480512619,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16016218066215515,
|
|
"drift/rejected_abs": 0.08440212905406952,
|
|
"epoch": 0.2860548271752086,
|
|
"grad_norm": 15.3125,
|
|
"learning_rate": 2.820528128971711e-13,
|
|
"logits/chosen": -2.104757308959961,
|
|
"logits/rejected": -2.0879600048065186,
|
|
"logps/chosen": -0.31782156229019165,
|
|
"logps/rejected": -0.3181937634944916,
|
|
"loss": 0.14113518595695496,
|
|
"loss/core": 0.13576209545135498,
|
|
"loss/preference_sft": 0.3204381763935089,
|
|
"loss/reference_anchor": 0.12147295475006104,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6012519598007202,
|
|
"rewards/margins": 0.888868510723114,
|
|
"rewards/rejected": 0.7123832106590271,
|
|
"ronpo/logit": 0.08888684213161469,
|
|
"ronpo/residual": 0.08423057198524475,
|
|
"ronpo/residual_abs": 0.14461413025856018,
|
|
"ronpo/target": 0.004656284116208553,
|
|
"ronpo/target_abs": 0.07319948077201843,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.2860548271752086,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.4220078429910872,
|
|
"train_runtime": 7042.2326,
|
|
"train_samples_per_second": 2.045,
|
|
"train_steps_per_second": 0.128
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|