ec51c3f261
Model: promotion/qwen3-8b-aaai27-flagship-ronpo-full-expect-s44 Source: Original Platform
4904 lines
186 KiB
JSON
4904 lines
186 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2860548271752086,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.26923736929893494,
|
|
"drift/rejected_abs": 0.12245669215917587,
|
|
"epoch": 0.0015891934843067143,
|
|
"grad_norm": 6.15625,
|
|
"learning_rate": 4.444444444444444e-09,
|
|
"logits/chosen": -1.8292624950408936,
|
|
"logits/rejected": -1.8292913436889648,
|
|
"logps/chosen": -0.25974613428115845,
|
|
"logps/rejected": -0.2631612718105316,
|
|
"loss": 0.3697437345981598,
|
|
"loss/core": 0.35558179020881653,
|
|
"loss/preference_sft": 0.25867244601249695,
|
|
"loss/reference_anchor": 0.25737133622169495,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6913039684295654,
|
|
"rewards/margins": 1.4922035932540894,
|
|
"rewards/rejected": 1.1991007328033447,
|
|
"ronpo/logit": 0.14922034740447998,
|
|
"ronpo/residual": 0.14157937467098236,
|
|
"ronpo/residual_abs": 0.20015816390514374,
|
|
"ronpo/target": 0.007640989031642675,
|
|
"ronpo/target_abs": 0.07512292265892029,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3896196186542511,
|
|
"drift/rejected_abs": 0.18290038406848907,
|
|
"epoch": 0.0031783869686134287,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 1e-08,
|
|
"logits/chosen": -1.8097797632217407,
|
|
"logits/rejected": -1.7882839441299438,
|
|
"logps/chosen": -0.2950737774372101,
|
|
"logps/rejected": -0.3086322844028473,
|
|
"loss": 0.6527813673019409,
|
|
"loss/core": 0.6042948365211487,
|
|
"loss/preference_sft": 0.2980811893939972,
|
|
"loss/reference_anchor": 0.9399248361587524,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.8948261737823486,
|
|
"rewards/margins": 2.0671069622039795,
|
|
"rewards/rejected": 1.8277194499969482,
|
|
"ronpo/logit": 0.20671074092388153,
|
|
"ronpo/residual": 0.18510529398918152,
|
|
"ronpo/residual_abs": 0.2420695573091507,
|
|
"ronpo/target": 0.021605418995022774,
|
|
"ronpo/target_abs": 0.07878313958644867,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3951796889305115,
|
|
"drift/rejected_abs": 0.09188535064458847,
|
|
"epoch": 0.004767580452920143,
|
|
"grad_norm": 0.96875,
|
|
"learning_rate": 1.5555555555555554e-08,
|
|
"logits/chosen": -1.7505464553833008,
|
|
"logits/rejected": -1.7822763919830322,
|
|
"logps/chosen": -0.29122844338417053,
|
|
"logps/rejected": -0.2962791323661804,
|
|
"loss": 1.0267797708511353,
|
|
"loss/core": 0.9949584007263184,
|
|
"loss/preference_sft": 0.28717607259750366,
|
|
"loss/reference_anchor": 0.6077091097831726,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.9513306617736816,
|
|
"rewards/margins": 3.043854236602783,
|
|
"rewards/rejected": 0.9074764251708984,
|
|
"ronpo/logit": 0.30438539385795593,
|
|
"ronpo/residual": 0.29969078302383423,
|
|
"ronpo/residual_abs": 0.35535189509391785,
|
|
"ronpo/target": 0.0046946206130087376,
|
|
"ronpo/target_abs": 0.07615546137094498,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2791958451271057,
|
|
"drift/rejected_abs": 0.13730254769325256,
|
|
"epoch": 0.006356773937226857,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 2.111111111111111e-08,
|
|
"logits/chosen": -1.9703651666641235,
|
|
"logits/rejected": -2.017350196838379,
|
|
"logps/chosen": -0.28874635696411133,
|
|
"logps/rejected": -0.2909610867500305,
|
|
"loss": 0.41066980361938477,
|
|
"loss/core": 0.3952859938144684,
|
|
"loss/preference_sft": 0.3018035292625427,
|
|
"loss/reference_anchor": 0.27749544382095337,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7919583320617676,
|
|
"rewards/margins": 1.4557034969329834,
|
|
"rewards/rejected": 1.3362548351287842,
|
|
"ronpo/logit": 0.14557036757469177,
|
|
"ronpo/residual": 0.14364346861839294,
|
|
"ronpo/residual_abs": 0.21150994300842285,
|
|
"ronpo/target": 0.001926881494000554,
|
|
"ronpo/target_abs": 0.0835525169968605,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32657527923583984,
|
|
"drift/rejected_abs": 0.14058224856853485,
|
|
"epoch": 0.007945967421533572,
|
|
"grad_norm": 5.71875,
|
|
"learning_rate": 2.6666666666666664e-08,
|
|
"logits/chosen": -1.6839513778686523,
|
|
"logits/rejected": -1.7249832153320312,
|
|
"logps/chosen": -0.2955150306224823,
|
|
"logps/rejected": -0.28480157256126404,
|
|
"loss": 0.48044338822364807,
|
|
"loss/core": 0.44645896553993225,
|
|
"loss/preference_sft": 0.3023730218410492,
|
|
"loss/reference_anchor": 0.6494489312171936,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.2632546424865723,
|
|
"rewards/margins": 1.8599933385849,
|
|
"rewards/rejected": 1.4032608270645142,
|
|
"ronpo/logit": 0.18599936366081238,
|
|
"ronpo/residual": 0.18265409767627716,
|
|
"ronpo/residual_abs": 0.22560951113700867,
|
|
"ronpo/target": 0.0033452813513576984,
|
|
"ronpo/target_abs": 0.07430481910705566,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37191981077194214,
|
|
"drift/rejected_abs": 0.11496677249670029,
|
|
"epoch": 0.009535160905840286,
|
|
"grad_norm": 408.0,
|
|
"learning_rate": 3.2222222222222224e-08,
|
|
"logits/chosen": -1.6651420593261719,
|
|
"logits/rejected": -1.615891695022583,
|
|
"logps/chosen": -0.2600121796131134,
|
|
"logps/rejected": -0.25948435068130493,
|
|
"loss": 1.2412470579147339,
|
|
"loss/core": 1.2034351825714111,
|
|
"loss/preference_sft": 0.2637467384338379,
|
|
"loss/reference_anchor": 0.7298651933670044,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.719198226928711,
|
|
"rewards/margins": 2.570418119430542,
|
|
"rewards/rejected": 1.148780107498169,
|
|
"ronpo/logit": 0.2570417821407318,
|
|
"ronpo/residual": 0.2595512270927429,
|
|
"ronpo/residual_abs": 0.322681725025177,
|
|
"ronpo/target": -0.0025094160810112953,
|
|
"ronpo/target_abs": 0.0828036218881607,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2374390810728073,
|
|
"drift/rejected_abs": 0.06636892259120941,
|
|
"epoch": 0.011124354390147001,
|
|
"grad_norm": 498.0,
|
|
"learning_rate": 3.7777777777777774e-08,
|
|
"logits/chosen": -1.8363662958145142,
|
|
"logits/rejected": -1.8149330615997314,
|
|
"logps/chosen": -0.322407066822052,
|
|
"logps/rejected": -0.31593409180641174,
|
|
"loss": 1.0479040145874023,
|
|
"loss/core": 1.019943118095398,
|
|
"loss/preference_sft": 0.3177246153354645,
|
|
"loss/reference_anchor": 0.5274423360824585,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3728532791137695,
|
|
"rewards/margins": 1.7133121490478516,
|
|
"rewards/rejected": 0.6595410108566284,
|
|
"ronpo/logit": 0.17133121192455292,
|
|
"ronpo/residual": 0.18483777344226837,
|
|
"ronpo/residual_abs": 0.2308831661939621,
|
|
"ronpo/target": -0.0135065708309412,
|
|
"ronpo/target_abs": 0.08307793736457825,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31959348917007446,
|
|
"drift/rejected_abs": 0.12583120167255402,
|
|
"epoch": 0.012713547874453715,
|
|
"grad_norm": 0.640625,
|
|
"learning_rate": 4.333333333333333e-08,
|
|
"logits/chosen": -1.7055801153182983,
|
|
"logits/rejected": -1.7161920070648193,
|
|
"logps/chosen": -0.2816811800003052,
|
|
"logps/rejected": -0.285296767950058,
|
|
"loss": 0.585159182548523,
|
|
"loss/core": 0.5509465932846069,
|
|
"loss/preference_sft": 0.2811506390571594,
|
|
"loss/reference_anchor": 0.6561352014541626,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.1953861713409424,
|
|
"rewards/margins": 1.939779281616211,
|
|
"rewards/rejected": 1.2556068897247314,
|
|
"ronpo/logit": 0.19397792220115662,
|
|
"ronpo/residual": 0.183498352766037,
|
|
"ronpo/residual_abs": 0.23382291197776794,
|
|
"ronpo/target": 0.010479571297764778,
|
|
"ronpo/target_abs": 0.08148641884326935,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.342174232006073,
|
|
"drift/rejected_abs": 0.17132499814033508,
|
|
"epoch": 0.014302741358760428,
|
|
"grad_norm": 9.1875,
|
|
"learning_rate": 4.888888888888889e-08,
|
|
"logits/chosen": -1.8917038440704346,
|
|
"logits/rejected": -1.9098457098007202,
|
|
"logps/chosen": -0.26641327142715454,
|
|
"logps/rejected": -0.25877052545547485,
|
|
"loss": 0.20708735287189484,
|
|
"loss/core": 0.16268716752529144,
|
|
"loss/preference_sft": 0.26177817583084106,
|
|
"loss/reference_anchor": 0.8618253469467163,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.419778347015381,
|
|
"rewards/margins": 1.7149407863616943,
|
|
"rewards/rejected": 1.7048375606536865,
|
|
"ronpo/logit": 0.17149408161640167,
|
|
"ronpo/residual": 0.1707213819026947,
|
|
"ronpo/residual_abs": 0.22870823740959167,
|
|
"ronpo/target": 0.0007726888870820403,
|
|
"ronpo/target_abs": 0.08850932121276855,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2557240426540375,
|
|
"drift/rejected_abs": 0.1332291066646576,
|
|
"epoch": 0.015891934843067144,
|
|
"grad_norm": 10.0625,
|
|
"learning_rate": 5.444444444444444e-08,
|
|
"logits/chosen": -1.7528432607650757,
|
|
"logits/rejected": -1.7131255865097046,
|
|
"logps/chosen": -0.25860145688056946,
|
|
"logps/rejected": -0.25767049193382263,
|
|
"loss": 0.14077365398406982,
|
|
"loss/core": 0.12432624399662018,
|
|
"loss/preference_sft": 0.2571946680545807,
|
|
"loss/reference_anchor": 0.30322855710983276,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.5567574501037598,
|
|
"rewards/margins": 1.230201005935669,
|
|
"rewards/rejected": 1.3265564441680908,
|
|
"ronpo/logit": 0.12302011251449585,
|
|
"ronpo/residual": 0.13287095725536346,
|
|
"ronpo/residual_abs": 0.17298848927021027,
|
|
"ronpo/target": -0.009850854054093361,
|
|
"ronpo/target_abs": 0.06360660493373871,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.186763733625412,
|
|
"drift/rejected_abs": 0.119536854326725,
|
|
"epoch": 0.01748112832737386,
|
|
"grad_norm": 2.65625,
|
|
"learning_rate": 6e-08,
|
|
"logits/chosen": -1.6005624532699585,
|
|
"logits/rejected": -1.5535829067230225,
|
|
"logps/chosen": -0.32774192094802856,
|
|
"logps/rejected": -0.3251161575317383,
|
|
"loss": 0.04407442361116409,
|
|
"loss/core": 0.034535981714725494,
|
|
"loss/preference_sft": 0.331603467464447,
|
|
"loss/reference_anchor": 0.15760840475559235,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.8662309646606445,
|
|
"rewards/margins": 0.6730316877365112,
|
|
"rewards/rejected": 1.1931991577148438,
|
|
"ronpo/logit": 0.06730317324399948,
|
|
"ronpo/residual": 0.05887698009610176,
|
|
"ronpo/residual_abs": 0.1300887018442154,
|
|
"ronpo/target": 0.008426194079220295,
|
|
"ronpo/target_abs": 0.0830519050359726,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.09217182546854019,
|
|
"drift/rejected_abs": 0.05582476407289505,
|
|
"epoch": 0.01907032181168057,
|
|
"grad_norm": 3.953125,
|
|
"learning_rate": 6.555555555555555e-08,
|
|
"logits/chosen": -2.019383668899536,
|
|
"logits/rejected": -2.0197396278381348,
|
|
"logps/chosen": -0.3089592456817627,
|
|
"logps/rejected": -0.30434226989746094,
|
|
"loss": 0.0220238845795393,
|
|
"loss/core": 0.019543960690498352,
|
|
"loss/preference_sft": 0.3074526786804199,
|
|
"loss/reference_anchor": 0.01885320618748665,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 0.9209924936294556,
|
|
"rewards/margins": 0.3633802831172943,
|
|
"rewards/rejected": 0.5576121211051941,
|
|
"ronpo/logit": 0.03633803129196167,
|
|
"ronpo/residual": 0.05237559229135513,
|
|
"ronpo/residual_abs": 0.1066572442650795,
|
|
"ronpo/target": -0.016037559136748314,
|
|
"ronpo/target_abs": 0.08774705976247787,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.38381367921829224,
|
|
"drift/rejected_abs": 0.1924135982990265,
|
|
"epoch": 0.020659515295987287,
|
|
"grad_norm": 241.0,
|
|
"learning_rate": 7.111111111111111e-08,
|
|
"logits/chosen": -1.7329849004745483,
|
|
"logits/rejected": -1.6644783020019531,
|
|
"logps/chosen": -0.2624332010746002,
|
|
"logps/rejected": -0.28014859557151794,
|
|
"loss": 0.7392578125,
|
|
"loss/core": 0.7066189646720886,
|
|
"loss/preference_sft": 0.26991426944732666,
|
|
"loss/reference_anchor": 0.6257854104042053,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.8338208198547363,
|
|
"rewards/margins": 1.931775689125061,
|
|
"rewards/rejected": 1.9020452499389648,
|
|
"ronpo/logit": 0.19317759573459625,
|
|
"ronpo/residual": 0.17849010229110718,
|
|
"ronpo/residual_abs": 0.30854010581970215,
|
|
"ronpo/target": 0.01468747854232788,
|
|
"ronpo/target_abs": 0.07874740660190582,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3801648020744324,
|
|
"drift/rejected_abs": 0.17235013842582703,
|
|
"epoch": 0.022248708780294002,
|
|
"grad_norm": 3.765625,
|
|
"learning_rate": 7.666666666666666e-08,
|
|
"logits/chosen": -1.6759541034698486,
|
|
"logits/rejected": -1.7048391103744507,
|
|
"logps/chosen": -0.28707024455070496,
|
|
"logps/rejected": -0.29981210827827454,
|
|
"loss": 1.3195468187332153,
|
|
"loss/core": 1.2741187810897827,
|
|
"loss/preference_sft": 0.30114036798477173,
|
|
"loss/reference_anchor": 0.878444492816925,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.8014285564422607,
|
|
"rewards/margins": 2.0898962020874023,
|
|
"rewards/rejected": 1.7115325927734375,
|
|
"ronpo/logit": 0.20898965001106262,
|
|
"ronpo/residual": 0.22612443566322327,
|
|
"ronpo/residual_abs": 0.3519369661808014,
|
|
"ronpo/target": -0.017134791240096092,
|
|
"ronpo/target_abs": 0.07949066162109375,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20290859043598175,
|
|
"drift/rejected_abs": 0.06069299578666687,
|
|
"epoch": 0.023837902264600714,
|
|
"grad_norm": 5.5,
|
|
"learning_rate": 8.222222222222221e-08,
|
|
"logits/chosen": -1.853276252746582,
|
|
"logits/rejected": -1.8313668966293335,
|
|
"logps/chosen": -0.2681577205657959,
|
|
"logps/rejected": -0.2761048674583435,
|
|
"loss": 0.13597606122493744,
|
|
"loss/core": 0.129965141415596,
|
|
"loss/preference_sft": 0.27459055185317993,
|
|
"loss/reference_anchor": 0.09275973588228226,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0285112857818604,
|
|
"rewards/margins": 1.4334367513656616,
|
|
"rewards/rejected": 0.5950745344161987,
|
|
"ronpo/logit": 0.14334365725517273,
|
|
"ronpo/residual": 0.14809009432792664,
|
|
"ronpo/residual_abs": 0.19457709789276123,
|
|
"ronpo/target": -0.004746442195028067,
|
|
"ronpo/target_abs": 0.08164028823375702,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19127903878688812,
|
|
"drift/rejected_abs": 0.10886410623788834,
|
|
"epoch": 0.02542709574890743,
|
|
"grad_norm": 13.8125,
|
|
"learning_rate": 8.777777777777778e-08,
|
|
"logits/chosen": -1.827038049697876,
|
|
"logits/rejected": -1.7601505517959595,
|
|
"logps/chosen": -0.302360475063324,
|
|
"logps/rejected": -0.29604336619377136,
|
|
"loss": 0.05623495578765869,
|
|
"loss/core": 0.049041394144296646,
|
|
"loss/preference_sft": 0.30537527799606323,
|
|
"loss/reference_anchor": 0.11333372443914413,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9121793508529663,
|
|
"rewards/margins": 0.8244398236274719,
|
|
"rewards/rejected": 1.0877394676208496,
|
|
"ronpo/logit": 0.08244398236274719,
|
|
"ronpo/residual": 0.0879630595445633,
|
|
"ronpo/residual_abs": 0.13678863644599915,
|
|
"ronpo/target": -0.005519069265574217,
|
|
"ronpo/target_abs": 0.07422248274087906,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.337031751871109,
|
|
"drift/rejected_abs": 0.1565742790699005,
|
|
"epoch": 0.027016289233214145,
|
|
"grad_norm": 22.25,
|
|
"learning_rate": 9.333333333333334e-08,
|
|
"logits/chosen": -1.9177894592285156,
|
|
"logits/rejected": -1.9806658029556274,
|
|
"logps/chosen": -0.27497977018356323,
|
|
"logps/rejected": -0.27717286348342896,
|
|
"loss": 0.6743783354759216,
|
|
"loss/core": 0.6322578191757202,
|
|
"loss/preference_sft": 0.27909091114997864,
|
|
"loss/reference_anchor": 0.8145014047622681,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.369861125946045,
|
|
"rewards/margins": 1.80503249168396,
|
|
"rewards/rejected": 1.5648287534713745,
|
|
"ronpo/logit": 0.1805032342672348,
|
|
"ronpo/residual": 0.1641511768102646,
|
|
"ronpo/residual_abs": 0.22668591141700745,
|
|
"ronpo/target": 0.016352057456970215,
|
|
"ronpo/target_abs": 0.07268329709768295,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22597479820251465,
|
|
"drift/rejected_abs": 0.10148023068904877,
|
|
"epoch": 0.028605482717520857,
|
|
"grad_norm": 4.1875,
|
|
"learning_rate": 9.888888888888889e-08,
|
|
"logits/chosen": -1.8360424041748047,
|
|
"logits/rejected": -1.9166383743286133,
|
|
"logps/chosen": -0.2628440856933594,
|
|
"logps/rejected": -0.2537669837474823,
|
|
"loss": 0.08646391332149506,
|
|
"loss/core": 0.08045277744531631,
|
|
"loss/preference_sft": 0.26596134901046753,
|
|
"loss/reference_anchor": 0.09362667798995972,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.257885456085205,
|
|
"rewards/margins": 1.2502182722091675,
|
|
"rewards/rejected": 1.0076671838760376,
|
|
"ronpo/logit": 0.125021830201149,
|
|
"ronpo/residual": 0.11178363859653473,
|
|
"ronpo/residual_abs": 0.1606568545103073,
|
|
"ronpo/target": 0.013238191604614258,
|
|
"ronpo/target_abs": 0.08457295596599579,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22616279125213623,
|
|
"drift/rejected_abs": 0.08887867629528046,
|
|
"epoch": 0.030194676201827572,
|
|
"grad_norm": 0.3984375,
|
|
"learning_rate": 9.999398298646737e-08,
|
|
"logits/chosen": -1.8040069341659546,
|
|
"logits/rejected": -1.828992486000061,
|
|
"logps/chosen": -0.2888103127479553,
|
|
"logps/rejected": -0.28947386145591736,
|
|
"loss": 0.18194636702537537,
|
|
"loss/core": 0.17117659747600555,
|
|
"loss/preference_sft": 0.2951456904411316,
|
|
"loss/reference_anchor": 0.18588101863861084,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 2.2616281509399414,
|
|
"rewards/margins": 1.3749189376831055,
|
|
"rewards/rejected": 0.8867092132568359,
|
|
"ronpo/logit": 0.1374918818473816,
|
|
"ronpo/residual": 0.12481904029846191,
|
|
"ronpo/residual_abs": 0.1675949990749359,
|
|
"ronpo/target": 0.012672854587435722,
|
|
"ronpo/target_abs": 0.07328122109174728,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3307506740093231,
|
|
"drift/rejected_abs": 0.16090306639671326,
|
|
"epoch": 0.03178386968613429,
|
|
"grad_norm": 92.5,
|
|
"learning_rate": 9.996954135095479e-08,
|
|
"logits/chosen": -1.7753002643585205,
|
|
"logits/rejected": -1.8347423076629639,
|
|
"logps/chosen": -0.2850727438926697,
|
|
"logps/rejected": -0.2890182435512543,
|
|
"loss": 0.7790960073471069,
|
|
"loss/core": 0.7361105680465698,
|
|
"loss/preference_sft": 0.2861769199371338,
|
|
"loss/reference_anchor": 0.8310915231704712,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3066513538360596,
|
|
"rewards/margins": 1.7068488597869873,
|
|
"rewards/rejected": 1.5998027324676514,
|
|
"ronpo/logit": 0.17068488895893097,
|
|
"ronpo/residual": 0.18404145538806915,
|
|
"ronpo/residual_abs": 0.2643963396549225,
|
|
"ronpo/target": -0.013356572017073631,
|
|
"ronpo/target_abs": 0.06924636662006378,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23268786072731018,
|
|
"drift/rejected_abs": 0.0795300155878067,
|
|
"epoch": 0.033373063170441,
|
|
"grad_norm": 12.375,
|
|
"learning_rate": 9.992630821454457e-08,
|
|
"logits/chosen": -1.9496657848358154,
|
|
"logits/rejected": -1.9096084833145142,
|
|
"logps/chosen": -0.2818007469177246,
|
|
"logps/rejected": -0.28615885972976685,
|
|
"loss": 0.29838305711746216,
|
|
"loss/core": 0.28360605239868164,
|
|
"loss/preference_sft": 0.28804826736450195,
|
|
"loss/reference_anchor": 0.2667350769042969,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.32478404045105,
|
|
"rewards/margins": 1.5362763404846191,
|
|
"rewards/rejected": 0.7885079383850098,
|
|
"ronpo/logit": 0.15362763404846191,
|
|
"ronpo/residual": 0.15497443079948425,
|
|
"ronpo/residual_abs": 0.20809760689735413,
|
|
"ronpo/target": -0.0013467819662764668,
|
|
"ronpo/target_abs": 0.06745518743991852,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.46602663397789,
|
|
"drift/rejected_abs": 0.2611501216888428,
|
|
"epoch": 0.03496225665474772,
|
|
"grad_norm": 936.0,
|
|
"learning_rate": 9.986429983545125e-08,
|
|
"logits/chosen": -1.918752670288086,
|
|
"logits/rejected": -1.8298696279525757,
|
|
"logps/chosen": -0.2680893540382385,
|
|
"logps/rejected": -0.3012104034423828,
|
|
"loss": 0.8778338432312012,
|
|
"loss/core": 0.8211013674736023,
|
|
"loss/preference_sft": 0.28689926862716675,
|
|
"loss/reference_anchor": 1.1059573888778687,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.65931510925293,
|
|
"rewards/margins": 2.0492444038391113,
|
|
"rewards/rejected": 2.6100707054138184,
|
|
"ronpo/logit": 0.20492446422576904,
|
|
"ronpo/residual": 0.19235225021839142,
|
|
"ronpo/residual_abs": 0.3394165635108948,
|
|
"ronpo/target": 0.01257222332060337,
|
|
"ronpo/target_abs": 0.0747305378317833,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20914717018604279,
|
|
"drift/rejected_abs": 0.10763482004404068,
|
|
"epoch": 0.03655145013905443,
|
|
"grad_norm": 7.125,
|
|
"learning_rate": 9.978353953249021e-08,
|
|
"logits/chosen": -1.7422930002212524,
|
|
"logits/rejected": -1.7559515237808228,
|
|
"logps/chosen": -0.2856474220752716,
|
|
"logps/rejected": -0.288565993309021,
|
|
"loss": 0.07564763724803925,
|
|
"loss/core": 0.06711847335100174,
|
|
"loss/preference_sft": 0.2876007556915283,
|
|
"loss/reference_anchor": 0.14182327687740326,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.089381456375122,
|
|
"rewards/margins": 1.014510154724121,
|
|
"rewards/rejected": 1.074871301651001,
|
|
"ronpo/logit": 0.10145101696252823,
|
|
"ronpo/residual": 0.0855717882514,
|
|
"ronpo/residual_abs": 0.15696951746940613,
|
|
"ronpo/target": 0.015879224985837936,
|
|
"ronpo/target_abs": 0.08091199398040771,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.38106095790863037,
|
|
"drift/rejected_abs": 0.1474742889404297,
|
|
"epoch": 0.03814064362336114,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 9.968405767630856e-08,
|
|
"logits/chosen": -1.8890774250030518,
|
|
"logits/rejected": -1.897849440574646,
|
|
"logps/chosen": -0.27462321519851685,
|
|
"logps/rejected": -0.287124902009964,
|
|
"loss": 0.8557225465774536,
|
|
"loss/core": 0.8139694333076477,
|
|
"loss/preference_sft": 0.28747329115867615,
|
|
"loss/reference_anchor": 0.8063166737556458,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.8102378845214844,
|
|
"rewards/margins": 2.3369901180267334,
|
|
"rewards/rejected": 1.4732472896575928,
|
|
"ronpo/logit": 0.2336990088224411,
|
|
"ronpo/residual": 0.22560825943946838,
|
|
"ronpo/residual_abs": 0.2692892253398895,
|
|
"ronpo/target": 0.008090745657682419,
|
|
"ronpo/target_abs": 0.07093735784292221,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23315230011940002,
|
|
"drift/rejected_abs": 0.05996812507510185,
|
|
"epoch": 0.03972983710766786,
|
|
"grad_norm": 12.25,
|
|
"learning_rate": 9.956589167796391e-08,
|
|
"logits/chosen": -1.9130750894546509,
|
|
"logits/rejected": -1.9361803531646729,
|
|
"logps/chosen": -0.2950516641139984,
|
|
"logps/rejected": -0.268522173166275,
|
|
"loss": 0.841772735118866,
|
|
"loss/core": 0.8178065419197083,
|
|
"loss/preference_sft": 0.2870258688926697,
|
|
"loss/reference_anchor": 0.45062151551246643,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.331399440765381,
|
|
"rewards/margins": 1.7358144521713257,
|
|
"rewards/rejected": 0.5955848097801208,
|
|
"ronpo/logit": 0.17358146607875824,
|
|
"ronpo/residual": 0.18443605303764343,
|
|
"ronpo/residual_abs": 0.2259272038936615,
|
|
"ronpo/target": -0.010854599997401237,
|
|
"ronpo/target_abs": 0.08973653614521027,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30845123529434204,
|
|
"drift/rejected_abs": 0.11910972744226456,
|
|
"epoch": 0.04131903059197457,
|
|
"grad_norm": 11.0625,
|
|
"learning_rate": 9.942908597485558e-08,
|
|
"logits/chosen": -1.8599214553833008,
|
|
"logits/rejected": -1.7708088159561157,
|
|
"logps/chosen": -0.25000277161598206,
|
|
"logps/rejected": -0.2528410851955414,
|
|
"loss": 0.5783834457397461,
|
|
"loss/core": 0.5539836883544922,
|
|
"loss/preference_sft": 0.25629931688308716,
|
|
"loss/reference_anchor": 0.46236711740493774,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0844438076019287,
|
|
"rewards/margins": 1.8948835134506226,
|
|
"rewards/rejected": 1.1895604133605957,
|
|
"ronpo/logit": 0.18948833644390106,
|
|
"ronpo/residual": 0.17439742386341095,
|
|
"ronpo/residual_abs": 0.2648027837276459,
|
|
"ronpo/target": 0.015090915374457836,
|
|
"ronpo/target_abs": 0.07385419309139252,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14000125229358673,
|
|
"drift/rejected_abs": 0.10091972351074219,
|
|
"epoch": 0.04290822407628129,
|
|
"grad_norm": 12.8125,
|
|
"learning_rate": 9.927369201401357e-08,
|
|
"logits/chosen": -1.9906437397003174,
|
|
"logits/rejected": -1.9516689777374268,
|
|
"logps/chosen": -0.2750958502292633,
|
|
"logps/rejected": -0.284817099571228,
|
|
"loss": 0.043532054871320724,
|
|
"loss/core": 0.03747930750250816,
|
|
"loss/preference_sft": 0.2810477614402771,
|
|
"loss/reference_anchor": 0.09295015782117844,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.398533821105957,
|
|
"rewards/margins": 0.38980793952941895,
|
|
"rewards/rejected": 1.0087257623672485,
|
|
"ronpo/logit": 0.038980789482593536,
|
|
"ronpo/residual": 0.042172592133283615,
|
|
"ronpo/residual_abs": 0.11995859444141388,
|
|
"ronpo/target": -0.0031918026506900787,
|
|
"ronpo/target_abs": 0.0830756425857544,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24642232060432434,
|
|
"drift/rejected_abs": 0.15418444573879242,
|
|
"epoch": 0.044497417560588004,
|
|
"grad_norm": 5.9375,
|
|
"learning_rate": 9.909976823275142e-08,
|
|
"logits/chosen": -1.6394424438476562,
|
|
"logits/rejected": -1.6369479894638062,
|
|
"logps/chosen": -0.26613327860832214,
|
|
"logps/rejected": -0.25486260652542114,
|
|
"loss": 0.07675577700138092,
|
|
"loss/core": 0.06679292768239975,
|
|
"loss/preference_sft": 0.26119264960289,
|
|
"loss/reference_anchor": 0.17313750088214874,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4627883434295654,
|
|
"rewards/margins": 0.9230669140815735,
|
|
"rewards/rejected": 1.5397213697433472,
|
|
"ronpo/logit": 0.09230668842792511,
|
|
"ronpo/residual": 0.0767742171883583,
|
|
"ronpo/residual_abs": 0.14399012923240662,
|
|
"ronpo/target": 0.01553247682750225,
|
|
"ronpo/target_abs": 0.0734403133392334,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.194285050034523,
|
|
"drift/rejected_abs": 0.10672137886285782,
|
|
"epoch": 0.04608661104489471,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 9.890738003669028e-08,
|
|
"logits/chosen": -2.0392231941223145,
|
|
"logits/rejected": -2.050145387649536,
|
|
"logps/chosen": -0.3017435669898987,
|
|
"logps/rejected": -0.3074425160884857,
|
|
"loss": 0.5806225538253784,
|
|
"loss/core": 0.5637769103050232,
|
|
"loss/preference_sft": 0.30560463666915894,
|
|
"loss/reference_anchor": 0.30635276436805725,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.942411184310913,
|
|
"rewards/margins": 0.8751977682113647,
|
|
"rewards/rejected": 1.067213773727417,
|
|
"ronpo/logit": 0.08751976490020752,
|
|
"ronpo/residual": 0.09972776472568512,
|
|
"ronpo/residual_abs": 0.25175055861473083,
|
|
"ronpo/target": -0.012207982130348682,
|
|
"ronpo/target_abs": 0.0755801871418953,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31181323528289795,
|
|
"drift/rejected_abs": 0.08847779780626297,
|
|
"epoch": 0.04767580452920143,
|
|
"grad_norm": 24.875,
|
|
"learning_rate": 9.86965997751626e-08,
|
|
"logits/chosen": -1.8684451580047607,
|
|
"logits/rejected": -1.8871656656265259,
|
|
"logps/chosen": -0.2658239006996155,
|
|
"logps/rejected": -0.2555927336215973,
|
|
"loss": 0.5834201574325562,
|
|
"loss/core": 0.5655571222305298,
|
|
"loss/preference_sft": 0.2657192051410675,
|
|
"loss/reference_anchor": 0.33068618178367615,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1168699264526367,
|
|
"rewards/margins": 2.2592427730560303,
|
|
"rewards/rejected": 0.8576269149780273,
|
|
"ronpo/logit": 0.2259242832660675,
|
|
"ronpo/residual": 0.22903215885162354,
|
|
"ronpo/residual_abs": 0.27631235122680664,
|
|
"ronpo/target": -0.003107901895418763,
|
|
"ronpo/target_abs": 0.08685997128486633,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12096400558948517,
|
|
"drift/rejected_abs": 0.07475753873586655,
|
|
"epoch": 0.04926499801350814,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 9.846750671400445e-08,
|
|
"logits/chosen": -1.976851463317871,
|
|
"logits/rejected": -1.939896583557129,
|
|
"logps/chosen": -0.2994900345802307,
|
|
"logps/rejected": -0.30892014503479004,
|
|
"loss": 0.03421011567115784,
|
|
"loss/core": 0.030272841453552246,
|
|
"loss/preference_sft": 0.31494635343551636,
|
|
"loss/reference_anchor": 0.04725093021988869,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.2083873748779297,
|
|
"rewards/margins": 0.46204623579978943,
|
|
"rewards/rejected": 0.7463411092758179,
|
|
"ronpo/logit": 0.04620462656021118,
|
|
"ronpo/residual": 0.03318546339869499,
|
|
"ronpo/residual_abs": 0.11656346172094345,
|
|
"ronpo/target": 0.013019162230193615,
|
|
"ronpo/target_abs": 0.08384847640991211,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16452792286872864,
|
|
"drift/rejected_abs": 0.1353653371334076,
|
|
"epoch": 0.05085419149781486,
|
|
"grad_norm": 8.4375,
|
|
"learning_rate": 9.822018700574695e-08,
|
|
"logits/chosen": -1.954433798789978,
|
|
"logits/rejected": -1.929897665977478,
|
|
"logps/chosen": -0.28900691866874695,
|
|
"logps/rejected": -0.291254460811615,
|
|
"loss": 0.2159826010465622,
|
|
"loss/core": 0.20637917518615723,
|
|
"loss/preference_sft": 0.2918491065502167,
|
|
"loss/reference_anchor": 0.16288307309150696,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.644606351852417,
|
|
"rewards/margins": 0.29627370834350586,
|
|
"rewards/rejected": 1.3483326435089111,
|
|
"ronpo/logit": 0.029627371579408646,
|
|
"ronpo/residual": 0.037656258791685104,
|
|
"ronpo/residual_abs": 0.1743960678577423,
|
|
"ronpo/target": -0.008028890937566757,
|
|
"ronpo/target_abs": 0.073066346347332,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3098263144493103,
|
|
"drift/rejected_abs": 0.1317211091518402,
|
|
"epoch": 0.052443384982121574,
|
|
"grad_norm": 5.40625,
|
|
"learning_rate": 9.79547336572177e-08,
|
|
"logits/chosen": -1.7146660089492798,
|
|
"logits/rejected": -1.6996208429336548,
|
|
"logps/chosen": -0.2845102846622467,
|
|
"logps/rejected": -0.2821315824985504,
|
|
"loss": 0.39138349890708923,
|
|
"loss/core": 0.3647650480270386,
|
|
"loss/preference_sft": 0.2856329381465912,
|
|
"loss/reference_anchor": 0.5038057565689087,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0979020595550537,
|
|
"rewards/margins": 1.7806909084320068,
|
|
"rewards/rejected": 1.3172109127044678,
|
|
"ronpo/logit": 0.1780690848827362,
|
|
"ronpo/residual": 0.1646689921617508,
|
|
"ronpo/residual_abs": 0.223629429936409,
|
|
"ronpo/target": 0.013400102965533733,
|
|
"ronpo/target_abs": 0.06691430509090424,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31989067792892456,
|
|
"drift/rejected_abs": 0.12814299762248993,
|
|
"epoch": 0.05403257846642829,
|
|
"grad_norm": 2.625,
|
|
"learning_rate": 9.767124649456482e-08,
|
|
"logits/chosen": -1.8619706630706787,
|
|
"logits/rejected": -1.8246967792510986,
|
|
"logps/chosen": -0.28764164447784424,
|
|
"logps/rejected": -0.2885397970676422,
|
|
"loss": 0.5198912024497986,
|
|
"loss/core": 0.4936496615409851,
|
|
"loss/preference_sft": 0.2862817645072937,
|
|
"loss/reference_anchor": 0.49620193243026733,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.1981494426727295,
|
|
"rewards/margins": 1.9242050647735596,
|
|
"rewards/rejected": 1.273944616317749,
|
|
"ronpo/logit": 0.19242048263549805,
|
|
"ronpo/residual": 0.18473686277866364,
|
|
"ronpo/residual_abs": 0.24073760211467743,
|
|
"ronpo/target": 0.007683630101382732,
|
|
"ronpo/target_abs": 0.0752769261598587,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22625796496868134,
|
|
"drift/rejected_abs": 0.08741482347249985,
|
|
"epoch": 0.055621771950735005,
|
|
"grad_norm": 38.25,
|
|
"learning_rate": 9.736983212571646e-08,
|
|
"logits/chosen": -1.825382947921753,
|
|
"logits/rejected": -1.8280212879180908,
|
|
"logps/chosen": -0.2802065908908844,
|
|
"logps/rejected": -0.2865963578224182,
|
|
"loss": 0.21462078392505646,
|
|
"loss/core": 0.2051103413105011,
|
|
"loss/preference_sft": 0.2825978994369507,
|
|
"loss/reference_anchor": 0.1619492471218109,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.261361598968506,
|
|
"rewards/margins": 1.3890936374664307,
|
|
"rewards/rejected": 0.8722679018974304,
|
|
"ronpo/logit": 0.13890936970710754,
|
|
"ronpo/residual": 0.14540444314479828,
|
|
"ronpo/residual_abs": 0.1883070468902588,
|
|
"ronpo/target": -0.006495094392448664,
|
|
"ronpo/target_abs": 0.07979901880025864,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31887388229370117,
|
|
"drift/rejected_abs": 0.14860941469669342,
|
|
"epoch": 0.057210965435041714,
|
|
"grad_norm": 137.0,
|
|
"learning_rate": 9.705060390028978e-08,
|
|
"logits/chosen": -1.8157069683074951,
|
|
"logits/rejected": -1.7937179803848267,
|
|
"logps/chosen": -0.2820594310760498,
|
|
"logps/rejected": -0.2725641131401062,
|
|
"loss": 0.30303138494491577,
|
|
"loss/core": 0.28699272871017456,
|
|
"loss/preference_sft": 0.26962658762931824,
|
|
"loss/reference_anchor": 0.2938117980957031,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.188739061355591,
|
|
"rewards/margins": 1.7026450634002686,
|
|
"rewards/rejected": 1.4860939979553223,
|
|
"ronpo/logit": 0.17026451230049133,
|
|
"ronpo/residual": 0.16811729967594147,
|
|
"ronpo/residual_abs": 0.21821363270282745,
|
|
"ronpo/target": 0.0021472182124853134,
|
|
"ronpo/target_abs": 0.07455535978078842,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3943818509578705,
|
|
"drift/rejected_abs": 0.16938166320323944,
|
|
"epoch": 0.05880015891934843,
|
|
"grad_norm": 13.9375,
|
|
"learning_rate": 9.671368186696487e-08,
|
|
"logits/chosen": -1.682560682296753,
|
|
"logits/rejected": -1.7234325408935547,
|
|
"logps/chosen": -0.29317352175712585,
|
|
"logps/rejected": -0.3039776384830475,
|
|
"loss": 0.7356561422348022,
|
|
"loss/core": 0.6865347623825073,
|
|
"loss/preference_sft": 0.3006138801574707,
|
|
"loss/reference_anchor": 0.9523669481277466,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.942030429840088,
|
|
"rewards/margins": 2.3751327991485596,
|
|
"rewards/rejected": 1.5668972730636597,
|
|
"ronpo/logit": 0.23751330375671387,
|
|
"ronpo/residual": 0.23533913493156433,
|
|
"ronpo/residual_abs": 0.28016549348831177,
|
|
"ronpo/target": 0.0021741650998592377,
|
|
"ronpo/target_abs": 0.07683144509792328,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21583101153373718,
|
|
"drift/rejected_abs": 0.14082249999046326,
|
|
"epoch": 0.060389352403655144,
|
|
"grad_norm": 1.84375,
|
|
"learning_rate": 9.635919272833936e-08,
|
|
"logits/chosen": -1.7084299325942993,
|
|
"logits/rejected": -1.7109653949737549,
|
|
"logps/chosen": -0.28803473711013794,
|
|
"logps/rejected": -0.29996567964553833,
|
|
"loss": 0.10773409903049469,
|
|
"loss/core": 0.09961052983999252,
|
|
"loss/preference_sft": 0.2932335436344147,
|
|
"loss/reference_anchor": 0.1331479549407959,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.157505750656128,
|
|
"rewards/margins": 0.7513991594314575,
|
|
"rewards/rejected": 1.4061065912246704,
|
|
"ronpo/logit": 0.07513991743326187,
|
|
"ronpo/residual": 0.061122387647628784,
|
|
"ronpo/residual_abs": 0.1610337495803833,
|
|
"ronpo/target": 0.014017535373568535,
|
|
"ronpo/target_abs": 0.08196841925382614,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18434445559978485,
|
|
"drift/rejected_abs": 0.11586540937423706,
|
|
"epoch": 0.06197854588796186,
|
|
"grad_norm": 1.765625,
|
|
"learning_rate": 9.598726979328077e-08,
|
|
"logits/chosen": -1.86000657081604,
|
|
"logits/rejected": -1.8386214971542358,
|
|
"logps/chosen": -0.3027092218399048,
|
|
"logps/rejected": -0.2923242449760437,
|
|
"loss": 0.20319950580596924,
|
|
"loss/core": 0.1940327286720276,
|
|
"loss/preference_sft": 0.2979516088962555,
|
|
"loss/reference_anchor": 0.15354016423225403,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8425309658050537,
|
|
"rewards/margins": 0.6865968704223633,
|
|
"rewards/rejected": 1.15593421459198,
|
|
"ronpo/logit": 0.06865968555212021,
|
|
"ronpo/residual": 0.06499691307544708,
|
|
"ronpo/residual_abs": 0.19421474635601044,
|
|
"ronpo/target": 0.0036627836525440216,
|
|
"ronpo/target_abs": 0.07378067076206207,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3092130422592163,
|
|
"drift/rejected_abs": 0.1502634435892105,
|
|
"epoch": 0.06356773937226858,
|
|
"grad_norm": 12.6875,
|
|
"learning_rate": 9.559805292679444e-08,
|
|
"logits/chosen": -1.878720998764038,
|
|
"logits/rejected": -1.853334665298462,
|
|
"logps/chosen": -0.28743189573287964,
|
|
"logps/rejected": -0.3008131980895996,
|
|
"loss": 0.38157254457473755,
|
|
"loss/core": 0.3494413197040558,
|
|
"loss/preference_sft": 0.2995978593826294,
|
|
"loss/reference_anchor": 0.6126652359962463,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.0919899940490723,
|
|
"rewards/margins": 1.5904356241226196,
|
|
"rewards/rejected": 1.5015544891357422,
|
|
"ronpo/logit": 0.1590435802936554,
|
|
"ronpo/residual": 0.16887164115905762,
|
|
"ronpo/residual_abs": 0.21642287075519562,
|
|
"ronpo/target": -0.009828063659369946,
|
|
"ronpo/target_abs": 0.09070595353841782,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2549411654472351,
|
|
"drift/rejected_abs": 0.10770858824253082,
|
|
"epoch": 0.06515693285657528,
|
|
"grad_norm": 9.0625,
|
|
"learning_rate": 9.519168849742603e-08,
|
|
"logits/chosen": -1.8176409006118774,
|
|
"logits/rejected": -1.7602897882461548,
|
|
"logps/chosen": -0.28294557332992554,
|
|
"logps/rejected": -0.2822961211204529,
|
|
"loss": 0.13457834720611572,
|
|
"loss/core": 0.1233421191573143,
|
|
"loss/preference_sft": 0.27934110164642334,
|
|
"loss/reference_anchor": 0.19679050147533417,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5482711791992188,
|
|
"rewards/margins": 1.4736816883087158,
|
|
"rewards/rejected": 1.074589490890503,
|
|
"ronpo/logit": 0.1473681777715683,
|
|
"ronpo/residual": 0.15736904740333557,
|
|
"ronpo/residual_abs": 0.1994774490594864,
|
|
"ronpo/target": -0.010000874288380146,
|
|
"ronpo/target_abs": 0.07206423580646515,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3109588325023651,
|
|
"drift/rejected_abs": 0.12311758100986481,
|
|
"epoch": 0.066746126340882,
|
|
"grad_norm": 19.75,
|
|
"learning_rate": 9.476832932221834e-08,
|
|
"logits/chosen": -1.8160145282745361,
|
|
"logits/rejected": -1.8787750005722046,
|
|
"logps/chosen": -0.2810691297054291,
|
|
"logps/rejected": -0.28845909237861633,
|
|
"loss": 0.5719379186630249,
|
|
"loss/core": 0.5511919260025024,
|
|
"loss/preference_sft": 0.2875823974609375,
|
|
"loss/reference_anchor": 0.38616234064102173,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.108250141143799,
|
|
"rewards/margins": 1.8811763525009155,
|
|
"rewards/rejected": 1.2270740270614624,
|
|
"ronpo/logit": 0.1881176233291626,
|
|
"ronpo/residual": 0.18612520396709442,
|
|
"ronpo/residual_abs": 0.2601304054260254,
|
|
"ronpo/target": 0.0019924379885196686,
|
|
"ronpo/target_abs": 0.07251577079296112,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.173118457198143,
|
|
"drift/rejected_abs": 0.11045239865779877,
|
|
"epoch": 0.06833531982518871,
|
|
"grad_norm": 4.3125,
|
|
"learning_rate": 9.432813460924306e-08,
|
|
"logits/chosen": -1.8905178308486938,
|
|
"logits/rejected": -1.8230892419815063,
|
|
"logps/chosen": -0.2746286988258362,
|
|
"logps/rejected": -0.28819310665130615,
|
|
"loss": 0.03434374928474426,
|
|
"loss/core": 0.02805573306977749,
|
|
"loss/preference_sft": 0.28005850315093994,
|
|
"loss/reference_anchor": 0.09775449335575104,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.731080412864685,
|
|
"rewards/margins": 0.6269391179084778,
|
|
"rewards/rejected": 1.1041414737701416,
|
|
"ronpo/logit": 0.06269390881061554,
|
|
"ronpo/residual": 0.05062420293688774,
|
|
"ronpo/residual_abs": 0.11113785207271576,
|
|
"ronpo/target": 0.012069707736372948,
|
|
"ronpo/target_abs": 0.07993054389953613,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3507832884788513,
|
|
"drift/rejected_abs": 0.13538460433483124,
|
|
"epoch": 0.06992451330949544,
|
|
"grad_norm": 1.5390625,
|
|
"learning_rate": 9.387126989772909e-08,
|
|
"logits/chosen": -1.713751196861267,
|
|
"logits/rejected": -1.7000278234481812,
|
|
"logps/chosen": -0.2693570852279663,
|
|
"logps/rejected": -0.288845419883728,
|
|
"loss": 1.1489449739456177,
|
|
"loss/core": 1.1118707656860352,
|
|
"loss/preference_sft": 0.28148311376571655,
|
|
"loss/reference_anchor": 0.713333010673523,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.5078330039978027,
|
|
"rewards/margins": 2.1564221382141113,
|
|
"rewards/rejected": 1.3514111042022705,
|
|
"ronpo/logit": 0.21564224362373352,
|
|
"ronpo/residual": 0.20636311173439026,
|
|
"ronpo/residual_abs": 0.3267207145690918,
|
|
"ronpo/target": 0.009279133751988411,
|
|
"ronpo/target_abs": 0.08725794404745102,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33888542652130127,
|
|
"drift/rejected_abs": 0.08225339651107788,
|
|
"epoch": 0.07151370679380215,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 9.339790699581003e-08,
|
|
"logits/chosen": -1.9155328273773193,
|
|
"logits/rejected": -1.8650407791137695,
|
|
"logps/chosen": -0.3122982084751129,
|
|
"logps/rejected": -0.30397704243659973,
|
|
"loss": 1.3468017578125,
|
|
"loss/core": 1.3081927299499512,
|
|
"loss/preference_sft": 0.3082147538661957,
|
|
"loss/reference_anchor": 0.741364061832428,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.388200044631958,
|
|
"rewards/margins": 2.5675928592681885,
|
|
"rewards/rejected": 0.8206073641777039,
|
|
"ronpo/logit": 0.25675931572914124,
|
|
"ronpo/residual": 0.24346081912517548,
|
|
"ronpo/residual_abs": 0.3151950538158417,
|
|
"ronpo/target": 0.01329845655709505,
|
|
"ronpo/target_abs": 0.08093789964914322,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2742893099784851,
|
|
"drift/rejected_abs": 0.09671930223703384,
|
|
"epoch": 0.07310290027810885,
|
|
"grad_norm": 1.9921875,
|
|
"learning_rate": 9.290822391591417e-08,
|
|
"logits/chosen": -1.6402956247329712,
|
|
"logits/rejected": -1.6450611352920532,
|
|
"logps/chosen": -0.2720198631286621,
|
|
"logps/rejected": -0.2679344117641449,
|
|
"loss": 0.3185911476612091,
|
|
"loss/core": 0.30630046129226685,
|
|
"loss/preference_sft": 0.26990702748298645,
|
|
"loss/reference_anchor": 0.21882298588752747,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.742389440536499,
|
|
"rewards/margins": 1.7752317190170288,
|
|
"rewards/rejected": 0.9671577215194702,
|
|
"ronpo/logit": 0.1775231808423996,
|
|
"ronpo/residual": 0.17394277453422546,
|
|
"ronpo/residual_abs": 0.22346839308738708,
|
|
"ronpo/target": 0.003580413293093443,
|
|
"ronpo/target_abs": 0.09092529118061066,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3355063498020172,
|
|
"drift/rejected_abs": 0.14276744425296783,
|
|
"epoch": 0.07469209376241558,
|
|
"grad_norm": 3.609375,
|
|
"learning_rate": 9.240240480782128e-08,
|
|
"logits/chosen": -1.8078244924545288,
|
|
"logits/rejected": -1.8107181787490845,
|
|
"logps/chosen": -0.2891586422920227,
|
|
"logps/rejected": -0.2673870921134949,
|
|
"loss": 0.7774661183357239,
|
|
"loss/core": 0.75029456615448,
|
|
"loss/preference_sft": 0.29623985290527344,
|
|
"loss/reference_anchor": 0.5138068795204163,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.3549816608428955,
|
|
"rewards/margins": 1.9295164346694946,
|
|
"rewards/rejected": 1.4254653453826904,
|
|
"ronpo/logit": 0.19295167922973633,
|
|
"ronpo/residual": 0.196090430021286,
|
|
"ronpo/residual_abs": 0.23866817355155945,
|
|
"ronpo/target": -0.0031387873459607363,
|
|
"ronpo/target_abs": 0.06865248829126358,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3238152265548706,
|
|
"drift/rejected_abs": 0.14746034145355225,
|
|
"epoch": 0.07628128724672228,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 9.188063988941146e-08,
|
|
"logits/chosen": -1.9462316036224365,
|
|
"logits/rejected": -1.9106372594833374,
|
|
"logps/chosen": -0.25194069743156433,
|
|
"logps/rejected": -0.26762017607688904,
|
|
"loss": 0.428089439868927,
|
|
"loss/core": 0.4055059850215912,
|
|
"loss/preference_sft": 0.25560957193374634,
|
|
"loss/reference_anchor": 0.4261084198951721,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.237283706665039,
|
|
"rewards/margins": 1.8325433731079102,
|
|
"rewards/rejected": 1.4047404527664185,
|
|
"ronpo/logit": 0.18325433135032654,
|
|
"ronpo/residual": 0.18264977633953094,
|
|
"ronpo/residual_abs": 0.21224486827850342,
|
|
"ronpo/target": 0.0006045631016604602,
|
|
"ronpo/target_abs": 0.055509041994810104,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24143171310424805,
|
|
"drift/rejected_abs": 0.09511907398700714,
|
|
"epoch": 0.07787048073102901,
|
|
"grad_norm": 4.53125,
|
|
"learning_rate": 9.134312537513186e-08,
|
|
"logits/chosen": -1.8824867010116577,
|
|
"logits/rejected": -1.8663095235824585,
|
|
"logps/chosen": -0.27780038118362427,
|
|
"logps/rejected": -0.27743133902549744,
|
|
"loss": 0.7090773582458496,
|
|
"loss/core": 0.6847109794616699,
|
|
"loss/preference_sft": 0.27157658338546753,
|
|
"loss/reference_anchor": 0.4601725935935974,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4126458168029785,
|
|
"rewards/margins": 1.462439775466919,
|
|
"rewards/rejected": 0.9502058029174805,
|
|
"ronpo/logit": 0.14624398946762085,
|
|
"ronpo/residual": 0.16018269956111908,
|
|
"ronpo/residual_abs": 0.220293328166008,
|
|
"ronpo/target": -0.013938702642917633,
|
|
"ronpo/target_abs": 0.08255701512098312,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23546819388866425,
|
|
"drift/rejected_abs": 0.12727732956409454,
|
|
"epoch": 0.07945967421533572,
|
|
"grad_norm": 3.34375,
|
|
"learning_rate": 9.07900634022086e-08,
|
|
"logits/chosen": -1.9230791330337524,
|
|
"logits/rejected": -1.8947263956069946,
|
|
"logps/chosen": -0.2548685669898987,
|
|
"logps/rejected": -0.25849586725234985,
|
|
"loss": 0.08169195055961609,
|
|
"loss/core": 0.06879390776157379,
|
|
"loss/preference_sft": 0.26240864396095276,
|
|
"loss/reference_anchor": 0.23172004520893097,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.35325288772583,
|
|
"rewards/margins": 1.0831009149551392,
|
|
"rewards/rejected": 1.2701518535614014,
|
|
"ronpo/logit": 0.10831008851528168,
|
|
"ronpo/residual": 0.09686598926782608,
|
|
"ronpo/residual_abs": 0.16113971173763275,
|
|
"ronpo/target": 0.01144410576671362,
|
|
"ronpo/target_abs": 0.07850413024425507,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22031930088996887,
|
|
"drift/rejected_abs": 0.14361494779586792,
|
|
"epoch": 0.08104886769964244,
|
|
"grad_norm": 0.6328125,
|
|
"learning_rate": 9.02216619546311e-08,
|
|
"logits/chosen": -1.790488600730896,
|
|
"logits/rejected": -1.882672667503357,
|
|
"logps/chosen": -0.3244701027870178,
|
|
"logps/rejected": -0.32260042428970337,
|
|
"loss": 0.12643373012542725,
|
|
"loss/core": 0.08407403528690338,
|
|
"loss/preference_sft": 0.326153427362442,
|
|
"loss/reference_anchor": 0.8145785331726074,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.202606678009033,
|
|
"rewards/margins": 0.766952395439148,
|
|
"rewards/rejected": 1.4356540441513062,
|
|
"ronpo/logit": 0.07669521868228912,
|
|
"ronpo/residual": 0.0710417702794075,
|
|
"ronpo/residual_abs": 0.12641890347003937,
|
|
"ronpo/target": 0.005653454922139645,
|
|
"ronpo/target_abs": 0.07296518236398697,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3037354350090027,
|
|
"drift/rejected_abs": 0.20296995341777802,
|
|
"epoch": 0.08263806118394915,
|
|
"grad_norm": 46.25,
|
|
"learning_rate": 8.963813478493788e-08,
|
|
"logits/chosen": -1.888965368270874,
|
|
"logits/rejected": -1.9428901672363281,
|
|
"logps/chosen": -0.27975988388061523,
|
|
"logps/rejected": -0.2818945348262787,
|
|
"loss": 0.22990703582763672,
|
|
"loss/core": 0.18613065779209137,
|
|
"loss/preference_sft": 0.27945324778556824,
|
|
"loss/reference_anchor": 0.8475823402404785,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.036375045776367,
|
|
"rewards/margins": 1.0084631443023682,
|
|
"rewards/rejected": 2.02791166305542,
|
|
"ronpo/logit": 0.1008463129401207,
|
|
"ronpo/residual": 0.10048683732748032,
|
|
"ronpo/residual_abs": 0.20321817696094513,
|
|
"ronpo/target": 0.00035947878495790064,
|
|
"ronpo/target_abs": 0.08794078230857849,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24098190665245056,
|
|
"drift/rejected_abs": 0.1554281860589981,
|
|
"epoch": 0.08422725466825585,
|
|
"grad_norm": 119.0,
|
|
"learning_rate": 8.903970133383297e-08,
|
|
"logits/chosen": -1.8848758935928345,
|
|
"logits/rejected": -1.8094570636749268,
|
|
"logps/chosen": -0.26744991540908813,
|
|
"logps/rejected": -0.2860262989997864,
|
|
"loss": 0.37246307730674744,
|
|
"loss/core": 0.35889145731925964,
|
|
"loss/preference_sft": 0.2836495339870453,
|
|
"loss/reference_anchor": 0.243066668510437,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4094414710998535,
|
|
"rewards/margins": 0.9786992073059082,
|
|
"rewards/rejected": 1.4307423830032349,
|
|
"ronpo/logit": 0.09786991775035858,
|
|
"ronpo/residual": 0.08945603668689728,
|
|
"ronpo/residual_abs": 0.22707316279411316,
|
|
"ronpo/target": 0.008413868956267834,
|
|
"ronpo/target_abs": 0.07881908118724823,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3145125210285187,
|
|
"drift/rejected_abs": 0.13519734144210815,
|
|
"epoch": 0.08581644815256258,
|
|
"grad_norm": 55.0,
|
|
"learning_rate": 8.842658664766316e-08,
|
|
"logits/chosen": -1.8931232690811157,
|
|
"logits/rejected": -1.8972790241241455,
|
|
"logps/chosen": -0.291258305311203,
|
|
"logps/rejected": -0.2826591730117798,
|
|
"loss": 0.6564851999282837,
|
|
"loss/core": 0.6233766078948975,
|
|
"loss/preference_sft": 0.2801192104816437,
|
|
"loss/reference_anchor": 0.634162187576294,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.143418312072754,
|
|
"rewards/margins": 1.7918039560317993,
|
|
"rewards/rejected": 1.3516143560409546,
|
|
"ronpo/logit": 0.17918041348457336,
|
|
"ronpo/residual": 0.1889977604150772,
|
|
"ronpo/residual_abs": 0.24317440390586853,
|
|
"ronpo/target": -0.009817345067858696,
|
|
"ronpo/target_abs": 0.07919232547283173,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14758369326591492,
|
|
"drift/rejected_abs": 0.06989634037017822,
|
|
"epoch": 0.08740564163686929,
|
|
"grad_norm": 13.3125,
|
|
"learning_rate": 8.779902129378738e-08,
|
|
"logits/chosen": -1.9660217761993408,
|
|
"logits/rejected": -1.9344068765640259,
|
|
"logps/chosen": -0.3128483295440674,
|
|
"logps/rejected": -0.3132597804069519,
|
|
"loss": 0.0763622298836708,
|
|
"loss/core": 0.07052332162857056,
|
|
"loss/preference_sft": 0.31526920199394226,
|
|
"loss/reference_anchor": 0.08525120466947556,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.475311517715454,
|
|
"rewards/margins": 0.7777661681175232,
|
|
"rewards/rejected": 0.6975454092025757,
|
|
"ronpo/logit": 0.07777661830186844,
|
|
"ronpo/residual": 0.07221326977014542,
|
|
"ronpo/residual_abs": 0.12839944660663605,
|
|
"ronpo/target": 0.005563343409448862,
|
|
"ronpo/target_abs": 0.06888459622859955,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25686928629875183,
|
|
"drift/rejected_abs": 0.12125973403453827,
|
|
"epoch": 0.08899483512117601,
|
|
"grad_norm": 1.71875,
|
|
"learning_rate": 8.715724127386971e-08,
|
|
"logits/chosen": -1.8945934772491455,
|
|
"logits/rejected": -1.8945395946502686,
|
|
"logps/chosen": -0.28814825415611267,
|
|
"logps/rejected": -0.27774301171302795,
|
|
"loss": 0.25721555948257446,
|
|
"loss/core": 0.24336083233356476,
|
|
"loss/preference_sft": 0.2840600907802582,
|
|
"loss/reference_anchor": 0.24868884682655334,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.568566083908081,
|
|
"rewards/margins": 1.357149600982666,
|
|
"rewards/rejected": 1.2114163637161255,
|
|
"ronpo/logit": 0.13571497797966003,
|
|
"ronpo/residual": 0.13394533097743988,
|
|
"ronpo/residual_abs": 0.2140486240386963,
|
|
"ronpo/target": 0.0017696395516395569,
|
|
"ronpo/target_abs": 0.08350421488285065,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22564442455768585,
|
|
"drift/rejected_abs": 0.10595756769180298,
|
|
"epoch": 0.09058402860548272,
|
|
"grad_norm": 1.359375,
|
|
"learning_rate": 8.650148793512872e-08,
|
|
"logits/chosen": -1.861046552658081,
|
|
"logits/rejected": -1.8655040264129639,
|
|
"logps/chosen": -0.2829212248325348,
|
|
"logps/rejected": -0.28727012872695923,
|
|
"loss": 0.11808571964502335,
|
|
"loss/core": 0.10924997180700302,
|
|
"loss/preference_sft": 0.28626394271850586,
|
|
"loss/reference_anchor": 0.14808858931064606,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.25600266456604,
|
|
"rewards/margins": 1.207207441329956,
|
|
"rewards/rejected": 1.0487949848175049,
|
|
"ronpo/logit": 0.1207207590341568,
|
|
"ronpo/residual": 0.14722566306591034,
|
|
"ronpo/residual_abs": 0.18361715972423553,
|
|
"ronpo/target": -0.026504909619688988,
|
|
"ronpo/target_abs": 0.08099932968616486,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3155427873134613,
|
|
"drift/rejected_abs": 0.13171601295471191,
|
|
"epoch": 0.09217322208978942,
|
|
"grad_norm": 45.5,
|
|
"learning_rate": 8.58320078795768e-08,
|
|
"logits/chosen": -1.9795125722885132,
|
|
"logits/rejected": -1.9921321868896484,
|
|
"logps/chosen": -0.2706143856048584,
|
|
"logps/rejected": -0.266631156206131,
|
|
"loss": 0.5211764574050903,
|
|
"loss/core": 0.4964214861392975,
|
|
"loss/preference_sft": 0.2676003873348236,
|
|
"loss/reference_anchor": 0.46833816170692444,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.1550698280334473,
|
|
"rewards/margins": 1.890966773033142,
|
|
"rewards/rejected": 1.2641032934188843,
|
|
"ronpo/logit": 0.18909668922424316,
|
|
"ronpo/residual": 0.1774296909570694,
|
|
"ronpo/residual_abs": 0.23044650256633759,
|
|
"ronpo/target": 0.011666996404528618,
|
|
"ronpo/target_abs": 0.07283170521259308,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3618084788322449,
|
|
"drift/rejected_abs": 0.14779672026634216,
|
|
"epoch": 0.09376241557409615,
|
|
"grad_norm": 64.5,
|
|
"learning_rate": 8.514905287128308e-08,
|
|
"logits/chosen": -1.6823537349700928,
|
|
"logits/rejected": -1.653702974319458,
|
|
"logps/chosen": -0.2833479344844818,
|
|
"logps/rejected": -0.30102211236953735,
|
|
"loss": 1.0796386003494263,
|
|
"loss/core": 1.046770691871643,
|
|
"loss/preference_sft": 0.29381901025772095,
|
|
"loss/reference_anchor": 0.6279757618904114,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.616563081741333,
|
|
"rewards/margins": 2.1403043270111084,
|
|
"rewards/rejected": 1.4762591123580933,
|
|
"ronpo/logit": 0.2140304297208786,
|
|
"ronpo/residual": 0.2224186658859253,
|
|
"ronpo/residual_abs": 0.360507071018219,
|
|
"ronpo/target": -0.008388230577111244,
|
|
"ronpo/target_abs": 0.08402254432439804,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25922662019729614,
|
|
"drift/rejected_abs": 0.12114284187555313,
|
|
"epoch": 0.09535160905840286,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 8.44528797416954e-08,
|
|
"logits/chosen": -1.9925893545150757,
|
|
"logits/rejected": -1.9968611001968384,
|
|
"logps/chosen": -0.2853473722934723,
|
|
"logps/rejected": -0.2928779125213623,
|
|
"loss": 0.6324478983879089,
|
|
"loss/core": 0.6148328185081482,
|
|
"loss/preference_sft": 0.2793324291706085,
|
|
"loss/reference_anchor": 0.3243691325187683,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.592266082763672,
|
|
"rewards/margins": 1.6641126871109009,
|
|
"rewards/rejected": 0.9281536340713501,
|
|
"ronpo/logit": 0.16641125082969666,
|
|
"ronpo/residual": 0.16771617531776428,
|
|
"ronpo/residual_abs": 0.2484259307384491,
|
|
"ronpo/target": -0.0013049064436927438,
|
|
"ronpo/target_abs": 0.07981012016534805,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33662670850753784,
|
|
"drift/rejected_abs": 0.15090534090995789,
|
|
"epoch": 0.09694080254270958,
|
|
"grad_norm": 9.1875,
|
|
"learning_rate": 8.374375029305638e-08,
|
|
"logits/chosen": -1.8022260665893555,
|
|
"logits/rejected": -1.7453006505966187,
|
|
"logps/chosen": -0.2850312888622284,
|
|
"logps/rejected": -0.29107189178466797,
|
|
"loss": 0.6764329075813293,
|
|
"loss/core": 0.6424604654312134,
|
|
"loss/preference_sft": 0.291935533285141,
|
|
"loss/reference_anchor": 0.6502560377120972,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.3646740913391113,
|
|
"rewards/margins": 1.873510718345642,
|
|
"rewards/rejected": 1.4911634922027588,
|
|
"ronpo/logit": 0.1873510777950287,
|
|
"ronpo/residual": 0.1918567419052124,
|
|
"ronpo/residual_abs": 0.3251708447933197,
|
|
"ronpo/target": -0.004505653399974108,
|
|
"ronpo/target_abs": 0.08541780710220337,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2991493344306946,
|
|
"drift/rejected_abs": 0.15711288154125214,
|
|
"epoch": 0.09852999602701629,
|
|
"grad_norm": 2.5625,
|
|
"learning_rate": 8.302193119995037e-08,
|
|
"logits/chosen": -1.8975536823272705,
|
|
"logits/rejected": -1.919425368309021,
|
|
"logps/chosen": -0.2862815260887146,
|
|
"logps/rejected": -0.27441829442977905,
|
|
"loss": 0.6102262735366821,
|
|
"loss/core": 0.576637327671051,
|
|
"loss/preference_sft": 0.27816590666770935,
|
|
"loss/reference_anchor": 0.643962025642395,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.991199016571045,
|
|
"rewards/margins": 1.4210230112075806,
|
|
"rewards/rejected": 1.570176362991333,
|
|
"ronpo/logit": 0.14210231602191925,
|
|
"ronpo/residual": 0.11606768518686295,
|
|
"ronpo/residual_abs": 0.18980509042739868,
|
|
"ronpo/target": 0.026034632697701454,
|
|
"ronpo/target_abs": 0.07823829352855682,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26028409600257874,
|
|
"drift/rejected_abs": 0.10342870652675629,
|
|
"epoch": 0.10011918951132301,
|
|
"grad_norm": 25.875,
|
|
"learning_rate": 8.22876939090181e-08,
|
|
"logits/chosen": -1.6694761514663696,
|
|
"logits/rejected": -1.6918532848358154,
|
|
"logps/chosen": -0.27476266026496887,
|
|
"logps/rejected": -0.2757488787174225,
|
|
"loss": 0.43638157844543457,
|
|
"loss/core": 0.4186526834964752,
|
|
"loss/preference_sft": 0.27721649408340454,
|
|
"loss/reference_anchor": 0.3268568515777588,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.6024768352508545,
|
|
"rewards/margins": 1.5710161924362183,
|
|
"rewards/rejected": 1.0314604043960571,
|
|
"ronpo/logit": 0.15710163116455078,
|
|
"ronpo/residual": 0.1434870809316635,
|
|
"ronpo/residual_abs": 0.2207198143005371,
|
|
"ronpo/target": 0.013614545576274395,
|
|
"ronpo/target_abs": 0.08071252703666687,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3579448461532593,
|
|
"drift/rejected_abs": 0.1325366348028183,
|
|
"epoch": 0.10170838299562972,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 8.154131453687656e-08,
|
|
"logits/chosen": -1.8659942150115967,
|
|
"logits/rejected": -1.9224580526351929,
|
|
"logps/chosen": -0.2523338794708252,
|
|
"logps/rejected": -0.24946343898773193,
|
|
"loss": 0.8073204159736633,
|
|
"loss/core": 0.7731778621673584,
|
|
"loss/preference_sft": 0.24916532635688782,
|
|
"loss/reference_anchor": 0.6579362154006958,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.579383373260498,
|
|
"rewards/margins": 2.2741141319274902,
|
|
"rewards/rejected": 1.3052693605422974,
|
|
"ronpo/logit": 0.2274114340543747,
|
|
"ronpo/residual": 0.22022266685962677,
|
|
"ronpo/residual_abs": 0.2674153447151184,
|
|
"ronpo/target": 0.007188776973634958,
|
|
"ronpo/target_abs": 0.07314587384462357,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23399193584918976,
|
|
"drift/rejected_abs": 0.09473086148500443,
|
|
"epoch": 0.10329757647993643,
|
|
"grad_norm": 364.0,
|
|
"learning_rate": 8.07830737662829e-08,
|
|
"logits/chosen": -1.743417739868164,
|
|
"logits/rejected": -1.7016518115997314,
|
|
"logps/chosen": -0.3154450058937073,
|
|
"logps/rejected": -0.3125259280204773,
|
|
"loss": 0.25524017214775085,
|
|
"loss/core": 0.23851685225963593,
|
|
"loss/preference_sft": 0.3184736371040344,
|
|
"loss/reference_anchor": 0.30261868238449097,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.336858034133911,
|
|
"rewards/margins": 1.3925182819366455,
|
|
"rewards/rejected": 0.9443395733833313,
|
|
"ronpo/logit": 0.13925182819366455,
|
|
"ronpo/residual": 0.15808895230293274,
|
|
"ronpo/residual_abs": 0.20290787518024445,
|
|
"ronpo/target": -0.018837137147784233,
|
|
"ronpo/target_abs": 0.08498185127973557,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2572224736213684,
|
|
"drift/rejected_abs": 0.1335063874721527,
|
|
"epoch": 0.10488676996424315,
|
|
"grad_norm": 2.078125,
|
|
"learning_rate": 8.001325674058123e-08,
|
|
"logits/chosen": -1.8432238101959229,
|
|
"logits/rejected": -1.8311374187469482,
|
|
"logps/chosen": -0.2789004445075989,
|
|
"logps/rejected": -0.2724122703075409,
|
|
"loss": 0.45331627130508423,
|
|
"loss/core": 0.4269883632659912,
|
|
"loss/preference_sft": 0.2750396132469177,
|
|
"loss/reference_anchor": 0.4990541338920593,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.5702948570251465,
|
|
"rewards/margins": 1.2371727228164673,
|
|
"rewards/rejected": 1.3331226110458374,
|
|
"ronpo/logit": 0.12371727079153061,
|
|
"ronpo/residual": 0.1249464750289917,
|
|
"ronpo/residual_abs": 0.22960133850574493,
|
|
"ronpo/target": -0.0012291899183765054,
|
|
"ronpo/target_abs": 0.07105612754821777,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3199670612812042,
|
|
"drift/rejected_abs": 0.1441265046596527,
|
|
"epoch": 0.10647596344854986,
|
|
"grad_norm": 59.5,
|
|
"learning_rate": 7.923215295647166e-08,
|
|
"logits/chosen": -1.8185367584228516,
|
|
"logits/rejected": -1.7632293701171875,
|
|
"logps/chosen": -0.28754284977912903,
|
|
"logps/rejected": -0.29002195596694946,
|
|
"loss": 0.5746418833732605,
|
|
"loss/core": 0.535438597202301,
|
|
"loss/preference_sft": 0.2888413071632385,
|
|
"loss/reference_anchor": 0.7551820874214172,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.1991424560546875,
|
|
"rewards/margins": 1.759937047958374,
|
|
"rewards/rejected": 1.439205527305603,
|
|
"ronpo/logit": 0.1759936809539795,
|
|
"ronpo/residual": 0.17700710892677307,
|
|
"ronpo/residual_abs": 0.2370648831129074,
|
|
"ronpo/target": -0.0010134152835235,
|
|
"ronpo/target_abs": 0.08590994030237198,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1872010976076126,
|
|
"drift/rejected_abs": 0.12332087755203247,
|
|
"epoch": 0.10806515693285658,
|
|
"grad_norm": 3.65625,
|
|
"learning_rate": 7.844005615514259e-08,
|
|
"logits/chosen": -1.8502109050750732,
|
|
"logits/rejected": -1.8752959966659546,
|
|
"logps/chosen": -0.30195698142051697,
|
|
"logps/rejected": -0.2972937226295471,
|
|
"loss": 0.05311671644449234,
|
|
"loss/core": 0.043439559638500214,
|
|
"loss/preference_sft": 0.30265796184539795,
|
|
"loss/reference_anchor": 0.1632772982120514,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8714481592178345,
|
|
"rewards/margins": 0.6399093866348267,
|
|
"rewards/rejected": 1.231539011001587,
|
|
"ronpo/logit": 0.06399093568325043,
|
|
"ronpo/residual": 0.07827167212963104,
|
|
"ronpo/residual_abs": 0.13036039471626282,
|
|
"ronpo/target": -0.014280728995800018,
|
|
"ronpo/target_abs": 0.07398774474859238,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21623043715953827,
|
|
"drift/rejected_abs": 0.08763010054826736,
|
|
"epoch": 0.10965435041716329,
|
|
"grad_norm": 5.53125,
|
|
"learning_rate": 7.763726421180662e-08,
|
|
"logits/chosen": -1.8741140365600586,
|
|
"logits/rejected": -1.8508151769638062,
|
|
"logps/chosen": -0.27523356676101685,
|
|
"logps/rejected": -0.2796024680137634,
|
|
"loss": 0.331326425075531,
|
|
"loss/core": 0.31637856364250183,
|
|
"loss/preference_sft": 0.28105422854423523,
|
|
"loss/reference_anchor": 0.270852655172348,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.160953998565674,
|
|
"rewards/margins": 1.2863576412200928,
|
|
"rewards/rejected": 0.8745959401130676,
|
|
"ronpo/logit": 0.12863577902317047,
|
|
"ronpo/residual": 0.13460756838321686,
|
|
"ronpo/residual_abs": 0.18582451343536377,
|
|
"ronpo/target": -0.005971794482320547,
|
|
"ronpo/target_abs": 0.08206266909837723,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4488181173801422,
|
|
"drift/rejected_abs": 0.1432909220457077,
|
|
"epoch": 0.11124354390147001,
|
|
"grad_norm": 744.0,
|
|
"learning_rate": 7.682407902368189e-08,
|
|
"logits/chosen": -1.8674952983856201,
|
|
"logits/rejected": -1.8539085388183594,
|
|
"logps/chosen": -0.2711220383644104,
|
|
"logps/rejected": -0.2666943073272705,
|
|
"loss": 1.336153268814087,
|
|
"loss/core": 1.280058741569519,
|
|
"loss/preference_sft": 0.2706068456172943,
|
|
"loss/reference_anchor": 1.094827651977539,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.485884666442871,
|
|
"rewards/margins": 3.054591178894043,
|
|
"rewards/rejected": 1.431293249130249,
|
|
"ronpo/logit": 0.3054591417312622,
|
|
"ronpo/residual": 0.3182205557823181,
|
|
"ronpo/residual_abs": 0.3483893871307373,
|
|
"ronpo/target": -0.012761471793055534,
|
|
"ronpo/target_abs": 0.08356453478336334,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4042721688747406,
|
|
"drift/rejected_abs": 0.18896692991256714,
|
|
"epoch": 0.11283273738577672,
|
|
"grad_norm": 2.0,
|
|
"learning_rate": 7.600080639646076e-08,
|
|
"logits/chosen": -1.8509187698364258,
|
|
"logits/rejected": -1.8338550329208374,
|
|
"logps/chosen": -0.2988142967224121,
|
|
"logps/rejected": -0.30226239562034607,
|
|
"loss": 0.6167121529579163,
|
|
"loss/core": 0.5703554749488831,
|
|
"loss/preference_sft": 0.2945895195007324,
|
|
"loss/reference_anchor": 0.8976724743843079,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.042208194732666,
|
|
"rewards/margins": 2.1540372371673584,
|
|
"rewards/rejected": 1.888170838356018,
|
|
"ronpo/logit": 0.21540376543998718,
|
|
"ronpo/residual": 0.20375964045524597,
|
|
"ronpo/residual_abs": 0.26894694566726685,
|
|
"ronpo/target": 0.011644105426967144,
|
|
"ronpo/target_abs": 0.06865202635526657,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24115343391895294,
|
|
"drift/rejected_abs": 0.13106033205986023,
|
|
"epoch": 0.11442193087008343,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 7.516775592930899e-08,
|
|
"logits/chosen": -1.953965425491333,
|
|
"logits/rejected": -2.0095295906066895,
|
|
"logps/chosen": -0.3034398555755615,
|
|
"logps/rejected": -0.308123916387558,
|
|
"loss": 0.12772902846336365,
|
|
"loss/core": 0.11105193197727203,
|
|
"loss/preference_sft": 0.3055204749107361,
|
|
"loss/reference_anchor": 0.3029900789260864,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.409573793411255,
|
|
"rewards/margins": 1.099807620048523,
|
|
"rewards/rejected": 1.3097662925720215,
|
|
"ronpo/logit": 0.1099807471036911,
|
|
"ronpo/residual": 0.1174292117357254,
|
|
"ronpo/residual_abs": 0.17334052920341492,
|
|
"ronpo/target": -0.007448455784469843,
|
|
"ronpo/target_abs": 0.0786883682012558,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3062785565853119,
|
|
"drift/rejected_abs": 0.130979984998703,
|
|
"epoch": 0.11601112435439015,
|
|
"grad_norm": 10.8125,
|
|
"learning_rate": 7.4325240898438e-08,
|
|
"logits/chosen": -1.8819477558135986,
|
|
"logits/rejected": -1.8994693756103516,
|
|
"logps/chosen": -0.2879257798194885,
|
|
"logps/rejected": -0.2965686619281769,
|
|
"loss": 0.4399788975715637,
|
|
"loss/core": 0.4211645722389221,
|
|
"loss/preference_sft": 0.296079158782959,
|
|
"loss/reference_anchor": 0.3466790020465851,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.0625298023223877,
|
|
"rewards/margins": 1.7536706924438477,
|
|
"rewards/rejected": 1.3088594675064087,
|
|
"ronpo/logit": 0.1753670871257782,
|
|
"ronpo/residual": 0.1744088977575302,
|
|
"ronpo/residual_abs": 0.22169923782348633,
|
|
"ronpo/target": 0.0009581688791513443,
|
|
"ronpo/target_abs": 0.07294227182865143,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4208388924598694,
|
|
"drift/rejected_abs": 0.19213302433490753,
|
|
"epoch": 0.11760031783869686,
|
|
"grad_norm": 9.0,
|
|
"learning_rate": 7.347357813929453e-08,
|
|
"logits/chosen": -1.6949617862701416,
|
|
"logits/rejected": -1.5868123769760132,
|
|
"logps/chosen": -0.28393587470054626,
|
|
"logps/rejected": -0.26977697014808655,
|
|
"loss": 0.9468614459037781,
|
|
"loss/core": 0.8858458399772644,
|
|
"loss/preference_sft": 0.2782512903213501,
|
|
"loss/reference_anchor": 1.192486047744751,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.20812463760376,
|
|
"rewards/margins": 2.3017334938049316,
|
|
"rewards/rejected": 1.9063911437988281,
|
|
"ronpo/logit": 0.23017334938049316,
|
|
"ronpo/residual": 0.22761540114879608,
|
|
"ronpo/residual_abs": 0.30848538875579834,
|
|
"ronpo/target": 0.0025579137727618217,
|
|
"ronpo/target_abs": 0.06729411333799362,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33695167303085327,
|
|
"drift/rejected_abs": 0.15755027532577515,
|
|
"epoch": 0.11918951132300358,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 7.261308792741188e-08,
|
|
"logits/chosen": -1.9283231496810913,
|
|
"logits/rejected": -1.898046851158142,
|
|
"logps/chosen": -0.26773491501808167,
|
|
"logps/rejected": -0.2634897232055664,
|
|
"loss": 0.5323135256767273,
|
|
"loss/core": 0.5038132667541504,
|
|
"loss/preference_sft": 0.2725459039211273,
|
|
"loss/reference_anchor": 0.5427502393722534,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.3691153526306152,
|
|
"rewards/margins": 1.7936127185821533,
|
|
"rewards/rejected": 1.575502872467041,
|
|
"ronpo/logit": 0.1793612688779831,
|
|
"ronpo/residual": 0.1783147007226944,
|
|
"ronpo/residual_abs": 0.2645108103752136,
|
|
"ronpo/target": 0.0010465823579579592,
|
|
"ronpo/target_abs": 0.07507561147212982,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24347706139087677,
|
|
"drift/rejected_abs": 0.0940445065498352,
|
|
"epoch": 0.12077870480731029,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 7.174409385796725e-08,
|
|
"logits/chosen": -1.512104868888855,
|
|
"logits/rejected": -1.5713586807250977,
|
|
"logps/chosen": -0.2921512722969055,
|
|
"logps/rejected": -0.288189172744751,
|
|
"loss": 0.7772008180618286,
|
|
"loss/core": 0.7538732290267944,
|
|
"loss/preference_sft": 0.2922476828098297,
|
|
"loss/reference_anchor": 0.43732795119285583,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.434725284576416,
|
|
"rewards/margins": 1.4957847595214844,
|
|
"rewards/rejected": 0.9389405250549316,
|
|
"ronpo/logit": 0.1495784968137741,
|
|
"ronpo/residual": 0.16179527342319489,
|
|
"ronpo/residual_abs": 0.20729383826255798,
|
|
"ronpo/target": -0.01221676729619503,
|
|
"ronpo/target_abs": 0.0761856958270073,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36468371748924255,
|
|
"drift/rejected_abs": 0.15189920365810394,
|
|
"epoch": 0.12236789829161701,
|
|
"grad_norm": 4.6875,
|
|
"learning_rate": 7.08669227240909e-08,
|
|
"logits/chosen": -1.9026060104370117,
|
|
"logits/rejected": -1.8286371231079102,
|
|
"logps/chosen": -0.3003026247024536,
|
|
"logps/rejected": -0.30434030294418335,
|
|
"loss": 0.5205832123756409,
|
|
"loss/core": 0.4895382523536682,
|
|
"loss/preference_sft": 0.30446332693099976,
|
|
"loss/reference_anchor": 0.5904520750045776,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.646836757659912,
|
|
"rewards/margins": 2.139817714691162,
|
|
"rewards/rejected": 1.5070189237594604,
|
|
"ronpo/logit": 0.21398182213306427,
|
|
"ronpo/residual": 0.21957902610301971,
|
|
"ronpo/residual_abs": 0.2609708309173584,
|
|
"ronpo/target": -0.005597242154181004,
|
|
"ronpo/target_abs": 0.07350078225135803,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18241234123706818,
|
|
"drift/rejected_abs": 0.07146935164928436,
|
|
"epoch": 0.12395709177592372,
|
|
"grad_norm": 2.03125,
|
|
"learning_rate": 6.99819043939726e-08,
|
|
"logits/chosen": -1.8736292123794556,
|
|
"logits/rejected": -1.856080412864685,
|
|
"logps/chosen": -0.2814716398715973,
|
|
"logps/rejected": -0.2780093252658844,
|
|
"loss": 0.18268391489982605,
|
|
"loss/core": 0.17545238137245178,
|
|
"loss/preference_sft": 0.2792362570762634,
|
|
"loss/reference_anchor": 0.1167074590921402,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8227360248565674,
|
|
"rewards/margins": 1.1081311702728271,
|
|
"rewards/rejected": 0.714604914188385,
|
|
"ronpo/logit": 0.11081312596797943,
|
|
"ronpo/residual": 0.11074452102184296,
|
|
"ronpo/residual_abs": 0.15610924363136292,
|
|
"ronpo/target": 6.859861605335027e-05,
|
|
"ronpo/target_abs": 0.07130326330661774,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23421840369701385,
|
|
"drift/rejected_abs": 0.12544074654579163,
|
|
"epoch": 0.12554628526023043,
|
|
"grad_norm": 144.0,
|
|
"learning_rate": 6.908937168681174e-08,
|
|
"logits/chosen": -1.8029218912124634,
|
|
"logits/rejected": -1.891135811805725,
|
|
"logps/chosen": -0.28115424513816833,
|
|
"logps/rejected": -0.28048890829086304,
|
|
"loss": 0.12526974081993103,
|
|
"loss/core": 0.10842245817184448,
|
|
"loss/preference_sft": 0.28288403153419495,
|
|
"loss/reference_anchor": 0.3086569011211395,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.341761827468872,
|
|
"rewards/margins": 1.0884926319122314,
|
|
"rewards/rejected": 1.2532694339752197,
|
|
"ronpo/logit": 0.10884924978017807,
|
|
"ronpo/residual": 0.10868626832962036,
|
|
"ronpo/residual_abs": 0.15744046866893768,
|
|
"ronpo/target": 0.0001629740872886032,
|
|
"ronpo/target_abs": 0.07715004682540894,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37465590238571167,
|
|
"drift/rejected_abs": 0.11181537806987762,
|
|
"epoch": 0.12713547874453715,
|
|
"grad_norm": 0.77734375,
|
|
"learning_rate": 6.818966024765757e-08,
|
|
"logits/chosen": -1.9342272281646729,
|
|
"logits/rejected": -1.9193241596221924,
|
|
"logps/chosen": -0.2912561595439911,
|
|
"logps/rejected": -0.2906317710876465,
|
|
"loss": 1.6204612255096436,
|
|
"loss/core": 1.5696020126342773,
|
|
"loss/preference_sft": 0.2945665419101715,
|
|
"loss/reference_anchor": 0.9877260327339172,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 3.7448832988739014,
|
|
"rewards/margins": 2.6269583702087402,
|
|
"rewards/rejected": 1.117925763130188,
|
|
"ronpo/logit": 0.2626958191394806,
|
|
"ronpo/residual": 0.27263596653938293,
|
|
"ronpo/residual_abs": 0.3709692656993866,
|
|
"ronpo/target": -0.009940169751644135,
|
|
"ronpo/target_abs": 0.08929356932640076,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24711406230926514,
|
|
"drift/rejected_abs": 0.12497474998235703,
|
|
"epoch": 0.12872467222884387,
|
|
"grad_norm": 4.59375,
|
|
"learning_rate": 6.728310842118683e-08,
|
|
"logits/chosen": -1.7766101360321045,
|
|
"logits/rejected": -1.727318525314331,
|
|
"logps/chosen": -0.2852860391139984,
|
|
"logps/rejected": -0.2805282771587372,
|
|
"loss": 0.2622772455215454,
|
|
"loss/core": 0.24621637165546417,
|
|
"loss/preference_sft": 0.287511944770813,
|
|
"loss/reference_anchor": 0.2924651503562927,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.46588134765625,
|
|
"rewards/margins": 1.2168688774108887,
|
|
"rewards/rejected": 1.2490125894546509,
|
|
"ronpo/logit": 0.1216868907213211,
|
|
"ronpo/residual": 0.11308858543634415,
|
|
"ronpo/residual_abs": 0.18690213561058044,
|
|
"ronpo/target": 0.00859830342233181,
|
|
"ronpo/target_abs": 0.08837716281414032,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15449269115924835,
|
|
"drift/rejected_abs": 0.07039482146501541,
|
|
"epoch": 0.13031386571315057,
|
|
"grad_norm": 0.7421875,
|
|
"learning_rate": 6.637005712446621e-08,
|
|
"logits/chosen": -1.8721539974212646,
|
|
"logits/rejected": -1.9462865591049194,
|
|
"logps/chosen": -0.28909945487976074,
|
|
"logps/rejected": -0.27632978558540344,
|
|
"loss": 0.08583154529333115,
|
|
"loss/core": 0.08015327155590057,
|
|
"loss/preference_sft": 0.283399373292923,
|
|
"loss/reference_anchor": 0.08522568643093109,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5443317890167236,
|
|
"rewards/margins": 0.842542827129364,
|
|
"rewards/rejected": 0.7017889022827148,
|
|
"ronpo/logit": 0.08425427973270416,
|
|
"ronpo/residual": 0.07985297590494156,
|
|
"ronpo/residual_abs": 0.14067772030830383,
|
|
"ronpo/target": 0.0044013154692947865,
|
|
"ronpo/target_abs": 0.07584073394536972,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1793230175971985,
|
|
"drift/rejected_abs": 0.08416859805583954,
|
|
"epoch": 0.1319030591974573,
|
|
"grad_norm": 2.109375,
|
|
"learning_rate": 6.545084971874738e-08,
|
|
"logits/chosen": -1.9961707592010498,
|
|
"logits/rejected": -1.9701541662216187,
|
|
"logps/chosen": -0.2783016562461853,
|
|
"logps/rejected": -0.27948951721191406,
|
|
"loss": 0.09949196875095367,
|
|
"loss/core": 0.09299661219120026,
|
|
"loss/preference_sft": 0.2811213433742523,
|
|
"loss/reference_anchor": 0.10179527848958969,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7914680242538452,
|
|
"rewards/margins": 0.9507063031196594,
|
|
"rewards/rejected": 0.8407617807388306,
|
|
"ronpo/logit": 0.09507062286138535,
|
|
"ronpo/residual": 0.08666277676820755,
|
|
"ronpo/residual_abs": 0.13639622926712036,
|
|
"ronpo/target": 0.008407856337726116,
|
|
"ronpo/target_abs": 0.06797736883163452,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1825791895389557,
|
|
"drift/rejected_abs": 0.07227766513824463,
|
|
"epoch": 0.133492252681764,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 6.452583188034274e-08,
|
|
"logits/chosen": -1.8393170833587646,
|
|
"logits/rejected": -1.8407024145126343,
|
|
"logps/chosen": -0.2888352870941162,
|
|
"logps/rejected": -0.2917371392250061,
|
|
"loss": 0.22071552276611328,
|
|
"loss/core": 0.20773322880268097,
|
|
"loss/preference_sft": 0.2941259443759918,
|
|
"loss/reference_anchor": 0.23023299872875214,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8255313634872437,
|
|
"rewards/margins": 1.1031744480133057,
|
|
"rewards/rejected": 0.7223570942878723,
|
|
"ronpo/logit": 0.11031744629144669,
|
|
"ronpo/residual": 0.09504842758178711,
|
|
"ronpo/residual_abs": 0.15620620548725128,
|
|
"ronpo/target": 0.015269016847014427,
|
|
"ronpo/target_abs": 0.07178054749965668,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3180219531059265,
|
|
"drift/rejected_abs": 0.08384044468402863,
|
|
"epoch": 0.1350814461660707,
|
|
"grad_norm": 4.5625,
|
|
"learning_rate": 6.35953514706309e-08,
|
|
"logits/chosen": -1.8810008764266968,
|
|
"logits/rejected": -1.9081312417984009,
|
|
"logps/chosen": -0.2878539562225342,
|
|
"logps/rejected": -0.2854660153388977,
|
|
"loss": 1.2196041345596313,
|
|
"loss/core": 1.1814826726913452,
|
|
"loss/preference_sft": 0.2875927686691284,
|
|
"loss/reference_anchor": 0.7336705327033997,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.178842544555664,
|
|
"rewards/margins": 2.3408150672912598,
|
|
"rewards/rejected": 0.8380276560783386,
|
|
"ronpo/logit": 0.23408150672912598,
|
|
"ronpo/residual": 0.2306959331035614,
|
|
"ronpo/residual_abs": 0.2859199643135071,
|
|
"ronpo/target": 0.0033855431247502565,
|
|
"ronpo/target_abs": 0.07320918142795563,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2660025954246521,
|
|
"drift/rejected_abs": 0.11940312385559082,
|
|
"epoch": 0.13667063965037743,
|
|
"grad_norm": 37.25,
|
|
"learning_rate": 6.26597584052401e-08,
|
|
"logits/chosen": -1.7275598049163818,
|
|
"logits/rejected": -1.7136932611465454,
|
|
"logps/chosen": -0.2784407138824463,
|
|
"logps/rejected": -0.2639455199241638,
|
|
"loss": 0.23619727790355682,
|
|
"loss/core": 0.22276940941810608,
|
|
"loss/preference_sft": 0.27869319915771484,
|
|
"loss/reference_anchor": 0.24068863689899445,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.6599392890930176,
|
|
"rewards/margins": 1.4684311151504517,
|
|
"rewards/rejected": 1.1915079355239868,
|
|
"ronpo/logit": 0.14684312045574188,
|
|
"ronpo/residual": 0.14823973178863525,
|
|
"ronpo/residual_abs": 0.19733932614326477,
|
|
"ronpo/target": -0.0013966044643893838,
|
|
"ronpo/target_abs": 0.0798254981637001,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.259042352437973,
|
|
"drift/rejected_abs": 0.1040254458785057,
|
|
"epoch": 0.13825983313468415,
|
|
"grad_norm": 1008.0,
|
|
"learning_rate": 6.171940452245923e-08,
|
|
"logits/chosen": -1.948926568031311,
|
|
"logits/rejected": -1.890885353088379,
|
|
"logps/chosen": -0.2998539209365845,
|
|
"logps/rejected": -0.2981581687927246,
|
|
"loss": 0.5641499757766724,
|
|
"loss/core": 0.5412354469299316,
|
|
"loss/preference_sft": 0.31242674589157104,
|
|
"loss/reference_anchor": 0.42705029249191284,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.5896012783050537,
|
|
"rewards/margins": 1.551031470298767,
|
|
"rewards/rejected": 1.0385698080062866,
|
|
"ronpo/logit": 0.1551031768321991,
|
|
"ronpo/residual": 0.1529240757226944,
|
|
"ronpo/residual_abs": 0.19178906083106995,
|
|
"ronpo/target": 0.002179093658924103,
|
|
"ronpo/target_abs": 0.06712304055690765,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16026827692985535,
|
|
"drift/rejected_abs": 0.1745794415473938,
|
|
"epoch": 0.13984902661899087,
|
|
"grad_norm": 25.125,
|
|
"learning_rate": 6.0774643450926e-08,
|
|
"logits/chosen": -2.0914130210876465,
|
|
"logits/rejected": -2.0466763973236084,
|
|
"logps/chosen": -0.2668331265449524,
|
|
"logps/rejected": -0.2701703906059265,
|
|
"loss": 0.4638034403324127,
|
|
"loss/core": 0.4482359290122986,
|
|
"loss/preference_sft": 0.26497453451156616,
|
|
"loss/reference_anchor": 0.28485244512557983,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6023889780044556,
|
|
"rewards/margins": -0.14340558648109436,
|
|
"rewards/rejected": 1.745794653892517,
|
|
"ronpo/logit": -0.01434055995196104,
|
|
"ronpo/residual": -0.01068189274519682,
|
|
"ronpo/residual_abs": 0.2119809091091156,
|
|
"ronpo/target": -0.00365866394713521,
|
|
"ronpo/target_abs": 0.07263807952404022,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.43698692321777344,
|
|
"drift/rejected_abs": 0.24082942306995392,
|
|
"epoch": 0.14143822010329757,
|
|
"grad_norm": 28.5,
|
|
"learning_rate": 5.982583047664151e-08,
|
|
"logits/chosen": -1.8886873722076416,
|
|
"logits/rejected": -1.890425443649292,
|
|
"logps/chosen": -0.27502498030662537,
|
|
"logps/rejected": -0.2749865651130676,
|
|
"loss": 0.6122740507125854,
|
|
"loss/core": 0.5536811947822571,
|
|
"loss/preference_sft": 0.27798959612846375,
|
|
"loss/reference_anchor": 1.1440602540969849,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.3676042556762695,
|
|
"rewards/margins": 1.9618676900863647,
|
|
"rewards/rejected": 2.4057364463806152,
|
|
"ronpo/logit": 0.19618678092956543,
|
|
"ronpo/residual": 0.18561862409114838,
|
|
"ronpo/residual_abs": 0.3030679523944855,
|
|
"ronpo/target": 0.010568159632384777,
|
|
"ronpo/target_abs": 0.0740106850862503,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3152461051940918,
|
|
"drift/rejected_abs": 0.1808178722858429,
|
|
"epoch": 0.1430274135876043,
|
|
"grad_norm": 116.5,
|
|
"learning_rate": 5.8873322409361756e-08,
|
|
"logits/chosen": -1.8608481884002686,
|
|
"logits/rejected": -1.8268924951553345,
|
|
"logps/chosen": -0.29395729303359985,
|
|
"logps/rejected": -0.2936895489692688,
|
|
"loss": 0.30116790533065796,
|
|
"loss/core": 0.26753267645835876,
|
|
"loss/preference_sft": 0.2933436632156372,
|
|
"loss/reference_anchor": 0.6433706879615784,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.1521642208099365,
|
|
"rewards/margins": 1.3444736003875732,
|
|
"rewards/rejected": 1.8076906204223633,
|
|
"ronpo/logit": 0.1344473659992218,
|
|
"ronpo/residual": 0.1293283849954605,
|
|
"ronpo/residual_abs": 0.19568553566932678,
|
|
"ronpo/target": 0.005118980538100004,
|
|
"ronpo/target_abs": 0.0815843939781189,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27072873711586,
|
|
"drift/rejected_abs": 0.13820311427116394,
|
|
"epoch": 0.144616607071911,
|
|
"grad_norm": 16.5,
|
|
"learning_rate": 5.7917477448416144e-08,
|
|
"logits/chosen": -1.8398069143295288,
|
|
"logits/rejected": -1.8747144937515259,
|
|
"logps/chosen": -0.29323771595954895,
|
|
"logps/rejected": -0.2729089856147766,
|
|
"loss": 0.1377260684967041,
|
|
"loss/core": 0.12696051597595215,
|
|
"loss/preference_sft": 0.28171589970588684,
|
|
"loss/reference_anchor": 0.1871393471956253,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.706803321838379,
|
|
"rewards/margins": 1.331863284111023,
|
|
"rewards/rejected": 1.3749401569366455,
|
|
"ronpo/logit": 0.13318632543087006,
|
|
"ronpo/residual": 0.12540921568870544,
|
|
"ronpo/residual_abs": 0.18542072176933289,
|
|
"ronpo/target": 0.007777099963277578,
|
|
"ronpo/target_abs": 0.08351351320743561,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3840409815311432,
|
|
"drift/rejected_abs": 0.16967609524726868,
|
|
"epoch": 0.1462058005562177,
|
|
"grad_norm": 12.6875,
|
|
"learning_rate": 5.695865504800327e-08,
|
|
"logits/chosen": -1.6525131464004517,
|
|
"logits/rejected": -1.67450749874115,
|
|
"logps/chosen": -0.3037227392196655,
|
|
"logps/rejected": -0.27811527252197266,
|
|
"loss": 0.7146202325820923,
|
|
"loss/core": 0.6745911240577698,
|
|
"loss/preference_sft": 0.3022574484348297,
|
|
"loss/reference_anchor": 0.7703584432601929,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.840378522872925,
|
|
"rewards/margins": 2.1443047523498535,
|
|
"rewards/rejected": 1.6960737705230713,
|
|
"ronpo/logit": 0.21443048119544983,
|
|
"ronpo/residual": 0.19126150012016296,
|
|
"ronpo/residual_abs": 0.2504216134548187,
|
|
"ronpo/target": 0.02316897362470627,
|
|
"ronpo/target_abs": 0.07098586112260818,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3566533029079437,
|
|
"drift/rejected_abs": 0.18750634789466858,
|
|
"epoch": 0.14779499404052443,
|
|
"grad_norm": 19.625,
|
|
"learning_rate": 5.5997215782014984e-08,
|
|
"logits/chosen": -1.612074851989746,
|
|
"logits/rejected": -1.6693141460418701,
|
|
"logps/chosen": -0.31581148505210876,
|
|
"logps/rejected": -0.2955818772315979,
|
|
"loss": 0.3881073594093323,
|
|
"loss/core": 0.3498273491859436,
|
|
"loss/preference_sft": 0.31020432710647583,
|
|
"loss/reference_anchor": 0.7345792651176453,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.5655674934387207,
|
|
"rewards/margins": 1.6920642852783203,
|
|
"rewards/rejected": 1.87350332736969,
|
|
"ronpo/logit": 0.1692064106464386,
|
|
"ronpo/residual": 0.15738491714000702,
|
|
"ronpo/residual_abs": 0.23218849301338196,
|
|
"ronpo/target": 0.011821477673947811,
|
|
"ronpo/target_abs": 0.08283017575740814,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2089032381772995,
|
|
"drift/rejected_abs": 0.1217624768614769,
|
|
"epoch": 0.14938418752483115,
|
|
"grad_norm": 7.96875,
|
|
"learning_rate": 5.503352120843923e-08,
|
|
"logits/chosen": -1.8853909969329834,
|
|
"logits/rejected": -1.8994200229644775,
|
|
"logps/chosen": -0.27790898084640503,
|
|
"logps/rejected": -0.26950299739837646,
|
|
"loss": 0.09184792637825012,
|
|
"loss/core": 0.08480606973171234,
|
|
"loss/preference_sft": 0.2741137444972992,
|
|
"loss/reference_anchor": 0.11342555284500122,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.0881950855255127,
|
|
"rewards/margins": 0.8740318417549133,
|
|
"rewards/rejected": 1.2141634225845337,
|
|
"ronpo/logit": 0.08740317821502686,
|
|
"ronpo/residual": 0.09419047832489014,
|
|
"ronpo/residual_abs": 0.16959841549396515,
|
|
"ronpo/target": -0.006787297315895557,
|
|
"ronpo/target_abs": 0.07587598264217377,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3570539653301239,
|
|
"drift/rejected_abs": 0.15286162495613098,
|
|
"epoch": 0.15097338100913787,
|
|
"grad_norm": 2.75,
|
|
"learning_rate": 5.406793373339291e-08,
|
|
"logits/chosen": -1.8066047430038452,
|
|
"logits/rejected": -1.767642617225647,
|
|
"logps/chosen": -0.26015806198120117,
|
|
"logps/rejected": -0.2632463574409485,
|
|
"loss": 0.34153276681900024,
|
|
"loss/core": 0.3111182451248169,
|
|
"loss/preference_sft": 0.264599084854126,
|
|
"loss/reference_anchor": 0.5818300843238831,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.570539951324463,
|
|
"rewards/margins": 2.0685620307922363,
|
|
"rewards/rejected": 1.5019776821136475,
|
|
"ronpo/logit": 0.20685617625713348,
|
|
"ronpo/residual": 0.20940890908241272,
|
|
"ronpo/residual_abs": 0.2527911961078644,
|
|
"ronpo/target": -0.0025527128018438816,
|
|
"ronpo/target_abs": 0.07948826253414154,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28908276557922363,
|
|
"drift/rejected_abs": 0.06378751993179321,
|
|
"epoch": 0.15256257449344457,
|
|
"grad_norm": 2.3125,
|
|
"learning_rate": 5.3100816474835766e-08,
|
|
"logits/chosen": -1.830521583557129,
|
|
"logits/rejected": -1.8023853302001953,
|
|
"logps/chosen": -0.291806161403656,
|
|
"logps/rejected": -0.2972370982170105,
|
|
"loss": 1.2332208156585693,
|
|
"loss/core": 1.1992995738983154,
|
|
"loss/preference_sft": 0.29961657524108887,
|
|
"loss/reference_anchor": 0.6484619379043579,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.889061450958252,
|
|
"rewards/margins": 2.2527663707733154,
|
|
"rewards/rejected": 0.6362951397895813,
|
|
"ronpo/logit": 0.2252766638994217,
|
|
"ronpo/residual": 0.2135426253080368,
|
|
"ronpo/residual_abs": 0.28047823905944824,
|
|
"ronpo/target": 0.011734018102288246,
|
|
"ronpo/target_abs": 0.08371776342391968,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28634947538375854,
|
|
"drift/rejected_abs": 0.09996393322944641,
|
|
"epoch": 0.1541517679777513,
|
|
"grad_norm": 4.0,
|
|
"learning_rate": 5.2132533126016534e-08,
|
|
"logits/chosen": -1.8586909770965576,
|
|
"logits/rejected": -1.8007152080535889,
|
|
"logps/chosen": -0.29752618074417114,
|
|
"logps/rejected": -0.2984181046485901,
|
|
"loss": 0.8611629605293274,
|
|
"loss/core": 0.83111971616745,
|
|
"loss/preference_sft": 0.29279690980911255,
|
|
"loss/reference_anchor": 0.5715832710266113,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8625049591064453,
|
|
"rewards/margins": 1.864074945449829,
|
|
"rewards/rejected": 0.9984301328659058,
|
|
"ronpo/logit": 0.18640747666358948,
|
|
"ronpo/residual": 0.19194810092449188,
|
|
"ronpo/residual_abs": 0.24163195490837097,
|
|
"ronpo/target": -0.0055406298488378525,
|
|
"ronpo/target_abs": 0.06841053813695908,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2852783799171448,
|
|
"drift/rejected_abs": 0.09443080425262451,
|
|
"epoch": 0.15574096146205801,
|
|
"grad_norm": 196.0,
|
|
"learning_rate": 5.116344781870281e-08,
|
|
"logits/chosen": -1.823598861694336,
|
|
"logits/rejected": -1.7952381372451782,
|
|
"logps/chosen": -0.30401426553726196,
|
|
"logps/rejected": -0.3061407506465912,
|
|
"loss": 0.7646321058273315,
|
|
"loss/core": 0.7394935488700867,
|
|
"loss/preference_sft": 0.3087760806083679,
|
|
"loss/reference_anchor": 0.4718933701515198,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.852783679962158,
|
|
"rewards/margins": 1.9092872142791748,
|
|
"rewards/rejected": 0.943496823310852,
|
|
"ronpo/logit": 0.19092872738838196,
|
|
"ronpo/residual": 0.19168706238269806,
|
|
"ronpo/residual_abs": 0.24266991019248962,
|
|
"ronpo/target": -0.0007583644473925233,
|
|
"ronpo/target_abs": 0.07142162322998047,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17704372107982635,
|
|
"drift/rejected_abs": 0.10839326679706573,
|
|
"epoch": 0.1573301549463647,
|
|
"grad_norm": 2.140625,
|
|
"learning_rate": 5.0193924986246017e-08,
|
|
"logits/chosen": -1.7229465246200562,
|
|
"logits/rejected": -1.6664669513702393,
|
|
"logps/chosen": -0.32384443283081055,
|
|
"logps/rejected": -0.31671226024627686,
|
|
"loss": 0.11827144771814346,
|
|
"loss/core": 0.10038117319345474,
|
|
"loss/preference_sft": 0.32083967328071594,
|
|
"loss/reference_anchor": 0.32572153210639954,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7700849771499634,
|
|
"rewards/margins": 0.6877198219299316,
|
|
"rewards/rejected": 1.0823650360107422,
|
|
"ronpo/logit": 0.06877198815345764,
|
|
"ronpo/residual": 0.07820957899093628,
|
|
"ronpo/residual_abs": 0.11904873698949814,
|
|
"ronpo/target": -0.009437596425414085,
|
|
"ronpo/target_abs": 0.06534653156995773,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3221167027950287,
|
|
"drift/rejected_abs": 0.16330395638942719,
|
|
"epoch": 0.15891934843067143,
|
|
"grad_norm": 1.5390625,
|
|
"learning_rate": 4.9224329226532834e-08,
|
|
"logits/chosen": -1.8693422079086304,
|
|
"logits/rejected": -1.8676942586898804,
|
|
"logps/chosen": -0.2746179401874542,
|
|
"logps/rejected": -0.2810618281364441,
|
|
"loss": 0.5076322555541992,
|
|
"loss/core": 0.4757213592529297,
|
|
"loss/preference_sft": 0.272911012172699,
|
|
"loss/reference_anchor": 0.610925555229187,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.2211670875549316,
|
|
"rewards/margins": 1.5898401737213135,
|
|
"rewards/rejected": 1.6313269138336182,
|
|
"ronpo/logit": 0.15898403525352478,
|
|
"ronpo/residual": 0.15926262736320496,
|
|
"ronpo/residual_abs": 0.23408159613609314,
|
|
"ronpo/target": -0.00027860701084136963,
|
|
"ronpo/target_abs": 0.07415109127759933,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22892192006111145,
|
|
"drift/rejected_abs": 0.15709178149700165,
|
|
"epoch": 0.16050854191497815,
|
|
"grad_norm": 8.5,
|
|
"learning_rate": 4.8255025164874966e-08,
|
|
"logits/chosen": -1.7380540370941162,
|
|
"logits/rejected": -1.744060754776001,
|
|
"logps/chosen": -0.3805738389492035,
|
|
"logps/rejected": -0.3870905041694641,
|
|
"loss": 0.4102400243282318,
|
|
"loss/core": 0.392947793006897,
|
|
"loss/preference_sft": 0.38436800241470337,
|
|
"loss/reference_anchor": 0.30740922689437866,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2868499755859375,
|
|
"rewards/margins": 1.78729248046875,
|
|
"rewards/rejected": 0.4995573163032532,
|
|
"ronpo/logit": 0.17872926592826843,
|
|
"ronpo/residual": 0.1838955581188202,
|
|
"ronpo/residual_abs": 0.22993488609790802,
|
|
"ronpo/target": -0.005166276823729277,
|
|
"ronpo/target_abs": 0.07546278089284897,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5437565445899963,
|
|
"drift/rejected_abs": 0.19869248569011688,
|
|
"epoch": 0.16209773539928488,
|
|
"grad_norm": 450.0,
|
|
"learning_rate": 4.7286377316888315e-08,
|
|
"logits/chosen": -1.8554989099502563,
|
|
"logits/rejected": -1.8838293552398682,
|
|
"logps/chosen": -0.2770172357559204,
|
|
"logps/rejected": -0.2674543261528015,
|
|
"loss": 2.1095359325408936,
|
|
"loss/core": 2.0345797538757324,
|
|
"loss/preference_sft": 0.28003743290901184,
|
|
"loss/reference_anchor": 1.471118450164795,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 5.4361066818237305,
|
|
"rewards/margins": 3.4505832195281982,
|
|
"rewards/rejected": 1.9855238199234009,
|
|
"ronpo/logit": 0.3450583517551422,
|
|
"ronpo/residual": 0.36223721504211426,
|
|
"ronpo/residual_abs": 0.4740299582481384,
|
|
"ronpo/target": -0.017178861424326897,
|
|
"ronpo/target_abs": 0.0916592925786972,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17446205019950867,
|
|
"drift/rejected_abs": 0.06303250789642334,
|
|
"epoch": 0.16368692888359157,
|
|
"grad_norm": 6.0,
|
|
"learning_rate": 4.631874995141376e-08,
|
|
"logits/chosen": -1.7710046768188477,
|
|
"logits/rejected": -1.81719172000885,
|
|
"logps/chosen": -0.311026394367218,
|
|
"logps/rejected": -0.31932035088539124,
|
|
"loss": 0.2285696566104889,
|
|
"loss/core": 0.21945735812187195,
|
|
"loss/preference_sft": 0.30587515234947205,
|
|
"loss/reference_anchor": 0.15165886282920837,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.744620680809021,
|
|
"rewards/margins": 1.1147360801696777,
|
|
"rewards/rejected": 0.6298844814300537,
|
|
"ronpo/logit": 0.11147361993789673,
|
|
"ronpo/residual": 0.12036045640707016,
|
|
"ronpo/residual_abs": 0.17754912376403809,
|
|
"ronpo/target": -0.00888682622462511,
|
|
"ronpo/target_abs": 0.08302748203277588,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2479497194290161,
|
|
"drift/rejected_abs": 0.15151138603687286,
|
|
"epoch": 0.1652761223678983,
|
|
"grad_norm": 95.5,
|
|
"learning_rate": 4.535250695353039e-08,
|
|
"logits/chosen": -2.097480058670044,
|
|
"logits/rejected": -2.106612205505371,
|
|
"logps/chosen": -0.238003209233284,
|
|
"logps/rejected": -0.2355692833662033,
|
|
"loss": 0.15418457984924316,
|
|
"loss/core": 0.14264442026615143,
|
|
"loss/preference_sft": 0.23966360092163086,
|
|
"loss/reference_anchor": 0.20683690905570984,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.4794869422912598,
|
|
"rewards/margins": 0.9650307893753052,
|
|
"rewards/rejected": 1.514456033706665,
|
|
"ronpo/logit": 0.09650308638811111,
|
|
"ronpo/residual": 0.08097958564758301,
|
|
"ronpo/residual_abs": 0.1965085119009018,
|
|
"ronpo/target": 0.015523498877882957,
|
|
"ronpo/target_abs": 0.07920856773853302,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2776569724082947,
|
|
"drift/rejected_abs": 0.15892525017261505,
|
|
"epoch": 0.16686531585220502,
|
|
"grad_norm": 54.0,
|
|
"learning_rate": 4.438801168771327e-08,
|
|
"logits/chosen": -2.0010154247283936,
|
|
"logits/rejected": -2.0862135887145996,
|
|
"logps/chosen": -0.3032376170158386,
|
|
"logps/rejected": -0.29709702730178833,
|
|
"loss": 0.501846194267273,
|
|
"loss/core": 0.4632810056209564,
|
|
"loss/preference_sft": 0.2949376106262207,
|
|
"loss/reference_anchor": 0.7418104410171509,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7760863304138184,
|
|
"rewards/margins": 1.1882919073104858,
|
|
"rewards/rejected": 1.587794303894043,
|
|
"ronpo/logit": 0.11882920563220978,
|
|
"ronpo/residual": 0.1192612275481224,
|
|
"ronpo/residual_abs": 0.2238401472568512,
|
|
"ronpo/target": -0.00043202488450333476,
|
|
"ronpo/target_abs": 0.0579981803894043,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28078198432922363,
|
|
"drift/rejected_abs": 0.10268757492303848,
|
|
"epoch": 0.1684545093365117,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 4.342562686118687e-08,
|
|
"logits/chosen": -1.7843738794326782,
|
|
"logits/rejected": -1.7553303241729736,
|
|
"logps/chosen": -0.2939331531524658,
|
|
"logps/rejected": -0.2914874851703644,
|
|
"loss": 0.7988781332969666,
|
|
"loss/core": 0.7687490582466125,
|
|
"loss/preference_sft": 0.2921197712421417,
|
|
"loss/reference_anchor": 0.5733689069747925,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.803372383117676,
|
|
"rewards/margins": 1.7815815210342407,
|
|
"rewards/rejected": 1.0217907428741455,
|
|
"ronpo/logit": 0.17815817892551422,
|
|
"ronpo/residual": 0.16743174195289612,
|
|
"ronpo/residual_abs": 0.23127658665180206,
|
|
"ronpo/target": 0.010726441629230976,
|
|
"ronpo/target_abs": 0.08158265054225922,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22689418494701385,
|
|
"drift/rejected_abs": 0.09927350282669067,
|
|
"epoch": 0.17004370282081843,
|
|
"grad_norm": 3.0625,
|
|
"learning_rate": 4.2465714387525845e-08,
|
|
"logits/chosen": -1.7310540676116943,
|
|
"logits/rejected": -1.8183752298355103,
|
|
"logps/chosen": -0.29117313027381897,
|
|
"logps/rejected": -0.29093536734580994,
|
|
"loss": 0.14975737035274506,
|
|
"loss/core": 0.13723108172416687,
|
|
"loss/preference_sft": 0.2861308157444,
|
|
"loss/reference_anchor": 0.2219126671552658,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2661190032958984,
|
|
"rewards/margins": 1.2744437456130981,
|
|
"rewards/rejected": 0.9916753768920898,
|
|
"ronpo/logit": 0.12744438648223877,
|
|
"ronpo/residual": 0.09695129096508026,
|
|
"ronpo/residual_abs": 0.17214377224445343,
|
|
"ronpo/target": 0.030493084341287613,
|
|
"ronpo/target_abs": 0.08196470886468887,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2985483407974243,
|
|
"drift/rejected_abs": 0.0866260677576065,
|
|
"epoch": 0.17163289630512515,
|
|
"grad_norm": 4.0625,
|
|
"learning_rate": 4.1508635250553965e-08,
|
|
"logits/chosen": -1.7881622314453125,
|
|
"logits/rejected": -1.8750550746917725,
|
|
"logps/chosen": -0.2710305452346802,
|
|
"logps/rejected": -0.26017826795578003,
|
|
"loss": 0.8527200818061829,
|
|
"loss/core": 0.8249381184577942,
|
|
"loss/preference_sft": 0.2635522186756134,
|
|
"loss/reference_anchor": 0.5292860269546509,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.984388828277588,
|
|
"rewards/margins": 2.1198816299438477,
|
|
"rewards/rejected": 0.8645073175430298,
|
|
"ronpo/logit": 0.211988165974617,
|
|
"ronpo/residual": 0.23462891578674316,
|
|
"ronpo/residual_abs": 0.26362109184265137,
|
|
"ronpo/target": -0.022640716284513474,
|
|
"ronpo/target_abs": 0.0714339092373848,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22164639830589294,
|
|
"drift/rejected_abs": 0.10531902313232422,
|
|
"epoch": 0.17322208978943188,
|
|
"grad_norm": 1.546875,
|
|
"learning_rate": 4.0554749368592995e-08,
|
|
"logits/chosen": -1.8922348022460938,
|
|
"logits/rejected": -1.908355712890625,
|
|
"logps/chosen": -0.28418028354644775,
|
|
"logps/rejected": -0.2719399929046631,
|
|
"loss": 0.0991106778383255,
|
|
"loss/core": 0.08887199312448502,
|
|
"loss/preference_sft": 0.27457016706466675,
|
|
"loss/reference_anchor": 0.1773165613412857,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.215404748916626,
|
|
"rewards/margins": 1.1632106304168701,
|
|
"rewards/rejected": 1.0521941184997559,
|
|
"ronpo/logit": 0.11632106453180313,
|
|
"ronpo/residual": 0.11025375127792358,
|
|
"ronpo/residual_abs": 0.15504197776317596,
|
|
"ronpo/target": 0.006067323964089155,
|
|
"ronpo/target_abs": 0.060761380940675735,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25841450691223145,
|
|
"drift/rejected_abs": 0.06605127453804016,
|
|
"epoch": 0.17481128327373857,
|
|
"grad_norm": 106.5,
|
|
"learning_rate": 3.960441545911204e-08,
|
|
"logits/chosen": -1.757881760597229,
|
|
"logits/rejected": -1.7778260707855225,
|
|
"logps/chosen": -0.2987707555294037,
|
|
"logps/rejected": -0.2908412516117096,
|
|
"loss": 0.6568024754524231,
|
|
"loss/core": 0.6380494832992554,
|
|
"loss/preference_sft": 0.2974953055381775,
|
|
"loss/reference_anchor": 0.3453090488910675,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.58390474319458,
|
|
"rewards/margins": 1.9925460815429688,
|
|
"rewards/rejected": 0.5913587808609009,
|
|
"ronpo/logit": 0.1992546170949936,
|
|
"ronpo/residual": 0.18910114467144012,
|
|
"ronpo/residual_abs": 0.2484695166349411,
|
|
"ronpo/target": 0.010153469629585743,
|
|
"ronpo/target_abs": 0.08087723702192307,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1866646111011505,
|
|
"drift/rejected_abs": 0.10433971881866455,
|
|
"epoch": 0.1764004767580453,
|
|
"grad_norm": 24.75,
|
|
"learning_rate": 3.8657990903828655e-08,
|
|
"logits/chosen": -1.8334095478057861,
|
|
"logits/rejected": -1.8890628814697266,
|
|
"logps/chosen": -0.29963818192481995,
|
|
"logps/rejected": -0.29799216985702515,
|
|
"loss": 0.08355274796485901,
|
|
"loss/core": 0.07509630918502808,
|
|
"loss/preference_sft": 0.30388861894607544,
|
|
"loss/reference_anchor": 0.13874009251594543,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8640848398208618,
|
|
"rewards/margins": 0.8224803805351257,
|
|
"rewards/rejected": 1.0416042804718018,
|
|
"ronpo/logit": 0.08224803954362869,
|
|
"ronpo/residual": 0.08422321081161499,
|
|
"ronpo/residual_abs": 0.1697530597448349,
|
|
"ronpo/target": -0.0019751801155507565,
|
|
"ronpo/target_abs": 0.09187708050012589,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23153360188007355,
|
|
"drift/rejected_abs": 0.13602390885353088,
|
|
"epoch": 0.17798967024235202,
|
|
"grad_norm": 7.09375,
|
|
"learning_rate": 3.7715831614312174e-08,
|
|
"logits/chosen": -1.848433256149292,
|
|
"logits/rejected": -1.8488585948944092,
|
|
"logps/chosen": -0.29612210392951965,
|
|
"logps/rejected": -0.29059985280036926,
|
|
"loss": 0.05996299535036087,
|
|
"loss/core": 0.051027536392211914,
|
|
"loss/preference_sft": 0.2972286343574524,
|
|
"loss/reference_anchor": 0.14898619055747986,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.315253734588623,
|
|
"rewards/margins": 0.9552527666091919,
|
|
"rewards/rejected": 1.3600012063980103,
|
|
"ronpo/logit": 0.09552527964115143,
|
|
"ronpo/residual": 0.09656206518411636,
|
|
"ronpo/residual_abs": 0.16236624121665955,
|
|
"ronpo/target": -0.001036791130900383,
|
|
"ronpo/target_abs": 0.08028535544872284,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13719969987869263,
|
|
"drift/rejected_abs": 0.06547123938798904,
|
|
"epoch": 0.1795788637266587,
|
|
"grad_norm": 16.75,
|
|
"learning_rate": 3.67782918981399e-08,
|
|
"logits/chosen": -1.8791453838348389,
|
|
"logits/rejected": -1.8915935754776,
|
|
"logps/chosen": -0.25503629446029663,
|
|
"logps/rejected": -0.2641448974609375,
|
|
"loss": 0.04704626649618149,
|
|
"loss/core": 0.04318038374185562,
|
|
"loss/preference_sft": 0.2602216601371765,
|
|
"loss/reference_anchor": 0.05129551887512207,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.3719971179962158,
|
|
"rewards/margins": 0.7177979350090027,
|
|
"rewards/rejected": 0.6541991829872131,
|
|
"ronpo/logit": 0.07177979499101639,
|
|
"ronpo/residual": 0.06935735046863556,
|
|
"ronpo/residual_abs": 0.13249507546424866,
|
|
"ronpo/target": 0.00242243567481637,
|
|
"ronpo/target_abs": 0.08676055818796158,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15917405486106873,
|
|
"drift/rejected_abs": 0.12015297263860703,
|
|
"epoch": 0.18116805721096543,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 3.584572432565648e-08,
|
|
"logits/chosen": -1.7565100193023682,
|
|
"logits/rejected": -1.7765508890151978,
|
|
"logps/chosen": -0.36137717962265015,
|
|
"logps/rejected": -0.29022398591041565,
|
|
"loss": 0.09365180879831314,
|
|
"loss/core": 0.0859026312828064,
|
|
"loss/preference_sft": 0.2855459153652191,
|
|
"loss/reference_anchor": 0.12642887234687805,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5901105403900146,
|
|
"rewards/margins": 0.3890073299407959,
|
|
"rewards/rejected": 1.2011032104492188,
|
|
"ronpo/logit": 0.03890075162053108,
|
|
"ronpo/residual": 0.04145831614732742,
|
|
"ronpo/residual_abs": 0.14707839488983154,
|
|
"ronpo/target": -0.00255756638944149,
|
|
"ronpo/target_abs": 0.08257381618022919,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2945803999900818,
|
|
"drift/rejected_abs": 0.22317442297935486,
|
|
"epoch": 0.18275725069527216,
|
|
"grad_norm": 161.0,
|
|
"learning_rate": 3.491847959738672e-08,
|
|
"logits/chosen": -1.6871364116668701,
|
|
"logits/rejected": -1.653978943824768,
|
|
"logps/chosen": -0.27193376421928406,
|
|
"logps/rejected": -0.2878838777542114,
|
|
"loss": 0.960303008556366,
|
|
"loss/core": 0.9258346557617188,
|
|
"loss/preference_sft": 0.28949350118637085,
|
|
"loss/reference_anchor": 0.6604174375534058,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9458041191101074,
|
|
"rewards/margins": 0.7202137112617493,
|
|
"rewards/rejected": 2.225590467453003,
|
|
"ronpo/logit": 0.07202135026454926,
|
|
"ronpo/residual": 0.07455171644687653,
|
|
"ronpo/residual_abs": 0.29728493094444275,
|
|
"ronpo/target": -0.002530353609472513,
|
|
"ronpo/target_abs": 0.07750637829303741,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2119152992963791,
|
|
"drift/rejected_abs": 0.1600765585899353,
|
|
"epoch": 0.18434644417957885,
|
|
"grad_norm": 1.546875,
|
|
"learning_rate": 3.3996906412151415e-08,
|
|
"logits/chosen": -1.7865766286849976,
|
|
"logits/rejected": -1.7664226293563843,
|
|
"logps/chosen": -0.28624483942985535,
|
|
"logps/rejected": -0.29210740327835083,
|
|
"loss": 0.2576879560947418,
|
|
"loss/core": 0.2448233664035797,
|
|
"loss/preference_sft": 0.3036908507347107,
|
|
"loss/reference_anchor": 0.2269231379032135,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.117682933807373,
|
|
"rewards/margins": 0.5200752019882202,
|
|
"rewards/rejected": 1.5976078510284424,
|
|
"ronpo/logit": 0.0520075187087059,
|
|
"ronpo/residual": 0.04997884854674339,
|
|
"ronpo/residual_abs": 0.2069791555404663,
|
|
"ronpo/target": 0.002028673654422164,
|
|
"ronpo/target_abs": 0.08073200285434723,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2667069733142853,
|
|
"drift/rejected_abs": 0.08909569680690765,
|
|
"epoch": 0.18593563766388557,
|
|
"grad_norm": 9.625,
|
|
"learning_rate": 3.3081351335935947e-08,
|
|
"logits/chosen": -1.7589290142059326,
|
|
"logits/rejected": -1.7354164123535156,
|
|
"logps/chosen": -0.276340514421463,
|
|
"logps/rejected": -0.2858516573905945,
|
|
"loss": 0.8393601179122925,
|
|
"loss/core": 0.8143278360366821,
|
|
"loss/preference_sft": 0.2846840023994446,
|
|
"loss/reference_anchor": 0.4721766412258148,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6665663719177246,
|
|
"rewards/margins": 1.7798312902450562,
|
|
"rewards/rejected": 0.8867348432540894,
|
|
"ronpo/logit": 0.1779831200838089,
|
|
"ronpo/residual": 0.18403668701648712,
|
|
"ronpo/residual_abs": 0.23977021872997284,
|
|
"ronpo/target": -0.006053552031517029,
|
|
"ronpo/target_abs": 0.08337777107954025,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20994123816490173,
|
|
"drift/rejected_abs": 0.10977955162525177,
|
|
"epoch": 0.1875248311481923,
|
|
"grad_norm": 32.25,
|
|
"learning_rate": 3.2172158671561e-08,
|
|
"logits/chosen": -1.847988486289978,
|
|
"logits/rejected": -1.746583342552185,
|
|
"logps/chosen": -0.27806663513183594,
|
|
"logps/rejected": -0.28961890935897827,
|
|
"loss": 0.10898953676223755,
|
|
"loss/core": 0.09721212089061737,
|
|
"loss/preference_sft": 0.27743273973464966,
|
|
"loss/reference_anchor": 0.2078050822019577,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.09867525100708,
|
|
"rewards/margins": 1.0014612674713135,
|
|
"rewards/rejected": 1.0972139835357666,
|
|
"ronpo/logit": 0.10014612972736359,
|
|
"ronpo/residual": 0.0981498658657074,
|
|
"ronpo/residual_abs": 0.16482563316822052,
|
|
"ronpo/target": 0.0019962673541158438,
|
|
"ronpo/target_abs": 0.08062820136547089,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4715874195098877,
|
|
"drift/rejected_abs": 0.19329656660556793,
|
|
"epoch": 0.18911402463249902,
|
|
"grad_norm": 384.0,
|
|
"learning_rate": 3.126967032920439e-08,
|
|
"logits/chosen": -1.7966649532318115,
|
|
"logits/rejected": -1.8083879947662354,
|
|
"logps/chosen": -0.277448832988739,
|
|
"logps/rejected": -0.2569620609283447,
|
|
"loss": 1.5912710428237915,
|
|
"loss/core": 1.5417354106903076,
|
|
"loss/preference_sft": 0.2574903070926666,
|
|
"loss/reference_anchor": 0.9649633169174194,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.715729236602783,
|
|
"rewards/margins": 2.7836599349975586,
|
|
"rewards/rejected": 1.932069182395935,
|
|
"ronpo/logit": 0.2783660292625427,
|
|
"ronpo/residual": 0.29117247462272644,
|
|
"ronpo/residual_abs": 0.3995015025138855,
|
|
"ronpo/target": -0.01280647236853838,
|
|
"ronpo/target_abs": 0.08523674309253693,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23165254294872284,
|
|
"drift/rejected_abs": 0.14367060363292694,
|
|
"epoch": 0.1907032181168057,
|
|
"grad_norm": 6.28125,
|
|
"learning_rate": 3.037422569782264e-08,
|
|
"logits/chosen": -1.9908279180526733,
|
|
"logits/rejected": -1.9903548955917358,
|
|
"logps/chosen": -0.2832784950733185,
|
|
"logps/rejected": -0.28507620096206665,
|
|
"loss": 0.03948095068335533,
|
|
"loss/core": 0.02752791717648506,
|
|
"loss/preference_sft": 0.2857871651649475,
|
|
"loss/reference_anchor": 0.21048197150230408,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3162155151367188,
|
|
"rewards/margins": 0.8797648549079895,
|
|
"rewards/rejected": 1.4364506006240845,
|
|
"ronpo/logit": 0.08797647804021835,
|
|
"ronpo/residual": 0.0719924122095108,
|
|
"ronpo/residual_abs": 0.10735388845205307,
|
|
"ronpo/target": 0.015984075143933296,
|
|
"ronpo/target_abs": 0.0789966955780983,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30132848024368286,
|
|
"drift/rejected_abs": 0.12802016735076904,
|
|
"epoch": 0.19229241160111243,
|
|
"grad_norm": 7.125,
|
|
"learning_rate": 2.9486161517520596e-08,
|
|
"logits/chosen": -1.7950258255004883,
|
|
"logits/rejected": -1.8050203323364258,
|
|
"logps/chosen": -0.24900193512439728,
|
|
"logps/rejected": -0.25392305850982666,
|
|
"loss": 0.19868217408657074,
|
|
"loss/core": 0.17862369120121002,
|
|
"loss/preference_sft": 0.24964216351509094,
|
|
"loss/reference_anchor": 0.3762053847312927,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0126354694366455,
|
|
"rewards/margins": 1.741882562637329,
|
|
"rewards/rejected": 1.2707531452178955,
|
|
"ronpo/logit": 0.17418824136257172,
|
|
"ronpo/residual": 0.17219789326190948,
|
|
"ronpo/residual_abs": 0.23744669556617737,
|
|
"ronpo/target": 0.0019903441425412893,
|
|
"ronpo/target_abs": 0.0929965078830719,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21064424514770508,
|
|
"drift/rejected_abs": 0.06865555047988892,
|
|
"epoch": 0.19388160508541916,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 2.8605811752917298e-08,
|
|
"logits/chosen": -1.791255235671997,
|
|
"logits/rejected": -1.8882793188095093,
|
|
"logps/chosen": -0.28656139969825745,
|
|
"logps/rejected": -0.28957584500312805,
|
|
"loss": 0.46913671493530273,
|
|
"loss/core": 0.4557432532310486,
|
|
"loss/preference_sft": 0.29006972908973694,
|
|
"loss/reference_anchor": 0.23886322975158691,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.106372356414795,
|
|
"rewards/margins": 1.4200407266616821,
|
|
"rewards/rejected": 0.6863317489624023,
|
|
"ronpo/logit": 0.14200405776500702,
|
|
"ronpo/residual": 0.15059545636177063,
|
|
"ronpo/residual_abs": 0.2058655023574829,
|
|
"ronpo/target": -0.00859138835221529,
|
|
"ronpo/target_abs": 0.08471350371837616,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20590801537036896,
|
|
"drift/rejected_abs": 0.12859569489955902,
|
|
"epoch": 0.19547079856972585,
|
|
"grad_norm": 47.5,
|
|
"learning_rate": 2.7733507467555527e-08,
|
|
"logits/chosen": -1.9091287851333618,
|
|
"logits/rejected": -1.945540189743042,
|
|
"logps/chosen": -0.2883964776992798,
|
|
"logps/rejected": -0.28995269536972046,
|
|
"loss": 0.06815268099308014,
|
|
"loss/core": 0.054342083632946014,
|
|
"loss/preference_sft": 0.28977853059768677,
|
|
"loss/reference_anchor": 0.24723413586616516,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0582265853881836,
|
|
"rewards/margins": 0.7743476629257202,
|
|
"rewards/rejected": 1.283879041671753,
|
|
"ronpo/logit": 0.07743476331233978,
|
|
"ronpo/residual": 0.08360174298286438,
|
|
"ronpo/residual_abs": 0.13559815287590027,
|
|
"ronpo/target": -0.006166977342218161,
|
|
"ronpo/target_abs": 0.06952059268951416,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1789097934961319,
|
|
"drift/rejected_abs": 0.08338747918605804,
|
|
"epoch": 0.19705999205403257,
|
|
"grad_norm": 0.63671875,
|
|
"learning_rate": 2.686957669940242e-08,
|
|
"logits/chosen": -1.8341875076293945,
|
|
"logits/rejected": -1.810105562210083,
|
|
"logps/chosen": -0.303658127784729,
|
|
"logps/rejected": -0.2884838283061981,
|
|
"loss": 0.13995635509490967,
|
|
"loss/core": 0.13397568464279175,
|
|
"loss/preference_sft": 0.2918221652507782,
|
|
"loss/reference_anchor": 0.09043111652135849,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7879546880722046,
|
|
"rewards/margins": 0.9541919827461243,
|
|
"rewards/rejected": 0.8337627649307251,
|
|
"ronpo/logit": 0.09541921317577362,
|
|
"ronpo/residual": 0.10469168424606323,
|
|
"ronpo/residual_abs": 0.16267558932304382,
|
|
"ronpo/target": -0.00927248690277338,
|
|
"ronpo/target_abs": 0.0726732388138771,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34457817673683167,
|
|
"drift/rejected_abs": 0.14021232724189758,
|
|
"epoch": 0.1986491855383393,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 2.6014344337487702e-08,
|
|
"logits/chosen": -1.959282636642456,
|
|
"logits/rejected": -1.9119141101837158,
|
|
"logps/chosen": -0.3181460499763489,
|
|
"logps/rejected": -0.2812593877315521,
|
|
"loss": 1.1086939573287964,
|
|
"loss/core": 1.0717629194259644,
|
|
"loss/preference_sft": 0.3202729821205139,
|
|
"loss/reference_anchor": 0.7065903544425964,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.445781707763672,
|
|
"rewards/margins": 2.05183482170105,
|
|
"rewards/rejected": 1.393946886062622,
|
|
"ronpo/logit": 0.2051835060119629,
|
|
"ronpo/residual": 0.19359295070171356,
|
|
"ronpo/residual_abs": 0.29983606934547424,
|
|
"ronpo/target": 0.011590538546442986,
|
|
"ronpo/target_abs": 0.07752411812543869,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28649866580963135,
|
|
"drift/rejected_abs": 0.10617027431726456,
|
|
"epoch": 0.20023837902264602,
|
|
"grad_norm": 26.125,
|
|
"learning_rate": 2.51681319997262e-08,
|
|
"logits/chosen": -1.5398986339569092,
|
|
"logits/rejected": -1.5675451755523682,
|
|
"logps/chosen": -0.2831787168979645,
|
|
"logps/rejected": -0.2901644706726074,
|
|
"loss": 0.3452921509742737,
|
|
"loss/core": 0.33066895604133606,
|
|
"loss/preference_sft": 0.2819228768348694,
|
|
"loss/reference_anchor": 0.26427116990089417,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.864680051803589,
|
|
"rewards/margins": 1.8066120147705078,
|
|
"rewards/rejected": 1.0580682754516602,
|
|
"ronpo/logit": 0.1806611865758896,
|
|
"ronpo/residual": 0.1922551691532135,
|
|
"ronpo/residual_abs": 0.23666691780090332,
|
|
"ronpo/target": -0.011593970470130444,
|
|
"ronpo/target_abs": 0.07857140153646469,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15038815140724182,
|
|
"drift/rejected_abs": 0.0819171741604805,
|
|
"epoch": 0.2018275725069527,
|
|
"grad_norm": 2.234375,
|
|
"learning_rate": 2.4331257911970626e-08,
|
|
"logits/chosen": -1.8283817768096924,
|
|
"logits/rejected": -1.9530715942382812,
|
|
"logps/chosen": -0.27565374970436096,
|
|
"logps/rejected": -0.2725347578525543,
|
|
"loss": 0.026056716218590736,
|
|
"loss/core": 0.02213389240205288,
|
|
"loss/preference_sft": 0.27325448393821716,
|
|
"loss/reference_anchor": 0.051131002604961395,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.5036888122558594,
|
|
"rewards/margins": 0.6845813989639282,
|
|
"rewards/rejected": 0.8191075325012207,
|
|
"ronpo/logit": 0.06845813244581223,
|
|
"ronpo/residual": 0.06282701343297958,
|
|
"ronpo/residual_abs": 0.11277289688587189,
|
|
"ronpo/target": 0.00563112273812294,
|
|
"ronpo/target_abs": 0.07500731945037842,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31310132145881653,
|
|
"drift/rejected_abs": 0.18400482833385468,
|
|
"epoch": 0.20341676599125944,
|
|
"grad_norm": 322.0,
|
|
"learning_rate": 2.350403678833976e-08,
|
|
"logits/chosen": -1.6889979839324951,
|
|
"logits/rejected": -1.654799222946167,
|
|
"logps/chosen": -0.29637211561203003,
|
|
"logps/rejected": -0.3162195682525635,
|
|
"loss": 0.19557903707027435,
|
|
"loss/core": 0.16963447630405426,
|
|
"loss/preference_sft": 0.3104371428489685,
|
|
"loss/reference_anchor": 0.4878477156162262,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.129647731781006,
|
|
"rewards/margins": 1.2897276878356934,
|
|
"rewards/rejected": 1.8399202823638916,
|
|
"ronpo/logit": 0.12897278368473053,
|
|
"ronpo/residual": 0.11268714815378189,
|
|
"ronpo/residual_abs": 0.1901034414768219,
|
|
"ronpo/target": 0.016285624355077744,
|
|
"ronpo/target_abs": 0.08053125441074371,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4701056480407715,
|
|
"drift/rejected_abs": 0.23759059607982635,
|
|
"epoch": 0.20500595947556616,
|
|
"grad_norm": 266.0,
|
|
"learning_rate": 2.2686779712867315e-08,
|
|
"logits/chosen": -1.9577453136444092,
|
|
"logits/rejected": -1.9497134685516357,
|
|
"logps/chosen": -0.23990201950073242,
|
|
"logps/rejected": -0.25109779834747314,
|
|
"loss": 1.5428487062454224,
|
|
"loss/core": 1.4728646278381348,
|
|
"loss/preference_sft": 0.24962405860424042,
|
|
"loss/reference_anchor": 1.3747169971466064,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.700854778289795,
|
|
"rewards/margins": 2.325613498687744,
|
|
"rewards/rejected": 2.375241756439209,
|
|
"ronpo/logit": 0.23256132006645203,
|
|
"ronpo/residual": 0.2318817675113678,
|
|
"ronpo/residual_abs": 0.38768142461776733,
|
|
"ronpo/target": 0.0006795950466766953,
|
|
"ronpo/target_abs": 0.06739381700754166,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28680965304374695,
|
|
"drift/rejected_abs": 0.09994098544120789,
|
|
"epoch": 0.20659515295987285,
|
|
"grad_norm": 5.78125,
|
|
"learning_rate": 2.1879794022516e-08,
|
|
"logits/chosen": -1.7325706481933594,
|
|
"logits/rejected": -1.8033367395401,
|
|
"logps/chosen": -0.2998145818710327,
|
|
"logps/rejected": -0.29500865936279297,
|
|
"loss": 0.3449249863624573,
|
|
"loss/core": 0.3255670964717865,
|
|
"loss/preference_sft": 0.29536116123199463,
|
|
"loss/reference_anchor": 0.35762161016464233,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8669979572296143,
|
|
"rewards/margins": 1.875012993812561,
|
|
"rewards/rejected": 0.991985023021698,
|
|
"ronpo/logit": 0.18750128149986267,
|
|
"ronpo/residual": 0.18114595115184784,
|
|
"ronpo/residual_abs": 0.23968927562236786,
|
|
"ronpo/target": 0.00635533919557929,
|
|
"ronpo/target_abs": 0.07919994741678238,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4130421280860901,
|
|
"drift/rejected_abs": 0.10792286694049835,
|
|
"epoch": 0.20818434644417957,
|
|
"grad_norm": 4.8125,
|
|
"learning_rate": 2.1083383191600673e-08,
|
|
"logits/chosen": -1.7494007349014282,
|
|
"logits/rejected": -1.7612907886505127,
|
|
"logps/chosen": -0.3074556589126587,
|
|
"logps/rejected": -0.3002517521381378,
|
|
"loss": 1.3415347337722778,
|
|
"loss/core": 1.3003575801849365,
|
|
"loss/preference_sft": 0.30100753903388977,
|
|
"loss/reference_anchor": 0.793440580368042,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.130122184753418,
|
|
"rewards/margins": 3.0868916511535645,
|
|
"rewards/rejected": 1.0432310104370117,
|
|
"ronpo/logit": 0.3086891770362854,
|
|
"ronpo/residual": 0.31742364168167114,
|
|
"ronpo/residual_abs": 0.3702516555786133,
|
|
"ronpo/target": -0.008734484203159809,
|
|
"ronpo/target_abs": 0.08217312395572662,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27077609300613403,
|
|
"drift/rejected_abs": 0.12126465141773224,
|
|
"epoch": 0.2097735399284863,
|
|
"grad_norm": 5.0,
|
|
"learning_rate": 2.029784671766404e-08,
|
|
"logits/chosen": -1.7809921503067017,
|
|
"logits/rejected": -1.7488667964935303,
|
|
"logps/chosen": -0.2880242168903351,
|
|
"logps/rejected": -0.2796364724636078,
|
|
"loss": 0.36306077241897583,
|
|
"loss/core": 0.33687418699264526,
|
|
"loss/preference_sft": 0.28567779064178467,
|
|
"loss/reference_anchor": 0.4951632022857666,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7072110176086426,
|
|
"rewards/margins": 1.4951145648956299,
|
|
"rewards/rejected": 1.2120963335037231,
|
|
"ronpo/logit": 0.1495114415884018,
|
|
"ronpo/residual": 0.14324429631233215,
|
|
"ronpo/residual_abs": 0.19393035769462585,
|
|
"ronpo/target": 0.006267151795327663,
|
|
"ronpo/target_abs": 0.0730602815747261,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34274253249168396,
|
|
"drift/rejected_abs": 0.11544802039861679,
|
|
"epoch": 0.21136273341279302,
|
|
"grad_norm": 102.5,
|
|
"learning_rate": 1.9523480008847854e-08,
|
|
"logits/chosen": -1.725452184677124,
|
|
"logits/rejected": -1.6947933435440063,
|
|
"logps/chosen": -0.2793700695037842,
|
|
"logps/rejected": -0.27260226011276245,
|
|
"loss": 0.6120738387107849,
|
|
"loss/core": 0.5833312273025513,
|
|
"loss/preference_sft": 0.27295297384262085,
|
|
"loss/reference_anchor": 0.5475563406944275,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.4263482093811035,
|
|
"rewards/margins": 2.2736732959747314,
|
|
"rewards/rejected": 1.152674913406372,
|
|
"ronpo/logit": 0.2273673117160797,
|
|
"ronpo/residual": 0.21752388775348663,
|
|
"ronpo/residual_abs": 0.290507972240448,
|
|
"ronpo/target": 0.00984344445168972,
|
|
"ronpo/target_abs": 0.07925274968147278,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17593860626220703,
|
|
"drift/rejected_abs": 0.08771608769893646,
|
|
"epoch": 0.2129519268970997,
|
|
"grad_norm": 2.640625,
|
|
"learning_rate": 1.8760574272802e-08,
|
|
"logits/chosen": -2.0317234992980957,
|
|
"logits/rejected": -1.96999990940094,
|
|
"logps/chosen": -0.2675436735153198,
|
|
"logps/rejected": -0.29036182165145874,
|
|
"loss": 0.06669995933771133,
|
|
"loss/core": 0.05987640097737312,
|
|
"loss/preference_sft": 0.29216253757476807,
|
|
"loss/reference_anchor": 0.1072549819946289,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7587525844573975,
|
|
"rewards/margins": 0.8839167356491089,
|
|
"rewards/rejected": 0.8748357892036438,
|
|
"ronpo/logit": 0.08839168399572372,
|
|
"ronpo/residual": 0.08926811814308167,
|
|
"ronpo/residual_abs": 0.1414795219898224,
|
|
"ronpo/target": -0.0008764358353801072,
|
|
"ronpo/target_abs": 0.07178516685962677,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19702181220054626,
|
|
"drift/rejected_abs": 0.10953919589519501,
|
|
"epoch": 0.21454112038140644,
|
|
"grad_norm": 5.375,
|
|
"learning_rate": 1.8009416407173257e-08,
|
|
"logits/chosen": -1.8570398092269897,
|
|
"logits/rejected": -1.8211997747421265,
|
|
"logps/chosen": -0.2882757782936096,
|
|
"logps/rejected": -0.29082340002059937,
|
|
"loss": 0.06841497123241425,
|
|
"loss/core": 0.06168912723660469,
|
|
"loss/preference_sft": 0.28591659665107727,
|
|
"loss/reference_anchor": 0.10592524707317352,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9673035144805908,
|
|
"rewards/margins": 0.8742406964302063,
|
|
"rewards/rejected": 1.0930628776550293,
|
|
"ronpo/logit": 0.08742406964302063,
|
|
"ronpo/residual": 0.08799617737531662,
|
|
"ronpo/residual_abs": 0.14206573367118835,
|
|
"ronpo/target": -0.0005720962071791291,
|
|
"ronpo/target_abs": 0.0792655348777771,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24524518847465515,
|
|
"drift/rejected_abs": 0.13547000288963318,
|
|
"epoch": 0.21613031386571316,
|
|
"grad_norm": 7.53125,
|
|
"learning_rate": 1.7270288891714813e-08,
|
|
"logits/chosen": -1.7146110534667969,
|
|
"logits/rejected": -1.7872194051742554,
|
|
"logps/chosen": -0.2838783264160156,
|
|
"logps/rejected": -0.27856138348579407,
|
|
"loss": 0.13877931237220764,
|
|
"loss/core": 0.1273031383752823,
|
|
"loss/preference_sft": 0.28666257858276367,
|
|
"loss/reference_anchor": 0.2008572518825531,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4522125720977783,
|
|
"rewards/margins": 1.1872022151947021,
|
|
"rewards/rejected": 1.2650099992752075,
|
|
"ronpo/logit": 0.11872021853923798,
|
|
"ronpo/residual": 0.1270017921924591,
|
|
"ronpo/residual_abs": 0.1994335949420929,
|
|
"ronpo/target": -0.00828153919428587,
|
|
"ronpo/target_abs": 0.07877428829669952,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2916144132614136,
|
|
"drift/rejected_abs": 0.11980894953012466,
|
|
"epoch": 0.21771950735001985,
|
|
"grad_norm": 8.9375,
|
|
"learning_rate": 1.6543469682057103e-08,
|
|
"logits/chosen": -1.750420331954956,
|
|
"logits/rejected": -1.7389971017837524,
|
|
"logps/chosen": -0.2631601393222809,
|
|
"logps/rejected": -0.2538334131240845,
|
|
"loss": 0.27099448442459106,
|
|
"loss/core": 0.25824078917503357,
|
|
"loss/preference_sft": 0.2480827122926712,
|
|
"loss/reference_anchor": 0.23026630282402039,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9157261848449707,
|
|
"rewards/margins": 1.7208629846572876,
|
|
"rewards/rejected": 1.1948630809783936,
|
|
"ronpo/logit": 0.17208631336688995,
|
|
"ronpo/residual": 0.16664986312389374,
|
|
"ronpo/residual_abs": 0.20796065032482147,
|
|
"ronpo/target": 0.005436430685222149,
|
|
"ronpo/target_abs": 0.07201899588108063,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2702964246273041,
|
|
"drift/rejected_abs": 0.07891695201396942,
|
|
"epoch": 0.21930870083432658,
|
|
"grad_norm": 1.484375,
|
|
"learning_rate": 1.582923210518014e-08,
|
|
"logits/chosen": -1.795771598815918,
|
|
"logits/rejected": -1.7635061740875244,
|
|
"logps/chosen": -0.29897016286849976,
|
|
"logps/rejected": -0.2939114570617676,
|
|
"loss": 0.30770987272262573,
|
|
"loss/core": 0.2953275740146637,
|
|
"loss/preference_sft": 0.28765055537223816,
|
|
"loss/reference_anchor": 0.21888060867786407,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.7028443813323975,
|
|
"rewards/margins": 1.9148327112197876,
|
|
"rewards/rejected": 0.788011908531189,
|
|
"ronpo/logit": 0.191483274102211,
|
|
"ronpo/residual": 0.18644031882286072,
|
|
"ronpo/residual_abs": 0.24509191513061523,
|
|
"ronpo/target": 0.0050429413095116615,
|
|
"ronpo/target_abs": 0.08253506571054459,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19787338376045227,
|
|
"drift/rejected_abs": 0.1597171276807785,
|
|
"epoch": 0.2208978943186333,
|
|
"grad_norm": 3.171875,
|
|
"learning_rate": 1.5127844756626436e-08,
|
|
"logits/chosen": -1.7620786428451538,
|
|
"logits/rejected": -1.723760962486267,
|
|
"logps/chosen": -0.3214377760887146,
|
|
"logps/rejected": -0.3080599308013916,
|
|
"loss": 0.4889402985572815,
|
|
"loss/core": 0.47200194001197815,
|
|
"loss/preference_sft": 0.32001546025276184,
|
|
"loss/reference_anchor": 0.30676406621932983,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9777981042861938,
|
|
"rewards/margins": 0.40713924169540405,
|
|
"rewards/rejected": 1.570658802986145,
|
|
"ronpo/logit": 0.040713924914598465,
|
|
"ronpo/residual": 0.03379645571112633,
|
|
"ronpo/residual_abs": 0.2298135757446289,
|
|
"ronpo/target": 0.0069174692034721375,
|
|
"ronpo/target_abs": 0.08368168771266937,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.267596572637558,
|
|
"drift/rejected_abs": 0.12422071397304535,
|
|
"epoch": 0.22248708780294002,
|
|
"grad_norm": 0.9296875,
|
|
"learning_rate": 1.4439571399493145e-08,
|
|
"logits/chosen": -1.9247478246688843,
|
|
"logits/rejected": -1.9092788696289062,
|
|
"logps/chosen": -0.30226415395736694,
|
|
"logps/rejected": -0.28551197052001953,
|
|
"loss": 0.23500370979309082,
|
|
"loss/core": 0.21412146091461182,
|
|
"loss/preference_sft": 0.2953455150127411,
|
|
"loss/reference_anchor": 0.3881102502346039,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.675750732421875,
|
|
"rewards/margins": 1.4344958066940308,
|
|
"rewards/rejected": 1.2412550449371338,
|
|
"ronpo/logit": 0.1434495896100998,
|
|
"ronpo/residual": 0.15175478160381317,
|
|
"ronpo/residual_abs": 0.19151520729064941,
|
|
"ronpo/target": -0.008305169641971588,
|
|
"ronpo/target_abs": 0.06857885420322418,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19850663840770721,
|
|
"drift/rejected_abs": 0.11235801875591278,
|
|
"epoch": 0.22407628128724671,
|
|
"grad_norm": 8.1875,
|
|
"learning_rate": 1.3764670865241557e-08,
|
|
"logits/chosen": -1.8514108657836914,
|
|
"logits/rejected": -1.7679784297943115,
|
|
"logps/chosen": -0.2819150388240814,
|
|
"logps/rejected": -0.28714990615844727,
|
|
"loss": 0.10464034229516983,
|
|
"loss/core": 0.09731554239988327,
|
|
"loss/preference_sft": 0.27761921286582947,
|
|
"loss/reference_anchor": 0.11873352527618408,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9850664138793945,
|
|
"rewards/margins": 0.8684585690498352,
|
|
"rewards/rejected": 1.1166080236434937,
|
|
"ronpo/logit": 0.08684585243463516,
|
|
"ronpo/residual": 0.09302155673503876,
|
|
"ronpo/residual_abs": 0.16833505034446716,
|
|
"ronpo/target": -0.006175708491355181,
|
|
"ronpo/target_abs": 0.08478256314992905,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16192565858364105,
|
|
"drift/rejected_abs": 0.0836598128080368,
|
|
"epoch": 0.22566547477155344,
|
|
"grad_norm": 7.28125,
|
|
"learning_rate": 1.310339695636118e-08,
|
|
"logits/chosen": -2.020831346511841,
|
|
"logits/rejected": -2.0246784687042236,
|
|
"logps/chosen": -0.2766228914260864,
|
|
"logps/rejected": -0.28187552094459534,
|
|
"loss": 0.03390822932124138,
|
|
"loss/core": 0.028157252818346024,
|
|
"loss/preference_sft": 0.2825476825237274,
|
|
"loss/reference_anchor": 0.08676482737064362,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6182422637939453,
|
|
"rewards/margins": 0.7829471826553345,
|
|
"rewards/rejected": 0.8352952003479004,
|
|
"ronpo/logit": 0.07829473167657852,
|
|
"ronpo/residual": 0.0662197396159172,
|
|
"ronpo/residual_abs": 0.11468762159347534,
|
|
"ronpo/target": 0.012074981816112995,
|
|
"ronpo/target_abs": 0.072537861764431,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.46716541051864624,
|
|
"drift/rejected_abs": 0.17989280819892883,
|
|
"epoch": 0.22725466825586016,
|
|
"grad_norm": 632.0,
|
|
"learning_rate": 1.245599835092504e-08,
|
|
"logits/chosen": -1.710375189781189,
|
|
"logits/rejected": -1.7550632953643799,
|
|
"logps/chosen": -0.2771041989326477,
|
|
"logps/rejected": -0.2840449810028076,
|
|
"loss": 1.0326440334320068,
|
|
"loss/core": 0.9840595126152039,
|
|
"loss/preference_sft": 0.2831014096736908,
|
|
"loss/reference_anchor": 0.9433791041374207,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.670206546783447,
|
|
"rewards/margins": 2.8777670860290527,
|
|
"rewards/rejected": 1.7924392223358154,
|
|
"ronpo/logit": 0.2877767086029053,
|
|
"ronpo/residual": 0.2957441806793213,
|
|
"ronpo/residual_abs": 0.35328811407089233,
|
|
"ronpo/target": -0.007967445999383926,
|
|
"ronpo/target_abs": 0.07957801222801208,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3361731469631195,
|
|
"drift/rejected_abs": 0.18497583270072937,
|
|
"epoch": 0.22884386174016685,
|
|
"grad_norm": 348.0,
|
|
"learning_rate": 1.1822718509071988e-08,
|
|
"logits/chosen": -1.8438173532485962,
|
|
"logits/rejected": -1.8709602355957031,
|
|
"logps/chosen": -0.2885790169239044,
|
|
"logps/rejected": -0.317440927028656,
|
|
"loss": 0.6532329320907593,
|
|
"loss/core": 0.6089370846748352,
|
|
"loss/preference_sft": 0.31347936391830444,
|
|
"loss/reference_anchor": 0.8545686602592468,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.3605964183807373,
|
|
"rewards/margins": 1.5140485763549805,
|
|
"rewards/rejected": 1.8465477228164673,
|
|
"ronpo/logit": 0.15140488743782043,
|
|
"ronpo/residual": 0.1639949083328247,
|
|
"ronpo/residual_abs": 0.25466206669807434,
|
|
"ronpo/target": -0.01259003859013319,
|
|
"ronpo/target_abs": 0.06609584391117096,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36866021156311035,
|
|
"drift/rejected_abs": 0.09367374330759048,
|
|
"epoch": 0.23043305522447358,
|
|
"grad_norm": 198.0,
|
|
"learning_rate": 1.1203795581451287e-08,
|
|
"logits/chosen": -1.7415307760238647,
|
|
"logits/rejected": -1.8097641468048096,
|
|
"logps/chosen": -0.26107972860336304,
|
|
"logps/rejected": -0.2517997622489929,
|
|
"loss": 1.3886072635650635,
|
|
"loss/core": 1.350977897644043,
|
|
"loss/preference_sft": 0.25317317247390747,
|
|
"loss/reference_anchor": 0.7272680997848511,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.6863365173339844,
|
|
"rewards/margins": 2.7501847743988037,
|
|
"rewards/rejected": 0.9361515045166016,
|
|
"ronpo/logit": 0.27501851320266724,
|
|
"ronpo/residual": 0.2745300233364105,
|
|
"ronpo/residual_abs": 0.33074140548706055,
|
|
"ronpo/target": 0.0004884883528575301,
|
|
"ronpo/target_abs": 0.07590387016534805,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5398408770561218,
|
|
"drift/rejected_abs": 0.16165503859519958,
|
|
"epoch": 0.2320222487087803,
|
|
"grad_norm": 3.734375,
|
|
"learning_rate": 1.0599462319663904e-08,
|
|
"logits/chosen": -1.5984643697738647,
|
|
"logits/rejected": -1.6125961542129517,
|
|
"logps/chosen": -0.28152695298194885,
|
|
"logps/rejected": -0.27059173583984375,
|
|
"loss": 1.8108030557632446,
|
|
"loss/core": 1.7431504726409912,
|
|
"loss/preference_sft": 0.2749939262866974,
|
|
"loss/reference_anchor": 1.3255517482757568,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 5.396024703979492,
|
|
"rewards/margins": 3.786198854446411,
|
|
"rewards/rejected": 1.609825849533081,
|
|
"ronpo/logit": 0.37861987948417664,
|
|
"ronpo/residual": 0.37991079688072205,
|
|
"ronpo/residual_abs": 0.4221607744693756,
|
|
"ronpo/target": -0.0012909111101180315,
|
|
"ronpo/target_abs": 0.07381873577833176,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3352048397064209,
|
|
"drift/rejected_abs": 0.058885980397462845,
|
|
"epoch": 0.23361144219308702,
|
|
"grad_norm": 3.828125,
|
|
"learning_rate": 1.0009945988734203e-08,
|
|
"logits/chosen": -1.7541921138763428,
|
|
"logits/rejected": -1.8055585622787476,
|
|
"logps/chosen": -0.2714094817638397,
|
|
"logps/rejected": -0.26239532232284546,
|
|
"loss": 0.8836636543273926,
|
|
"loss/core": 0.8595617413520813,
|
|
"loss/preference_sft": 0.2709152102470398,
|
|
"loss/reference_anchor": 0.4549466669559479,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.3511319160461426,
|
|
"rewards/margins": 2.791016101837158,
|
|
"rewards/rejected": 0.5601160526275635,
|
|
"ronpo/logit": 0.2791016399860382,
|
|
"ronpo/residual": 0.28273439407348633,
|
|
"ronpo/residual_abs": 0.32099446654319763,
|
|
"ronpo/target": -0.0036327368579804897,
|
|
"ronpo/target_abs": 0.061633963137865067,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2550628185272217,
|
|
"drift/rejected_abs": 0.10268987715244293,
|
|
"epoch": 0.23520063567739372,
|
|
"grad_norm": 12.4375,
|
|
"learning_rate": 9.435468281644798e-09,
|
|
"logits/chosen": -1.7330278158187866,
|
|
"logits/rejected": -1.7721294164657593,
|
|
"logps/chosen": -0.2473641335964203,
|
|
"logps/rejected": -0.24295814335346222,
|
|
"loss": 0.14805634319782257,
|
|
"loss/core": 0.14048026502132416,
|
|
"loss/preference_sft": 0.24413037300109863,
|
|
"loss/reference_anchor": 0.12710848450660706,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5494306087493896,
|
|
"rewards/margins": 1.5241954326629639,
|
|
"rewards/rejected": 1.0252354145050049,
|
|
"ronpo/logit": 0.1524195522069931,
|
|
"ronpo/residual": 0.17534348368644714,
|
|
"ronpo/residual_abs": 0.20049890875816345,
|
|
"ronpo/target": -0.02292392961680889,
|
|
"ronpo/target_abs": 0.06671784818172455,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22789403796195984,
|
|
"drift/rejected_abs": 0.15626075863838196,
|
|
"epoch": 0.23678982916170044,
|
|
"grad_norm": 31.875,
|
|
"learning_rate": 8.876245235966883e-09,
|
|
"logits/chosen": -1.8382848501205444,
|
|
"logits/rejected": -1.8339512348175049,
|
|
"logps/chosen": -0.2761487364768982,
|
|
"logps/rejected": -0.282112181186676,
|
|
"loss": 0.2007966786623001,
|
|
"loss/core": 0.1876274198293686,
|
|
"loss/preference_sft": 0.28972652554512024,
|
|
"loss/reference_anchor": 0.23441258072853088,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2789406776428223,
|
|
"rewards/margins": 0.7250955104827881,
|
|
"rewards/rejected": 1.5538451671600342,
|
|
"ronpo/logit": 0.07250954210758209,
|
|
"ronpo/residual": 0.06640531122684479,
|
|
"ronpo/residual_abs": 0.1977664977312088,
|
|
"ronpo/target": 0.00610423693433404,
|
|
"ronpo/target_abs": 0.07519592344760895,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17921927571296692,
|
|
"drift/rejected_abs": 0.10521320253610611,
|
|
"epoch": 0.23837902264600716,
|
|
"grad_norm": 2.578125,
|
|
"learning_rate": 8.332487152617423e-09,
|
|
"logits/chosen": -1.672652244567871,
|
|
"logits/rejected": -1.641906976699829,
|
|
"logps/chosen": -0.29797834157943726,
|
|
"logps/rejected": -0.2950153052806854,
|
|
"loss": 0.05657457560300827,
|
|
"loss/core": 0.05042596533894539,
|
|
"loss/preference_sft": 0.3005446791648865,
|
|
"loss/reference_anchor": 0.09291768074035645,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.790204644203186,
|
|
"rewards/margins": 0.7381783723831177,
|
|
"rewards/rejected": 1.0520262718200684,
|
|
"ronpo/logit": 0.07381784170866013,
|
|
"ronpo/residual": 0.05989062041044235,
|
|
"ronpo/residual_abs": 0.14463061094284058,
|
|
"ronpo/target": 0.013927215710282326,
|
|
"ronpo/target_abs": 0.08233248442411423,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.40166425704956055,
|
|
"drift/rejected_abs": 0.09812676906585693,
|
|
"epoch": 0.23996821613031386,
|
|
"grad_norm": 2320.0,
|
|
"learning_rate": 7.804398516773464e-09,
|
|
"logits/chosen": -1.9683334827423096,
|
|
"logits/rejected": -1.9788726568222046,
|
|
"logps/chosen": -0.283616840839386,
|
|
"logps/rejected": -0.2929161489009857,
|
|
"loss": 1.8589715957641602,
|
|
"loss/core": 1.8126780986785889,
|
|
"loss/preference_sft": 0.2952941060066223,
|
|
"loss/reference_anchor": 0.8963397741317749,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.014865875244141,
|
|
"rewards/margins": 3.2176384925842285,
|
|
"rewards/rejected": 0.7972274422645569,
|
|
"ronpo/logit": 0.3217638432979584,
|
|
"ronpo/residual": 0.2891455590724945,
|
|
"ronpo/residual_abs": 0.35307732224464417,
|
|
"ronpo/target": 0.03261827304959297,
|
|
"ronpo/target_abs": 0.08080680668354034,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23729880154132843,
|
|
"drift/rejected_abs": 0.11338727176189423,
|
|
"epoch": 0.24155740961462058,
|
|
"grad_norm": 11.3125,
|
|
"learning_rate": 7.292177920973724e-09,
|
|
"logits/chosen": -2.0670156478881836,
|
|
"logits/rejected": -1.978662133216858,
|
|
"logps/chosen": -0.2790946364402771,
|
|
"logps/rejected": -0.28680047392845154,
|
|
"loss": 0.584754228591919,
|
|
"loss/core": 0.5627850890159607,
|
|
"loss/preference_sft": 0.28632253408432007,
|
|
"loss/reference_anchor": 0.4107509255409241,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.371988534927368,
|
|
"rewards/margins": 1.239213228225708,
|
|
"rewards/rejected": 1.1327753067016602,
|
|
"ronpo/logit": 0.12392131984233856,
|
|
"ronpo/residual": 0.1416681855916977,
|
|
"ronpo/residual_abs": 0.21293732523918152,
|
|
"ronpo/target": -0.01774686574935913,
|
|
"ronpo/target_abs": 0.08494757860898972,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16878637671470642,
|
|
"drift/rejected_abs": 0.11679929494857788,
|
|
"epoch": 0.2431466030989273,
|
|
"grad_norm": 0.7109375,
|
|
"learning_rate": 6.796017990435976e-09,
|
|
"logits/chosen": -1.8809000253677368,
|
|
"logits/rejected": -1.863707184791565,
|
|
"logps/chosen": -0.2994479835033417,
|
|
"logps/rejected": -0.2976071238517761,
|
|
"loss": 0.05739004537463188,
|
|
"loss/core": 0.049158744513988495,
|
|
"loss/preference_sft": 0.29969534277915955,
|
|
"loss/reference_anchor": 0.13465645909309387,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6868946552276611,
|
|
"rewards/margins": 0.5216689109802246,
|
|
"rewards/rejected": 1.1652257442474365,
|
|
"ronpo/logit": 0.0521668903529644,
|
|
"ronpo/residual": 0.06433439254760742,
|
|
"ronpo/residual_abs": 0.12495573610067368,
|
|
"ronpo/target": -0.012167500331997871,
|
|
"ronpo/target_abs": 0.0812489464879036,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16920286417007446,
|
|
"drift/rejected_abs": 0.10078708082437515,
|
|
"epoch": 0.24473579658323402,
|
|
"grad_norm": 0.921875,
|
|
"learning_rate": 6.316105310618664e-09,
|
|
"logits/chosen": -1.8295981884002686,
|
|
"logits/rejected": -1.8808387517929077,
|
|
"logps/chosen": -0.2839786112308502,
|
|
"logps/rejected": -0.288639098405838,
|
|
"loss": 0.039382454007864,
|
|
"loss/core": 0.03182164579629898,
|
|
"loss/preference_sft": 0.28843414783477783,
|
|
"loss/reference_anchor": 0.12237273156642914,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6907180547714233,
|
|
"rewards/margins": 0.6828474402427673,
|
|
"rewards/rejected": 1.0078706741333008,
|
|
"ronpo/logit": 0.06828473508358002,
|
|
"ronpo/residual": 0.06084071844816208,
|
|
"ronpo/residual_abs": 0.1161794438958168,
|
|
"ronpo/target": 0.00744402501732111,
|
|
"ronpo/target_abs": 0.07908720523118973,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2419697493314743,
|
|
"drift/rejected_abs": 0.11826857179403305,
|
|
"epoch": 0.24632499006754072,
|
|
"grad_norm": 5.9375,
|
|
"learning_rate": 5.8526203570536504e-09,
|
|
"logits/chosen": -1.7400401830673218,
|
|
"logits/rejected": -1.7736377716064453,
|
|
"logps/chosen": -0.2894227206707001,
|
|
"logps/rejected": -0.2918685972690582,
|
|
"loss": 0.08226810395717621,
|
|
"loss/core": 0.07246662676334381,
|
|
"loss/preference_sft": 0.29761046171188354,
|
|
"loss/reference_anchor": 0.16626867651939392,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.4190304279327393,
|
|
"rewards/margins": 1.2379631996154785,
|
|
"rewards/rejected": 1.1810672283172607,
|
|
"ronpo/logit": 0.12379632890224457,
|
|
"ronpo/residual": 0.11871888488531113,
|
|
"ronpo/residual_abs": 0.17133726179599762,
|
|
"ronpo/target": 0.0050774430856108665,
|
|
"ronpo/target_abs": 0.09046323597431183,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26481837034225464,
|
|
"drift/rejected_abs": 0.0806707888841629,
|
|
"epoch": 0.24791418355184744,
|
|
"grad_norm": 6.53125,
|
|
"learning_rate": 5.405737427476853e-09,
|
|
"logits/chosen": -1.9032214879989624,
|
|
"logits/rejected": -1.9158647060394287,
|
|
"logps/chosen": -0.275672972202301,
|
|
"logps/rejected": -0.27686625719070435,
|
|
"loss": 1.127648115158081,
|
|
"loss/core": 1.096915364265442,
|
|
"loss/preference_sft": 0.2758302688598633,
|
|
"loss/reference_anchor": 0.5870689749717712,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6463587284088135,
|
|
"rewards/margins": 1.8401899337768555,
|
|
"rewards/rejected": 0.8061690330505371,
|
|
"ronpo/logit": 0.18401899933815002,
|
|
"ronpo/residual": 0.19550299644470215,
|
|
"ronpo/residual_abs": 0.2645988464355469,
|
|
"ronpo/target": -0.011484002694487572,
|
|
"ronpo/target_abs": 0.09347420930862427,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18544304370880127,
|
|
"drift/rejected_abs": 0.06540529429912567,
|
|
"epoch": 0.24950337703615416,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 4.975624576281889e-09,
|
|
"logits/chosen": -1.76251220703125,
|
|
"logits/rejected": -1.7324647903442383,
|
|
"logps/chosen": -0.2766936719417572,
|
|
"logps/rejected": -0.2744191288948059,
|
|
"loss": 0.19484534859657288,
|
|
"loss/core": 0.18755874037742615,
|
|
"loss/preference_sft": 0.27636733651161194,
|
|
"loss/reference_anchor": 0.11809553951025009,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8540092706680298,
|
|
"rewards/margins": 1.2056849002838135,
|
|
"rewards/rejected": 0.6483243703842163,
|
|
"ronpo/logit": 0.12056849896907806,
|
|
"ronpo/residual": 0.12910667061805725,
|
|
"ronpo/residual_abs": 0.16958020627498627,
|
|
"ronpo/target": -0.00853817630559206,
|
|
"ronpo/target_abs": 0.06679105758666992,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23630964756011963,
|
|
"drift/rejected_abs": 0.12132765352725983,
|
|
"epoch": 0.25109257052046086,
|
|
"grad_norm": 16.125,
|
|
"learning_rate": 4.562443551321787e-09,
|
|
"logits/chosen": -1.8636810779571533,
|
|
"logits/rejected": -1.7788467407226562,
|
|
"logps/chosen": -0.2716495394706726,
|
|
"logps/rejected": -0.2747117877006531,
|
|
"loss": 0.19760456681251526,
|
|
"loss/core": 0.1853138655424118,
|
|
"loss/preference_sft": 0.2780051827430725,
|
|
"loss/reference_anchor": 0.21801309287548065,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3603432178497314,
|
|
"rewards/margins": 1.1492493152618408,
|
|
"rewards/rejected": 1.211093783378601,
|
|
"ronpo/logit": 0.11492495238780975,
|
|
"ronpo/residual": 0.12750717997550964,
|
|
"ronpo/residual_abs": 0.19350960850715637,
|
|
"ronpo/target": -0.01258223969489336,
|
|
"ronpo/target_abs": 0.07843288779258728,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2949658930301666,
|
|
"drift/rejected_abs": 0.15656772255897522,
|
|
"epoch": 0.2526817640047676,
|
|
"grad_norm": 5.03125,
|
|
"learning_rate": 4.166349733082153e-09,
|
|
"logits/chosen": -1.9113426208496094,
|
|
"logits/rejected": -1.931244134902954,
|
|
"logps/chosen": -0.28906121850013733,
|
|
"logps/rejected": -0.30459824204444885,
|
|
"loss": 0.7576580047607422,
|
|
"loss/core": 0.732965886592865,
|
|
"loss/preference_sft": 0.28880181908607483,
|
|
"loss/reference_anchor": 0.46496111154556274,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9488015174865723,
|
|
"rewards/margins": 1.3835405111312866,
|
|
"rewards/rejected": 1.565260648727417,
|
|
"ronpo/logit": 0.13835404813289642,
|
|
"ronpo/residual": 0.17754340171813965,
|
|
"ronpo/residual_abs": 0.3039698004722595,
|
|
"ronpo/target": -0.03918936103582382,
|
|
"ronpo/target_abs": 0.08568587899208069,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4828338027000427,
|
|
"drift/rejected_abs": 0.11110155284404755,
|
|
"epoch": 0.2542709574890743,
|
|
"grad_norm": 136.0,
|
|
"learning_rate": 3.7874920762489935e-09,
|
|
"logits/chosen": -1.885493278503418,
|
|
"logits/rejected": -1.9146902561187744,
|
|
"logps/chosen": -0.2795495390892029,
|
|
"logps/rejected": -0.2741713523864746,
|
|
"loss": 1.7612409591674805,
|
|
"loss/core": 1.7040965557098389,
|
|
"loss/preference_sft": 0.27895671129226685,
|
|
"loss/reference_anchor": 1.1149905920028687,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.827905178070068,
|
|
"rewards/margins": 3.717952251434326,
|
|
"rewards/rejected": 1.109952688217163,
|
|
"ronpo/logit": 0.37179526686668396,
|
|
"ronpo/residual": 0.3783615827560425,
|
|
"ronpo/residual_abs": 0.4227665960788727,
|
|
"ronpo/target": -0.006566292606294155,
|
|
"ronpo/target_abs": 0.07662634551525116,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18141977488994598,
|
|
"drift/rejected_abs": 0.08246532082557678,
|
|
"epoch": 0.255860150973381,
|
|
"grad_norm": 1.6484375,
|
|
"learning_rate": 3.426013053692878e-09,
|
|
"logits/chosen": -1.7892440557479858,
|
|
"logits/rejected": -1.836334466934204,
|
|
"logps/chosen": -0.2862665057182312,
|
|
"logps/rejected": -0.2810719907283783,
|
|
"loss": 0.12512250244617462,
|
|
"loss/core": 0.11690256744623184,
|
|
"loss/preference_sft": 0.2856505513191223,
|
|
"loss/reference_anchor": 0.13583365082740784,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8133046627044678,
|
|
"rewards/margins": 0.9899392127990723,
|
|
"rewards/rejected": 0.8233655691146851,
|
|
"ronpo/logit": 0.0989939272403717,
|
|
"ronpo/residual": 0.10105105489492416,
|
|
"ronpo/residual_abs": 0.15274189412593842,
|
|
"ronpo/target": -0.0020571290515363216,
|
|
"ronpo/target_abs": 0.0786915272474289,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2540569007396698,
|
|
"drift/rejected_abs": 0.10452580451965332,
|
|
"epoch": 0.25744934445768775,
|
|
"grad_norm": 2.15625,
|
|
"learning_rate": 3.0820486028908077e-09,
|
|
"logits/chosen": -1.997826337814331,
|
|
"logits/rejected": -2.0715103149414062,
|
|
"logps/chosen": -0.2701925039291382,
|
|
"logps/rejected": -0.26402124762535095,
|
|
"loss": 0.43526023626327515,
|
|
"loss/core": 0.41667813062667847,
|
|
"loss/preference_sft": 0.2734178304672241,
|
|
"loss/reference_anchor": 0.3443005383014679,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.5394034385681152,
|
|
"rewards/margins": 1.49575674533844,
|
|
"rewards/rejected": 1.0436466932296753,
|
|
"ronpo/logit": 0.14957566559314728,
|
|
"ronpo/residual": 0.16587159037590027,
|
|
"ronpo/residual_abs": 0.2100309580564499,
|
|
"ronpo/target": -0.01629592850804329,
|
|
"ronpo/target_abs": 0.07158873975276947,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26953545212745667,
|
|
"drift/rejected_abs": 0.11704172194004059,
|
|
"epoch": 0.25903853794199444,
|
|
"grad_norm": 0.8203125,
|
|
"learning_rate": 2.7557280748055966e-09,
|
|
"logits/chosen": -1.7852869033813477,
|
|
"logits/rejected": -1.8557426929473877,
|
|
"logps/chosen": -0.2673250436782837,
|
|
"logps/rejected": -0.2700131833553314,
|
|
"loss": 0.40478211641311646,
|
|
"loss/core": 0.385967880487442,
|
|
"loss/preference_sft": 0.27039602398872375,
|
|
"loss/reference_anchor": 0.3492446839809418,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.6953542232513428,
|
|
"rewards/margins": 1.5257197618484497,
|
|
"rewards/rejected": 1.1696348190307617,
|
|
"ronpo/logit": 0.15257197618484497,
|
|
"ronpo/residual": 0.15629687905311584,
|
|
"ronpo/residual_abs": 0.19490864872932434,
|
|
"ronpo/target": -0.003724911715835333,
|
|
"ronpo/target_abs": 0.07523742318153381,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2987804412841797,
|
|
"drift/rejected_abs": 0.1348761022090912,
|
|
"epoch": 0.26062773142630113,
|
|
"grad_norm": 205.0,
|
|
"learning_rate": 2.4471741852423233e-09,
|
|
"logits/chosen": -1.8530000448226929,
|
|
"logits/rejected": -1.8922631740570068,
|
|
"logps/chosen": -0.2706072926521301,
|
|
"logps/rejected": -0.26072853803634644,
|
|
"loss": 0.3187785744667053,
|
|
"loss/core": 0.30332642793655396,
|
|
"loss/preference_sft": 0.2643357813358307,
|
|
"loss/reference_anchor": 0.2826094925403595,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.9860172271728516,
|
|
"rewards/margins": 1.6376988887786865,
|
|
"rewards/rejected": 1.348318338394165,
|
|
"ronpo/logit": 0.1637699007987976,
|
|
"ronpo/residual": 0.16065004467964172,
|
|
"ronpo/residual_abs": 0.21420709788799286,
|
|
"ronpo/target": 0.0031198528595268726,
|
|
"ronpo/target_abs": 0.08196929842233658,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22662219405174255,
|
|
"drift/rejected_abs": 0.09158599376678467,
|
|
"epoch": 0.2622169249106079,
|
|
"grad_norm": 9.5,
|
|
"learning_rate": 2.1565029686999305e-09,
|
|
"logits/chosen": -1.7125442028045654,
|
|
"logits/rejected": -1.758292555809021,
|
|
"logps/chosen": -0.2971535921096802,
|
|
"logps/rejected": -0.30583325028419495,
|
|
"loss": 0.2096571922302246,
|
|
"loss/core": 0.19694137573242188,
|
|
"loss/preference_sft": 0.30101680755615234,
|
|
"loss/reference_anchor": 0.2242145836353302,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.266221761703491,
|
|
"rewards/margins": 1.3520539999008179,
|
|
"rewards/rejected": 0.9141677618026733,
|
|
"ronpo/logit": 0.13520540297031403,
|
|
"ronpo/residual": 0.11602624505758286,
|
|
"ronpo/residual_abs": 0.17035934329032898,
|
|
"ronpo/target": 0.019179154187440872,
|
|
"ronpo/target_abs": 0.07527647167444229,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15426619350910187,
|
|
"drift/rejected_abs": 0.07910831272602081,
|
|
"epoch": 0.2638061183949146,
|
|
"grad_norm": 2.59375,
|
|
"learning_rate": 1.8838237347353845e-09,
|
|
"logits/chosen": -1.9536901712417603,
|
|
"logits/rejected": -1.982287049293518,
|
|
"logps/chosen": -0.30930250883102417,
|
|
"logps/rejected": -0.31085312366485596,
|
|
"loss": 0.08624901622533798,
|
|
"loss/core": 0.07954810559749603,
|
|
"loss/preference_sft": 0.31143519282341003,
|
|
"loss/reference_anchor": 0.10287465900182724,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.5410267114639282,
|
|
"rewards/margins": 0.749943733215332,
|
|
"rewards/rejected": 0.791083037853241,
|
|
"ronpo/logit": 0.07499437034130096,
|
|
"ronpo/residual": 0.05749637633562088,
|
|
"ronpo/residual_abs": 0.13403594493865967,
|
|
"ronpo/target": 0.01749800331890583,
|
|
"ronpo/target_abs": 0.07980482280254364,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34234294295310974,
|
|
"drift/rejected_abs": 0.15316882729530334,
|
|
"epoch": 0.2653953118792213,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 1.6292390268568101e-09,
|
|
"logits/chosen": -1.5117677450180054,
|
|
"logits/rejected": -1.5660102367401123,
|
|
"logps/chosen": -0.27989453077316284,
|
|
"logps/rejected": -0.30721816420555115,
|
|
"loss": 0.3620285987854004,
|
|
"loss/core": 0.34146106243133545,
|
|
"loss/preference_sft": 0.30074799060821533,
|
|
"loss/reference_anchor": 0.3812769651412964,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4228203296661377,
|
|
"rewards/margins": 1.891760230064392,
|
|
"rewards/rejected": 1.5310604572296143,
|
|
"ronpo/logit": 0.1891760379076004,
|
|
"ronpo/residual": 0.17069289088249207,
|
|
"ronpo/residual_abs": 0.2289830446243286,
|
|
"ronpo/target": 0.01848311722278595,
|
|
"ronpo/target_abs": 0.07700731605291367,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3780483305454254,
|
|
"drift/rejected_abs": 0.12148091942071915,
|
|
"epoch": 0.266984505363528,
|
|
"grad_norm": 7.28125,
|
|
"learning_rate": 1.392844583961078e-09,
|
|
"logits/chosen": -1.8261111974716187,
|
|
"logits/rejected": -1.9145349264144897,
|
|
"logps/chosen": -0.2696583867073059,
|
|
"logps/rejected": -0.26150408387184143,
|
|
"loss": 1.2962982654571533,
|
|
"loss/core": 1.2566306591033936,
|
|
"loss/preference_sft": 0.2642626464366913,
|
|
"loss/reference_anchor": 0.7669227123260498,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.7794418334960938,
|
|
"rewards/margins": 2.5657799243927,
|
|
"rewards/rejected": 1.213661789894104,
|
|
"ronpo/logit": 0.2565780282020569,
|
|
"ronpo/residual": 0.2539408504962921,
|
|
"ronpo/residual_abs": 0.3109070360660553,
|
|
"ronpo/target": 0.002637143013998866,
|
|
"ronpo/target_abs": 0.0665135607123375,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1283138245344162,
|
|
"drift/rejected_abs": 0.066379114985466,
|
|
"epoch": 0.2685736988478347,
|
|
"grad_norm": 17.125,
|
|
"learning_rate": 1.1747293043303518e-09,
|
|
"logits/chosen": -1.858109474182129,
|
|
"logits/rejected": -1.7954213619232178,
|
|
"logps/chosen": -0.28292885422706604,
|
|
"logps/rejected": -0.2912057936191559,
|
|
"loss": 0.03620520606637001,
|
|
"loss/core": 0.032505668699741364,
|
|
"loss/preference_sft": 0.28797465562820435,
|
|
"loss/reference_anchor": 0.04519306868314743,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.2819868326187134,
|
|
"rewards/margins": 0.6205378174781799,
|
|
"rewards/rejected": 0.6614489555358887,
|
|
"ronpo/logit": 0.062053781002759933,
|
|
"ronpo/residual": 0.05858888477087021,
|
|
"ronpo/residual_abs": 0.11494974046945572,
|
|
"ronpo/target": 0.0034649022854864597,
|
|
"ronpo/target_abs": 0.08666227012872696,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15765869617462158,
|
|
"drift/rejected_abs": 0.1053333505988121,
|
|
"epoch": 0.2701628923321414,
|
|
"grad_norm": 5.5625,
|
|
"learning_rate": 9.749752122010346e-10,
|
|
"logits/chosen": -1.743290901184082,
|
|
"logits/rejected": -1.6410249471664429,
|
|
"logps/chosen": -0.3139035701751709,
|
|
"logps/rejected": -0.31750422716140747,
|
|
"loss": 0.059708237648010254,
|
|
"loss/core": 0.047553762793540955,
|
|
"loss/preference_sft": 0.3167984187602997,
|
|
"loss/reference_anchor": 0.2114095687866211,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5753216743469238,
|
|
"rewards/margins": 0.5241764783859253,
|
|
"rewards/rejected": 1.0511451959609985,
|
|
"ronpo/logit": 0.05241764709353447,
|
|
"ronpo/residual": 0.039160002022981644,
|
|
"ronpo/residual_abs": 0.13094462454319,
|
|
"ronpo/target": 0.013257652521133423,
|
|
"ronpo/target_abs": 0.08758213371038437,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20687925815582275,
|
|
"drift/rejected_abs": 0.10394798219203949,
|
|
"epoch": 0.27175208581644816,
|
|
"grad_norm": 9.0625,
|
|
"learning_rate": 7.936574269178375e-10,
|
|
"logits/chosen": -1.7965866327285767,
|
|
"logits/rejected": -1.8572895526885986,
|
|
"logps/chosen": -0.2916758358478546,
|
|
"logps/rejected": -0.2980891466140747,
|
|
"loss": 0.13103294372558594,
|
|
"loss/core": 0.12218041718006134,
|
|
"loss/preference_sft": 0.30287644267082214,
|
|
"loss/reference_anchor": 0.14676304161548615,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.067711353302002,
|
|
"rewards/margins": 1.0313072204589844,
|
|
"rewards/rejected": 1.0364038944244385,
|
|
"ronpo/logit": 0.10313073545694351,
|
|
"ronpo/residual": 0.12047480046749115,
|
|
"ronpo/residual_abs": 0.19301636517047882,
|
|
"ronpo/target": -0.017344076186418533,
|
|
"ronpo/target_abs": 0.08581249415874481,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23877234756946564,
|
|
"drift/rejected_abs": 0.16189390420913696,
|
|
"epoch": 0.27334127930075486,
|
|
"grad_norm": 14.6875,
|
|
"learning_rate": 6.308441346844384e-10,
|
|
"logits/chosen": -1.7404086589813232,
|
|
"logits/rejected": -1.6830250024795532,
|
|
"logps/chosen": -0.2857043445110321,
|
|
"logps/rejected": -0.2822682857513428,
|
|
"loss": 0.7747726440429688,
|
|
"loss/core": 0.7492650747299194,
|
|
"loss/preference_sft": 0.2928890585899353,
|
|
"loss/reference_anchor": 0.48086386919021606,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.387723445892334,
|
|
"rewards/margins": 0.7728492617607117,
|
|
"rewards/rejected": 1.614874243736267,
|
|
"ronpo/logit": 0.07728491723537445,
|
|
"ronpo/residual": 0.09040971845388412,
|
|
"ronpo/residual_abs": 0.27997887134552,
|
|
"ronpo/target": -0.013124806806445122,
|
|
"ronpo/target_abs": 0.07721184939146042,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35751527547836304,
|
|
"drift/rejected_abs": 0.18375881016254425,
|
|
"epoch": 0.2749304727850616,
|
|
"grad_norm": 2.03125,
|
|
"learning_rate": 4.865965629214819e-10,
|
|
"logits/chosen": -1.853266716003418,
|
|
"logits/rejected": -1.793908715248108,
|
|
"logps/chosen": -0.2659180164337158,
|
|
"logps/rejected": -0.2806767523288727,
|
|
"loss": 0.9706258773803711,
|
|
"loss/core": 0.9136497378349304,
|
|
"loss/preference_sft": 0.27038851380348206,
|
|
"loss/reference_anchor": 1.1124818325042725,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.574227809906006,
|
|
"rewards/margins": 1.7369954586029053,
|
|
"rewards/rejected": 1.8372325897216797,
|
|
"ronpo/logit": 0.1736995428800583,
|
|
"ronpo/residual": 0.18940187990665436,
|
|
"ronpo/residual_abs": 0.29147475957870483,
|
|
"ronpo/target": -0.015702344477176666,
|
|
"ronpo/target_abs": 0.0712803453207016,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2339034080505371,
|
|
"drift/rejected_abs": 0.11022868007421494,
|
|
"epoch": 0.2765196662693683,
|
|
"grad_norm": 2.9375,
|
|
"learning_rate": 3.609689572414143e-10,
|
|
"logits/chosen": -1.794655203819275,
|
|
"logits/rejected": -1.8671070337295532,
|
|
"logps/chosen": -0.27023810148239136,
|
|
"logps/rejected": -0.27533429861068726,
|
|
"loss": 0.15046367049217224,
|
|
"loss/core": 0.14148980379104614,
|
|
"loss/preference_sft": 0.26623964309692383,
|
|
"loss/reference_anchor": 0.15285304188728333,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.3387210369110107,
|
|
"rewards/margins": 1.2503958940505981,
|
|
"rewards/rejected": 1.0883252620697021,
|
|
"ronpo/logit": 0.12503957748413086,
|
|
"ronpo/residual": 0.12151853740215302,
|
|
"ronpo/residual_abs": 0.17508959770202637,
|
|
"ronpo/target": 0.0035210344940423965,
|
|
"ronpo/target_abs": 0.08289609849452972,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2378336638212204,
|
|
"drift/rejected_abs": 0.08225777000188828,
|
|
"epoch": 0.278108859753675,
|
|
"grad_norm": 2.140625,
|
|
"learning_rate": 2.5400856104894063e-10,
|
|
"logits/chosen": -1.9827836751937866,
|
|
"logits/rejected": -2.0595450401306152,
|
|
"logps/chosen": -0.2802538275718689,
|
|
"logps/rejected": -0.2741009593009949,
|
|
"loss": 0.7382990717887878,
|
|
"loss/core": 0.7156741619110107,
|
|
"loss/preference_sft": 0.2707212567329407,
|
|
"loss/reference_anchor": 0.42542606592178345,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3760526180267334,
|
|
"rewards/margins": 1.553762435913086,
|
|
"rewards/rejected": 0.8222904205322266,
|
|
"ronpo/logit": 0.15537627041339874,
|
|
"ronpo/residual": 0.15069438517093658,
|
|
"ronpo/residual_abs": 0.20385237038135529,
|
|
"ronpo/target": 0.004681874066591263,
|
|
"ronpo/target_abs": 0.06999818235635757,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2282506674528122,
|
|
"drift/rejected_abs": 0.14202216267585754,
|
|
"epoch": 0.27969805323798175,
|
|
"grad_norm": 11.5625,
|
|
"learning_rate": 1.6575559777469716e-10,
|
|
"logits/chosen": -1.6084506511688232,
|
|
"logits/rejected": -1.652144432067871,
|
|
"logps/chosen": -0.28672975301742554,
|
|
"logps/rejected": -0.2819688022136688,
|
|
"loss": 0.3919622004032135,
|
|
"loss/core": 0.37796106934547424,
|
|
"loss/preference_sft": 0.28091609477996826,
|
|
"loss/reference_anchor": 0.2519303262233734,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2814536094665527,
|
|
"rewards/margins": 0.8872292637825012,
|
|
"rewards/rejected": 1.3942242860794067,
|
|
"ronpo/logit": 0.08872292190790176,
|
|
"ronpo/residual": 0.10213188827037811,
|
|
"ronpo/residual_abs": 0.2572005093097687,
|
|
"ronpo/target": -0.013408951461315155,
|
|
"ronpo/target_abs": 0.07671914249658585,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2979094684123993,
|
|
"drift/rejected_abs": 0.12212695926427841,
|
|
"epoch": 0.28128724672228844,
|
|
"grad_norm": 0.765625,
|
|
"learning_rate": 9.624325574890125e-11,
|
|
"logits/chosen": -1.799660325050354,
|
|
"logits/rejected": -1.7860395908355713,
|
|
"logps/chosen": -0.2907426357269287,
|
|
"logps/rejected": -0.29116326570510864,
|
|
"loss": 0.39276254177093506,
|
|
"loss/core": 0.3697052299976349,
|
|
"loss/preference_sft": 0.2975885272026062,
|
|
"loss/reference_anchor": 0.4313863217830658,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.97800874710083,
|
|
"rewards/margins": 1.7583093643188477,
|
|
"rewards/rejected": 1.2196992635726929,
|
|
"ronpo/logit": 0.1758309304714203,
|
|
"ronpo/residual": 0.20061039924621582,
|
|
"ronpo/residual_abs": 0.22602948546409607,
|
|
"ronpo/target": -0.024779465049505234,
|
|
"ronpo/target_abs": 0.07615292817354202,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13840210437774658,
|
|
"drift/rejected_abs": 0.10734204202890396,
|
|
"epoch": 0.28287644020659514,
|
|
"grad_norm": 8.3125,
|
|
"learning_rate": 4.5497675720540527e-11,
|
|
"logits/chosen": -1.923963189125061,
|
|
"logits/rejected": -1.902496099472046,
|
|
"logps/chosen": -0.28208020329475403,
|
|
"logps/rejected": -0.2937772274017334,
|
|
"loss": 0.12399367988109589,
|
|
"loss/core": 0.11801999807357788,
|
|
"loss/preference_sft": 0.2783675789833069,
|
|
"loss/reference_anchor": 0.0916367918252945,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.3840210437774658,
|
|
"rewards/margins": 0.3155991733074188,
|
|
"rewards/rejected": 1.0684218406677246,
|
|
"ronpo/logit": 0.03155992180109024,
|
|
"ronpo/residual": 0.055709730833768845,
|
|
"ronpo/residual_abs": 0.15228456258773804,
|
|
"ronpo/target": -0.024149809032678604,
|
|
"ronpo/target_abs": 0.08036401867866516,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1609112173318863,
|
|
"drift/rejected_abs": 0.08355027437210083,
|
|
"epoch": 0.2844656336909019,
|
|
"grad_norm": 12.6875,
|
|
"learning_rate": 1.3537941026914302e-11,
|
|
"logits/chosen": -1.9804353713989258,
|
|
"logits/rejected": -1.975773572921753,
|
|
"logps/chosen": -0.30720216035842896,
|
|
"logps/rejected": -0.30572324991226196,
|
|
"loss": 0.1231997162103653,
|
|
"loss/core": 0.11622112989425659,
|
|
"loss/preference_sft": 0.3030904531478882,
|
|
"loss/reference_anchor": 0.10926266759634018,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.608919382095337,
|
|
"rewards/margins": 0.7737261652946472,
|
|
"rewards/rejected": 0.8351932764053345,
|
|
"ronpo/logit": 0.07737261801958084,
|
|
"ronpo/residual": 0.07732744514942169,
|
|
"ronpo/residual_abs": 0.12864170968532562,
|
|
"ronpo/target": 4.518199784797616e-05,
|
|
"ronpo/target_abs": 0.059388406574726105,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23922479152679443,
|
|
"drift/rejected_abs": 0.13327732682228088,
|
|
"epoch": 0.2860548271752086,
|
|
"grad_norm": 15.75,
|
|
"learning_rate": 3.7607041719622813e-13,
|
|
"logits/chosen": -1.8160746097564697,
|
|
"logits/rejected": -1.8043482303619385,
|
|
"logps/chosen": -0.2887292504310608,
|
|
"logps/rejected": -0.28216832876205444,
|
|
"loss": 0.0885263979434967,
|
|
"loss/core": 0.0769236832857132,
|
|
"loss/preference_sft": 0.285482257604599,
|
|
"loss/reference_anchor": 0.20350602269172668,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.3910775184631348,
|
|
"rewards/margins": 1.061415433883667,
|
|
"rewards/rejected": 1.3296619653701782,
|
|
"ronpo/logit": 0.10614154487848282,
|
|
"ronpo/residual": 0.1308242827653885,
|
|
"ronpo/residual_abs": 0.15298447012901306,
|
|
"ronpo/target": -0.024682745337486267,
|
|
"ronpo/target_abs": 0.07124222069978714,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.2860548271752086,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.48311558212671013,
|
|
"train_runtime": 8488.8129,
|
|
"train_samples_per_second": 1.696,
|
|
"train_steps_per_second": 0.106
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|