17d80b4a2f
Model: promotion/qwen3-8b-aaai27-flagship-ronpo-full-expect-s43 Source: Original Platform
4904 lines
186 KiB
JSON
4904 lines
186 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2860548271752086,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.19945812225341797,
|
|
"drift/rejected_abs": 0.08849310874938965,
|
|
"epoch": 0.0015891934843067143,
|
|
"grad_norm": 4.5625,
|
|
"learning_rate": 4.444444444444444e-09,
|
|
"logits/chosen": -1.5555963516235352,
|
|
"logits/rejected": -1.5219662189483643,
|
|
"logps/chosen": -0.3077252507209778,
|
|
"logps/rejected": -0.29369106888771057,
|
|
"loss": 0.131231427192688,
|
|
"loss/core": 0.12094147503376007,
|
|
"loss/preference_sft": 0.3084653615951538,
|
|
"loss/reference_anchor": 0.17495320737361908,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9923360347747803,
|
|
"rewards/margins": 1.110528826713562,
|
|
"rewards/rejected": 0.8818074464797974,
|
|
"ronpo/logit": 0.11105290800333023,
|
|
"ronpo/residual": 0.1199965700507164,
|
|
"ronpo/residual_abs": 0.1629294753074646,
|
|
"ronpo/target": -0.008943679742515087,
|
|
"ronpo/target_abs": 0.07324962317943573,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2019089162349701,
|
|
"drift/rejected_abs": 0.095692940056324,
|
|
"epoch": 0.0031783869686134287,
|
|
"grad_norm": 76.5,
|
|
"learning_rate": 1e-08,
|
|
"logits/chosen": -1.7638843059539795,
|
|
"logits/rejected": -1.8241863250732422,
|
|
"logps/chosen": -0.30898669362068176,
|
|
"logps/rejected": -0.2883630692958832,
|
|
"loss": 0.1183585375547409,
|
|
"loss/core": 0.1104976087808609,
|
|
"loss/preference_sft": 0.29389849305152893,
|
|
"loss/reference_anchor": 0.12782886624336243,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0182459354400635,
|
|
"rewards/margins": 1.062389850616455,
|
|
"rewards/rejected": 0.9558561444282532,
|
|
"ronpo/logit": 0.10623898357152939,
|
|
"ronpo/residual": 0.11770063638687134,
|
|
"ronpo/residual_abs": 0.1581176221370697,
|
|
"ronpo/target": -0.011461647227406502,
|
|
"ronpo/target_abs": 0.07125519216060638,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36563432216644287,
|
|
"drift/rejected_abs": 0.12579265236854553,
|
|
"epoch": 0.004767580452920143,
|
|
"grad_norm": 0.640625,
|
|
"learning_rate": 1.5555555555555554e-08,
|
|
"logits/chosen": -1.926892876625061,
|
|
"logits/rejected": -1.9229015111923218,
|
|
"logps/chosen": -0.27894189953804016,
|
|
"logps/rejected": -0.29260116815567017,
|
|
"loss": 0.8448435068130493,
|
|
"loss/core": 0.8088053464889526,
|
|
"loss/preference_sft": 0.2909303307533264,
|
|
"loss/reference_anchor": 0.6916698813438416,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.655860185623169,
|
|
"rewards/margins": 2.3997905254364014,
|
|
"rewards/rejected": 1.2560697793960571,
|
|
"ronpo/logit": 0.2399790734052658,
|
|
"ronpo/residual": 0.256378710269928,
|
|
"ronpo/residual_abs": 0.3004401624202728,
|
|
"ronpo/target": -0.016399653628468513,
|
|
"ronpo/target_abs": 0.0730489045381546,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26957765221595764,
|
|
"drift/rejected_abs": 0.06649773567914963,
|
|
"epoch": 0.006356773937226857,
|
|
"grad_norm": 0.79296875,
|
|
"learning_rate": 2.111111111111111e-08,
|
|
"logits/chosen": -1.8648866415023804,
|
|
"logits/rejected": -1.967498779296875,
|
|
"logps/chosen": -0.30245983600616455,
|
|
"logps/rejected": -0.2743304967880249,
|
|
"loss": 0.7822330594062805,
|
|
"loss/core": 0.7592211365699768,
|
|
"loss/preference_sft": 0.27905479073524475,
|
|
"loss/reference_anchor": 0.43233203887939453,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6956257820129395,
|
|
"rewards/margins": 2.0322680473327637,
|
|
"rewards/rejected": 0.6633577942848206,
|
|
"ronpo/logit": 0.20322680473327637,
|
|
"ronpo/residual": 0.2019917517900467,
|
|
"ronpo/residual_abs": 0.2696748375892639,
|
|
"ronpo/target": 0.0012350535253062844,
|
|
"ronpo/target_abs": 0.0822242870926857,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29503053426742554,
|
|
"drift/rejected_abs": 0.09023113548755646,
|
|
"epoch": 0.007945967421533572,
|
|
"grad_norm": 1.828125,
|
|
"learning_rate": 2.6666666666666664e-08,
|
|
"logits/chosen": -1.8297017812728882,
|
|
"logits/rejected": -1.94186532497406,
|
|
"logps/chosen": -0.27752918004989624,
|
|
"logps/rejected": -0.29865169525146484,
|
|
"loss": 0.48796534538269043,
|
|
"loss/core": 0.47148722410202026,
|
|
"loss/preference_sft": 0.29656797647476196,
|
|
"loss/reference_anchor": 0.2999069392681122,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9502077102661133,
|
|
"rewards/margins": 2.120311737060547,
|
|
"rewards/rejected": 0.8298957943916321,
|
|
"ronpo/logit": 0.21203120052814484,
|
|
"ronpo/residual": 0.21126720309257507,
|
|
"ronpo/residual_abs": 0.2526918649673462,
|
|
"ronpo/target": 0.0007639724644832313,
|
|
"ronpo/target_abs": 0.0715799331665039,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26316240429878235,
|
|
"drift/rejected_abs": 0.1573808193206787,
|
|
"epoch": 0.009535160905840286,
|
|
"grad_norm": 84.0,
|
|
"learning_rate": 3.2222222222222224e-08,
|
|
"logits/chosen": -1.9381355047225952,
|
|
"logits/rejected": -1.8474152088165283,
|
|
"logps/chosen": -0.2904534339904785,
|
|
"logps/rejected": -0.3180440664291382,
|
|
"loss": 0.5810376405715942,
|
|
"loss/core": 0.5595085024833679,
|
|
"loss/preference_sft": 0.30132001638412476,
|
|
"loss/reference_anchor": 0.4004506468772888,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6302835941314697,
|
|
"rewards/margins": 1.1446410417556763,
|
|
"rewards/rejected": 1.485642671585083,
|
|
"ronpo/logit": 0.11446411907672882,
|
|
"ronpo/residual": 0.12280688434839249,
|
|
"ronpo/residual_abs": 0.26302680373191833,
|
|
"ronpo/target": -0.008342771790921688,
|
|
"ronpo/target_abs": 0.07446413487195969,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25724148750305176,
|
|
"drift/rejected_abs": 0.1849718689918518,
|
|
"epoch": 0.011124354390147001,
|
|
"grad_norm": 6.40625,
|
|
"learning_rate": 3.7777777777777774e-08,
|
|
"logits/chosen": -1.877733588218689,
|
|
"logits/rejected": -1.8011366128921509,
|
|
"logps/chosen": -0.2792748808860779,
|
|
"logps/rejected": -0.2892039120197296,
|
|
"loss": 0.25435659289360046,
|
|
"loss/core": 0.23435644805431366,
|
|
"loss/preference_sft": 0.28339701890945435,
|
|
"loss/reference_anchor": 0.37166398763656616,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.5721800327301025,
|
|
"rewards/margins": 0.7232538461685181,
|
|
"rewards/rejected": 1.8489259481430054,
|
|
"ronpo/logit": 0.07232538610696793,
|
|
"ronpo/residual": 0.07871496677398682,
|
|
"ronpo/residual_abs": 0.19634081423282623,
|
|
"ronpo/target": -0.0063895778730511665,
|
|
"ronpo/target_abs": 0.06378191709518433,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2774297595024109,
|
|
"drift/rejected_abs": 0.10878102481365204,
|
|
"epoch": 0.012713547874453715,
|
|
"grad_norm": 3.296875,
|
|
"learning_rate": 4.333333333333333e-08,
|
|
"logits/chosen": -1.8935270309448242,
|
|
"logits/rejected": -1.898699164390564,
|
|
"logps/chosen": -0.25949639081954956,
|
|
"logps/rejected": -0.26641005277633667,
|
|
"loss": 0.20093131065368652,
|
|
"loss/core": 0.18912899494171143,
|
|
"loss/preference_sft": 0.2602488696575165,
|
|
"loss/reference_anchor": 0.2100217044353485,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.7733113765716553,
|
|
"rewards/margins": 1.7004846334457397,
|
|
"rewards/rejected": 1.0728265047073364,
|
|
"ronpo/logit": 0.17004846036434174,
|
|
"ronpo/residual": 0.1733754575252533,
|
|
"ronpo/residual_abs": 0.21049460768699646,
|
|
"ronpo/target": -0.003326986450701952,
|
|
"ronpo/target_abs": 0.07367634028196335,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2615305185317993,
|
|
"drift/rejected_abs": 0.0843420997262001,
|
|
"epoch": 0.014302741358760428,
|
|
"grad_norm": 1.3515625,
|
|
"learning_rate": 4.888888888888889e-08,
|
|
"logits/chosen": -1.9090964794158936,
|
|
"logits/rejected": -2.0183889865875244,
|
|
"logps/chosen": -0.28550487756729126,
|
|
"logps/rejected": -0.26880985498428345,
|
|
"loss": 0.8131304979324341,
|
|
"loss/core": 0.7812657356262207,
|
|
"loss/preference_sft": 0.28103402256965637,
|
|
"loss/reference_anchor": 0.6091889142990112,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6137125492095947,
|
|
"rewards/margins": 1.7702915668487549,
|
|
"rewards/rejected": 0.8434208631515503,
|
|
"ronpo/logit": 0.17702916264533997,
|
|
"ronpo/residual": 0.17782527208328247,
|
|
"ronpo/residual_abs": 0.23299217224121094,
|
|
"ronpo/target": -0.0007961191004142165,
|
|
"ronpo/target_abs": 0.07125305384397507,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2804063558578491,
|
|
"drift/rejected_abs": 0.14204956591129303,
|
|
"epoch": 0.015891934843067144,
|
|
"grad_norm": 1.703125,
|
|
"learning_rate": 5.444444444444444e-08,
|
|
"logits/chosen": -1.752661943435669,
|
|
"logits/rejected": -1.7710239887237549,
|
|
"logps/chosen": -0.2862663269042969,
|
|
"logps/rejected": -0.2904655933380127,
|
|
"loss": 0.3519284725189209,
|
|
"loss/core": 0.31644538044929504,
|
|
"loss/preference_sft": 0.2913067936897278,
|
|
"loss/reference_anchor": 0.6805309653282166,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.80353045463562,
|
|
"rewards/margins": 1.38631272315979,
|
|
"rewards/rejected": 1.41721773147583,
|
|
"ronpo/logit": 0.13863126933574677,
|
|
"ronpo/residual": 0.14011706411838531,
|
|
"ronpo/residual_abs": 0.1854199469089508,
|
|
"ronpo/target": -0.0014857802307233214,
|
|
"ronpo/target_abs": 0.06841117143630981,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.298065185546875,
|
|
"drift/rejected_abs": 0.15730881690979004,
|
|
"epoch": 0.01748112832737386,
|
|
"grad_norm": 56.0,
|
|
"learning_rate": 6e-08,
|
|
"logits/chosen": -1.8553667068481445,
|
|
"logits/rejected": -1.7833325862884521,
|
|
"logps/chosen": -0.28672200441360474,
|
|
"logps/rejected": -0.28446197509765625,
|
|
"loss": 0.23183119297027588,
|
|
"loss/core": 0.21043534576892853,
|
|
"loss/preference_sft": 0.28606119751930237,
|
|
"loss/reference_anchor": 0.3993109464645386,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.98065185546875,
|
|
"rewards/margins": 1.4075638055801392,
|
|
"rewards/rejected": 1.57308828830719,
|
|
"ronpo/logit": 0.14075636863708496,
|
|
"ronpo/residual": 0.1200438141822815,
|
|
"ronpo/residual_abs": 0.1800813376903534,
|
|
"ronpo/target": 0.020712558180093765,
|
|
"ronpo/target_abs": 0.08266838639974594,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18171152472496033,
|
|
"drift/rejected_abs": 0.06555067002773285,
|
|
"epoch": 0.01907032181168057,
|
|
"grad_norm": 81.5,
|
|
"learning_rate": 6.555555555555555e-08,
|
|
"logits/chosen": -1.7977211475372314,
|
|
"logits/rejected": -1.767542839050293,
|
|
"logps/chosen": -0.29876524209976196,
|
|
"logps/rejected": -0.3064151406288147,
|
|
"loss": 0.10823305696249008,
|
|
"loss/core": 0.10275810956954956,
|
|
"loss/preference_sft": 0.3003951907157898,
|
|
"loss/reference_anchor": 0.07945962250232697,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8157427310943604,
|
|
"rewards/margins": 1.168804407119751,
|
|
"rewards/rejected": 0.6469382047653198,
|
|
"ronpo/logit": 0.11688046157360077,
|
|
"ronpo/residual": 0.10604210197925568,
|
|
"ronpo/residual_abs": 0.1665903478860855,
|
|
"ronpo/target": 0.010838357731699944,
|
|
"ronpo/target_abs": 0.0728057250380516,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2507414221763611,
|
|
"drift/rejected_abs": 0.07751794159412384,
|
|
"epoch": 0.020659515295987287,
|
|
"grad_norm": 19.625,
|
|
"learning_rate": 7.111111111111111e-08,
|
|
"logits/chosen": -1.930026650428772,
|
|
"logits/rejected": -1.948189377784729,
|
|
"logps/chosen": -0.28983187675476074,
|
|
"logps/rejected": -0.28243488073349,
|
|
"loss": 0.34065335988998413,
|
|
"loss/core": 0.32895928621292114,
|
|
"loss/preference_sft": 0.2844352424144745,
|
|
"loss/reference_anchor": 0.20543844997882843,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5072410106658936,
|
|
"rewards/margins": 1.733149766921997,
|
|
"rewards/rejected": 0.7740914225578308,
|
|
"ronpo/logit": 0.17331495881080627,
|
|
"ronpo/residual": 0.15960010886192322,
|
|
"ronpo/residual_abs": 0.21261218190193176,
|
|
"ronpo/target": 0.013714875094592571,
|
|
"ronpo/target_abs": 0.07702931016683578,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2543380856513977,
|
|
"drift/rejected_abs": 0.10286681354045868,
|
|
"epoch": 0.022248708780294002,
|
|
"grad_norm": 6.0,
|
|
"learning_rate": 7.666666666666666e-08,
|
|
"logits/chosen": -1.7886531352996826,
|
|
"logits/rejected": -1.9062068462371826,
|
|
"logps/chosen": -0.2821660041809082,
|
|
"logps/rejected": -0.2808881998062134,
|
|
"loss": 0.3984472155570984,
|
|
"loss/core": 0.38331031799316406,
|
|
"loss/preference_sft": 0.2853568494319916,
|
|
"loss/reference_anchor": 0.2742030918598175,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5432658195495605,
|
|
"rewards/margins": 1.5179535150527954,
|
|
"rewards/rejected": 1.0253121852874756,
|
|
"ronpo/logit": 0.15179535746574402,
|
|
"ronpo/residual": 0.14931924641132355,
|
|
"ronpo/residual_abs": 0.20238187909126282,
|
|
"ronpo/target": 0.0024761161766946316,
|
|
"ronpo/target_abs": 0.08015619218349457,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24210648238658905,
|
|
"drift/rejected_abs": 0.10418188571929932,
|
|
"epoch": 0.023837902264600714,
|
|
"grad_norm": 19.375,
|
|
"learning_rate": 8.222222222222221e-08,
|
|
"logits/chosen": -1.8596198558807373,
|
|
"logits/rejected": -1.8323400020599365,
|
|
"logps/chosen": -0.25651809573173523,
|
|
"logps/rejected": -0.2563713490962982,
|
|
"loss": 0.15341153740882874,
|
|
"loss/core": 0.14641332626342773,
|
|
"loss/preference_sft": 0.2539079785346985,
|
|
"loss/reference_anchor": 0.11457308381795883,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4205851554870605,
|
|
"rewards/margins": 1.379667043685913,
|
|
"rewards/rejected": 1.0409181118011475,
|
|
"ronpo/logit": 0.13796670734882355,
|
|
"ronpo/residual": 0.14936383068561554,
|
|
"ronpo/residual_abs": 0.1982424557209015,
|
|
"ronpo/target": -0.01139712892472744,
|
|
"ronpo/target_abs": 0.07977505028247833,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2583889365196228,
|
|
"drift/rejected_abs": 0.1300758719444275,
|
|
"epoch": 0.02542709574890743,
|
|
"grad_norm": 508.0,
|
|
"learning_rate": 8.777777777777778e-08,
|
|
"logits/chosen": -1.824342966079712,
|
|
"logits/rejected": -1.8747689723968506,
|
|
"logps/chosen": -0.2892422080039978,
|
|
"logps/rejected": -0.28287678956985474,
|
|
"loss": 0.6578446626663208,
|
|
"loss/core": 0.638018012046814,
|
|
"loss/preference_sft": 0.28955796360969543,
|
|
"loss/reference_anchor": 0.3675774931907654,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.583889961242676,
|
|
"rewards/margins": 1.3049442768096924,
|
|
"rewards/rejected": 1.2789456844329834,
|
|
"ronpo/logit": 0.13049443066120148,
|
|
"ronpo/residual": 0.12255732715129852,
|
|
"ronpo/residual_abs": 0.27486568689346313,
|
|
"ronpo/target": 0.007937087677419186,
|
|
"ronpo/target_abs": 0.08260422199964523,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24885332584381104,
|
|
"drift/rejected_abs": 0.11546947062015533,
|
|
"epoch": 0.027016289233214145,
|
|
"grad_norm": 14.25,
|
|
"learning_rate": 9.333333333333334e-08,
|
|
"logits/chosen": -1.8859790563583374,
|
|
"logits/rejected": -1.8343837261199951,
|
|
"logps/chosen": -0.32009655237197876,
|
|
"logps/rejected": -0.31681352853775024,
|
|
"loss": 0.40035802125930786,
|
|
"loss/core": 0.38102349638938904,
|
|
"loss/preference_sft": 0.3166871666908264,
|
|
"loss/reference_anchor": 0.35502058267593384,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.4881718158721924,
|
|
"rewards/margins": 1.3343416452407837,
|
|
"rewards/rejected": 1.1538300514221191,
|
|
"ronpo/logit": 0.13343417644500732,
|
|
"ronpo/residual": 0.1329519897699356,
|
|
"ronpo/residual_abs": 0.18393748998641968,
|
|
"ronpo/target": 0.00048218583106063306,
|
|
"ronpo/target_abs": 0.07522345334291458,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22070515155792236,
|
|
"drift/rejected_abs": 0.16914622485637665,
|
|
"epoch": 0.028605482717520857,
|
|
"grad_norm": 1.21875,
|
|
"learning_rate": 9.888888888888889e-08,
|
|
"logits/chosen": -1.731783151626587,
|
|
"logits/rejected": -1.822945237159729,
|
|
"logps/chosen": -0.2804669737815857,
|
|
"logps/rejected": -0.2766118347644806,
|
|
"loss": 0.7024124264717102,
|
|
"loss/core": 0.6772509813308716,
|
|
"loss/preference_sft": 0.2828295826911926,
|
|
"loss/reference_anchor": 0.4749454855918884,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2057957649230957,
|
|
"rewards/margins": 0.515635073184967,
|
|
"rewards/rejected": 1.6901605129241943,
|
|
"ronpo/logit": 0.051563508808612823,
|
|
"ronpo/residual": 0.048208534717559814,
|
|
"ronpo/residual_abs": 0.24764589965343475,
|
|
"ronpo/target": 0.003354982938617468,
|
|
"ronpo/target_abs": 0.07700905948877335,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34500211477279663,
|
|
"drift/rejected_abs": 0.1328619420528412,
|
|
"epoch": 0.030194676201827572,
|
|
"grad_norm": 21.0,
|
|
"learning_rate": 9.999398298646737e-08,
|
|
"logits/chosen": -1.738399863243103,
|
|
"logits/rejected": -1.7765659093856812,
|
|
"logps/chosen": -0.28980618715286255,
|
|
"logps/rejected": -0.31215035915374756,
|
|
"loss": 0.6481006741523743,
|
|
"loss/core": 0.6118947267532349,
|
|
"loss/preference_sft": 0.2964279055595398,
|
|
"loss/reference_anchor": 0.6944752335548401,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.4496827125549316,
|
|
"rewards/margins": 2.1238207817077637,
|
|
"rewards/rejected": 1.3258618116378784,
|
|
"ronpo/logit": 0.21238207817077637,
|
|
"ronpo/residual": 0.19465920329093933,
|
|
"ronpo/residual_abs": 0.24489042162895203,
|
|
"ronpo/target": 0.017722856253385544,
|
|
"ronpo/target_abs": 0.07983126491308212,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26026713848114014,
|
|
"drift/rejected_abs": 0.09202776104211807,
|
|
"epoch": 0.03178386968613429,
|
|
"grad_norm": 2.0,
|
|
"learning_rate": 9.996954135095479e-08,
|
|
"logits/chosen": -1.9308345317840576,
|
|
"logits/rejected": -1.9035898447036743,
|
|
"logps/chosen": -0.2800833582878113,
|
|
"logps/rejected": -0.2895572781562805,
|
|
"loss": 0.4711598753929138,
|
|
"loss/core": 0.451214462518692,
|
|
"loss/preference_sft": 0.28193235397338867,
|
|
"loss/reference_anchor": 0.37071505188941956,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.6015172004699707,
|
|
"rewards/margins": 1.6833524703979492,
|
|
"rewards/rejected": 0.9181646108627319,
|
|
"ronpo/logit": 0.16833525896072388,
|
|
"ronpo/residual": 0.1610206663608551,
|
|
"ronpo/residual_abs": 0.2178463488817215,
|
|
"ronpo/target": 0.0073145912028849125,
|
|
"ronpo/target_abs": 0.08214585483074188,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27864447236061096,
|
|
"drift/rejected_abs": 0.12608464062213898,
|
|
"epoch": 0.033373063170441,
|
|
"grad_norm": 9.75,
|
|
"learning_rate": 9.992630821454457e-08,
|
|
"logits/chosen": -2.042860984802246,
|
|
"logits/rejected": -1.9781955480575562,
|
|
"logps/chosen": -0.28923875093460083,
|
|
"logps/rejected": -0.2902441620826721,
|
|
"loss": 0.3296965956687927,
|
|
"loss/core": 0.3039691150188446,
|
|
"loss/preference_sft": 0.29211506247520447,
|
|
"loss/reference_anchor": 0.4853378236293793,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.786428928375244,
|
|
"rewards/margins": 1.5330383777618408,
|
|
"rewards/rejected": 1.2533905506134033,
|
|
"ronpo/logit": 0.1533038467168808,
|
|
"ronpo/residual": 0.14926132559776306,
|
|
"ronpo/residual_abs": 0.2045602798461914,
|
|
"ronpo/target": 0.004042509943246841,
|
|
"ronpo/target_abs": 0.07107819616794586,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18290773034095764,
|
|
"drift/rejected_abs": 0.10879548639059067,
|
|
"epoch": 0.03496225665474772,
|
|
"grad_norm": 3.71875,
|
|
"learning_rate": 9.986429983545125e-08,
|
|
"logits/chosen": -1.8636986017227173,
|
|
"logits/rejected": -1.8569608926773071,
|
|
"logps/chosen": -0.31769293546676636,
|
|
"logps/rejected": -0.318257600069046,
|
|
"loss": 0.056669317185878754,
|
|
"loss/core": 0.04444245249032974,
|
|
"loss/preference_sft": 0.31768161058425903,
|
|
"loss/reference_anchor": 0.21276922523975372,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8289499282836914,
|
|
"rewards/margins": 0.741577684879303,
|
|
"rewards/rejected": 1.0873721837997437,
|
|
"ronpo/logit": 0.07415778189897537,
|
|
"ronpo/residual": 0.08273956924676895,
|
|
"ronpo/residual_abs": 0.13138622045516968,
|
|
"ronpo/target": -0.008581787347793579,
|
|
"ronpo/target_abs": 0.07794088870286942,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27672436833381653,
|
|
"drift/rejected_abs": 0.2216198742389679,
|
|
"epoch": 0.03655145013905443,
|
|
"grad_norm": 2.65625,
|
|
"learning_rate": 9.978353953249021e-08,
|
|
"logits/chosen": -1.8198753595352173,
|
|
"logits/rejected": -1.8555314540863037,
|
|
"logps/chosen": -0.29371920228004456,
|
|
"logps/rejected": -0.28360024094581604,
|
|
"loss": 0.3811081051826477,
|
|
"loss/core": 0.3532530963420868,
|
|
"loss/preference_sft": 0.2940063774585724,
|
|
"loss/reference_anchor": 0.5276997089385986,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.7668750286102295,
|
|
"rewards/margins": 0.5547674894332886,
|
|
"rewards/rejected": 2.2121076583862305,
|
|
"ronpo/logit": 0.05547674745321274,
|
|
"ronpo/residual": 0.051227767020463943,
|
|
"ronpo/residual_abs": 0.20360875129699707,
|
|
"ronpo/target": 0.004248985089361668,
|
|
"ronpo/target_abs": 0.07594992220401764,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3134176731109619,
|
|
"drift/rejected_abs": 0.17082719504833221,
|
|
"epoch": 0.03814064362336114,
|
|
"grad_norm": 40.75,
|
|
"learning_rate": 9.968405767630856e-08,
|
|
"logits/chosen": -1.703507423400879,
|
|
"logits/rejected": -1.6853641271591187,
|
|
"logps/chosen": -0.2665267586708069,
|
|
"logps/rejected": -0.2603232264518738,
|
|
"loss": 0.5027941465377808,
|
|
"loss/core": 0.470782607793808,
|
|
"loss/preference_sft": 0.26472344994544983,
|
|
"loss/reference_anchor": 0.6137583255767822,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.134176731109619,
|
|
"rewards/margins": 1.4267246723175049,
|
|
"rewards/rejected": 1.7074520587921143,
|
|
"ronpo/logit": 0.14267246425151825,
|
|
"ronpo/residual": 0.12057846784591675,
|
|
"ronpo/residual_abs": 0.21640296280384064,
|
|
"ronpo/target": 0.02209400199353695,
|
|
"ronpo/target_abs": 0.08593425899744034,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2831875681877136,
|
|
"drift/rejected_abs": 0.1215093731880188,
|
|
"epoch": 0.03972983710766786,
|
|
"grad_norm": 7.09375,
|
|
"learning_rate": 9.956589167796391e-08,
|
|
"logits/chosen": -1.780076026916504,
|
|
"logits/rejected": -1.7246229648590088,
|
|
"logps/chosen": -0.29332637786865234,
|
|
"logps/rejected": -0.31054961681365967,
|
|
"loss": 0.4214431643486023,
|
|
"loss/core": 0.38739684224128723,
|
|
"loss/preference_sft": 0.30493617057800293,
|
|
"loss/reference_anchor": 0.6504331231117249,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8314144611358643,
|
|
"rewards/margins": 1.6172332763671875,
|
|
"rewards/rejected": 1.2141809463500977,
|
|
"ronpo/logit": 0.161723330616951,
|
|
"ronpo/residual": 0.15208038687705994,
|
|
"ronpo/residual_abs": 0.20944802463054657,
|
|
"ronpo/target": 0.009642958641052246,
|
|
"ronpo/target_abs": 0.07323791086673737,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35686972737312317,
|
|
"drift/rejected_abs": 0.12469740957021713,
|
|
"epoch": 0.04131903059197457,
|
|
"grad_norm": 5.59375,
|
|
"learning_rate": 9.942908597485558e-08,
|
|
"logits/chosen": -1.721999168395996,
|
|
"logits/rejected": -1.697299599647522,
|
|
"logps/chosen": -0.25378936529159546,
|
|
"logps/rejected": -0.2636905908584595,
|
|
"loss": 0.6324118971824646,
|
|
"loss/core": 0.606758713722229,
|
|
"loss/preference_sft": 0.2626436650753021,
|
|
"loss/reference_anchor": 0.4867998957633972,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.5683834552764893,
|
|
"rewards/margins": 2.3224399089813232,
|
|
"rewards/rejected": 1.245943307876587,
|
|
"ronpo/logit": 0.23224398493766785,
|
|
"ronpo/residual": 0.2395981252193451,
|
|
"ronpo/residual_abs": 0.2776317298412323,
|
|
"ronpo/target": -0.007354182191193104,
|
|
"ronpo/target_abs": 0.07926900684833527,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34960705041885376,
|
|
"drift/rejected_abs": 0.13111598789691925,
|
|
"epoch": 0.04290822407628129,
|
|
"grad_norm": 5.9375,
|
|
"learning_rate": 9.927369201401357e-08,
|
|
"logits/chosen": -1.7188808917999268,
|
|
"logits/rejected": -1.7290832996368408,
|
|
"logps/chosen": -0.2801087200641632,
|
|
"logps/rejected": -0.2907566428184509,
|
|
"loss": 0.7031916379928589,
|
|
"loss/core": 0.6620404720306396,
|
|
"loss/preference_sft": 0.27922695875167847,
|
|
"loss/reference_anchor": 0.7951001524925232,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.4959259033203125,
|
|
"rewards/margins": 2.1855595111846924,
|
|
"rewards/rejected": 1.3103667497634888,
|
|
"ronpo/logit": 0.21855595707893372,
|
|
"ronpo/residual": 0.2089838683605194,
|
|
"ronpo/residual_abs": 0.273912638425827,
|
|
"ronpo/target": 0.009572053328156471,
|
|
"ronpo/target_abs": 0.09021146595478058,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28669261932373047,
|
|
"drift/rejected_abs": 0.23457542061805725,
|
|
"epoch": 0.044497417560588004,
|
|
"grad_norm": 26.875,
|
|
"learning_rate": 9.909976823275142e-08,
|
|
"logits/chosen": -1.638810396194458,
|
|
"logits/rejected": -1.6302299499511719,
|
|
"logps/chosen": -0.31148478388786316,
|
|
"logps/rejected": -0.3387099504470825,
|
|
"loss": 0.41511544585227966,
|
|
"loss/core": 0.3812304139137268,
|
|
"loss/preference_sft": 0.3309209644794464,
|
|
"loss/reference_anchor": 0.6446089744567871,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8666906356811523,
|
|
"rewards/margins": 0.520958662033081,
|
|
"rewards/rejected": 2.3457322120666504,
|
|
"ronpo/logit": 0.05209586024284363,
|
|
"ronpo/residual": 0.04535352438688278,
|
|
"ronpo/residual_abs": 0.25816768407821655,
|
|
"ronpo/target": 0.0067423381842672825,
|
|
"ronpo/target_abs": 0.08022363483905792,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.40737858414649963,
|
|
"drift/rejected_abs": 0.19810840487480164,
|
|
"epoch": 0.04608661104489471,
|
|
"grad_norm": 10.625,
|
|
"learning_rate": 9.890738003669028e-08,
|
|
"logits/chosen": -1.8872385025024414,
|
|
"logits/rejected": -1.8847707509994507,
|
|
"logps/chosen": -0.25984078645706177,
|
|
"logps/rejected": -0.25426381826400757,
|
|
"loss": 0.8179662823677063,
|
|
"loss/core": 0.7683526277542114,
|
|
"loss/preference_sft": 0.25788503885269165,
|
|
"loss/reference_anchor": 0.9664837121963501,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.073483467102051,
|
|
"rewards/margins": 2.0932326316833496,
|
|
"rewards/rejected": 1.9802510738372803,
|
|
"ronpo/logit": 0.20932325720787048,
|
|
"ronpo/residual": 0.1968086212873459,
|
|
"ronpo/residual_abs": 0.3058253526687622,
|
|
"ronpo/target": 0.012514638714492321,
|
|
"ronpo/target_abs": 0.0701758936047554,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.261055052280426,
|
|
"drift/rejected_abs": 0.11264361441135406,
|
|
"epoch": 0.04767580452920143,
|
|
"grad_norm": 2.046875,
|
|
"learning_rate": 9.86965997751626e-08,
|
|
"logits/chosen": -1.7386753559112549,
|
|
"logits/rejected": -1.7452685832977295,
|
|
"logps/chosen": -0.2891419529914856,
|
|
"logps/rejected": -0.2879791855812073,
|
|
"loss": 0.33339086174964905,
|
|
"loss/core": 0.30610546469688416,
|
|
"loss/preference_sft": 0.28957223892211914,
|
|
"loss/reference_anchor": 0.5167508125305176,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.6090314388275146,
|
|
"rewards/margins": 1.486400842666626,
|
|
"rewards/rejected": 1.1226304769515991,
|
|
"ronpo/logit": 0.14864009618759155,
|
|
"ronpo/residual": 0.14336484670639038,
|
|
"ronpo/residual_abs": 0.18963593244552612,
|
|
"ronpo/target": 0.005275248549878597,
|
|
"ronpo/target_abs": 0.06922867149114609,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22688674926757812,
|
|
"drift/rejected_abs": 0.0957280695438385,
|
|
"epoch": 0.04926499801350814,
|
|
"grad_norm": 552.0,
|
|
"learning_rate": 9.846750671400445e-08,
|
|
"logits/chosen": -1.7219197750091553,
|
|
"logits/rejected": -1.7531629800796509,
|
|
"logps/chosen": -0.2790243923664093,
|
|
"logps/rejected": -0.29044419527053833,
|
|
"loss": 0.3570828139781952,
|
|
"loss/core": 0.3397815227508545,
|
|
"loss/preference_sft": 0.2791074812412262,
|
|
"loss/reference_anchor": 0.3181149959564209,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2685012817382812,
|
|
"rewards/margins": 1.3128230571746826,
|
|
"rewards/rejected": 0.9556782841682434,
|
|
"ronpo/logit": 0.13128229975700378,
|
|
"ronpo/residual": 0.11850941181182861,
|
|
"ronpo/residual_abs": 0.17623624205589294,
|
|
"ronpo/target": 0.012772892601788044,
|
|
"ronpo/target_abs": 0.07684291899204254,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3770247995853424,
|
|
"drift/rejected_abs": 0.15572576224803925,
|
|
"epoch": 0.05085419149781486,
|
|
"grad_norm": 51.25,
|
|
"learning_rate": 9.822018700574695e-08,
|
|
"logits/chosen": -1.5965344905853271,
|
|
"logits/rejected": -1.5869042873382568,
|
|
"logps/chosen": -0.29843515157699585,
|
|
"logps/rejected": -0.2949294447898865,
|
|
"loss": 0.4583340287208557,
|
|
"loss/core": 0.4307394027709961,
|
|
"loss/preference_sft": 0.30313509702682495,
|
|
"loss/reference_anchor": 0.5215789079666138,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.7689895629882812,
|
|
"rewards/margins": 2.212876558303833,
|
|
"rewards/rejected": 1.5561130046844482,
|
|
"ronpo/logit": 0.22128765285015106,
|
|
"ronpo/residual": 0.22782309353351593,
|
|
"ronpo/residual_abs": 0.266335129737854,
|
|
"ronpo/target": -0.0065354108810424805,
|
|
"ronpo/target_abs": 0.08122516423463821,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2590554654598236,
|
|
"drift/rejected_abs": 0.10995769500732422,
|
|
"epoch": 0.052443384982121574,
|
|
"grad_norm": 46.25,
|
|
"learning_rate": 9.79547336572177e-08,
|
|
"logits/chosen": -1.6693989038467407,
|
|
"logits/rejected": -1.6664832830429077,
|
|
"logps/chosen": -0.3052573502063751,
|
|
"logps/rejected": -0.30682119727134705,
|
|
"loss": 0.22655808925628662,
|
|
"loss/core": 0.21496911346912384,
|
|
"loss/preference_sft": 0.30755677819252014,
|
|
"loss/reference_anchor": 0.2010241001844406,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5874056816101074,
|
|
"rewards/margins": 1.5105947256088257,
|
|
"rewards/rejected": 1.0768110752105713,
|
|
"ronpo/logit": 0.15105947852134705,
|
|
"ronpo/residual": 0.1594732105731964,
|
|
"ronpo/residual_abs": 0.21000726521015167,
|
|
"ronpo/target": -0.008413726463913918,
|
|
"ronpo/target_abs": 0.08734320104122162,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26380234956741333,
|
|
"drift/rejected_abs": 0.1027008444070816,
|
|
"epoch": 0.05403257846642829,
|
|
"grad_norm": 124.0,
|
|
"learning_rate": 9.767124649456482e-08,
|
|
"logits/chosen": -1.8073135614395142,
|
|
"logits/rejected": -1.8926242589950562,
|
|
"logps/chosen": -0.25851204991340637,
|
|
"logps/rejected": -0.25199228525161743,
|
|
"loss": 0.32083290815353394,
|
|
"loss/core": 0.30648404359817505,
|
|
"loss/preference_sft": 0.25841936469078064,
|
|
"loss/reference_anchor": 0.2611349821090698,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6353073120117188,
|
|
"rewards/margins": 1.6092818975448608,
|
|
"rewards/rejected": 1.0260252952575684,
|
|
"ronpo/logit": 0.16092820465564728,
|
|
"ronpo/residual": 0.14506474137306213,
|
|
"ronpo/residual_abs": 0.2043898105621338,
|
|
"ronpo/target": 0.015863457694649696,
|
|
"ronpo/target_abs": 0.08150150626897812,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30660122632980347,
|
|
"drift/rejected_abs": 0.16423708200454712,
|
|
"epoch": 0.055621771950735005,
|
|
"grad_norm": 5.59375,
|
|
"learning_rate": 9.736983212571646e-08,
|
|
"logits/chosen": -1.4690876007080078,
|
|
"logits/rejected": -1.5064828395843506,
|
|
"logps/chosen": -0.2989944815635681,
|
|
"logps/rejected": -0.2933068871498108,
|
|
"loss": 0.18233861029148102,
|
|
"loss/core": 0.15927359461784363,
|
|
"loss/preference_sft": 0.29980307817459106,
|
|
"loss/reference_anchor": 0.43132057785987854,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.0638973712921143,
|
|
"rewards/margins": 1.4231058359146118,
|
|
"rewards/rejected": 1.640791654586792,
|
|
"ronpo/logit": 0.14231057465076447,
|
|
"ronpo/residual": 0.14812180399894714,
|
|
"ronpo/residual_abs": 0.19247175753116608,
|
|
"ronpo/target": -0.00581121863797307,
|
|
"ronpo/target_abs": 0.0764647126197815,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21774089336395264,
|
|
"drift/rejected_abs": 0.12308623641729355,
|
|
"epoch": 0.057210965435041714,
|
|
"grad_norm": 2.0625,
|
|
"learning_rate": 9.705060390028978e-08,
|
|
"logits/chosen": -1.8982244729995728,
|
|
"logits/rejected": -1.8753488063812256,
|
|
"logps/chosen": -0.2724835276603699,
|
|
"logps/rejected": -0.26976484060287476,
|
|
"loss": 0.2106136530637741,
|
|
"loss/core": 0.19985875487327576,
|
|
"loss/preference_sft": 0.2746678590774536,
|
|
"loss/reference_anchor": 0.18763145804405212,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.176549196243286,
|
|
"rewards/margins": 0.9472295045852661,
|
|
"rewards/rejected": 1.22931969165802,
|
|
"ronpo/logit": 0.09472296386957169,
|
|
"ronpo/residual": 0.09877835214138031,
|
|
"ronpo/residual_abs": 0.19839002192020416,
|
|
"ronpo/target": -0.004055389203131199,
|
|
"ronpo/target_abs": 0.07911963760852814,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20750541985034943,
|
|
"drift/rejected_abs": 0.07880739867687225,
|
|
"epoch": 0.05880015891934843,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 9.671368186696487e-08,
|
|
"logits/chosen": -1.7366540431976318,
|
|
"logits/rejected": -1.6725021600723267,
|
|
"logps/chosen": -0.2818494439125061,
|
|
"logps/rejected": -0.29854708909988403,
|
|
"loss": 0.2648107707500458,
|
|
"loss/core": 0.25557440519332886,
|
|
"loss/preference_sft": 0.29618388414382935,
|
|
"loss/reference_anchor": 0.1551087200641632,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.072450876235962,
|
|
"rewards/margins": 1.4119017124176025,
|
|
"rewards/rejected": 0.6605492830276489,
|
|
"ronpo/logit": 0.14119017124176025,
|
|
"ronpo/residual": 0.13928617537021637,
|
|
"ronpo/residual_abs": 0.18467354774475098,
|
|
"ronpo/target": 0.0019040137995034456,
|
|
"ronpo/target_abs": 0.06882323324680328,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3288821876049042,
|
|
"drift/rejected_abs": 0.08692674338817596,
|
|
"epoch": 0.060389352403655144,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 9.635919272833936e-08,
|
|
"logits/chosen": -1.8826234340667725,
|
|
"logits/rejected": -1.8909105062484741,
|
|
"logps/chosen": -0.28138142824172974,
|
|
"logps/rejected": -0.2874371409416199,
|
|
"loss": 0.9378984570503235,
|
|
"loss/core": 0.9062137603759766,
|
|
"loss/preference_sft": 0.28439387679100037,
|
|
"loss/reference_anchor": 0.6052535772323608,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2882328033447266,
|
|
"rewards/margins": 2.4242477416992188,
|
|
"rewards/rejected": 0.8639850616455078,
|
|
"ronpo/logit": 0.24242480099201202,
|
|
"ronpo/residual": 0.24681024253368378,
|
|
"ronpo/residual_abs": 0.28807663917541504,
|
|
"ronpo/target": -0.004385476466268301,
|
|
"ronpo/target_abs": 0.06821522861719131,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.48508280515670776,
|
|
"drift/rejected_abs": 0.12983950972557068,
|
|
"epoch": 0.06197854588796186,
|
|
"grad_norm": 1.3203125,
|
|
"learning_rate": 9.598726979328077e-08,
|
|
"logits/chosen": -1.7950727939605713,
|
|
"logits/rejected": -1.7613496780395508,
|
|
"logps/chosen": -0.2707488238811493,
|
|
"logps/rejected": -0.2625698447227478,
|
|
"loss": 1.6498689651489258,
|
|
"loss/core": 1.5984039306640625,
|
|
"loss/preference_sft": 0.27121761441230774,
|
|
"loss/reference_anchor": 1.002181887626648,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.849929332733154,
|
|
"rewards/margins": 3.582291841506958,
|
|
"rewards/rejected": 1.267637848854065,
|
|
"ronpo/logit": 0.3582291901111603,
|
|
"ronpo/residual": 0.3614763617515564,
|
|
"ronpo/residual_abs": 0.3961934447288513,
|
|
"ronpo/target": -0.0032471846789121628,
|
|
"ronpo/target_abs": 0.06749024987220764,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3213449716567993,
|
|
"drift/rejected_abs": 0.18218696117401123,
|
|
"epoch": 0.06356773937226858,
|
|
"grad_norm": 209.0,
|
|
"learning_rate": 9.559805292679444e-08,
|
|
"logits/chosen": -1.7774794101715088,
|
|
"logits/rejected": -1.779119849205017,
|
|
"logps/chosen": -0.28203001618385315,
|
|
"logps/rejected": -0.28085026144981384,
|
|
"loss": 0.3768664002418518,
|
|
"loss/core": 0.3414820730686188,
|
|
"loss/preference_sft": 0.2905542552471161,
|
|
"loss/reference_anchor": 0.6786319017410278,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2131171226501465,
|
|
"rewards/margins": 1.3922719955444336,
|
|
"rewards/rejected": 1.8208452463150024,
|
|
"ronpo/logit": 0.13922719657421112,
|
|
"ronpo/residual": 0.14935089647769928,
|
|
"ronpo/residual_abs": 0.21834442019462585,
|
|
"ronpo/target": -0.010123697109520435,
|
|
"ronpo/target_abs": 0.0792742595076561,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30102676153182983,
|
|
"drift/rejected_abs": 0.12649984657764435,
|
|
"epoch": 0.06515693285657528,
|
|
"grad_norm": 7.59375,
|
|
"learning_rate": 9.519168849742603e-08,
|
|
"logits/chosen": -1.7655588388442993,
|
|
"logits/rejected": -1.8515663146972656,
|
|
"logps/chosen": -0.28426095843315125,
|
|
"logps/rejected": -0.2876798212528229,
|
|
"loss": 0.44624027609825134,
|
|
"loss/core": 0.4282476007938385,
|
|
"loss/preference_sft": 0.28018221259117126,
|
|
"loss/reference_anchor": 0.33183637261390686,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.0090479850769043,
|
|
"rewards/margins": 1.7447162866592407,
|
|
"rewards/rejected": 1.2643316984176636,
|
|
"ronpo/logit": 0.1744716465473175,
|
|
"ronpo/residual": 0.17375977337360382,
|
|
"ronpo/residual_abs": 0.23645329475402832,
|
|
"ronpo/target": 0.0007118784124031663,
|
|
"ronpo/target_abs": 0.09003230184316635,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13736221194267273,
|
|
"drift/rejected_abs": 0.07152147591114044,
|
|
"epoch": 0.066746126340882,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 9.476832932221834e-08,
|
|
"logits/chosen": -1.9468104839324951,
|
|
"logits/rejected": -2.017134666442871,
|
|
"logps/chosen": -0.2806920111179352,
|
|
"logps/rejected": -0.28426095843315125,
|
|
"loss": 0.03696361929178238,
|
|
"loss/core": 0.033381301909685135,
|
|
"loss/preference_sft": 0.2832663953304291,
|
|
"loss/reference_anchor": 0.0433196984231472,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.3729757070541382,
|
|
"rewards/margins": 0.6626027822494507,
|
|
"rewards/rejected": 0.7103729248046875,
|
|
"ronpo/logit": 0.06626029312610626,
|
|
"ronpo/residual": 0.06426443159580231,
|
|
"ronpo/residual_abs": 0.12241494655609131,
|
|
"ronpo/target": 0.001995860366150737,
|
|
"ronpo/target_abs": 0.0830467939376831,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23734159767627716,
|
|
"drift/rejected_abs": 0.12650033831596375,
|
|
"epoch": 0.06833531982518871,
|
|
"grad_norm": 2.3125,
|
|
"learning_rate": 9.432813460924306e-08,
|
|
"logits/chosen": -1.6526628732681274,
|
|
"logits/rejected": -1.7219336032867432,
|
|
"logps/chosen": -0.2975338399410248,
|
|
"logps/rejected": -0.30417123436927795,
|
|
"loss": 0.06939711421728134,
|
|
"loss/core": 0.057345591485500336,
|
|
"loss/preference_sft": 0.301188588142395,
|
|
"loss/reference_anchor": 0.21091172099113464,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3707804679870605,
|
|
"rewards/margins": 1.1078386306762695,
|
|
"rewards/rejected": 1.262941837310791,
|
|
"ronpo/logit": 0.11078387498855591,
|
|
"ronpo/residual": 0.09446224570274353,
|
|
"ronpo/residual_abs": 0.14788493514060974,
|
|
"ronpo/target": 0.01632162369787693,
|
|
"ronpo/target_abs": 0.07581789791584015,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.199172705411911,
|
|
"drift/rejected_abs": 0.09683258831501007,
|
|
"epoch": 0.06992451330949544,
|
|
"grad_norm": 7.71875,
|
|
"learning_rate": 9.387126989772909e-08,
|
|
"logits/chosen": -1.8001289367675781,
|
|
"logits/rejected": -1.7968250513076782,
|
|
"logps/chosen": -0.27282461524009705,
|
|
"logps/rejected": -0.27572038769721985,
|
|
"loss": 0.20236146450042725,
|
|
"loss/core": 0.19408978521823883,
|
|
"loss/preference_sft": 0.27387744188308716,
|
|
"loss/reference_anchor": 0.13804566860198975,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9908561706542969,
|
|
"rewards/margins": 1.023787260055542,
|
|
"rewards/rejected": 0.9670687913894653,
|
|
"ronpo/logit": 0.10237874835729599,
|
|
"ronpo/residual": 0.10155370086431503,
|
|
"ronpo/residual_abs": 0.16425423324108124,
|
|
"ronpo/target": 0.0008250333485193551,
|
|
"ronpo/target_abs": 0.08938291668891907,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16761744022369385,
|
|
"drift/rejected_abs": 0.09081226587295532,
|
|
"epoch": 0.07151370679380215,
|
|
"grad_norm": 3.21875,
|
|
"learning_rate": 9.339790699581003e-08,
|
|
"logits/chosen": -1.9558331966400146,
|
|
"logits/rejected": -1.9702494144439697,
|
|
"logps/chosen": -0.24233689904212952,
|
|
"logps/rejected": -0.24989719688892365,
|
|
"loss": 0.055875737220048904,
|
|
"loss/core": 0.050802744925022125,
|
|
"loss/preference_sft": 0.24640336632728577,
|
|
"loss/reference_anchor": 0.07681950181722641,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.673455834388733,
|
|
"rewards/margins": 0.7707704901695251,
|
|
"rewards/rejected": 0.9026853442192078,
|
|
"ronpo/logit": 0.07707705348730087,
|
|
"ronpo/residual": 0.07403306663036346,
|
|
"ronpo/residual_abs": 0.1344565600156784,
|
|
"ronpo/target": 0.003043986391276121,
|
|
"ronpo/target_abs": 0.08832766115665436,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4040397107601166,
|
|
"drift/rejected_abs": 0.20574426651000977,
|
|
"epoch": 0.07310290027810885,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 9.290822391591417e-08,
|
|
"logits/chosen": -1.8052107095718384,
|
|
"logits/rejected": -1.7864511013031006,
|
|
"logps/chosen": -0.2899695038795471,
|
|
"logps/rejected": -0.30955272912979126,
|
|
"loss": 0.5364000797271729,
|
|
"loss/core": 0.4829653203487396,
|
|
"loss/preference_sft": 0.3120691180229187,
|
|
"loss/reference_anchor": 1.0374867916107178,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.037346839904785,
|
|
"rewards/margins": 2.1194443702697754,
|
|
"rewards/rejected": 1.9179027080535889,
|
|
"ronpo/logit": 0.21194446086883545,
|
|
"ronpo/residual": 0.20892982184886932,
|
|
"ronpo/residual_abs": 0.2797432839870453,
|
|
"ronpo/target": 0.0030146308708935976,
|
|
"ronpo/target_abs": 0.07361957430839539,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35165148973464966,
|
|
"drift/rejected_abs": 0.18592815101146698,
|
|
"epoch": 0.07469209376241558,
|
|
"grad_norm": 1.4609375,
|
|
"learning_rate": 9.240240480782128e-08,
|
|
"logits/chosen": -1.792009711265564,
|
|
"logits/rejected": -1.8102452754974365,
|
|
"logps/chosen": -0.29723426699638367,
|
|
"logps/rejected": -0.28721755743026733,
|
|
"loss": 0.3889918923377991,
|
|
"loss/core": 0.3515408933162689,
|
|
"loss/preference_sft": 0.28965941071510315,
|
|
"loss/reference_anchor": 0.7200536131858826,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.5157008171081543,
|
|
"rewards/margins": 1.6589634418487549,
|
|
"rewards/rejected": 1.856737494468689,
|
|
"ronpo/logit": 0.16589637100696564,
|
|
"ronpo/residual": 0.158092200756073,
|
|
"ronpo/residual_abs": 0.2130337655544281,
|
|
"ronpo/target": 0.007804171647876501,
|
|
"ronpo/target_abs": 0.07572085410356522,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23401710391044617,
|
|
"drift/rejected_abs": 0.11872808635234833,
|
|
"epoch": 0.07628128724672228,
|
|
"grad_norm": 4.40625,
|
|
"learning_rate": 9.188063988941146e-08,
|
|
"logits/chosen": -1.817143201828003,
|
|
"logits/rejected": -1.893754243850708,
|
|
"logps/chosen": -0.2975711226463318,
|
|
"logps/rejected": -0.2947074770927429,
|
|
"loss": 0.11131559312343597,
|
|
"loss/core": 0.09855598956346512,
|
|
"loss/preference_sft": 0.2999657392501831,
|
|
"loss/reference_anchor": 0.22519561648368835,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.339799642562866,
|
|
"rewards/margins": 1.156066656112671,
|
|
"rewards/rejected": 1.1837332248687744,
|
|
"ronpo/logit": 0.1156066507101059,
|
|
"ronpo/residual": 0.10619938373565674,
|
|
"ronpo/residual_abs": 0.15284030139446259,
|
|
"ronpo/target": 0.009407281875610352,
|
|
"ronpo/target_abs": 0.07423986494541168,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3331848680973053,
|
|
"drift/rejected_abs": 0.11539693921804428,
|
|
"epoch": 0.07787048073102901,
|
|
"grad_norm": 234.0,
|
|
"learning_rate": 9.134312537513186e-08,
|
|
"logits/chosen": -1.8041841983795166,
|
|
"logits/rejected": -1.761364221572876,
|
|
"logps/chosen": -0.3000238239765167,
|
|
"logps/rejected": -0.30252355337142944,
|
|
"loss": 0.8759735822677612,
|
|
"loss/core": 0.8312212228775024,
|
|
"loss/preference_sft": 0.30587834119796753,
|
|
"loss/reference_anchor": 0.8644579648971558,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.3306591510772705,
|
|
"rewards/margins": 2.1822597980499268,
|
|
"rewards/rejected": 1.1483994722366333,
|
|
"ronpo/logit": 0.21822600066661835,
|
|
"ronpo/residual": 0.21011772751808167,
|
|
"ronpo/residual_abs": 0.2682834267616272,
|
|
"ronpo/target": 0.008108270354568958,
|
|
"ronpo/target_abs": 0.09494705498218536,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35135117173194885,
|
|
"drift/rejected_abs": 0.15672887861728668,
|
|
"epoch": 0.07945967421533572,
|
|
"grad_norm": 352.0,
|
|
"learning_rate": 9.07900634022086e-08,
|
|
"logits/chosen": -1.749436378479004,
|
|
"logits/rejected": -1.7206599712371826,
|
|
"logps/chosen": -0.2767244577407837,
|
|
"logps/rejected": -0.2613469660282135,
|
|
"loss": 0.6066776514053345,
|
|
"loss/core": 0.5774033069610596,
|
|
"loss/preference_sft": 0.27615392208099365,
|
|
"loss/reference_anchor": 0.5578705072402954,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.513512134552002,
|
|
"rewards/margins": 1.9805538654327393,
|
|
"rewards/rejected": 1.5329577922821045,
|
|
"ronpo/logit": 0.19805540144443512,
|
|
"ronpo/residual": 0.19628280401229858,
|
|
"ronpo/residual_abs": 0.2545327842235565,
|
|
"ronpo/target": 0.0017725772922858596,
|
|
"ronpo/target_abs": 0.07362736016511917,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.09521497786045074,
|
|
"drift/rejected_abs": 0.04314413666725159,
|
|
"epoch": 0.08104886769964244,
|
|
"grad_norm": 1.703125,
|
|
"learning_rate": 9.02216619546311e-08,
|
|
"logits/chosen": -2.127336263656616,
|
|
"logits/rejected": -2.1303391456604004,
|
|
"logps/chosen": -0.2789131999015808,
|
|
"logps/rejected": -0.2843419015407562,
|
|
"loss": 0.02066163904964924,
|
|
"loss/core": 0.018253901973366737,
|
|
"loss/preference_sft": 0.28216642141342163,
|
|
"loss/reference_anchor": 0.019938113167881966,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.9484003186225891,
|
|
"rewards/margins": 0.5196825861930847,
|
|
"rewards/rejected": 0.428717702627182,
|
|
"ronpo/logit": 0.05196825787425041,
|
|
"ronpo/residual": 0.041401177644729614,
|
|
"ronpo/residual_abs": 0.0943097323179245,
|
|
"ronpo/target": 0.01056708488613367,
|
|
"ronpo/target_abs": 0.0819949060678482,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37315139174461365,
|
|
"drift/rejected_abs": 0.16794715821743011,
|
|
"epoch": 0.08263806118394915,
|
|
"grad_norm": 11.5625,
|
|
"learning_rate": 8.963813478493788e-08,
|
|
"logits/chosen": -1.6915229558944702,
|
|
"logits/rejected": -1.7035410404205322,
|
|
"logps/chosen": -0.2909337878227234,
|
|
"logps/rejected": -0.2909685969352722,
|
|
"loss": 0.5750402808189392,
|
|
"loss/core": 0.5253722071647644,
|
|
"loss/preference_sft": 0.2933681607246399,
|
|
"loss/reference_anchor": 0.9640249013900757,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.7303051948547363,
|
|
"rewards/margins": 2.0620627403259277,
|
|
"rewards/rejected": 1.66824209690094,
|
|
"ronpo/logit": 0.2062062919139862,
|
|
"ronpo/residual": 0.1919330209493637,
|
|
"ronpo/residual_abs": 0.2425989806652069,
|
|
"ronpo/target": 0.014273243956267834,
|
|
"ronpo/target_abs": 0.06451570987701416,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37269216775894165,
|
|
"drift/rejected_abs": 0.09161754697561264,
|
|
"epoch": 0.08422725466825585,
|
|
"grad_norm": 21.125,
|
|
"learning_rate": 8.903970133383297e-08,
|
|
"logits/chosen": -1.644796371459961,
|
|
"logits/rejected": -1.5450862646102905,
|
|
"logps/chosen": -0.280166894197464,
|
|
"logps/rejected": -0.2835793197154999,
|
|
"loss": 0.9720993041992188,
|
|
"loss/core": 0.943614661693573,
|
|
"loss/preference_sft": 0.28308966755867004,
|
|
"loss/reference_anchor": 0.5413841009140015,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.7253494262695312,
|
|
"rewards/margins": 2.8579189777374268,
|
|
"rewards/rejected": 0.8674303293228149,
|
|
"ronpo/logit": 0.28579190373420715,
|
|
"ronpo/residual": 0.2972434461116791,
|
|
"ronpo/residual_abs": 0.3468155562877655,
|
|
"ronpo/target": -0.011451545171439648,
|
|
"ronpo/target_abs": 0.08932532370090485,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1571657955646515,
|
|
"drift/rejected_abs": 0.07838908582925797,
|
|
"epoch": 0.08581644815256258,
|
|
"grad_norm": 10.375,
|
|
"learning_rate": 8.842658664766316e-08,
|
|
"logits/chosen": -1.829146146774292,
|
|
"logits/rejected": -1.841604471206665,
|
|
"logps/chosen": -0.2962016463279724,
|
|
"logps/rejected": -0.29258397221565247,
|
|
"loss": 0.0605291910469532,
|
|
"loss/core": 0.054931849241256714,
|
|
"loss/preference_sft": 0.2943229079246521,
|
|
"loss/reference_anchor": 0.08251471817493439,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5701178312301636,
|
|
"rewards/margins": 0.78829425573349,
|
|
"rewards/rejected": 0.7818235158920288,
|
|
"ronpo/logit": 0.07882943749427795,
|
|
"ronpo/residual": 0.09931907802820206,
|
|
"ronpo/residual_abs": 0.13743585348129272,
|
|
"ronpo/target": -0.0204896442592144,
|
|
"ronpo/target_abs": 0.08406566083431244,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34716716408729553,
|
|
"drift/rejected_abs": 0.14563432335853577,
|
|
"epoch": 0.08740564163686929,
|
|
"grad_norm": 6.90625,
|
|
"learning_rate": 8.779902129378738e-08,
|
|
"logits/chosen": -1.930598497390747,
|
|
"logits/rejected": -1.9004106521606445,
|
|
"logps/chosen": -0.29537269473075867,
|
|
"logps/rejected": -0.28550130128860474,
|
|
"loss": 0.7920290231704712,
|
|
"loss/core": 0.7438088655471802,
|
|
"loss/preference_sft": 0.2878211736679077,
|
|
"loss/reference_anchor": 0.9356187582015991,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.469741106033325,
|
|
"rewards/margins": 2.0159237384796143,
|
|
"rewards/rejected": 1.4538179636001587,
|
|
"ronpo/logit": 0.201592355966568,
|
|
"ronpo/residual": 0.20089873671531677,
|
|
"ronpo/residual_abs": 0.24797526001930237,
|
|
"ronpo/target": 0.0006936040008440614,
|
|
"ronpo/target_abs": 0.0743836909532547,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3208922743797302,
|
|
"drift/rejected_abs": 0.11808327585458755,
|
|
"epoch": 0.08899483512117601,
|
|
"grad_norm": 21.375,
|
|
"learning_rate": 8.715724127386971e-08,
|
|
"logits/chosen": -1.851441740989685,
|
|
"logits/rejected": -1.7920417785644531,
|
|
"logps/chosen": -0.291858971118927,
|
|
"logps/rejected": -0.27598926424980164,
|
|
"loss": 0.43859606981277466,
|
|
"loss/core": 0.4198855459690094,
|
|
"loss/preference_sft": 0.27710169553756714,
|
|
"loss/reference_anchor": 0.3464996814727783,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2073960304260254,
|
|
"rewards/margins": 2.0731959342956543,
|
|
"rewards/rejected": 1.1342003345489502,
|
|
"ronpo/logit": 0.20731958746910095,
|
|
"ronpo/residual": 0.21950101852416992,
|
|
"ronpo/residual_abs": 0.2542910575866699,
|
|
"ronpo/target": -0.012181434780359268,
|
|
"ronpo/target_abs": 0.07466322928667068,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.422764390707016,
|
|
"drift/rejected_abs": 0.2523038387298584,
|
|
"epoch": 0.09058402860548272,
|
|
"grad_norm": 1.7890625,
|
|
"learning_rate": 8.650148793512872e-08,
|
|
"logits/chosen": -1.6715549230575562,
|
|
"logits/rejected": -1.730615258216858,
|
|
"logps/chosen": -0.27390456199645996,
|
|
"logps/rejected": -0.27246493101119995,
|
|
"loss": 0.37580451369285583,
|
|
"loss/core": 0.3173265755176544,
|
|
"loss/preference_sft": 0.27324944734573364,
|
|
"loss/reference_anchor": 1.1422338485717773,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.225140571594238,
|
|
"rewards/margins": 1.8235841989517212,
|
|
"rewards/rejected": 2.4015564918518066,
|
|
"ronpo/logit": 0.18235842883586884,
|
|
"ronpo/residual": 0.18450120091438293,
|
|
"ronpo/residual_abs": 0.24420492351055145,
|
|
"ronpo/target": -0.0021427671890705824,
|
|
"ronpo/target_abs": 0.07870659232139587,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.339019238948822,
|
|
"drift/rejected_abs": 0.1414244920015335,
|
|
"epoch": 0.09217322208978942,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 8.58320078795768e-08,
|
|
"logits/chosen": -1.9007600545883179,
|
|
"logits/rejected": -1.8737331628799438,
|
|
"logps/chosen": -0.2887403964996338,
|
|
"logps/rejected": -0.3162103295326233,
|
|
"loss": 0.5720577239990234,
|
|
"loss/core": 0.5303006768226624,
|
|
"loss/preference_sft": 0.3076707720756531,
|
|
"loss/reference_anchor": 0.8043742179870605,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.390192747116089,
|
|
"rewards/margins": 2.156050682067871,
|
|
"rewards/rejected": 1.2341418266296387,
|
|
"ronpo/logit": 0.21560506522655487,
|
|
"ronpo/residual": 0.21163058280944824,
|
|
"ronpo/residual_abs": 0.25786668062210083,
|
|
"ronpo/target": 0.003974485211074352,
|
|
"ronpo/target_abs": 0.08457241952419281,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.38282856345176697,
|
|
"drift/rejected_abs": 0.10466931760311127,
|
|
"epoch": 0.09376241557409615,
|
|
"grad_norm": 11.125,
|
|
"learning_rate": 8.514905287128308e-08,
|
|
"logits/chosen": -1.9592187404632568,
|
|
"logits/rejected": -1.918391466140747,
|
|
"logps/chosen": -0.29651257395744324,
|
|
"logps/rejected": -0.29091230034828186,
|
|
"loss": 1.480013132095337,
|
|
"loss/core": 1.4311615228652954,
|
|
"loss/preference_sft": 0.29933270812034607,
|
|
"loss/reference_anchor": 0.9471020698547363,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.8279099464416504,
|
|
"rewards/margins": 2.7820181846618652,
|
|
"rewards/rejected": 1.0458917617797852,
|
|
"ronpo/logit": 0.27820178866386414,
|
|
"ronpo/residual": 0.2564352750778198,
|
|
"ronpo/residual_abs": 0.34638240933418274,
|
|
"ronpo/target": 0.0217665396630764,
|
|
"ronpo/target_abs": 0.0853576809167862,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23551969230175018,
|
|
"drift/rejected_abs": 0.11991935968399048,
|
|
"epoch": 0.09535160905840286,
|
|
"grad_norm": 4.71875,
|
|
"learning_rate": 8.44528797416954e-08,
|
|
"logits/chosen": -1.7564899921417236,
|
|
"logits/rejected": -1.8135544061660767,
|
|
"logps/chosen": -0.2883635461330414,
|
|
"logps/rejected": -0.2851993441581726,
|
|
"loss": 0.13555917143821716,
|
|
"loss/core": 0.12408057600259781,
|
|
"loss/preference_sft": 0.28471457958221436,
|
|
"loss/reference_anchor": 0.20110058784484863,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.353811740875244,
|
|
"rewards/margins": 1.1569559574127197,
|
|
"rewards/rejected": 1.1968556642532349,
|
|
"ronpo/logit": 0.11569561064243317,
|
|
"ronpo/residual": 0.11366703361272812,
|
|
"ronpo/residual_abs": 0.1557668149471283,
|
|
"ronpo/target": 0.0020285695791244507,
|
|
"ronpo/target_abs": 0.06852469593286514,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3189460337162018,
|
|
"drift/rejected_abs": 0.1352815330028534,
|
|
"epoch": 0.09694080254270958,
|
|
"grad_norm": 31.5,
|
|
"learning_rate": 8.374375029305638e-08,
|
|
"logits/chosen": -1.7944324016571045,
|
|
"logits/rejected": -1.7328087091445923,
|
|
"logps/chosen": -0.27897724509239197,
|
|
"logps/rejected": -0.2867986559867859,
|
|
"loss": 0.629349946975708,
|
|
"loss/core": 0.5993045568466187,
|
|
"loss/preference_sft": 0.2911568880081177,
|
|
"loss/reference_anchor": 0.5717930197715759,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.189460277557373,
|
|
"rewards/margins": 1.8377918004989624,
|
|
"rewards/rejected": 1.3516688346862793,
|
|
"ronpo/logit": 0.18377919495105743,
|
|
"ronpo/residual": 0.16819463670253754,
|
|
"ronpo/residual_abs": 0.22840599715709686,
|
|
"ronpo/target": 0.015584555454552174,
|
|
"ronpo/target_abs": 0.08353091031312943,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22331972420215607,
|
|
"drift/rejected_abs": 0.10017452389001846,
|
|
"epoch": 0.09852999602701629,
|
|
"grad_norm": 0.9765625,
|
|
"learning_rate": 8.302193119995037e-08,
|
|
"logits/chosen": -1.83052659034729,
|
|
"logits/rejected": -1.8760980367660522,
|
|
"logps/chosen": -0.3000250458717346,
|
|
"logps/rejected": -0.302760511636734,
|
|
"loss": 0.1551820933818817,
|
|
"loss/core": 0.1419788897037506,
|
|
"loss/preference_sft": 0.3033260107040405,
|
|
"loss/reference_anchor": 0.23373158276081085,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2327847480773926,
|
|
"rewards/margins": 1.2316186428070068,
|
|
"rewards/rejected": 1.0011662244796753,
|
|
"ronpo/logit": 0.12316185235977173,
|
|
"ronpo/residual": 0.12490953505039215,
|
|
"ronpo/residual_abs": 0.15779349207878113,
|
|
"ronpo/target": -0.0017476793145760894,
|
|
"ronpo/target_abs": 0.0755293220281601,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32748138904571533,
|
|
"drift/rejected_abs": 0.12331084161996841,
|
|
"epoch": 0.10011918951132301,
|
|
"grad_norm": 330.0,
|
|
"learning_rate": 8.22876939090181e-08,
|
|
"logits/chosen": -1.8843028545379639,
|
|
"logits/rejected": -1.8397438526153564,
|
|
"logps/chosen": -0.2662331163883209,
|
|
"logps/rejected": -0.2638329565525055,
|
|
"loss": 0.6274744272232056,
|
|
"loss/core": 0.6013913750648499,
|
|
"loss/preference_sft": 0.2718098759651184,
|
|
"loss/reference_anchor": 0.4944801330566406,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.274233341217041,
|
|
"rewards/margins": 2.0433292388916016,
|
|
"rewards/rejected": 1.2309037446975708,
|
|
"ronpo/logit": 0.20433291792869568,
|
|
"ronpo/residual": 0.2113933265209198,
|
|
"ronpo/residual_abs": 0.2690233290195465,
|
|
"ronpo/target": -0.007060416974127293,
|
|
"ronpo/target_abs": 0.08037460595369339,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31982502341270447,
|
|
"drift/rejected_abs": 0.1502150595188141,
|
|
"epoch": 0.10170838299562972,
|
|
"grad_norm": 9.0,
|
|
"learning_rate": 8.154131453687656e-08,
|
|
"logits/chosen": -1.6950571537017822,
|
|
"logits/rejected": -1.6967027187347412,
|
|
"logps/chosen": -0.29967036843299866,
|
|
"logps/rejected": -0.2906130254268646,
|
|
"loss": 0.22242435812950134,
|
|
"loss/core": 0.20424409210681915,
|
|
"loss/preference_sft": 0.29998618364334106,
|
|
"loss/reference_anchor": 0.3336065411567688,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.1972603797912598,
|
|
"rewards/margins": 1.7026844024658203,
|
|
"rewards/rejected": 1.4945757389068604,
|
|
"ronpo/logit": 0.1702684462070465,
|
|
"ronpo/residual": 0.16520681977272034,
|
|
"ronpo/residual_abs": 0.21789643168449402,
|
|
"ronpo/target": 0.005061624571681023,
|
|
"ronpo/target_abs": 0.07731436938047409,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32239073514938354,
|
|
"drift/rejected_abs": 0.14487099647521973,
|
|
"epoch": 0.10329757647993643,
|
|
"grad_norm": 2.515625,
|
|
"learning_rate": 8.07830737662829e-08,
|
|
"logits/chosen": -1.7711395025253296,
|
|
"logits/rejected": -1.7771180868148804,
|
|
"logps/chosen": -0.27507781982421875,
|
|
"logps/rejected": -0.26983487606048584,
|
|
"loss": 0.46221646666526794,
|
|
"loss/core": 0.43381762504577637,
|
|
"loss/preference_sft": 0.274006187915802,
|
|
"loss/reference_anchor": 0.5405768156051636,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.223078489303589,
|
|
"rewards/margins": 1.7749700546264648,
|
|
"rewards/rejected": 1.4481085538864136,
|
|
"ronpo/logit": 0.177496999502182,
|
|
"ronpo/residual": 0.17691360414028168,
|
|
"ronpo/residual_abs": 0.22765477001667023,
|
|
"ronpo/target": 0.0005833914619870484,
|
|
"ronpo/target_abs": 0.07749475538730621,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2092551738023758,
|
|
"drift/rejected_abs": 0.05679240822792053,
|
|
"epoch": 0.10488676996424315,
|
|
"grad_norm": 13.8125,
|
|
"learning_rate": 8.001325674058123e-08,
|
|
"logits/chosen": -1.9151840209960938,
|
|
"logits/rejected": -1.9076945781707764,
|
|
"logps/chosen": -0.3010541498661041,
|
|
"logps/rejected": -0.298443078994751,
|
|
"loss": 0.29586261510849,
|
|
"loss/core": 0.28491804003715515,
|
|
"loss/preference_sft": 0.3068598210811615,
|
|
"loss/reference_anchor": 0.18820573389530182,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0916950702667236,
|
|
"rewards/margins": 1.525160312652588,
|
|
"rewards/rejected": 0.5665348172187805,
|
|
"ronpo/logit": 0.15251603722572327,
|
|
"ronpo/residual": 0.1487692892551422,
|
|
"ronpo/residual_abs": 0.19916026294231415,
|
|
"ronpo/target": 0.003746759844943881,
|
|
"ronpo/target_abs": 0.07705475389957428,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2681005001068115,
|
|
"drift/rejected_abs": 0.13281159102916718,
|
|
"epoch": 0.10647596344854986,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 7.923215295647166e-08,
|
|
"logits/chosen": -1.8540802001953125,
|
|
"logits/rejected": -1.8404872417449951,
|
|
"logps/chosen": -0.28104037046432495,
|
|
"logps/rejected": -0.28956103324890137,
|
|
"loss": 0.3578251004219055,
|
|
"loss/core": 0.33799657225608826,
|
|
"loss/preference_sft": 0.28577661514282227,
|
|
"loss/reference_anchor": 0.36799150705337524,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.680506944656372,
|
|
"rewards/margins": 1.3567620515823364,
|
|
"rewards/rejected": 1.323744773864746,
|
|
"ronpo/logit": 0.13567620515823364,
|
|
"ronpo/residual": 0.1216505616903305,
|
|
"ronpo/residual_abs": 0.2040475606918335,
|
|
"ronpo/target": 0.014025667682290077,
|
|
"ronpo/target_abs": 0.05643507093191147,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20863750576972961,
|
|
"drift/rejected_abs": 0.23069891333580017,
|
|
"epoch": 0.10806515693285658,
|
|
"grad_norm": 416.0,
|
|
"learning_rate": 7.844005615514259e-08,
|
|
"logits/chosen": -1.861107587814331,
|
|
"logits/rejected": -1.8314059972763062,
|
|
"logps/chosen": -0.26891106367111206,
|
|
"logps/rejected": -0.2669699788093567,
|
|
"loss": 1.0711219310760498,
|
|
"loss/core": 1.038293480873108,
|
|
"loss/preference_sft": 0.2746712565422058,
|
|
"loss/reference_anchor": 0.6291025280952454,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0860025882720947,
|
|
"rewards/margins": -0.2168906182050705,
|
|
"rewards/rejected": 2.3028931617736816,
|
|
"ronpo/logit": -0.02168908901512623,
|
|
"ronpo/residual": -0.020689811557531357,
|
|
"ronpo/residual_abs": 0.2519962787628174,
|
|
"ronpo/target": -0.00099927035626024,
|
|
"ronpo/target_abs": 0.07252907007932663,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3522050678730011,
|
|
"drift/rejected_abs": 0.1813865453004837,
|
|
"epoch": 0.10965435041716329,
|
|
"grad_norm": 812.0,
|
|
"learning_rate": 7.763726421180662e-08,
|
|
"logits/chosen": -1.597299337387085,
|
|
"logits/rejected": -1.547257423400879,
|
|
"logps/chosen": -0.28551051020622253,
|
|
"logps/rejected": -0.3178768754005432,
|
|
"loss": 0.6678337454795837,
|
|
"loss/core": 0.6344470381736755,
|
|
"loss/preference_sft": 0.289370059967041,
|
|
"loss/reference_anchor": 0.6387966871261597,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.520669937133789,
|
|
"rewards/margins": 1.727455496788025,
|
|
"rewards/rejected": 1.7932144403457642,
|
|
"ronpo/logit": 0.17274554073810577,
|
|
"ronpo/residual": 0.14491984248161316,
|
|
"ronpo/residual_abs": 0.2273443192243576,
|
|
"ronpo/target": 0.027825692668557167,
|
|
"ronpo/target_abs": 0.07946719974279404,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3626919686794281,
|
|
"drift/rejected_abs": 0.11279211193323135,
|
|
"epoch": 0.11124354390147001,
|
|
"grad_norm": 288.0,
|
|
"learning_rate": 7.682407902368189e-08,
|
|
"logits/chosen": -1.664306640625,
|
|
"logits/rejected": -1.6896919012069702,
|
|
"logps/chosen": -0.2869907021522522,
|
|
"logps/rejected": -0.2831929326057434,
|
|
"loss": 1.0064929723739624,
|
|
"loss/core": 0.9767097234725952,
|
|
"loss/preference_sft": 0.284060537815094,
|
|
"loss/reference_anchor": 0.5672638416290283,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.626801013946533,
|
|
"rewards/margins": 2.499552011489868,
|
|
"rewards/rejected": 1.127249002456665,
|
|
"ronpo/logit": 0.2499551773071289,
|
|
"ronpo/residual": 0.2459290474653244,
|
|
"ronpo/residual_abs": 0.3209651708602905,
|
|
"ronpo/target": 0.004026124719530344,
|
|
"ronpo/target_abs": 0.07355201244354248,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26780807971954346,
|
|
"drift/rejected_abs": 0.08546146005392075,
|
|
"epoch": 0.11283273738577672,
|
|
"grad_norm": 1.890625,
|
|
"learning_rate": 7.600080639646076e-08,
|
|
"logits/chosen": -1.7517549991607666,
|
|
"logits/rejected": -1.759873390197754,
|
|
"logps/chosen": -0.3038514256477356,
|
|
"logps/rejected": -0.2689318358898163,
|
|
"loss": 0.7741256952285767,
|
|
"loss/core": 0.749835193157196,
|
|
"loss/preference_sft": 0.30647626519203186,
|
|
"loss/reference_anchor": 0.45516258478164673,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6770687103271484,
|
|
"rewards/margins": 1.8246443271636963,
|
|
"rewards/rejected": 0.8524240255355835,
|
|
"ronpo/logit": 0.18246445059776306,
|
|
"ronpo/residual": 0.15374180674552917,
|
|
"ronpo/residual_abs": 0.23904690146446228,
|
|
"ronpo/target": 0.02872266247868538,
|
|
"ronpo/target_abs": 0.0866011381149292,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16474132239818573,
|
|
"drift/rejected_abs": 0.12225530296564102,
|
|
"epoch": 0.11442193087008343,
|
|
"grad_norm": 106.0,
|
|
"learning_rate": 7.516775592930899e-08,
|
|
"logits/chosen": -1.9887319803237915,
|
|
"logits/rejected": -1.9524673223495483,
|
|
"logps/chosen": -0.27265065908432007,
|
|
"logps/rejected": -0.2855183184146881,
|
|
"loss": 0.09760759770870209,
|
|
"loss/core": 0.08684544265270233,
|
|
"loss/preference_sft": 0.2860459089279175,
|
|
"loss/reference_anchor": 0.18663868308067322,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.647254228591919,
|
|
"rewards/margins": 0.4279909133911133,
|
|
"rewards/rejected": 1.2192633152008057,
|
|
"ronpo/logit": 0.04279909282922745,
|
|
"ronpo/residual": 0.05301178619265556,
|
|
"ronpo/residual_abs": 0.15181580185890198,
|
|
"ronpo/target": -0.010212697088718414,
|
|
"ronpo/target_abs": 0.08592395484447479,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3150689899921417,
|
|
"drift/rejected_abs": 0.11829020828008652,
|
|
"epoch": 0.11601112435439015,
|
|
"grad_norm": 114.0,
|
|
"learning_rate": 7.4325240898438e-08,
|
|
"logits/chosen": -1.7951710224151611,
|
|
"logits/rejected": -1.8196980953216553,
|
|
"logps/chosen": -0.3030449450016022,
|
|
"logps/rejected": -0.3088149428367615,
|
|
"loss": 0.6980522871017456,
|
|
"loss/core": 0.6711905598640442,
|
|
"loss/preference_sft": 0.29767075181007385,
|
|
"loss/reference_anchor": 0.5074678659439087,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.149803876876831,
|
|
"rewards/margins": 1.9678528308868408,
|
|
"rewards/rejected": 1.1819511651992798,
|
|
"ronpo/logit": 0.19678528606891632,
|
|
"ronpo/residual": 0.2056920975446701,
|
|
"ronpo/residual_abs": 0.2885230481624603,
|
|
"ronpo/target": -0.00890684686601162,
|
|
"ronpo/target_abs": 0.08591794967651367,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2871250510215759,
|
|
"drift/rejected_abs": 0.19906646013259888,
|
|
"epoch": 0.11760031783869686,
|
|
"grad_norm": 6.375,
|
|
"learning_rate": 7.347357813929453e-08,
|
|
"logits/chosen": -1.6759065389633179,
|
|
"logits/rejected": -1.616633415222168,
|
|
"logps/chosen": -0.2482568770647049,
|
|
"logps/rejected": -0.2896507978439331,
|
|
"loss": 0.6905705332756042,
|
|
"loss/core": 0.6696237325668335,
|
|
"loss/preference_sft": 0.28166383504867554,
|
|
"loss/reference_anchor": 0.39076724648475647,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.871251106262207,
|
|
"rewards/margins": 0.9647668600082397,
|
|
"rewards/rejected": 1.9064840078353882,
|
|
"ronpo/logit": 0.09647669643163681,
|
|
"ronpo/residual": 0.09969191998243332,
|
|
"ronpo/residual_abs": 0.2865365743637085,
|
|
"ronpo/target": -0.003215217497199774,
|
|
"ronpo/target_abs": 0.08571871370077133,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5225389003753662,
|
|
"drift/rejected_abs": 0.21518293023109436,
|
|
"epoch": 0.11918951132300358,
|
|
"grad_norm": 33.0,
|
|
"learning_rate": 7.261308792741188e-08,
|
|
"logits/chosen": -1.641985297203064,
|
|
"logits/rejected": -1.6296402215957642,
|
|
"logps/chosen": -0.3135181665420532,
|
|
"logps/rejected": -0.31274524331092834,
|
|
"loss": 1.3733961582183838,
|
|
"loss/core": 1.2828857898712158,
|
|
"loss/preference_sft": 0.314228355884552,
|
|
"loss/reference_anchor": 1.7787834405899048,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 5.22520112991333,
|
|
"rewards/margins": 3.074826717376709,
|
|
"rewards/rejected": 2.150374412536621,
|
|
"ronpo/logit": 0.30748268961906433,
|
|
"ronpo/residual": 0.3128319978713989,
|
|
"ronpo/residual_abs": 0.36880940198898315,
|
|
"ronpo/target": -0.005349296145141125,
|
|
"ronpo/target_abs": 0.07712045311927795,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35133180022239685,
|
|
"drift/rejected_abs": 0.23328383266925812,
|
|
"epoch": 0.12077870480731029,
|
|
"grad_norm": 16.875,
|
|
"learning_rate": 7.174409385796725e-08,
|
|
"logits/chosen": -1.8136361837387085,
|
|
"logits/rejected": -1.7627443075180054,
|
|
"logps/chosen": -0.26291412115097046,
|
|
"logps/rejected": -0.26428166031837463,
|
|
"loss": 0.9764726758003235,
|
|
"loss/core": 0.9314364194869995,
|
|
"loss/preference_sft": 0.2695896327495575,
|
|
"loss/reference_anchor": 0.8737646341323853,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.5123963356018066,
|
|
"rewards/margins": 1.181367039680481,
|
|
"rewards/rejected": 2.3310296535491943,
|
|
"ronpo/logit": 0.11813666671514511,
|
|
"ronpo/residual": 0.12999659776687622,
|
|
"ronpo/residual_abs": 0.31305280327796936,
|
|
"ronpo/target": -0.011859911493957043,
|
|
"ronpo/target_abs": 0.0706958919763565,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36728110909461975,
|
|
"drift/rejected_abs": 0.14616873860359192,
|
|
"epoch": 0.12236789829161701,
|
|
"grad_norm": 7.125,
|
|
"learning_rate": 7.08669227240909e-08,
|
|
"logits/chosen": -1.8517367839813232,
|
|
"logits/rejected": -1.8667278289794922,
|
|
"logps/chosen": -0.28371182084083557,
|
|
"logps/rejected": -0.30156999826431274,
|
|
"loss": 0.6238721609115601,
|
|
"loss/core": 0.5842751860618591,
|
|
"loss/preference_sft": 0.2967758774757385,
|
|
"loss/reference_anchor": 0.7622602581977844,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.6717803478240967,
|
|
"rewards/margins": 2.2115306854248047,
|
|
"rewards/rejected": 1.4602497816085815,
|
|
"ronpo/logit": 0.22115305066108704,
|
|
"ronpo/residual": 0.2303275614976883,
|
|
"ronpo/residual_abs": 0.2650774121284485,
|
|
"ronpo/target": -0.009174478240311146,
|
|
"ronpo/target_abs": 0.06511794030666351,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22626559436321259,
|
|
"drift/rejected_abs": 0.08026406168937683,
|
|
"epoch": 0.12395709177592372,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 6.99819043939726e-08,
|
|
"logits/chosen": -1.9849250316619873,
|
|
"logits/rejected": -1.9792630672454834,
|
|
"logps/chosen": -0.2899254560470581,
|
|
"logps/rejected": -0.2959898114204407,
|
|
"loss": 0.16416332125663757,
|
|
"loss/core": 0.15644721686840057,
|
|
"loss/preference_sft": 0.2958739101886749,
|
|
"loss/reference_anchor": 0.12473498284816742,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.262193202972412,
|
|
"rewards/margins": 1.4670231342315674,
|
|
"rewards/rejected": 0.7951699495315552,
|
|
"ronpo/logit": 0.1467023342847824,
|
|
"ronpo/residual": 0.15477152168750763,
|
|
"ronpo/residual_abs": 0.1925136297941208,
|
|
"ronpo/target": -0.00806919950991869,
|
|
"ronpo/target_abs": 0.07871512323617935,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36781710386276245,
|
|
"drift/rejected_abs": 0.17324690520763397,
|
|
"epoch": 0.12554628526023043,
|
|
"grad_norm": 44.25,
|
|
"learning_rate": 6.908937168681174e-08,
|
|
"logits/chosen": -1.6767114400863647,
|
|
"logits/rejected": -1.7050321102142334,
|
|
"logps/chosen": -0.26027780771255493,
|
|
"logps/rejected": -0.25346577167510986,
|
|
"loss": 0.45207443833351135,
|
|
"loss/core": 0.40612488985061646,
|
|
"loss/preference_sft": 0.25639814138412476,
|
|
"loss/reference_anchor": 0.893352210521698,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.676452159881592,
|
|
"rewards/margins": 1.9461982250213623,
|
|
"rewards/rejected": 1.73025381565094,
|
|
"ronpo/logit": 0.19461986422538757,
|
|
"ronpo/residual": 0.19026371836662292,
|
|
"ronpo/residual_abs": 0.24404847621917725,
|
|
"ronpo/target": 0.0043561337515711784,
|
|
"ronpo/target_abs": 0.056930769234895706,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3046533465385437,
|
|
"drift/rejected_abs": 0.11980541050434113,
|
|
"epoch": 0.12713547874453715,
|
|
"grad_norm": 92.5,
|
|
"learning_rate": 6.818966024765757e-08,
|
|
"logits/chosen": -1.7593889236450195,
|
|
"logits/rejected": -1.7481002807617188,
|
|
"logps/chosen": -0.3030829429626465,
|
|
"logps/rejected": -0.29993414878845215,
|
|
"loss": 0.5101957321166992,
|
|
"loss/core": 0.48600858449935913,
|
|
"loss/preference_sft": 0.29708653688430786,
|
|
"loss/reference_anchor": 0.4540334641933441,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0456247329711914,
|
|
"rewards/margins": 1.8476349115371704,
|
|
"rewards/rejected": 1.1979897022247314,
|
|
"ronpo/logit": 0.18476352095603943,
|
|
"ronpo/residual": 0.19976118206977844,
|
|
"ronpo/residual_abs": 0.23570862412452698,
|
|
"ronpo/target": -0.014997666701674461,
|
|
"ronpo/target_abs": 0.07206665724515915,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2052384316921234,
|
|
"drift/rejected_abs": 0.11188337951898575,
|
|
"epoch": 0.12872467222884387,
|
|
"grad_norm": 21.75,
|
|
"learning_rate": 6.728310842118683e-08,
|
|
"logits/chosen": -2.14799427986145,
|
|
"logits/rejected": -2.0824573040008545,
|
|
"logps/chosen": -0.286243736743927,
|
|
"logps/rejected": -0.27005332708358765,
|
|
"loss": 0.06200665980577469,
|
|
"loss/core": 0.05436018109321594,
|
|
"loss/preference_sft": 0.27878865599632263,
|
|
"loss/reference_anchor": 0.1250506341457367,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.052384614944458,
|
|
"rewards/margins": 0.9343948364257812,
|
|
"rewards/rejected": 1.1179895401000977,
|
|
"ronpo/logit": 0.0934394896030426,
|
|
"ronpo/residual": 0.10573802143335342,
|
|
"ronpo/residual_abs": 0.15494385361671448,
|
|
"ronpo/target": -0.012298518791794777,
|
|
"ronpo/target_abs": 0.09242352098226547,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35525864362716675,
|
|
"drift/rejected_abs": 0.17476989328861237,
|
|
"epoch": 0.13031386571315057,
|
|
"grad_norm": 18.125,
|
|
"learning_rate": 6.637005712446621e-08,
|
|
"logits/chosen": -1.7883316278457642,
|
|
"logits/rejected": -1.7869304418563843,
|
|
"logps/chosen": -0.25297173857688904,
|
|
"logps/rejected": -0.2491494119167328,
|
|
"loss": 0.45122259855270386,
|
|
"loss/core": 0.42534032464027405,
|
|
"loss/preference_sft": 0.25509124994277954,
|
|
"loss/reference_anchor": 0.4921363890171051,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.551823377609253,
|
|
"rewards/margins": 1.8160899877548218,
|
|
"rewards/rejected": 1.7357333898544312,
|
|
"ronpo/logit": 0.18160900473594666,
|
|
"ronpo/residual": 0.1837209165096283,
|
|
"ronpo/residual_abs": 0.25326791405677795,
|
|
"ronpo/target": -0.002111931797116995,
|
|
"ronpo/target_abs": 0.06821735203266144,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25623852014541626,
|
|
"drift/rejected_abs": 0.1311134397983551,
|
|
"epoch": 0.1319030591974573,
|
|
"grad_norm": 45.5,
|
|
"learning_rate": 6.545084971874738e-08,
|
|
"logits/chosen": -1.7289890050888062,
|
|
"logits/rejected": -1.7614386081695557,
|
|
"logps/chosen": -0.2858942747116089,
|
|
"logps/rejected": -0.2846941351890564,
|
|
"loss": 0.18270006775856018,
|
|
"loss/core": 0.1690942347049713,
|
|
"loss/preference_sft": 0.2852400839328766,
|
|
"loss/reference_anchor": 0.24359269440174103,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.5602011680603027,
|
|
"rewards/margins": 1.2526819705963135,
|
|
"rewards/rejected": 1.307519555091858,
|
|
"ronpo/logit": 0.12526817619800568,
|
|
"ronpo/residual": 0.1053306832909584,
|
|
"ronpo/residual_abs": 0.18119731545448303,
|
|
"ronpo/target": 0.01993749849498272,
|
|
"ronpo/target_abs": 0.07559109479188919,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1505512297153473,
|
|
"drift/rejected_abs": 0.08253350853919983,
|
|
"epoch": 0.133492252681764,
|
|
"grad_norm": 12.125,
|
|
"learning_rate": 6.452583188034274e-08,
|
|
"logits/chosen": -1.7700755596160889,
|
|
"logits/rejected": -1.791124701499939,
|
|
"logps/chosen": -0.2845797538757324,
|
|
"logps/rejected": -0.2848334014415741,
|
|
"loss": 0.03450245410203934,
|
|
"loss/core": 0.030069872736930847,
|
|
"loss/preference_sft": 0.2898898720741272,
|
|
"loss/reference_anchor": 0.05966266989707947,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.5050227642059326,
|
|
"rewards/margins": 0.6809515953063965,
|
|
"rewards/rejected": 0.8240710496902466,
|
|
"ronpo/logit": 0.06809516996145248,
|
|
"ronpo/residual": 0.06789997965097427,
|
|
"ronpo/residual_abs": 0.11400087177753448,
|
|
"ronpo/target": 0.0001951886370079592,
|
|
"ronpo/target_abs": 0.07576659321784973,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2125011682510376,
|
|
"drift/rejected_abs": 0.09349431842565536,
|
|
"epoch": 0.1350814461660707,
|
|
"grad_norm": 2.640625,
|
|
"learning_rate": 6.35953514706309e-08,
|
|
"logits/chosen": -1.9663327932357788,
|
|
"logits/rejected": -1.9798023700714111,
|
|
"logps/chosen": -0.3189515769481659,
|
|
"logps/rejected": -0.3072357177734375,
|
|
"loss": 0.19334067404270172,
|
|
"loss/core": 0.1795506328344345,
|
|
"loss/preference_sft": 0.3102140724658966,
|
|
"loss/reference_anchor": 0.24477925896644592,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.124784231185913,
|
|
"rewards/margins": 1.1916717290878296,
|
|
"rewards/rejected": 0.933112621307373,
|
|
"ronpo/logit": 0.11916716396808624,
|
|
"ronpo/residual": 0.0928797796368599,
|
|
"ronpo/residual_abs": 0.16781887412071228,
|
|
"ronpo/target": 0.026287391781806946,
|
|
"ronpo/target_abs": 0.08866839855909348,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3195541203022003,
|
|
"drift/rejected_abs": 0.15836499631404877,
|
|
"epoch": 0.13667063965037743,
|
|
"grad_norm": 2.296875,
|
|
"learning_rate": 6.26597584052401e-08,
|
|
"logits/chosen": -1.5612452030181885,
|
|
"logits/rejected": -1.5029213428497314,
|
|
"logps/chosen": -0.285647451877594,
|
|
"logps/rejected": -0.29933539032936096,
|
|
"loss": 0.45964616537094116,
|
|
"loss/core": 0.4238843321800232,
|
|
"loss/preference_sft": 0.29544249176979065,
|
|
"loss/reference_anchor": 0.6856938004493713,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.19382905960083,
|
|
"rewards/margins": 1.6618378162384033,
|
|
"rewards/rejected": 1.5319918394088745,
|
|
"ronpo/logit": 0.166183739900589,
|
|
"ronpo/residual": 0.16860680282115936,
|
|
"ronpo/residual_abs": 0.22841337323188782,
|
|
"ronpo/target": -0.0024230708368122578,
|
|
"ronpo/target_abs": 0.08164582401514053,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5011860132217407,
|
|
"drift/rejected_abs": 0.2851537764072418,
|
|
"epoch": 0.13825983313468415,
|
|
"grad_norm": 1.859375,
|
|
"learning_rate": 6.171940452245923e-08,
|
|
"logits/chosen": -1.9892600774765015,
|
|
"logits/rejected": -1.9781463146209717,
|
|
"logps/chosen": -0.2973264157772064,
|
|
"logps/rejected": -0.28821566700935364,
|
|
"loss": 0.9089182019233704,
|
|
"loss/core": 0.8096088171005249,
|
|
"loss/preference_sft": 0.29011955857276917,
|
|
"loss/reference_anchor": 1.9571746587753296,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 5.010129451751709,
|
|
"rewards/margins": 2.1612067222595215,
|
|
"rewards/rejected": 2.8489227294921875,
|
|
"ronpo/logit": 0.2161206752061844,
|
|
"ronpo/residual": 0.20811843872070312,
|
|
"ronpo/residual_abs": 0.3104029595851898,
|
|
"ronpo/target": 0.008002226240932941,
|
|
"ronpo/target_abs": 0.0776105746626854,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34103599190711975,
|
|
"drift/rejected_abs": 0.1386711746454239,
|
|
"epoch": 0.13984902661899087,
|
|
"grad_norm": 480.0,
|
|
"learning_rate": 6.0774643450926e-08,
|
|
"logits/chosen": -1.8233058452606201,
|
|
"logits/rejected": -1.8174107074737549,
|
|
"logps/chosen": -0.278222918510437,
|
|
"logps/rejected": -0.28023761510849,
|
|
"loss": 0.6035203337669373,
|
|
"loss/core": 0.5771490335464478,
|
|
"loss/preference_sft": 0.28037887811660767,
|
|
"loss/reference_anchor": 0.49938860535621643,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.4100043773651123,
|
|
"rewards/margins": 2.0244264602661133,
|
|
"rewards/rejected": 1.3855777978897095,
|
|
"ronpo/logit": 0.20244267582893372,
|
|
"ronpo/residual": 0.2062930315732956,
|
|
"ronpo/residual_abs": 0.2548952102661133,
|
|
"ronpo/target": -0.003850378794595599,
|
|
"ronpo/target_abs": 0.07118948549032211,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15881577134132385,
|
|
"drift/rejected_abs": 0.07861863076686859,
|
|
"epoch": 0.14143822010329757,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 5.982583047664151e-08,
|
|
"logits/chosen": -1.8153941631317139,
|
|
"logits/rejected": -1.7571932077407837,
|
|
"logps/chosen": -0.29266512393951416,
|
|
"logps/rejected": -0.2830495536327362,
|
|
"loss": 0.06146319955587387,
|
|
"loss/core": 0.056963998824357986,
|
|
"loss/preference_sft": 0.2907244563102722,
|
|
"loss/reference_anchor": 0.06091168522834778,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5873725414276123,
|
|
"rewards/margins": 0.804255485534668,
|
|
"rewards/rejected": 0.7831171154975891,
|
|
"ronpo/logit": 0.08042554557323456,
|
|
"ronpo/residual": 0.06820248067378998,
|
|
"ronpo/residual_abs": 0.13618895411491394,
|
|
"ronpo/target": 0.01222306489944458,
|
|
"ronpo/target_abs": 0.0686340257525444,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27281489968299866,
|
|
"drift/rejected_abs": 0.15323896706104279,
|
|
"epoch": 0.1430274135876043,
|
|
"grad_norm": 5.6875,
|
|
"learning_rate": 5.8873322409361756e-08,
|
|
"logits/chosen": -1.9215360879898071,
|
|
"logits/rejected": -1.7960364818572998,
|
|
"logps/chosen": -0.2579748034477234,
|
|
"logps/rejected": -0.2565074563026428,
|
|
"loss": 0.11640655994415283,
|
|
"loss/core": 0.10340286791324615,
|
|
"loss/preference_sft": 0.2626219391822815,
|
|
"loss/reference_anchor": 0.2338119000196457,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.728148937225342,
|
|
"rewards/margins": 1.1970012187957764,
|
|
"rewards/rejected": 1.5311477184295654,
|
|
"ronpo/logit": 0.1197001114487648,
|
|
"ronpo/residual": 0.1095530167222023,
|
|
"ronpo/residual_abs": 0.1741444170475006,
|
|
"ronpo/target": 0.010147105902433395,
|
|
"ronpo/target_abs": 0.08469612896442413,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4300481677055359,
|
|
"drift/rejected_abs": 0.09889129549264908,
|
|
"epoch": 0.144616607071911,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 5.7917477448416144e-08,
|
|
"logits/chosen": -1.7946687936782837,
|
|
"logits/rejected": -1.744091272354126,
|
|
"logps/chosen": -0.28207752108573914,
|
|
"logps/rejected": -0.28602561354637146,
|
|
"loss": 1.9361851215362549,
|
|
"loss/core": 1.8742101192474365,
|
|
"loss/preference_sft": 0.2911958396434784,
|
|
"loss/reference_anchor": 1.2103819847106934,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.30048131942749,
|
|
"rewards/margins": 3.311803102493286,
|
|
"rewards/rejected": 0.9886780977249146,
|
|
"ronpo/logit": 0.3311803340911865,
|
|
"ronpo/residual": 0.3193929195404053,
|
|
"ronpo/residual_abs": 0.3614575266838074,
|
|
"ronpo/target": 0.011787394993007183,
|
|
"ronpo/target_abs": 0.06393725425004959,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2632841169834137,
|
|
"drift/rejected_abs": 0.06613259017467499,
|
|
"epoch": 0.1462058005562177,
|
|
"grad_norm": 26.625,
|
|
"learning_rate": 5.695865504800327e-08,
|
|
"logits/chosen": -1.7865066528320312,
|
|
"logits/rejected": -1.81333327293396,
|
|
"logps/chosen": -0.3158157467842102,
|
|
"logps/rejected": -0.3114406168460846,
|
|
"loss": 0.8839119076728821,
|
|
"loss/core": 0.8581841588020325,
|
|
"loss/preference_sft": 0.31136512756347656,
|
|
"loss/reference_anchor": 0.48341670632362366,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6306302547454834,
|
|
"rewards/margins": 1.9706141948699951,
|
|
"rewards/rejected": 0.6600159406661987,
|
|
"ronpo/logit": 0.1970614641904831,
|
|
"ronpo/residual": 0.20879793167114258,
|
|
"ronpo/residual_abs": 0.2626114785671234,
|
|
"ronpo/target": -0.011736475862562656,
|
|
"ronpo/target_abs": 0.08205690234899521,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18584024906158447,
|
|
"drift/rejected_abs": 0.1095375269651413,
|
|
"epoch": 0.14779499404052443,
|
|
"grad_norm": 3.09375,
|
|
"learning_rate": 5.5997215782014984e-08,
|
|
"logits/chosen": -1.861520767211914,
|
|
"logits/rejected": -1.8145278692245483,
|
|
"logps/chosen": -0.3011169731616974,
|
|
"logps/rejected": -0.30149930715560913,
|
|
"loss": 0.086545430123806,
|
|
"loss/core": 0.07661891728639603,
|
|
"loss/preference_sft": 0.3033149540424347,
|
|
"loss/reference_anchor": 0.1681988537311554,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8580764532089233,
|
|
"rewards/margins": 0.7628976106643677,
|
|
"rewards/rejected": 1.0951789617538452,
|
|
"ronpo/logit": 0.07628976553678513,
|
|
"ronpo/residual": 0.07566403597593307,
|
|
"ronpo/residual_abs": 0.14903251826763153,
|
|
"ronpo/target": 0.0006257258355617523,
|
|
"ronpo/target_abs": 0.08669845014810562,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2513282001018524,
|
|
"drift/rejected_abs": 0.14881765842437744,
|
|
"epoch": 0.14938418752483115,
|
|
"grad_norm": 1.421875,
|
|
"learning_rate": 5.503352120843923e-08,
|
|
"logits/chosen": -1.9602091312408447,
|
|
"logits/rejected": -1.9950196743011475,
|
|
"logps/chosen": -0.32254284620285034,
|
|
"logps/rejected": -0.3045181632041931,
|
|
"loss": 0.2901255190372467,
|
|
"loss/core": 0.25516438484191895,
|
|
"loss/preference_sft": 0.31007856130599976,
|
|
"loss/reference_anchor": 0.6682136654853821,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.511671304702759,
|
|
"rewards/margins": 1.0251829624176025,
|
|
"rewards/rejected": 1.4864884614944458,
|
|
"ronpo/logit": 0.10251828283071518,
|
|
"ronpo/residual": 0.10790930688381195,
|
|
"ronpo/residual_abs": 0.15721961855888367,
|
|
"ronpo/target": -0.0053910380229353905,
|
|
"ronpo/target_abs": 0.06892824172973633,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2127540111541748,
|
|
"drift/rejected_abs": 0.11244404315948486,
|
|
"epoch": 0.15097338100913787,
|
|
"grad_norm": 9.4375,
|
|
"learning_rate": 5.406793373339291e-08,
|
|
"logits/chosen": -1.941626787185669,
|
|
"logits/rejected": -1.9039710760116577,
|
|
"logps/chosen": -0.271634042263031,
|
|
"logps/rejected": -0.25962454080581665,
|
|
"loss": 0.07151190936565399,
|
|
"loss/core": 0.06459919363260269,
|
|
"loss/preference_sft": 0.267228901386261,
|
|
"loss/reference_anchor": 0.11153163760900497,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.12636137008667,
|
|
"rewards/margins": 1.0033315420150757,
|
|
"rewards/rejected": 1.1230299472808838,
|
|
"ronpo/logit": 0.10033313184976578,
|
|
"ronpo/residual": 0.10426007211208344,
|
|
"ronpo/residual_abs": 0.15038594603538513,
|
|
"ronpo/target": -0.003926927223801613,
|
|
"ronpo/target_abs": 0.06808660924434662,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14452792704105377,
|
|
"drift/rejected_abs": 0.05248863250017166,
|
|
"epoch": 0.15256257449344457,
|
|
"grad_norm": 3.234375,
|
|
"learning_rate": 5.3100816474835766e-08,
|
|
"logits/chosen": -1.8934004306793213,
|
|
"logits/rejected": -1.9814293384552002,
|
|
"logps/chosen": -0.316048800945282,
|
|
"logps/rejected": -0.3190820813179016,
|
|
"loss": 0.05717045068740845,
|
|
"loss/core": 0.05321327596902847,
|
|
"loss/preference_sft": 0.3178454637527466,
|
|
"loss/reference_anchor": 0.04735885187983513,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.4452793598175049,
|
|
"rewards/margins": 0.9231868982315063,
|
|
"rewards/rejected": 0.5220924019813538,
|
|
"ronpo/logit": 0.09231869131326675,
|
|
"ronpo/residual": 0.07715944945812225,
|
|
"ronpo/residual_abs": 0.14198298752307892,
|
|
"ronpo/target": 0.015159247443079948,
|
|
"ronpo/target_abs": 0.09165415912866592,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2202528715133667,
|
|
"drift/rejected_abs": 0.1520129144191742,
|
|
"epoch": 0.1541517679777513,
|
|
"grad_norm": 43.5,
|
|
"learning_rate": 5.2132533126016534e-08,
|
|
"logits/chosen": -1.76580810546875,
|
|
"logits/rejected": -1.7142798900604248,
|
|
"logps/chosen": -0.28421875834465027,
|
|
"logps/rejected": -0.2967900335788727,
|
|
"loss": 0.08789822459220886,
|
|
"loss/core": 0.06753416359424591,
|
|
"loss/preference_sft": 0.30561891198158264,
|
|
"loss/reference_anchor": 0.376719206571579,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.201965808868408,
|
|
"rewards/margins": 0.6839362978935242,
|
|
"rewards/rejected": 1.5180293321609497,
|
|
"ronpo/logit": 0.06839364767074585,
|
|
"ronpo/residual": 0.0788511410355568,
|
|
"ronpo/residual_abs": 0.14319899678230286,
|
|
"ronpo/target": -0.01045750267803669,
|
|
"ronpo/target_abs": 0.07381303608417511,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3343612253665924,
|
|
"drift/rejected_abs": 0.13808700442314148,
|
|
"epoch": 0.15574096146205801,
|
|
"grad_norm": 24.25,
|
|
"learning_rate": 5.116344781870281e-08,
|
|
"logits/chosen": -1.6695568561553955,
|
|
"logits/rejected": -1.6447023153305054,
|
|
"logps/chosen": -0.24842624366283417,
|
|
"logps/rejected": -0.24762670695781708,
|
|
"loss": 0.2929847836494446,
|
|
"loss/core": 0.2731058895587921,
|
|
"loss/preference_sft": 0.2550978660583496,
|
|
"loss/reference_anchor": 0.37206801772117615,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.3418400287628174,
|
|
"rewards/margins": 1.9618446826934814,
|
|
"rewards/rejected": 1.3799952268600464,
|
|
"ronpo/logit": 0.19618447124958038,
|
|
"ronpo/residual": 0.1800888329744339,
|
|
"ronpo/residual_abs": 0.23794646561145782,
|
|
"ronpo/target": 0.016095636412501335,
|
|
"ronpo/target_abs": 0.07803904265165329,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.08378258347511292,
|
|
"drift/rejected_abs": 0.03460133820772171,
|
|
"epoch": 0.1573301549463647,
|
|
"grad_norm": 21.625,
|
|
"learning_rate": 5.0193924986246017e-08,
|
|
"logits/chosen": -1.8778587579727173,
|
|
"logits/rejected": -1.8371680974960327,
|
|
"logps/chosen": -0.30237627029418945,
|
|
"logps/rejected": -0.3027265667915344,
|
|
"loss": 0.032751940190792084,
|
|
"loss/core": 0.03037204220890999,
|
|
"loss/preference_sft": 0.30147770047187805,
|
|
"loss/reference_anchor": 0.017450321465730667,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.8357017636299133,
|
|
"rewards/margins": 0.496019184589386,
|
|
"rewards/rejected": 0.3396826386451721,
|
|
"ronpo/logit": 0.04960191622376442,
|
|
"ronpo/residual": 0.06800566613674164,
|
|
"ronpo/residual_abs": 0.10949333012104034,
|
|
"ronpo/target": -0.01840374991297722,
|
|
"ronpo/target_abs": 0.07336892932653427,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3098334074020386,
|
|
"drift/rejected_abs": 0.20627370476722717,
|
|
"epoch": 0.15891934843067143,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 4.9224329226532834e-08,
|
|
"logits/chosen": -1.7887645959854126,
|
|
"logits/rejected": -1.8265126943588257,
|
|
"logps/chosen": -0.30098414421081543,
|
|
"logps/rejected": -0.28809601068496704,
|
|
"loss": 0.17970222234725952,
|
|
"loss/core": 0.1251877248287201,
|
|
"loss/preference_sft": 0.29191064834594727,
|
|
"loss/reference_anchor": 1.0610988140106201,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.0972111225128174,
|
|
"rewards/margins": 1.035546898841858,
|
|
"rewards/rejected": 2.06166410446167,
|
|
"ronpo/logit": 0.10355468839406967,
|
|
"ronpo/residual": 0.11979351192712784,
|
|
"ronpo/residual_abs": 0.17769846320152283,
|
|
"ronpo/target": -0.01623881608247757,
|
|
"ronpo/target_abs": 0.08119514584541321,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4369746744632721,
|
|
"drift/rejected_abs": 0.14313063025474548,
|
|
"epoch": 0.16050854191497815,
|
|
"grad_norm": 0.73046875,
|
|
"learning_rate": 4.8255025164874966e-08,
|
|
"logits/chosen": -1.9329090118408203,
|
|
"logits/rejected": -1.8906736373901367,
|
|
"logps/chosen": -0.2954661250114441,
|
|
"logps/rejected": -0.2762017846107483,
|
|
"loss": 1.6327224969863892,
|
|
"loss/core": 1.5709314346313477,
|
|
"loss/preference_sft": 0.2684187591075897,
|
|
"loss/reference_anchor": 1.2089756727218628,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.366913795471191,
|
|
"rewards/margins": 2.9390738010406494,
|
|
"rewards/rejected": 1.427840232849121,
|
|
"ronpo/logit": 0.29390740394592285,
|
|
"ronpo/residual": 0.29641279578208923,
|
|
"ronpo/residual_abs": 0.33722934126853943,
|
|
"ronpo/target": -0.0025054272264242172,
|
|
"ronpo/target_abs": 0.06659742444753647,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19818530976772308,
|
|
"drift/rejected_abs": 0.09616676717996597,
|
|
"epoch": 0.16209773539928488,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 4.7286377316888315e-08,
|
|
"logits/chosen": -1.8848209381103516,
|
|
"logits/rejected": -1.9005283117294312,
|
|
"logps/chosen": -0.2789894938468933,
|
|
"logps/rejected": -0.29110604524612427,
|
|
"loss": 0.12926748394966125,
|
|
"loss/core": 0.1197998970746994,
|
|
"loss/preference_sft": 0.27562230825424194,
|
|
"loss/reference_anchor": 0.16178929805755615,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.980821967124939,
|
|
"rewards/margins": 1.0209791660308838,
|
|
"rewards/rejected": 0.9598425626754761,
|
|
"ronpo/logit": 0.10209791362285614,
|
|
"ronpo/residual": 0.10504169762134552,
|
|
"ronpo/residual_abs": 0.1421099603176117,
|
|
"ronpo/target": -0.002943767700344324,
|
|
"ronpo/target_abs": 0.0699705258011818,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33460456132888794,
|
|
"drift/rejected_abs": 0.1129327192902565,
|
|
"epoch": 0.16368692888359157,
|
|
"grad_norm": 6.21875,
|
|
"learning_rate": 4.631874995141376e-08,
|
|
"logits/chosen": -1.8654136657714844,
|
|
"logits/rejected": -1.799949288368225,
|
|
"logps/chosen": -0.2905055284500122,
|
|
"logps/rejected": -0.328683078289032,
|
|
"loss": 0.9674808382987976,
|
|
"loss/core": 0.9366046190261841,
|
|
"loss/preference_sft": 0.32833415269851685,
|
|
"loss/reference_anchor": 0.5846928954124451,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.344911575317383,
|
|
"rewards/margins": 2.230219602584839,
|
|
"rewards/rejected": 1.1146920919418335,
|
|
"ronpo/logit": 0.2230219542980194,
|
|
"ronpo/residual": 0.25300055742263794,
|
|
"ronpo/residual_abs": 0.2966272830963135,
|
|
"ronpo/target": -0.02997860684990883,
|
|
"ronpo/target_abs": 0.08517836779356003,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28033939003944397,
|
|
"drift/rejected_abs": 0.10343112051486969,
|
|
"epoch": 0.1652761223678983,
|
|
"grad_norm": 1.4375,
|
|
"learning_rate": 4.535250695353039e-08,
|
|
"logits/chosen": -1.6896569728851318,
|
|
"logits/rejected": -1.6575727462768555,
|
|
"logps/chosen": -0.3145434856414795,
|
|
"logps/rejected": -0.30273348093032837,
|
|
"loss": 0.3629056215286255,
|
|
"loss/core": 0.34335726499557495,
|
|
"loss/preference_sft": 0.30437609553337097,
|
|
"loss/reference_anchor": 0.3605298101902008,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.802736759185791,
|
|
"rewards/margins": 1.7711127996444702,
|
|
"rewards/rejected": 1.0316243171691895,
|
|
"ronpo/logit": 0.17711128294467926,
|
|
"ronpo/residual": 0.19178500771522522,
|
|
"ronpo/residual_abs": 0.23747721314430237,
|
|
"ronpo/target": -0.014673741534352303,
|
|
"ronpo/target_abs": 0.07631038129329681,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23427505791187286,
|
|
"drift/rejected_abs": 0.19274473190307617,
|
|
"epoch": 0.16686531585220502,
|
|
"grad_norm": 0.458984375,
|
|
"learning_rate": 4.438801168771327e-08,
|
|
"logits/chosen": -1.811059594154358,
|
|
"logits/rejected": -1.7457149028778076,
|
|
"logps/chosen": -0.2789601981639862,
|
|
"logps/rejected": -0.2822405695915222,
|
|
"loss": 0.6040679216384888,
|
|
"loss/core": 0.5695381760597229,
|
|
"loss/preference_sft": 0.28496214747428894,
|
|
"loss/reference_anchor": 0.6620967984199524,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.342750310897827,
|
|
"rewards/margins": 0.4186655580997467,
|
|
"rewards/rejected": 1.9240849018096924,
|
|
"ronpo/logit": 0.04186655953526497,
|
|
"ronpo/residual": 0.058896519243717194,
|
|
"ronpo/residual_abs": 0.2368089258670807,
|
|
"ronpo/target": -0.017029952257871628,
|
|
"ronpo/target_abs": 0.065000519156456,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27734851837158203,
|
|
"drift/rejected_abs": 0.15080618858337402,
|
|
"epoch": 0.1684545093365117,
|
|
"grad_norm": 18.25,
|
|
"learning_rate": 4.342562686118687e-08,
|
|
"logits/chosen": -1.7826459407806396,
|
|
"logits/rejected": -1.716897964477539,
|
|
"logps/chosen": -0.2615172564983368,
|
|
"logps/rejected": -0.2668178975582123,
|
|
"loss": 0.23560409247875214,
|
|
"loss/core": 0.22138634324073792,
|
|
"loss/preference_sft": 0.2622404992580414,
|
|
"loss/reference_anchor": 0.25813111662864685,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.771690845489502,
|
|
"rewards/margins": 1.2636291980743408,
|
|
"rewards/rejected": 1.5080616474151611,
|
|
"ronpo/logit": 0.12636291980743408,
|
|
"ronpo/residual": 0.13701412081718445,
|
|
"ronpo/residual_abs": 0.22391745448112488,
|
|
"ronpo/target": -0.010651206597685814,
|
|
"ronpo/target_abs": 0.08514300733804703,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3595048785209656,
|
|
"drift/rejected_abs": 0.1292239874601364,
|
|
"epoch": 0.17004370282081843,
|
|
"grad_norm": 450.0,
|
|
"learning_rate": 4.2465714387525845e-08,
|
|
"logits/chosen": -1.6385431289672852,
|
|
"logits/rejected": -1.6657865047454834,
|
|
"logps/chosen": -0.2958716154098511,
|
|
"logps/rejected": -0.3102319836616516,
|
|
"loss": 1.2631784677505493,
|
|
"loss/core": 1.2100608348846436,
|
|
"loss/preference_sft": 0.29885178804397583,
|
|
"loss/reference_anchor": 1.0324665307998657,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.5940539836883545,
|
|
"rewards/margins": 2.3138844966888428,
|
|
"rewards/rejected": 1.2801692485809326,
|
|
"ronpo/logit": 0.23138847947120667,
|
|
"ronpo/residual": 0.22915032505989075,
|
|
"ronpo/residual_abs": 0.2899910509586334,
|
|
"ronpo/target": 0.002238139044493437,
|
|
"ronpo/target_abs": 0.07926853001117706,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3161032199859619,
|
|
"drift/rejected_abs": 0.10872749239206314,
|
|
"epoch": 0.17163289630512515,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 4.1508635250553965e-08,
|
|
"logits/chosen": -1.6535122394561768,
|
|
"logits/rejected": -1.7463783025741577,
|
|
"logps/chosen": -0.27848881483078003,
|
|
"logps/rejected": -0.271538108587265,
|
|
"loss": 0.9267969131469727,
|
|
"loss/core": 0.8974257707595825,
|
|
"loss/preference_sft": 0.27611082792282104,
|
|
"loss/reference_anchor": 0.5598097443580627,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.160801410675049,
|
|
"rewards/margins": 2.073526620864868,
|
|
"rewards/rejected": 1.0872750282287598,
|
|
"ronpo/logit": 0.20735260844230652,
|
|
"ronpo/residual": 0.19190707802772522,
|
|
"ronpo/residual_abs": 0.25170207023620605,
|
|
"ronpo/target": 0.01544556487351656,
|
|
"ronpo/target_abs": 0.08709506690502167,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3257554769515991,
|
|
"drift/rejected_abs": 0.13702979683876038,
|
|
"epoch": 0.17322208978943188,
|
|
"grad_norm": 32.25,
|
|
"learning_rate": 4.0554749368592995e-08,
|
|
"logits/chosen": -1.6410595178604126,
|
|
"logits/rejected": -1.6608667373657227,
|
|
"logps/chosen": -0.3060685992240906,
|
|
"logps/rejected": -0.29526740312576294,
|
|
"loss": 0.48791199922561646,
|
|
"loss/core": 0.46756306290626526,
|
|
"loss/preference_sft": 0.30378422141075134,
|
|
"loss/reference_anchor": 0.3766012191772461,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.256690263748169,
|
|
"rewards/margins": 1.887966513633728,
|
|
"rewards/rejected": 1.368723750114441,
|
|
"ronpo/logit": 0.18879660964012146,
|
|
"ronpo/residual": 0.17176969349384308,
|
|
"ronpo/residual_abs": 0.2844948470592499,
|
|
"ronpo/target": 0.017026938498020172,
|
|
"ronpo/target_abs": 0.08832515776157379,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22399315237998962,
|
|
"drift/rejected_abs": 0.10357632488012314,
|
|
"epoch": 0.17481128327373857,
|
|
"grad_norm": 22.0,
|
|
"learning_rate": 3.960441545911204e-08,
|
|
"logits/chosen": -1.787977933883667,
|
|
"logits/rejected": -1.812804937362671,
|
|
"logps/chosen": -0.2800469994544983,
|
|
"logps/rejected": -0.27289098501205444,
|
|
"loss": 0.11282701790332794,
|
|
"loss/core": 0.10225673019886017,
|
|
"loss/preference_sft": 0.28042763471603394,
|
|
"loss/reference_anchor": 0.18336261808872223,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.2391538619995117,
|
|
"rewards/margins": 1.2041218280792236,
|
|
"rewards/rejected": 1.035032033920288,
|
|
"ronpo/logit": 0.12041217088699341,
|
|
"ronpo/residual": 0.10191146284341812,
|
|
"ronpo/residual_abs": 0.15801844000816345,
|
|
"ronpo/target": 0.018500715494155884,
|
|
"ronpo/target_abs": 0.08216994255781174,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22719137370586395,
|
|
"drift/rejected_abs": 0.0889752134680748,
|
|
"epoch": 0.1764004767580453,
|
|
"grad_norm": 7.78125,
|
|
"learning_rate": 3.8657990903828655e-08,
|
|
"logits/chosen": -1.6774036884307861,
|
|
"logits/rejected": -1.6927173137664795,
|
|
"logps/chosen": -0.28917351365089417,
|
|
"logps/rejected": -0.28399351239204407,
|
|
"loss": 0.5557877421379089,
|
|
"loss/core": 0.531394898891449,
|
|
"loss/preference_sft": 0.2850340008735657,
|
|
"loss/reference_anchor": 0.45935481786727905,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.271148204803467,
|
|
"rewards/margins": 1.386685848236084,
|
|
"rewards/rejected": 0.8844623565673828,
|
|
"ronpo/logit": 0.13866856694221497,
|
|
"ronpo/residual": 0.12557737529277802,
|
|
"ronpo/residual_abs": 0.18068963289260864,
|
|
"ronpo/target": 0.01309121958911419,
|
|
"ronpo/target_abs": 0.08298710733652115,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32667970657348633,
|
|
"drift/rejected_abs": 0.13467362523078918,
|
|
"epoch": 0.17798967024235202,
|
|
"grad_norm": 23.125,
|
|
"learning_rate": 3.7715831614312174e-08,
|
|
"logits/chosen": -1.8457578420639038,
|
|
"logits/rejected": -1.8797744512557983,
|
|
"logps/chosen": -0.2537483274936676,
|
|
"logps/rejected": -0.25457605719566345,
|
|
"loss": 0.4204334616661072,
|
|
"loss/core": 0.3994172215461731,
|
|
"loss/preference_sft": 0.2603244185447693,
|
|
"loss/reference_anchor": 0.3942931294441223,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.265937328338623,
|
|
"rewards/margins": 1.921112298965454,
|
|
"rewards/rejected": 1.3448251485824585,
|
|
"ronpo/logit": 0.19211125373840332,
|
|
"ronpo/residual": 0.20365771651268005,
|
|
"ronpo/residual_abs": 0.2606772780418396,
|
|
"ronpo/target": -0.011546471156179905,
|
|
"ronpo/target_abs": 0.08113471418619156,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18193916976451874,
|
|
"drift/rejected_abs": 0.08163003623485565,
|
|
"epoch": 0.1795788637266587,
|
|
"grad_norm": 2.234375,
|
|
"learning_rate": 3.67782918981399e-08,
|
|
"logits/chosen": -1.9215097427368164,
|
|
"logits/rejected": -1.9562021493911743,
|
|
"logps/chosen": -0.2751021981239319,
|
|
"logps/rejected": -0.27950841188430786,
|
|
"loss": 0.14423240721225739,
|
|
"loss/core": 0.13524220883846283,
|
|
"loss/preference_sft": 0.27696993947029114,
|
|
"loss/reference_anchor": 0.15210723876953125,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.818294882774353,
|
|
"rewards/margins": 1.004157304763794,
|
|
"rewards/rejected": 0.8141376376152039,
|
|
"ronpo/logit": 0.10041572898626328,
|
|
"ronpo/residual": 0.11555007845163345,
|
|
"ronpo/residual_abs": 0.1486656218767166,
|
|
"ronpo/target": -0.01513433177024126,
|
|
"ronpo/target_abs": 0.07270049303770065,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30289047956466675,
|
|
"drift/rejected_abs": 0.1840471625328064,
|
|
"epoch": 0.18116805721096543,
|
|
"grad_norm": 6.96875,
|
|
"learning_rate": 3.584572432565648e-08,
|
|
"logits/chosen": -1.8963152170181274,
|
|
"logits/rejected": -1.8974689245224,
|
|
"logps/chosen": -0.2739769220352173,
|
|
"logps/rejected": -0.26712292432785034,
|
|
"loss": 0.3248220384120941,
|
|
"loss/core": 0.3049502372741699,
|
|
"loss/preference_sft": 0.2694453001022339,
|
|
"loss/reference_anchor": 0.3704921007156372,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.028883934020996,
|
|
"rewards/margins": 1.1917301416397095,
|
|
"rewards/rejected": 1.837153673171997,
|
|
"ronpo/logit": 0.11917302757501602,
|
|
"ronpo/residual": 0.1130208745598793,
|
|
"ronpo/residual_abs": 0.2363136261701584,
|
|
"ronpo/target": 0.00615213718265295,
|
|
"ronpo/target_abs": 0.08978766947984695,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24188709259033203,
|
|
"drift/rejected_abs": 0.09579716622829437,
|
|
"epoch": 0.18275725069527216,
|
|
"grad_norm": 81.5,
|
|
"learning_rate": 3.491847959738672e-08,
|
|
"logits/chosen": -1.8621025085449219,
|
|
"logits/rejected": -1.7978849411010742,
|
|
"logps/chosen": -0.2750481963157654,
|
|
"logps/rejected": -0.27491137385368347,
|
|
"loss": 0.15762899816036224,
|
|
"loss/core": 0.14700618386268616,
|
|
"loss/preference_sft": 0.2757367789745331,
|
|
"loss/reference_anchor": 0.18488280475139618,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4188714027404785,
|
|
"rewards/margins": 1.4699872732162476,
|
|
"rewards/rejected": 0.9488840103149414,
|
|
"ronpo/logit": 0.14699873328208923,
|
|
"ronpo/residual": 0.12900659441947937,
|
|
"ronpo/residual_abs": 0.19406011700630188,
|
|
"ronpo/target": 0.01799212209880352,
|
|
"ronpo/target_abs": 0.0778588205575943,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.42244258522987366,
|
|
"drift/rejected_abs": 0.14777334034442902,
|
|
"epoch": 0.18434644417957885,
|
|
"grad_norm": 85.5,
|
|
"learning_rate": 3.3996906412151415e-08,
|
|
"logits/chosen": -1.7508312463760376,
|
|
"logits/rejected": -1.7235362529754639,
|
|
"logps/chosen": -0.2758569121360779,
|
|
"logps/rejected": -0.2773497700691223,
|
|
"loss": 0.6645770072937012,
|
|
"loss/core": 0.6348112225532532,
|
|
"loss/preference_sft": 0.2726094722747803,
|
|
"loss/reference_anchor": 0.5680543780326843,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.22373104095459,
|
|
"rewards/margins": 2.7611031532287598,
|
|
"rewards/rejected": 1.4626275300979614,
|
|
"ronpo/logit": 0.27611032128334045,
|
|
"ronpo/residual": 0.27662983536720276,
|
|
"ronpo/residual_abs": 0.3160479664802551,
|
|
"ronpo/target": -0.0005195157718844712,
|
|
"ronpo/target_abs": 0.0704670175909996,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2685272991657257,
|
|
"drift/rejected_abs": 0.12745855748653412,
|
|
"epoch": 0.18593563766388557,
|
|
"grad_norm": 8.25,
|
|
"learning_rate": 3.3081351335935947e-08,
|
|
"logits/chosen": -1.864542007446289,
|
|
"logits/rejected": -1.882070541381836,
|
|
"logps/chosen": -0.2766827344894409,
|
|
"logps/rejected": -0.2663271725177765,
|
|
"loss": 0.3190165162086487,
|
|
"loss/core": 0.29371339082717896,
|
|
"loss/preference_sft": 0.27629172801971436,
|
|
"loss/reference_anchor": 0.47843486070632935,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.683490753173828,
|
|
"rewards/margins": 1.414543867111206,
|
|
"rewards/rejected": 1.268946886062622,
|
|
"ronpo/logit": 0.14145438373088837,
|
|
"ronpo/residual": 0.11811937391757965,
|
|
"ronpo/residual_abs": 0.18924079835414886,
|
|
"ronpo/target": 0.02333502098917961,
|
|
"ronpo/target_abs": 0.07983358204364777,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28528568148612976,
|
|
"drift/rejected_abs": 0.1476709544658661,
|
|
"epoch": 0.1875248311481923,
|
|
"grad_norm": 2.765625,
|
|
"learning_rate": 3.2172158671561e-08,
|
|
"logits/chosen": -1.8741655349731445,
|
|
"logits/rejected": -1.9201557636260986,
|
|
"logps/chosen": -0.29076990485191345,
|
|
"logps/rejected": -0.28063392639160156,
|
|
"loss": 0.3805859088897705,
|
|
"loss/core": 0.36150652170181274,
|
|
"loss/preference_sft": 0.2858158349990845,
|
|
"loss/reference_anchor": 0.3530064523220062,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8528571128845215,
|
|
"rewards/margins": 1.3815311193466187,
|
|
"rewards/rejected": 1.4713258743286133,
|
|
"ronpo/logit": 0.1381531059741974,
|
|
"ronpo/residual": 0.14053115248680115,
|
|
"ronpo/residual_abs": 0.2531353533267975,
|
|
"ronpo/target": -0.002378053730353713,
|
|
"ronpo/target_abs": 0.07857644557952881,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1880657821893692,
|
|
"drift/rejected_abs": 0.08449472486972809,
|
|
"epoch": 0.18911402463249902,
|
|
"grad_norm": 5.9375,
|
|
"learning_rate": 3.126967032920439e-08,
|
|
"logits/chosen": -1.7578903436660767,
|
|
"logits/rejected": -1.783585786819458,
|
|
"logps/chosen": -0.28262588381767273,
|
|
"logps/rejected": -0.2848055958747864,
|
|
"loss": 0.07938707619905472,
|
|
"loss/core": 0.07369506359100342,
|
|
"loss/preference_sft": 0.28935831785202026,
|
|
"loss/reference_anchor": 0.08490435779094696,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8778693675994873,
|
|
"rewards/margins": 1.0337949991226196,
|
|
"rewards/rejected": 0.8440744280815125,
|
|
"ronpo/logit": 0.1033795103430748,
|
|
"ronpo/residual": 0.09837718307971954,
|
|
"ronpo/residual_abs": 0.14326299726963043,
|
|
"ronpo/target": 0.005002323072403669,
|
|
"ronpo/target_abs": 0.06280878931283951,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.38983526825904846,
|
|
"drift/rejected_abs": 0.1542014479637146,
|
|
"epoch": 0.1907032181168057,
|
|
"grad_norm": 6.0625,
|
|
"learning_rate": 3.037422569782264e-08,
|
|
"logits/chosen": -1.7056328058242798,
|
|
"logits/rejected": -1.6530392169952393,
|
|
"logps/chosen": -0.2509880065917969,
|
|
"logps/rejected": -0.2619604170322418,
|
|
"loss": 0.701583206653595,
|
|
"loss/core": 0.663438081741333,
|
|
"loss/preference_sft": 0.25877368450164795,
|
|
"loss/reference_anchor": 0.7370229959487915,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.898181200027466,
|
|
"rewards/margins": 2.3588569164276123,
|
|
"rewards/rejected": 1.5393238067626953,
|
|
"ronpo/logit": 0.23588573932647705,
|
|
"ronpo/residual": 0.23551622033119202,
|
|
"ronpo/residual_abs": 0.29292362928390503,
|
|
"ronpo/target": 0.0003694974002428353,
|
|
"ronpo/target_abs": 0.08346446603536606,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.38783174753189087,
|
|
"drift/rejected_abs": 0.19440807402133942,
|
|
"epoch": 0.19229241160111243,
|
|
"grad_norm": 944.0,
|
|
"learning_rate": 2.9486161517520596e-08,
|
|
"logits/chosen": -1.8185880184173584,
|
|
"logits/rejected": -1.771431565284729,
|
|
"logps/chosen": -0.28466832637786865,
|
|
"logps/rejected": -0.28105106949806213,
|
|
"loss": 0.8451844453811646,
|
|
"loss/core": 0.7918490171432495,
|
|
"loss/preference_sft": 0.2889874577522278,
|
|
"loss/reference_anchor": 1.0378073453903198,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.8768718242645264,
|
|
"rewards/margins": 1.943351149559021,
|
|
"rewards/rejected": 1.9335205554962158,
|
|
"ronpo/logit": 0.19433513283729553,
|
|
"ronpo/residual": 0.17206700146198273,
|
|
"ronpo/residual_abs": 0.29958412051200867,
|
|
"ronpo/target": 0.02226812019944191,
|
|
"ronpo/target_abs": 0.08312074840068817,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3009849190711975,
|
|
"drift/rejected_abs": 0.11645524203777313,
|
|
"epoch": 0.19388160508541916,
|
|
"grad_norm": 177.0,
|
|
"learning_rate": 2.8605811752917298e-08,
|
|
"logits/chosen": -1.9999271631240845,
|
|
"logits/rejected": -2.000976085662842,
|
|
"logps/chosen": -0.27779993414878845,
|
|
"logps/rejected": -0.277484655380249,
|
|
"loss": 0.6784787178039551,
|
|
"loss/core": 0.6391760110855103,
|
|
"loss/preference_sft": 0.2794664800167084,
|
|
"loss/reference_anchor": 0.7581073641777039,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.006049394607544,
|
|
"rewards/margins": 1.8458268642425537,
|
|
"rewards/rejected": 1.1602224111557007,
|
|
"ronpo/logit": 0.1845826953649521,
|
|
"ronpo/residual": 0.18958279490470886,
|
|
"ronpo/residual_abs": 0.23554134368896484,
|
|
"ronpo/target": -0.005000096745789051,
|
|
"ronpo/target_abs": 0.07321993261575699,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27367550134658813,
|
|
"drift/rejected_abs": 0.08648597449064255,
|
|
"epoch": 0.19547079856972585,
|
|
"grad_norm": 1.4296875,
|
|
"learning_rate": 2.7733507467555527e-08,
|
|
"logits/chosen": -1.92385733127594,
|
|
"logits/rejected": -1.9347591400146484,
|
|
"logps/chosen": -0.294109970331192,
|
|
"logps/rejected": -0.27960658073425293,
|
|
"loss": 0.4980928897857666,
|
|
"loss/core": 0.48153653740882874,
|
|
"loss/preference_sft": 0.2906952500343323,
|
|
"loss/reference_anchor": 0.30205675959587097,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.735163688659668,
|
|
"rewards/margins": 1.8733736276626587,
|
|
"rewards/rejected": 0.8617900609970093,
|
|
"ronpo/logit": 0.18733736872673035,
|
|
"ronpo/residual": 0.18723754584789276,
|
|
"ronpo/residual_abs": 0.24290697276592255,
|
|
"ronpo/target": 9.980481263482943e-05,
|
|
"ronpo/target_abs": 0.07993032038211823,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11056061089038849,
|
|
"drift/rejected_abs": 0.0920964851975441,
|
|
"epoch": 0.19705999205403257,
|
|
"grad_norm": 197.0,
|
|
"learning_rate": 2.686957669940242e-08,
|
|
"logits/chosen": -1.8443567752838135,
|
|
"logits/rejected": -1.8728549480438232,
|
|
"logps/chosen": -0.3104780912399292,
|
|
"logps/rejected": -0.30508318543434143,
|
|
"loss": 0.1757979691028595,
|
|
"loss/core": 0.16883358359336853,
|
|
"loss/preference_sft": 0.303710013628006,
|
|
"loss/reference_anchor": 0.108916737139225,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.1045570373535156,
|
|
"rewards/margins": 0.18372417986392975,
|
|
"rewards/rejected": 0.9208329319953918,
|
|
"ronpo/logit": 0.018372418358922005,
|
|
"ronpo/residual": 0.047217488288879395,
|
|
"ronpo/residual_abs": 0.1612224578857422,
|
|
"ronpo/target": -0.02884506806731224,
|
|
"ronpo/target_abs": 0.08857301622629166,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24403142929077148,
|
|
"drift/rejected_abs": 0.1381409615278244,
|
|
"epoch": 0.1986491855383393,
|
|
"grad_norm": 3.203125,
|
|
"learning_rate": 2.6014344337487702e-08,
|
|
"logits/chosen": -1.6158983707427979,
|
|
"logits/rejected": -1.693192720413208,
|
|
"logps/chosen": -0.29910343885421753,
|
|
"logps/rejected": -0.2826142907142639,
|
|
"loss": 0.07637186348438263,
|
|
"loss/core": 0.06501691043376923,
|
|
"loss/preference_sft": 0.29273831844329834,
|
|
"loss/reference_anchor": 0.1978251039981842,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4398252964019775,
|
|
"rewards/margins": 1.059569001197815,
|
|
"rewards/rejected": 1.3802564144134521,
|
|
"ronpo/logit": 0.10595688968896866,
|
|
"ronpo/residual": 0.10343287140130997,
|
|
"ronpo/residual_abs": 0.15932440757751465,
|
|
"ronpo/target": 0.0025240257382392883,
|
|
"ronpo/target_abs": 0.08598555624485016,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.263141006231308,
|
|
"drift/rejected_abs": 0.11765114963054657,
|
|
"epoch": 0.20023837902264602,
|
|
"grad_norm": 6.4375,
|
|
"learning_rate": 2.51681319997262e-08,
|
|
"logits/chosen": -2.0076048374176025,
|
|
"logits/rejected": -1.9558988809585571,
|
|
"logps/chosen": -0.27460214495658875,
|
|
"logps/rejected": -0.26767414808273315,
|
|
"loss": 0.26412975788116455,
|
|
"loss/core": 0.24851731956005096,
|
|
"loss/preference_sft": 0.2709651291370392,
|
|
"loss/reference_anchor": 0.2851526141166687,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6304500102996826,
|
|
"rewards/margins": 1.4610058069229126,
|
|
"rewards/rejected": 1.1694444417953491,
|
|
"ronpo/logit": 0.14610058069229126,
|
|
"ronpo/residual": 0.13935787975788116,
|
|
"ronpo/residual_abs": 0.1994233876466751,
|
|
"ronpo/target": 0.006742709316313267,
|
|
"ronpo/target_abs": 0.0730559304356575,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16716927289962769,
|
|
"drift/rejected_abs": 0.07392685115337372,
|
|
"epoch": 0.2018275725069527,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 2.4331257911970626e-08,
|
|
"logits/chosen": -1.8764339685440063,
|
|
"logits/rejected": -1.900081992149353,
|
|
"logps/chosen": -0.3134706914424896,
|
|
"logps/rejected": -0.3003923296928406,
|
|
"loss": 0.12685158848762512,
|
|
"loss/core": 0.12000010162591934,
|
|
"loss/preference_sft": 0.312440425157547,
|
|
"loss/reference_anchor": 0.10578552633523941,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.670975923538208,
|
|
"rewards/margins": 0.9369431734085083,
|
|
"rewards/rejected": 0.7340327501296997,
|
|
"ronpo/logit": 0.09369431436061859,
|
|
"ronpo/residual": 0.08684712648391724,
|
|
"ronpo/residual_abs": 0.1398160457611084,
|
|
"ronpo/target": 0.0068471841514110565,
|
|
"ronpo/target_abs": 0.0784897580742836,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34248116612434387,
|
|
"drift/rejected_abs": 0.08185670524835587,
|
|
"epoch": 0.20341676599125944,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 2.350403678833976e-08,
|
|
"logits/chosen": -1.7811704874038696,
|
|
"logits/rejected": -1.7476812601089478,
|
|
"logps/chosen": -0.316486656665802,
|
|
"logps/rejected": -0.3071419894695282,
|
|
"loss": 1.2131757736206055,
|
|
"loss/core": 1.1782453060150146,
|
|
"loss/preference_sft": 0.30921900272369385,
|
|
"loss/reference_anchor": 0.667685866355896,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4235236644744873,
|
|
"rewards/margins": 2.605436086654663,
|
|
"rewards/rejected": 0.8180872797966003,
|
|
"ronpo/logit": 0.2605436146259308,
|
|
"ronpo/residual": 0.2597568929195404,
|
|
"ronpo/residual_abs": 0.3174801170825958,
|
|
"ronpo/target": 0.0007867075619287789,
|
|
"ronpo/target_abs": 0.0725146010518074,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3487434983253479,
|
|
"drift/rejected_abs": 0.1786487102508545,
|
|
"epoch": 0.20500595947556616,
|
|
"grad_norm": 31.875,
|
|
"learning_rate": 2.2686779712867315e-08,
|
|
"logits/chosen": -1.9565203189849854,
|
|
"logits/rejected": -1.954850435256958,
|
|
"logps/chosen": -0.29713040590286255,
|
|
"logps/rejected": -0.29292789101600647,
|
|
"loss": 0.609022319316864,
|
|
"loss/core": 0.548900842666626,
|
|
"loss/preference_sft": 0.2938844561576843,
|
|
"loss/reference_anchor": 1.1730408668518066,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.4874351024627686,
|
|
"rewards/margins": 1.7022377252578735,
|
|
"rewards/rejected": 1.7851974964141846,
|
|
"ronpo/logit": 0.17022375762462616,
|
|
"ronpo/residual": 0.17084741592407227,
|
|
"ronpo/residual_abs": 0.2177102118730545,
|
|
"ronpo/target": -0.0006236616754904389,
|
|
"ronpo/target_abs": 0.0669097825884819,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2442498654127121,
|
|
"drift/rejected_abs": 0.09864034503698349,
|
|
"epoch": 0.20659515295987285,
|
|
"grad_norm": 35.5,
|
|
"learning_rate": 2.1879794022516e-08,
|
|
"logits/chosen": -1.7124214172363281,
|
|
"logits/rejected": -1.7760320901870728,
|
|
"logps/chosen": -0.2731096148490906,
|
|
"logps/rejected": -0.28077295422554016,
|
|
"loss": 0.12899014353752136,
|
|
"loss/core": 0.11936305463314056,
|
|
"loss/preference_sft": 0.2820122241973877,
|
|
"loss/reference_anchor": 0.16434051096439362,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.4420547485351562,
|
|
"rewards/margins": 1.4615581035614014,
|
|
"rewards/rejected": 0.9804967641830444,
|
|
"ronpo/logit": 0.14615580439567566,
|
|
"ronpo/residual": 0.1428360939025879,
|
|
"ronpo/residual_abs": 0.18947355449199677,
|
|
"ronpo/target": 0.003319724230095744,
|
|
"ronpo/target_abs": 0.0828058049082756,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18947014212608337,
|
|
"drift/rejected_abs": 0.0865272730588913,
|
|
"epoch": 0.20818434644417957,
|
|
"grad_norm": 48.25,
|
|
"learning_rate": 2.1083383191600673e-08,
|
|
"logits/chosen": -1.6375888586044312,
|
|
"logits/rejected": -1.6597034931182861,
|
|
"logps/chosen": -0.2796572744846344,
|
|
"logps/rejected": -0.2848132252693176,
|
|
"loss": 0.2575990557670593,
|
|
"loss/core": 0.2444666624069214,
|
|
"loss/preference_sft": 0.28200775384902954,
|
|
"loss/reference_anchor": 0.23444685339927673,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8931496143341064,
|
|
"rewards/margins": 1.028186321258545,
|
|
"rewards/rejected": 0.8649630546569824,
|
|
"ronpo/logit": 0.10281863063573837,
|
|
"ronpo/residual": 0.09756249189376831,
|
|
"ronpo/residual_abs": 0.17437264323234558,
|
|
"ronpo/target": 0.005256150849163532,
|
|
"ronpo/target_abs": 0.0813509151339531,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2058045119047165,
|
|
"drift/rejected_abs": 0.0914444550871849,
|
|
"epoch": 0.2097735399284863,
|
|
"grad_norm": 192.0,
|
|
"learning_rate": 2.029784671766404e-08,
|
|
"logits/chosen": -1.7744375467300415,
|
|
"logits/rejected": -1.7810022830963135,
|
|
"logps/chosen": -0.2940472662448883,
|
|
"logps/rejected": -0.29379576444625854,
|
|
"loss": 0.24170386791229248,
|
|
"loss/core": 0.23180270195007324,
|
|
"loss/preference_sft": 0.2991836667060852,
|
|
"loss/reference_anchor": 0.1681046038866043,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.056238889694214,
|
|
"rewards/margins": 1.14398193359375,
|
|
"rewards/rejected": 0.9122568368911743,
|
|
"ronpo/logit": 0.11439820379018784,
|
|
"ronpo/residual": 0.10298512876033783,
|
|
"ronpo/residual_abs": 0.164739191532135,
|
|
"ronpo/target": 0.01141307596117258,
|
|
"ronpo/target_abs": 0.07934392988681793,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.38957634568214417,
|
|
"drift/rejected_abs": 0.11693358421325684,
|
|
"epoch": 0.21136273341279302,
|
|
"grad_norm": 5.28125,
|
|
"learning_rate": 1.9523480008847854e-08,
|
|
"logits/chosen": -1.8623239994049072,
|
|
"logits/rejected": -1.796321153640747,
|
|
"logps/chosen": -0.27734166383743286,
|
|
"logps/rejected": -0.2643659710884094,
|
|
"loss": 0.7925857305526733,
|
|
"loss/core": 0.7652046084403992,
|
|
"loss/preference_sft": 0.2695058584213257,
|
|
"loss/reference_anchor": 0.5206728577613831,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.895763397216797,
|
|
"rewards/margins": 2.7316036224365234,
|
|
"rewards/rejected": 1.1641600131988525,
|
|
"ronpo/logit": 0.2731603682041168,
|
|
"ronpo/residual": 0.28218016028404236,
|
|
"ronpo/residual_abs": 0.3127972483634949,
|
|
"ronpo/target": -0.009019814431667328,
|
|
"ronpo/target_abs": 0.07987822592258453,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14632907509803772,
|
|
"drift/rejected_abs": 0.0827428549528122,
|
|
"epoch": 0.2129519268970997,
|
|
"grad_norm": 6.8125,
|
|
"learning_rate": 1.8760574272802e-08,
|
|
"logits/chosen": -1.893527626991272,
|
|
"logits/rejected": -1.864617943763733,
|
|
"logps/chosen": -0.2893826961517334,
|
|
"logps/rejected": -0.28469884395599365,
|
|
"loss": 0.048207398504018784,
|
|
"loss/core": 0.04376941919326782,
|
|
"loss/preference_sft": 0.29032936692237854,
|
|
"loss/reference_anchor": 0.05972672253847122,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.463290810585022,
|
|
"rewards/margins": 0.6385823488235474,
|
|
"rewards/rejected": 0.8247084617614746,
|
|
"ronpo/logit": 0.06385823339223862,
|
|
"ronpo/residual": 0.04385846480727196,
|
|
"ronpo/residual_abs": 0.13213634490966797,
|
|
"ronpo/target": 0.019999774172902107,
|
|
"ronpo/target_abs": 0.09107968956232071,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1413107067346573,
|
|
"drift/rejected_abs": 0.06459660083055496,
|
|
"epoch": 0.21454112038140644,
|
|
"grad_norm": 3.078125,
|
|
"learning_rate": 1.8009416407173257e-08,
|
|
"logits/chosen": -1.669947624206543,
|
|
"logits/rejected": -1.8324556350708008,
|
|
"logps/chosen": -0.31188154220581055,
|
|
"logps/rejected": -0.3008914887905121,
|
|
"loss": 0.0633026510477066,
|
|
"loss/core": 0.05928937345743179,
|
|
"loss/preference_sft": 0.30907323956489563,
|
|
"loss/reference_anchor": 0.04935837537050247,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.4124871492385864,
|
|
"rewards/margins": 0.767764687538147,
|
|
"rewards/rejected": 0.6447224020957947,
|
|
"ronpo/logit": 0.07677648216485977,
|
|
"ronpo/residual": 0.06589711457490921,
|
|
"ronpo/residual_abs": 0.1341782808303833,
|
|
"ronpo/target": 0.010879358276724815,
|
|
"ronpo/target_abs": 0.07572633028030396,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25216197967529297,
|
|
"drift/rejected_abs": 0.13002386689186096,
|
|
"epoch": 0.21613031386571316,
|
|
"grad_norm": 9.875,
|
|
"learning_rate": 1.7270288891714813e-08,
|
|
"logits/chosen": -1.821359634399414,
|
|
"logits/rejected": -1.7911430597305298,
|
|
"logps/chosen": -0.2525402307510376,
|
|
"logps/rejected": -0.26482558250427246,
|
|
"loss": 0.3033674359321594,
|
|
"loss/core": 0.2915551960468292,
|
|
"loss/preference_sft": 0.2592369616031647,
|
|
"loss/reference_anchor": 0.2103213369846344,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5206844806671143,
|
|
"rewards/margins": 1.2219209671020508,
|
|
"rewards/rejected": 1.2987635135650635,
|
|
"ronpo/logit": 0.12219210714101791,
|
|
"ronpo/residual": 0.10521011054515839,
|
|
"ronpo/residual_abs": 0.25379353761672974,
|
|
"ronpo/target": 0.016981983557343483,
|
|
"ronpo/target_abs": 0.08764125406742096,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2599492371082306,
|
|
"drift/rejected_abs": 0.15403486788272858,
|
|
"epoch": 0.21771950735001985,
|
|
"grad_norm": 7.5,
|
|
"learning_rate": 1.6543469682057103e-08,
|
|
"logits/chosen": -1.9823789596557617,
|
|
"logits/rejected": -1.9565290212631226,
|
|
"logps/chosen": -0.267892062664032,
|
|
"logps/rejected": -0.2650378346443176,
|
|
"loss": 0.14402912557125092,
|
|
"loss/core": 0.1269291490316391,
|
|
"loss/preference_sft": 0.26532235741615295,
|
|
"loss/reference_anchor": 0.3154672384262085,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5976312160491943,
|
|
"rewards/margins": 1.058511734008789,
|
|
"rewards/rejected": 1.5391194820404053,
|
|
"ronpo/logit": 0.1058511734008789,
|
|
"ronpo/residual": 0.12076152861118317,
|
|
"ronpo/residual_abs": 0.1801002025604248,
|
|
"ronpo/target": -0.014910360798239708,
|
|
"ronpo/target_abs": 0.08301638811826706,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18989941477775574,
|
|
"drift/rejected_abs": 0.08928252756595612,
|
|
"epoch": 0.21930870083432658,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 1.582923210518014e-08,
|
|
"logits/chosen": -1.8569204807281494,
|
|
"logits/rejected": -1.8808708190917969,
|
|
"logps/chosen": -0.2738683819770813,
|
|
"logps/rejected": -0.277884304523468,
|
|
"loss": 0.1312539279460907,
|
|
"loss/core": 0.12408691644668579,
|
|
"loss/preference_sft": 0.2749662697315216,
|
|
"loss/reference_anchor": 0.11584333330392838,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.898453950881958,
|
|
"rewards/margins": 1.0071128606796265,
|
|
"rewards/rejected": 0.8913409113883972,
|
|
"ronpo/logit": 0.10071128606796265,
|
|
"ronpo/residual": 0.09246021509170532,
|
|
"ronpo/residual_abs": 0.14917293190956116,
|
|
"ronpo/target": 0.0082510681822896,
|
|
"ronpo/target_abs": 0.07735707610845566,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2127593457698822,
|
|
"drift/rejected_abs": 0.1324995905160904,
|
|
"epoch": 0.2208978943186333,
|
|
"grad_norm": 1.7734375,
|
|
"learning_rate": 1.5127844756626436e-08,
|
|
"logits/chosen": -1.8886617422103882,
|
|
"logits/rejected": -1.866326093673706,
|
|
"logps/chosen": -0.30845409631729126,
|
|
"logps/rejected": -0.311231791973114,
|
|
"loss": 0.07940120995044708,
|
|
"loss/core": 0.06841110438108444,
|
|
"loss/preference_sft": 0.3118094503879547,
|
|
"loss/reference_anchor": 0.18862099945545197,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.124119281768799,
|
|
"rewards/margins": 0.8049348592758179,
|
|
"rewards/rejected": 1.31918466091156,
|
|
"ronpo/logit": 0.0804934874176979,
|
|
"ronpo/residual": 0.08477647602558136,
|
|
"ronpo/residual_abs": 0.15998438000679016,
|
|
"ronpo/target": -0.004282998386770487,
|
|
"ronpo/target_abs": 0.08868735283613205,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2849193811416626,
|
|
"drift/rejected_abs": 0.15217965841293335,
|
|
"epoch": 0.22248708780294002,
|
|
"grad_norm": 3.828125,
|
|
"learning_rate": 1.4439571399493145e-08,
|
|
"logits/chosen": -1.804674506187439,
|
|
"logits/rejected": -1.758469581604004,
|
|
"logps/chosen": -0.268559992313385,
|
|
"logps/rejected": -0.25594258308410645,
|
|
"loss": 0.3818897306919098,
|
|
"loss/core": 0.36068207025527954,
|
|
"loss/preference_sft": 0.27696743607521057,
|
|
"loss/reference_anchor": 0.3964560031890869,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.849193811416626,
|
|
"rewards/margins": 1.3282945156097412,
|
|
"rewards/rejected": 1.5208990573883057,
|
|
"ronpo/logit": 0.132829487323761,
|
|
"ronpo/residual": 0.14115296304225922,
|
|
"ronpo/residual_abs": 0.24943992495536804,
|
|
"ronpo/target": -0.008323474787175655,
|
|
"ronpo/target_abs": 0.07197760045528412,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32905009388923645,
|
|
"drift/rejected_abs": 0.09508834779262543,
|
|
"epoch": 0.22407628128724671,
|
|
"grad_norm": 25.125,
|
|
"learning_rate": 1.3764670865241557e-08,
|
|
"logits/chosen": -1.7177156209945679,
|
|
"logits/rejected": -1.6808580160140991,
|
|
"logps/chosen": -0.26480594277381897,
|
|
"logps/rejected": -0.27780020236968994,
|
|
"loss": 1.043757677078247,
|
|
"loss/core": 1.009283423423767,
|
|
"loss/preference_sft": 0.2664128839969635,
|
|
"loss/reference_anchor": 0.6628433465957642,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.290501117706299,
|
|
"rewards/margins": 2.3396174907684326,
|
|
"rewards/rejected": 0.9508835673332214,
|
|
"ronpo/logit": 0.2339617758989334,
|
|
"ronpo/residual": 0.24024538695812225,
|
|
"ronpo/residual_abs": 0.28473344445228577,
|
|
"ronpo/target": -0.00628363573923707,
|
|
"ronpo/target_abs": 0.07632976025342941,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4384216368198395,
|
|
"drift/rejected_abs": 0.1377977877855301,
|
|
"epoch": 0.22566547477155344,
|
|
"grad_norm": 41.5,
|
|
"learning_rate": 1.310339695636118e-08,
|
|
"logits/chosen": -1.8576635122299194,
|
|
"logits/rejected": -1.8414652347564697,
|
|
"logps/chosen": -0.30472978949546814,
|
|
"logps/rejected": -0.3077087104320526,
|
|
"loss": 1.5751392841339111,
|
|
"loss/core": 1.5171544551849365,
|
|
"loss/preference_sft": 0.3141063153743744,
|
|
"loss/reference_anchor": 1.1282850503921509,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.384188652038574,
|
|
"rewards/margins": 3.014103651046753,
|
|
"rewards/rejected": 1.37008535861969,
|
|
"ronpo/logit": 0.30141040682792664,
|
|
"ronpo/residual": 0.3095789849758148,
|
|
"ronpo/residual_abs": 0.35598164796829224,
|
|
"ronpo/target": -0.008168593980371952,
|
|
"ronpo/target_abs": 0.08209376782178879,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2829729914665222,
|
|
"drift/rejected_abs": 0.10287122428417206,
|
|
"epoch": 0.22725466825586016,
|
|
"grad_norm": 3.40625,
|
|
"learning_rate": 1.245599835092504e-08,
|
|
"logits/chosen": -1.7704976797103882,
|
|
"logits/rejected": -1.8326141834259033,
|
|
"logps/chosen": -0.29713183641433716,
|
|
"logps/rejected": -0.3228703439235687,
|
|
"loss": 0.8810839653015137,
|
|
"loss/core": 0.8518657684326172,
|
|
"loss/preference_sft": 0.30574506521224976,
|
|
"loss/reference_anchor": 0.5537921786308289,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.8288185596466064,
|
|
"rewards/margins": 1.8033968210220337,
|
|
"rewards/rejected": 1.0254218578338623,
|
|
"ronpo/logit": 0.18033969402313232,
|
|
"ronpo/residual": 0.15812399983406067,
|
|
"ronpo/residual_abs": 0.22622855007648468,
|
|
"ronpo/target": 0.02221568673849106,
|
|
"ronpo/target_abs": 0.07745952159166336,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4834979474544525,
|
|
"drift/rejected_abs": 0.14388954639434814,
|
|
"epoch": 0.22884386174016685,
|
|
"grad_norm": 10.0625,
|
|
"learning_rate": 1.1822718509071988e-08,
|
|
"logits/chosen": -1.8110580444335938,
|
|
"logits/rejected": -1.8221015930175781,
|
|
"logps/chosen": -0.2708401083946228,
|
|
"logps/rejected": -0.2723672091960907,
|
|
"loss": 2.1321358680725098,
|
|
"loss/core": 2.061873435974121,
|
|
"loss/preference_sft": 0.2761738896369934,
|
|
"loss/reference_anchor": 1.3776265382766724,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.834713459014893,
|
|
"rewards/margins": 3.3967089653015137,
|
|
"rewards/rejected": 1.4380043745040894,
|
|
"ronpo/logit": 0.33967092633247375,
|
|
"ronpo/residual": 0.3569709062576294,
|
|
"ronpo/residual_abs": 0.4112846255302429,
|
|
"ronpo/target": -0.017299985513091087,
|
|
"ronpo/target_abs": 0.08841768652200699,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24089869856834412,
|
|
"drift/rejected_abs": 0.11333000659942627,
|
|
"epoch": 0.23043305522447358,
|
|
"grad_norm": 9.3125,
|
|
"learning_rate": 1.1203795581451287e-08,
|
|
"logits/chosen": -1.7944285869598389,
|
|
"logits/rejected": -1.7804477214813232,
|
|
"logps/chosen": -0.3111926019191742,
|
|
"logps/rejected": -0.3086337447166443,
|
|
"loss": 0.23198828101158142,
|
|
"loss/core": 0.2124747484922409,
|
|
"loss/preference_sft": 0.31827235221862793,
|
|
"loss/reference_anchor": 0.35844409465789795,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.3976962566375732,
|
|
"rewards/margins": 1.275817632675171,
|
|
"rewards/rejected": 1.121878743171692,
|
|
"ronpo/logit": 0.12758177518844604,
|
|
"ronpo/residual": 0.1312047243118286,
|
|
"ronpo/residual_abs": 0.1830698698759079,
|
|
"ronpo/target": -0.0036229542456567287,
|
|
"ronpo/target_abs": 0.07441584765911102,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18596777319908142,
|
|
"drift/rejected_abs": 0.09318926930427551,
|
|
"epoch": 0.2320222487087803,
|
|
"grad_norm": 91.5,
|
|
"learning_rate": 1.0599462319663904e-08,
|
|
"logits/chosen": -1.8789104223251343,
|
|
"logits/rejected": -1.7996902465820312,
|
|
"logps/chosen": -0.28050804138183594,
|
|
"logps/rejected": -0.28717318177223206,
|
|
"loss": 0.09919708967208862,
|
|
"loss/core": 0.08663090318441391,
|
|
"loss/preference_sft": 0.28598660230636597,
|
|
"loss/reference_anchor": 0.22272494435310364,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.859379768371582,
|
|
"rewards/margins": 0.9287394285202026,
|
|
"rewards/rejected": 0.9306403398513794,
|
|
"ronpo/logit": 0.0928739532828331,
|
|
"ronpo/residual": 0.10799574851989746,
|
|
"ronpo/residual_abs": 0.14936821162700653,
|
|
"ronpo/target": -0.015121802687644958,
|
|
"ronpo/target_abs": 0.07535012811422348,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18436908721923828,
|
|
"drift/rejected_abs": 0.08182942867279053,
|
|
"epoch": 0.23361144219308702,
|
|
"grad_norm": 2.96875,
|
|
"learning_rate": 1.0009945988734203e-08,
|
|
"logits/chosen": -1.8670095205307007,
|
|
"logits/rejected": -1.9019635915756226,
|
|
"logps/chosen": -0.2799551486968994,
|
|
"logps/rejected": -0.27838417887687683,
|
|
"loss": 0.13352563977241516,
|
|
"loss/core": 0.12648555636405945,
|
|
"loss/preference_sft": 0.2852904796600342,
|
|
"loss/reference_anchor": 0.11227299273014069,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.843457818031311,
|
|
"rewards/margins": 1.0267233848571777,
|
|
"rewards/rejected": 0.8167344331741333,
|
|
"ronpo/logit": 0.10267233848571777,
|
|
"ronpo/residual": 0.10309547185897827,
|
|
"ronpo/residual_abs": 0.14217627048492432,
|
|
"ronpo/target": -0.00042313747690059245,
|
|
"ronpo/target_abs": 0.07198002189397812,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18157462775707245,
|
|
"drift/rejected_abs": 0.05101984739303589,
|
|
"epoch": 0.23520063567739372,
|
|
"grad_norm": 3.703125,
|
|
"learning_rate": 9.435468281644798e-09,
|
|
"logits/chosen": -1.789648413658142,
|
|
"logits/rejected": -1.8189513683319092,
|
|
"logps/chosen": -0.2933686375617981,
|
|
"logps/rejected": -0.2890208959579468,
|
|
"loss": 0.39400720596313477,
|
|
"loss/core": 0.38300806283950806,
|
|
"loss/preference_sft": 0.2970552444458008,
|
|
"loss/reference_anchor": 0.19027724862098694,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.814532995223999,
|
|
"rewards/margins": 1.3092584609985352,
|
|
"rewards/rejected": 0.5052744150161743,
|
|
"ronpo/logit": 0.13092584908008575,
|
|
"ronpo/residual": 0.12319651991128922,
|
|
"ronpo/residual_abs": 0.16983821988105774,
|
|
"ronpo/target": 0.0077293342910707,
|
|
"ronpo/target_abs": 0.06624205410480499,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3139849007129669,
|
|
"drift/rejected_abs": 0.12468956410884857,
|
|
"epoch": 0.23678982916170044,
|
|
"grad_norm": 7.1875,
|
|
"learning_rate": 8.876245235966883e-09,
|
|
"logits/chosen": -1.5578440427780151,
|
|
"logits/rejected": -1.514190673828125,
|
|
"logps/chosen": -0.2828544080257416,
|
|
"logps/rejected": -0.29736337065696716,
|
|
"loss": 0.2802901864051819,
|
|
"loss/core": 0.265177845954895,
|
|
"loss/preference_sft": 0.29184380173683167,
|
|
"loss/reference_anchor": 0.27306291460990906,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1378657817840576,
|
|
"rewards/margins": 1.8915119171142578,
|
|
"rewards/rejected": 1.2463538646697998,
|
|
"ronpo/logit": 0.18915119767189026,
|
|
"ronpo/residual": 0.19307444989681244,
|
|
"ronpo/residual_abs": 0.24125421047210693,
|
|
"ronpo/target": -0.0039232466369867325,
|
|
"ronpo/target_abs": 0.08785430341959,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3995332419872284,
|
|
"drift/rejected_abs": 0.20359914004802704,
|
|
"epoch": 0.23837902264600716,
|
|
"grad_norm": 12.625,
|
|
"learning_rate": 8.332487152617423e-09,
|
|
"logits/chosen": -1.831875205039978,
|
|
"logits/rejected": -1.8069343566894531,
|
|
"logps/chosen": -0.29953733086586,
|
|
"logps/rejected": -0.282090961933136,
|
|
"loss": 1.5924561023712158,
|
|
"loss/core": 1.5354974269866943,
|
|
"loss/preference_sft": 0.28296080231666565,
|
|
"loss/reference_anchor": 1.1108777523040771,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.993727207183838,
|
|
"rewards/margins": 1.9586591720581055,
|
|
"rewards/rejected": 2.035067558288574,
|
|
"ronpo/logit": 0.19586597383022308,
|
|
"ronpo/residual": 0.18231366574764252,
|
|
"ronpo/residual_abs": 0.3800063729286194,
|
|
"ronpo/target": 0.013552306219935417,
|
|
"ronpo/target_abs": 0.06323491036891937,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4130985140800476,
|
|
"drift/rejected_abs": 0.10734666883945465,
|
|
"epoch": 0.23996821613031386,
|
|
"grad_norm": 37.25,
|
|
"learning_rate": 7.804398516773464e-09,
|
|
"logits/chosen": -1.844090461730957,
|
|
"logits/rejected": -1.9883720874786377,
|
|
"logps/chosen": -0.298724889755249,
|
|
"logps/rejected": -0.29675233364105225,
|
|
"loss": 1.6366679668426514,
|
|
"loss/core": 1.5810120105743408,
|
|
"loss/preference_sft": 0.3001899719238281,
|
|
"loss/reference_anchor": 1.083099365234375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.13097620010376,
|
|
"rewards/margins": 3.073918342590332,
|
|
"rewards/rejected": 1.0570578575134277,
|
|
"ronpo/logit": 0.30739185214042664,
|
|
"ronpo/residual": 0.29710453748703003,
|
|
"ronpo/residual_abs": 0.3430529236793518,
|
|
"ronpo/target": 0.01028729323297739,
|
|
"ronpo/target_abs": 0.06950847059488297,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3958708643913269,
|
|
"drift/rejected_abs": 0.11953407526016235,
|
|
"epoch": 0.24155740961462058,
|
|
"grad_norm": 17.75,
|
|
"learning_rate": 7.292177920973724e-09,
|
|
"logits/chosen": -1.7642419338226318,
|
|
"logits/rejected": -1.7521655559539795,
|
|
"logps/chosen": -0.2835713028907776,
|
|
"logps/rejected": -0.2796153426170349,
|
|
"loss": 1.3200641870498657,
|
|
"loss/core": 1.2784984111785889,
|
|
"loss/preference_sft": 0.2840017080307007,
|
|
"loss/reference_anchor": 0.8029199838638306,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.9578804969787598,
|
|
"rewards/margins": 2.7641587257385254,
|
|
"rewards/rejected": 1.1937220096588135,
|
|
"ronpo/logit": 0.2764158248901367,
|
|
"ronpo/residual": 0.26121872663497925,
|
|
"ronpo/residual_abs": 0.325788676738739,
|
|
"ronpo/target": 0.015197129920125008,
|
|
"ronpo/target_abs": 0.07956832647323608,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2987801730632782,
|
|
"drift/rejected_abs": 0.1019306555390358,
|
|
"epoch": 0.2431466030989273,
|
|
"grad_norm": 1.65625,
|
|
"learning_rate": 6.796017990435976e-09,
|
|
"logits/chosen": -1.7064396142959595,
|
|
"logits/rejected": -1.7161552906036377,
|
|
"logps/chosen": -0.27210524678230286,
|
|
"logps/rejected": -0.26053616404533386,
|
|
"loss": 0.3539239466190338,
|
|
"loss/core": 0.3408985733985901,
|
|
"loss/preference_sft": 0.26584452390670776,
|
|
"loss/reference_anchor": 0.23392312228679657,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.987363338470459,
|
|
"rewards/margins": 1.9795618057250977,
|
|
"rewards/rejected": 1.0078017711639404,
|
|
"ronpo/logit": 0.1979561746120453,
|
|
"ronpo/residual": 0.19465027749538422,
|
|
"ronpo/residual_abs": 0.2738838493824005,
|
|
"ronpo/target": 0.003305905032902956,
|
|
"ronpo/target_abs": 0.06634985655546188,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2944992780685425,
|
|
"drift/rejected_abs": 0.08568915724754333,
|
|
"epoch": 0.24473579658323402,
|
|
"grad_norm": 1.5546875,
|
|
"learning_rate": 6.316105310618664e-09,
|
|
"logits/chosen": -1.618821144104004,
|
|
"logits/rejected": -1.687339186668396,
|
|
"logps/chosen": -0.2868710160255432,
|
|
"logps/rejected": -0.28085970878601074,
|
|
"loss": 0.5253114700317383,
|
|
"loss/core": 0.5071266889572144,
|
|
"loss/preference_sft": 0.28473153710365295,
|
|
"loss/reference_anchor": 0.33522337675094604,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.94403076171875,
|
|
"rewards/margins": 2.089409351348877,
|
|
"rewards/rejected": 0.8546212315559387,
|
|
"ronpo/logit": 0.20894095301628113,
|
|
"ronpo/residual": 0.21501250565052032,
|
|
"ronpo/residual_abs": 0.251351535320282,
|
|
"ronpo/target": -0.006071551237255335,
|
|
"ronpo/target_abs": 0.0737655907869339,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.39996713399887085,
|
|
"drift/rejected_abs": 0.18100953102111816,
|
|
"epoch": 0.24632499006754072,
|
|
"grad_norm": 4.5625,
|
|
"learning_rate": 5.8526203570536504e-09,
|
|
"logits/chosen": -1.798459768295288,
|
|
"logits/rejected": -1.7976763248443604,
|
|
"logps/chosen": -0.26244863867759705,
|
|
"logps/rejected": -0.2599298357963562,
|
|
"loss": 0.3279760181903839,
|
|
"loss/core": 0.30524200201034546,
|
|
"loss/preference_sft": 0.26279181241989136,
|
|
"loss/reference_anchor": 0.4284016191959381,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.99951434135437,
|
|
"rewards/margins": 2.1924331188201904,
|
|
"rewards/rejected": 1.807080864906311,
|
|
"ronpo/logit": 0.21924333274364471,
|
|
"ronpo/residual": 0.22038576006889343,
|
|
"ronpo/residual_abs": 0.2682816684246063,
|
|
"ronpo/target": -0.0011424478143453598,
|
|
"ronpo/target_abs": 0.0722413957118988,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1589912623167038,
|
|
"drift/rejected_abs": 0.09040515124797821,
|
|
"epoch": 0.24791418355184744,
|
|
"grad_norm": 1.2578125,
|
|
"learning_rate": 5.405737427476853e-09,
|
|
"logits/chosen": -1.9486103057861328,
|
|
"logits/rejected": -1.923584222793579,
|
|
"logps/chosen": -0.2780265510082245,
|
|
"logps/rejected": -0.27566081285476685,
|
|
"loss": 0.03590798377990723,
|
|
"loss/core": 0.030683238059282303,
|
|
"loss/preference_sft": 0.27917927503585815,
|
|
"loss/reference_anchor": 0.07657701522111893,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.58791184425354,
|
|
"rewards/margins": 0.684416651725769,
|
|
"rewards/rejected": 0.9034950137138367,
|
|
"ronpo/logit": 0.06844166666269302,
|
|
"ronpo/residual": 0.07022113353013992,
|
|
"ronpo/residual_abs": 0.12148270756006241,
|
|
"ronpo/target": -0.0017794653540477157,
|
|
"ronpo/target_abs": 0.07558373361825943,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21656222641468048,
|
|
"drift/rejected_abs": 0.09968177229166031,
|
|
"epoch": 0.24950337703615416,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 4.975624576281889e-09,
|
|
"logits/chosen": -1.7334492206573486,
|
|
"logits/rejected": -1.745598554611206,
|
|
"logps/chosen": -0.2670922577381134,
|
|
"logps/rejected": -0.28659647703170776,
|
|
"loss": 0.13521121442317963,
|
|
"loss/core": 0.12704432010650635,
|
|
"loss/preference_sft": 0.2725936770439148,
|
|
"loss/reference_anchor": 0.1360786259174347,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1648879051208496,
|
|
"rewards/margins": 1.1697208881378174,
|
|
"rewards/rejected": 0.9951672554016113,
|
|
"ronpo/logit": 0.11697210371494293,
|
|
"ronpo/residual": 0.13287900388240814,
|
|
"ronpo/residual_abs": 0.17512091994285583,
|
|
"ronpo/target": -0.015906909480690956,
|
|
"ronpo/target_abs": 0.08019174635410309,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27706044912338257,
|
|
"drift/rejected_abs": 0.08440014719963074,
|
|
"epoch": 0.25109257052046086,
|
|
"grad_norm": 145.0,
|
|
"learning_rate": 4.562443551321787e-09,
|
|
"logits/chosen": -1.738996148109436,
|
|
"logits/rejected": -1.7702322006225586,
|
|
"logps/chosen": -0.2960914671421051,
|
|
"logps/rejected": -0.2777342200279236,
|
|
"loss": 0.7593822479248047,
|
|
"loss/core": 0.7336170077323914,
|
|
"loss/preference_sft": 0.3006787896156311,
|
|
"loss/reference_anchor": 0.48523634672164917,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7688331604003906,
|
|
"rewards/margins": 1.9260284900665283,
|
|
"rewards/rejected": 0.8428047895431519,
|
|
"ronpo/logit": 0.19260284304618835,
|
|
"ronpo/residual": 0.18357378244400024,
|
|
"ronpo/residual_abs": 0.22910535335540771,
|
|
"ronpo/target": 0.009029053151607513,
|
|
"ronpo/target_abs": 0.0732058733701706,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23308022320270538,
|
|
"drift/rejected_abs": 0.11013714224100113,
|
|
"epoch": 0.2526817640047676,
|
|
"grad_norm": 3.625,
|
|
"learning_rate": 4.166349733082153e-09,
|
|
"logits/chosen": -1.8755409717559814,
|
|
"logits/rejected": -1.8500206470489502,
|
|
"logps/chosen": -0.2593986392021179,
|
|
"logps/rejected": -0.2735549807548523,
|
|
"loss": 0.09857200086116791,
|
|
"loss/core": 0.09173043817281723,
|
|
"loss/preference_sft": 0.2679574191570282,
|
|
"loss/reference_anchor": 0.11003559827804565,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.330761671066284,
|
|
"rewards/margins": 1.2306572198867798,
|
|
"rewards/rejected": 1.1001043319702148,
|
|
"ronpo/logit": 0.12306573241949081,
|
|
"ronpo/residual": 0.12092306464910507,
|
|
"ronpo/residual_abs": 0.1728290170431137,
|
|
"ronpo/target": 0.002142664510756731,
|
|
"ronpo/target_abs": 0.08366601169109344,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30347901582717896,
|
|
"drift/rejected_abs": 0.14543643593788147,
|
|
"epoch": 0.2542709574890743,
|
|
"grad_norm": 3.921875,
|
|
"learning_rate": 3.7874920762489935e-09,
|
|
"logits/chosen": -1.8720595836639404,
|
|
"logits/rejected": -1.7962238788604736,
|
|
"logps/chosen": -0.28576621413230896,
|
|
"logps/rejected": -0.2803238034248352,
|
|
"loss": 0.264794260263443,
|
|
"loss/core": 0.2447708547115326,
|
|
"loss/preference_sft": 0.28448182344436646,
|
|
"loss/reference_anchor": 0.37201887369155884,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.034790515899658,
|
|
"rewards/margins": 1.5887119770050049,
|
|
"rewards/rejected": 1.4460780620574951,
|
|
"ronpo/logit": 0.15887121856212616,
|
|
"ronpo/residual": 0.1484646499156952,
|
|
"ronpo/residual_abs": 0.22516998648643494,
|
|
"ronpo/target": 0.010406563989818096,
|
|
"ronpo/target_abs": 0.07933439314365387,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2623865604400635,
|
|
"drift/rejected_abs": 0.13297392427921295,
|
|
"epoch": 0.255860150973381,
|
|
"grad_norm": 6.15625,
|
|
"learning_rate": 3.426013053692878e-09,
|
|
"logits/chosen": -1.765254020690918,
|
|
"logits/rejected": -1.6951442956924438,
|
|
"logps/chosen": -0.25918513536453247,
|
|
"logps/rejected": -0.2669827938079834,
|
|
"loss": 0.13117735087871552,
|
|
"loss/core": 0.1169406920671463,
|
|
"loss/preference_sft": 0.26121383905410767,
|
|
"loss/reference_anchor": 0.25861161947250366,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6238656044006348,
|
|
"rewards/margins": 1.2945048809051514,
|
|
"rewards/rejected": 1.3293604850769043,
|
|
"ronpo/logit": 0.1294504851102829,
|
|
"ronpo/residual": 0.1414770781993866,
|
|
"ronpo/residual_abs": 0.17369873821735382,
|
|
"ronpo/target": -0.012026568874716759,
|
|
"ronpo/target_abs": 0.07064296305179596,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4997061789035797,
|
|
"drift/rejected_abs": 0.19101104140281677,
|
|
"epoch": 0.25744934445768775,
|
|
"grad_norm": 832.0,
|
|
"learning_rate": 3.0820486028908077e-09,
|
|
"logits/chosen": -1.7124788761138916,
|
|
"logits/rejected": -1.7720760107040405,
|
|
"logps/chosen": -0.25494056940078735,
|
|
"logps/rejected": -0.24831128120422363,
|
|
"loss": 0.960421085357666,
|
|
"loss/core": 0.9222633242607117,
|
|
"loss/preference_sft": 0.26125359535217285,
|
|
"loss/reference_anchor": 0.737030029296875,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 4.997061729431152,
|
|
"rewards/margins": 3.0881500244140625,
|
|
"rewards/rejected": 1.9089114665985107,
|
|
"ronpo/logit": 0.30881503224372864,
|
|
"ronpo/residual": 0.30170923471450806,
|
|
"ronpo/residual_abs": 0.3625412583351135,
|
|
"ronpo/target": 0.007105761673301458,
|
|
"ronpo/target_abs": 0.06471297889947891,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3279181122779846,
|
|
"drift/rejected_abs": 0.13609564304351807,
|
|
"epoch": 0.25903853794199444,
|
|
"grad_norm": 1504.0,
|
|
"learning_rate": 2.7557280748055966e-09,
|
|
"logits/chosen": -1.8881208896636963,
|
|
"logits/rejected": -1.9525415897369385,
|
|
"logps/chosen": -0.2766019403934479,
|
|
"logps/rejected": -0.2734753489494324,
|
|
"loss": 0.871407687664032,
|
|
"loss/core": 0.8373653292655945,
|
|
"loss/preference_sft": 0.2795865535736084,
|
|
"loss/reference_anchor": 0.6528871655464172,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.26275634765625,
|
|
"rewards/margins": 1.9969438314437866,
|
|
"rewards/rejected": 1.2658125162124634,
|
|
"ronpo/logit": 0.19969438016414642,
|
|
"ronpo/residual": 0.20564095675945282,
|
|
"ronpo/residual_abs": 0.27914053201675415,
|
|
"ronpo/target": -0.005946573801338673,
|
|
"ronpo/target_abs": 0.07861017435789108,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12528498470783234,
|
|
"drift/rejected_abs": 0.08170704543590546,
|
|
"epoch": 0.26062773142630113,
|
|
"grad_norm": 2.03125,
|
|
"learning_rate": 2.4471741852423233e-09,
|
|
"logits/chosen": -2.0222911834716797,
|
|
"logits/rejected": -2.054588556289673,
|
|
"logps/chosen": -0.2888307273387909,
|
|
"logps/rejected": -0.2907455861568451,
|
|
"loss": 0.06904357671737671,
|
|
"loss/core": 0.06414641439914703,
|
|
"loss/preference_sft": 0.2814570367336273,
|
|
"loss/reference_anchor": 0.06979776173830032,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.252849817276001,
|
|
"rewards/margins": 0.436107337474823,
|
|
"rewards/rejected": 0.8167423009872437,
|
|
"ronpo/logit": 0.04361073672771454,
|
|
"ronpo/residual": 0.04019293561577797,
|
|
"ronpo/residual_abs": 0.13318975269794464,
|
|
"ronpo/target": 0.0034177973866462708,
|
|
"ronpo/target_abs": 0.07740087062120438,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1749141365289688,
|
|
"drift/rejected_abs": 0.09744749963283539,
|
|
"epoch": 0.2622169249106079,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 2.1565029686999305e-09,
|
|
"logits/chosen": -2.02028226852417,
|
|
"logits/rejected": -1.9485108852386475,
|
|
"logps/chosen": -0.29039934277534485,
|
|
"logps/rejected": -0.30477070808410645,
|
|
"loss": 0.08298453688621521,
|
|
"loss/core": 0.07695572078227997,
|
|
"loss/preference_sft": 0.30129221081733704,
|
|
"loss/reference_anchor": 0.09044675529003143,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7491413354873657,
|
|
"rewards/margins": 0.7976366281509399,
|
|
"rewards/rejected": 0.9515047073364258,
|
|
"ronpo/logit": 0.07976364344358444,
|
|
"ronpo/residual": 0.09047956764698029,
|
|
"ronpo/residual_abs": 0.13659612834453583,
|
|
"ronpo/target": -0.010715911164879799,
|
|
"ronpo/target_abs": 0.06951193511486053,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.39292028546333313,
|
|
"drift/rejected_abs": 0.15564313530921936,
|
|
"epoch": 0.2638061183949146,
|
|
"grad_norm": 51.75,
|
|
"learning_rate": 1.8838237347353845e-09,
|
|
"logits/chosen": -1.829711675643921,
|
|
"logits/rejected": -1.8036315441131592,
|
|
"logps/chosen": -0.28607892990112305,
|
|
"logps/rejected": -0.2774202823638916,
|
|
"loss": 1.1372624635696411,
|
|
"loss/core": 1.0995932817459106,
|
|
"loss/preference_sft": 0.2805403172969818,
|
|
"loss/reference_anchor": 0.7253289222717285,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.927274703979492,
|
|
"rewards/margins": 2.3715097904205322,
|
|
"rewards/rejected": 1.5557647943496704,
|
|
"ronpo/logit": 0.23715098202228546,
|
|
"ronpo/residual": 0.2534126341342926,
|
|
"ronpo/residual_abs": 0.40109115839004517,
|
|
"ronpo/target": -0.01626167818903923,
|
|
"ronpo/target_abs": 0.08010418713092804,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11015255749225616,
|
|
"drift/rejected_abs": 0.08351143449544907,
|
|
"epoch": 0.2653953118792213,
|
|
"grad_norm": 3.609375,
|
|
"learning_rate": 1.6292390268568101e-09,
|
|
"logits/chosen": -1.9036916494369507,
|
|
"logits/rejected": -1.8151781558990479,
|
|
"logps/chosen": -0.388921856880188,
|
|
"logps/rejected": -0.2949334681034088,
|
|
"loss": 0.0698942169547081,
|
|
"loss/core": 0.0649130567908287,
|
|
"loss/preference_sft": 0.30027008056640625,
|
|
"loss/reference_anchor": 0.06959625333547592,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.099797010421753,
|
|
"rewards/margins": 0.264737993478775,
|
|
"rewards/rejected": 0.8350589871406555,
|
|
"ronpo/logit": 0.02647380903363228,
|
|
"ronpo/residual": 0.028390701860189438,
|
|
"ronpo/residual_abs": 0.12346197664737701,
|
|
"ronpo/target": -0.0019168935250490904,
|
|
"ronpo/target_abs": 0.07617279142141342,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2376348078250885,
|
|
"drift/rejected_abs": 0.11902894079685211,
|
|
"epoch": 0.266984505363528,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 1.392844583961078e-09,
|
|
"logits/chosen": -1.9641727209091187,
|
|
"logits/rejected": -1.9484821557998657,
|
|
"logps/chosen": -0.29593294858932495,
|
|
"logps/rejected": -0.29610633850097656,
|
|
"loss": 0.33072429895401,
|
|
"loss/core": 0.30467796325683594,
|
|
"loss/preference_sft": 0.291830450296402,
|
|
"loss/reference_anchor": 0.49174419045448303,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3760135173797607,
|
|
"rewards/margins": 1.187029480934143,
|
|
"rewards/rejected": 1.1889841556549072,
|
|
"ronpo/logit": 0.1187029629945755,
|
|
"ronpo/residual": 0.14209163188934326,
|
|
"ronpo/residual_abs": 0.18299010396003723,
|
|
"ronpo/target": -0.023388683795928955,
|
|
"ronpo/target_abs": 0.0768231675028801,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1240852028131485,
|
|
"drift/rejected_abs": 0.0827094167470932,
|
|
"epoch": 0.2685736988478347,
|
|
"grad_norm": 1.6953125,
|
|
"learning_rate": 1.1747293043303518e-09,
|
|
"logits/chosen": -1.8987678289413452,
|
|
"logits/rejected": -1.8802201747894287,
|
|
"logps/chosen": -0.30965709686279297,
|
|
"logps/rejected": -0.2994539141654968,
|
|
"loss": 0.02463197335600853,
|
|
"loss/core": 0.02076641097664833,
|
|
"loss/preference_sft": 0.3030204176902771,
|
|
"loss/reference_anchor": 0.04700922966003418,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2399028539657593,
|
|
"rewards/margins": 0.4139002859592438,
|
|
"rewards/rejected": 0.8260027170181274,
|
|
"ronpo/logit": 0.041390031576156616,
|
|
"ronpo/residual": 0.043523531407117844,
|
|
"ronpo/residual_abs": 0.10440732538700104,
|
|
"ronpo/target": -0.002133502857759595,
|
|
"ronpo/target_abs": 0.072882741689682,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2770349085330963,
|
|
"drift/rejected_abs": 0.1225806251168251,
|
|
"epoch": 0.2701628923321414,
|
|
"grad_norm": 59.0,
|
|
"learning_rate": 9.749752122010346e-10,
|
|
"logits/chosen": -1.6999680995941162,
|
|
"logits/rejected": -1.6244916915893555,
|
|
"logps/chosen": -0.28973299264907837,
|
|
"logps/rejected": -0.3002259433269501,
|
|
"loss": 0.36320897936820984,
|
|
"loss/core": 0.34727710485458374,
|
|
"loss/preference_sft": 0.2909989058971405,
|
|
"loss/reference_anchor": 0.2895377278327942,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.769200086593628,
|
|
"rewards/margins": 1.6400721073150635,
|
|
"rewards/rejected": 1.1291282176971436,
|
|
"ronpo/logit": 0.16400720179080963,
|
|
"ronpo/residual": 0.15951409935951233,
|
|
"ronpo/residual_abs": 0.21055129170417786,
|
|
"ronpo/target": 0.004493103362619877,
|
|
"ronpo/target_abs": 0.07943262159824371,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32434701919555664,
|
|
"drift/rejected_abs": 0.10581743717193604,
|
|
"epoch": 0.27175208581644816,
|
|
"grad_norm": 74.5,
|
|
"learning_rate": 7.936574269178375e-10,
|
|
"logits/chosen": -1.7718331813812256,
|
|
"logits/rejected": -1.7748457193374634,
|
|
"logps/chosen": -0.3256663680076599,
|
|
"logps/rejected": -0.32961875200271606,
|
|
"loss": 0.8090022206306458,
|
|
"loss/core": 0.7803620100021362,
|
|
"loss/preference_sft": 0.33879441022872925,
|
|
"loss/reference_anchor": 0.538924515247345,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.241380214691162,
|
|
"rewards/margins": 2.1839098930358887,
|
|
"rewards/rejected": 1.0574705600738525,
|
|
"ronpo/logit": 0.21839098632335663,
|
|
"ronpo/residual": 0.23008199036121368,
|
|
"ronpo/residual_abs": 0.28760650753974915,
|
|
"ronpo/target": -0.011690998449921608,
|
|
"ronpo/target_abs": 0.09034421294927597,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3449726402759552,
|
|
"drift/rejected_abs": 0.1498272866010666,
|
|
"epoch": 0.27334127930075486,
|
|
"grad_norm": 190.0,
|
|
"learning_rate": 6.308441346844384e-10,
|
|
"logits/chosen": -1.9813814163208008,
|
|
"logits/rejected": -1.9928051233291626,
|
|
"logps/chosen": -0.28107166290283203,
|
|
"logps/rejected": -0.2793976366519928,
|
|
"loss": 0.5188241004943848,
|
|
"loss/core": 0.4884120523929596,
|
|
"loss/preference_sft": 0.28123125433921814,
|
|
"loss/reference_anchor": 0.5801182985305786,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.4484634399414062,
|
|
"rewards/margins": 1.9525508880615234,
|
|
"rewards/rejected": 1.4959125518798828,
|
|
"ronpo/logit": 0.1952550858259201,
|
|
"ronpo/residual": 0.2171076238155365,
|
|
"ronpo/residual_abs": 0.2710626721382141,
|
|
"ronpo/target": -0.02185254916548729,
|
|
"ronpo/target_abs": 0.0946681946516037,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34945374727249146,
|
|
"drift/rejected_abs": 0.13005147874355316,
|
|
"epoch": 0.2749304727850616,
|
|
"grad_norm": 68.5,
|
|
"learning_rate": 4.865965629214819e-10,
|
|
"logits/chosen": -1.6880762577056885,
|
|
"logits/rejected": -1.679848074913025,
|
|
"logps/chosen": -0.28330764174461365,
|
|
"logps/rejected": -0.28511470556259155,
|
|
"loss": 0.8294771909713745,
|
|
"loss/core": 0.7987702488899231,
|
|
"loss/preference_sft": 0.28860875964164734,
|
|
"loss/reference_anchor": 0.5852810740470886,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.494537353515625,
|
|
"rewards/margins": 2.194039821624756,
|
|
"rewards/rejected": 1.3004978895187378,
|
|
"ronpo/logit": 0.2194039523601532,
|
|
"ronpo/residual": 0.21074934303760529,
|
|
"ronpo/residual_abs": 0.2737117409706116,
|
|
"ronpo/target": 0.008654648438096046,
|
|
"ronpo/target_abs": 0.09077721834182739,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2341894656419754,
|
|
"drift/rejected_abs": 0.08986641466617584,
|
|
"epoch": 0.2765196662693683,
|
|
"grad_norm": 10.375,
|
|
"learning_rate": 3.609689572414143e-10,
|
|
"logits/chosen": -1.6340396404266357,
|
|
"logits/rejected": -1.6042693853378296,
|
|
"logps/chosen": -0.3055315911769867,
|
|
"logps/rejected": -0.30593135952949524,
|
|
"loss": 0.21940772235393524,
|
|
"loss/core": 0.20771245658397675,
|
|
"loss/preference_sft": 0.3087262213230133,
|
|
"loss/reference_anchor": 0.20303305983543396,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.340512990951538,
|
|
"rewards/margins": 1.4420377016067505,
|
|
"rewards/rejected": 0.8984752893447876,
|
|
"ronpo/logit": 0.14420375227928162,
|
|
"ronpo/residual": 0.12460936605930328,
|
|
"ronpo/residual_abs": 0.19094470143318176,
|
|
"ronpo/target": 0.019594397395849228,
|
|
"ronpo/target_abs": 0.06982195377349854,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3996649384498596,
|
|
"drift/rejected_abs": 0.1653774082660675,
|
|
"epoch": 0.278108859753675,
|
|
"grad_norm": 3.75,
|
|
"learning_rate": 2.5400856104894063e-10,
|
|
"logits/chosen": -1.6849534511566162,
|
|
"logits/rejected": -1.678696870803833,
|
|
"logps/chosen": -0.2519184947013855,
|
|
"logps/rejected": -0.2652212083339691,
|
|
"loss": 1.347104787826538,
|
|
"loss/core": 1.2939437627792358,
|
|
"loss/preference_sft": 0.254986047744751,
|
|
"loss/reference_anchor": 1.0377271175384521,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.9966492652893066,
|
|
"rewards/margins": 2.35422420501709,
|
|
"rewards/rejected": 1.6424251794815063,
|
|
"ronpo/logit": 0.23542241752147675,
|
|
"ronpo/residual": 0.24257414042949677,
|
|
"ronpo/residual_abs": 0.3515390455722809,
|
|
"ronpo/target": -0.0071517289616167545,
|
|
"ronpo/target_abs": 0.07295659929513931,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2877359986305237,
|
|
"drift/rejected_abs": 0.21151041984558105,
|
|
"epoch": 0.27969805323798175,
|
|
"grad_norm": 52.0,
|
|
"learning_rate": 1.6575559777469716e-10,
|
|
"logits/chosen": -1.8132593631744385,
|
|
"logits/rejected": -1.8345884084701538,
|
|
"logps/chosen": -0.27794936299324036,
|
|
"logps/rejected": -0.29504913091659546,
|
|
"loss": 0.7968089580535889,
|
|
"loss/core": 0.7706011533737183,
|
|
"loss/preference_sft": 0.2765906751155853,
|
|
"loss/reference_anchor": 0.49649912118911743,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8772411346435547,
|
|
"rewards/margins": 0.7623518109321594,
|
|
"rewards/rejected": 2.11488938331604,
|
|
"ronpo/logit": 0.07623519748449326,
|
|
"ronpo/residual": 0.08102894574403763,
|
|
"ronpo/residual_abs": 0.29983019828796387,
|
|
"ronpo/target": -0.004793758969753981,
|
|
"ronpo/target_abs": 0.08000515401363373,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3133094906806946,
|
|
"drift/rejected_abs": 0.1380651891231537,
|
|
"epoch": 0.28128724672228844,
|
|
"grad_norm": 4.8125,
|
|
"learning_rate": 9.624325574890125e-11,
|
|
"logits/chosen": -1.596097707748413,
|
|
"logits/rejected": -1.5967873334884644,
|
|
"logps/chosen": -0.3078146278858185,
|
|
"logps/rejected": -0.31876230239868164,
|
|
"loss": 0.26423799991607666,
|
|
"loss/core": 0.2500300407409668,
|
|
"loss/preference_sft": 0.311987042427063,
|
|
"loss/reference_anchor": 0.2529599070549011,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.131915330886841,
|
|
"rewards/margins": 1.7639577388763428,
|
|
"rewards/rejected": 1.367957353591919,
|
|
"ronpo/logit": 0.17639580368995667,
|
|
"ronpo/residual": 0.17563918232917786,
|
|
"ronpo/residual_abs": 0.22658184170722961,
|
|
"ronpo/target": 0.0007566086715087295,
|
|
"ronpo/target_abs": 0.0707857608795166,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10474882274866104,
|
|
"drift/rejected_abs": 0.07787434756755829,
|
|
"epoch": 0.28287644020659514,
|
|
"grad_norm": 0.73828125,
|
|
"learning_rate": 4.5497675720540527e-11,
|
|
"logits/chosen": -1.9180591106414795,
|
|
"logits/rejected": -1.8720309734344482,
|
|
"logps/chosen": -0.32056906819343567,
|
|
"logps/rejected": -0.31805619597435,
|
|
"loss": 0.020853275433182716,
|
|
"loss/core": 0.01702974922955036,
|
|
"loss/preference_sft": 0.31593579053878784,
|
|
"loss/reference_anchor": 0.04487697035074234,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.0474880933761597,
|
|
"rewards/margins": 0.2687447667121887,
|
|
"rewards/rejected": 0.7787435054779053,
|
|
"ronpo/logit": 0.026874477043747902,
|
|
"ronpo/residual": 0.041672371327877045,
|
|
"ronpo/residual_abs": 0.09372963011264801,
|
|
"ronpo/target": -0.014797890558838844,
|
|
"ronpo/target_abs": 0.07123903930187225,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4191473424434662,
|
|
"drift/rejected_abs": 0.1661919206380844,
|
|
"epoch": 0.2844656336909019,
|
|
"grad_norm": 61.25,
|
|
"learning_rate": 1.3537941026914302e-11,
|
|
"logits/chosen": -1.8258793354034424,
|
|
"logits/rejected": -1.7830054759979248,
|
|
"logps/chosen": -0.27327242493629456,
|
|
"logps/rejected": -0.28399670124053955,
|
|
"loss": 0.7437120079994202,
|
|
"loss/core": 0.7049421668052673,
|
|
"loss/preference_sft": 0.27699244022369385,
|
|
"loss/reference_anchor": 0.7476966381072998,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.191155433654785,
|
|
"rewards/margins": 2.578888177871704,
|
|
"rewards/rejected": 1.6122677326202393,
|
|
"ronpo/logit": 0.2578887939453125,
|
|
"ronpo/residual": 0.2486531287431717,
|
|
"ronpo/residual_abs": 0.30036187171936035,
|
|
"ronpo/target": 0.009235644713044167,
|
|
"ronpo/target_abs": 0.07194842398166656,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18127210438251495,
|
|
"drift/rejected_abs": 0.08103598654270172,
|
|
"epoch": 0.2860548271752086,
|
|
"grad_norm": 23.25,
|
|
"learning_rate": 3.7607041719622813e-13,
|
|
"logits/chosen": -1.8266232013702393,
|
|
"logits/rejected": -1.9079567193984985,
|
|
"logps/chosen": -0.2764757573604584,
|
|
"logps/rejected": -0.2799293100833893,
|
|
"loss": 0.06943485885858536,
|
|
"loss/core": 0.06367450952529907,
|
|
"loss/preference_sft": 0.28113025426864624,
|
|
"loss/reference_anchor": 0.08709382265806198,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.8119440078735352,
|
|
"rewards/margins": 1.002147912979126,
|
|
"rewards/rejected": 0.8097962141036987,
|
|
"ronpo/logit": 0.10021479427814484,
|
|
"ronpo/residual": 0.08235163241624832,
|
|
"ronpo/residual_abs": 0.14560458064079285,
|
|
"ronpo/target": 0.017863161861896515,
|
|
"ronpo/target_abs": 0.07489818334579468,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.2860548271752086,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.48261189058423043,
|
|
"train_runtime": 8509.445,
|
|
"train_samples_per_second": 1.692,
|
|
"train_steps_per_second": 0.106
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|