{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.2860548271752086, "eval_steps": 500, "global_step": 900, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "drift/chosen_abs": 0.2888891100883484, "drift/rejected_abs": 0.17491090297698975, "epoch": 0.0015891934843067143, "grad_norm": 0.90625, "learning_rate": 3.3333333333333334e-09, "logits/chosen": -2.22796893119812, "logits/rejected": -2.1537139415740967, "logps/chosen": -0.25039365887641907, "logps/rejected": -0.2549845278263092, "loss": 0.2590796649456024, "loss/core": 0.24525848031044006, "loss/preference_sft": 0.2586572766304016, "loss/reference_anchor": 0.3690255582332611, "rewards/accuracies": 0.875, "rewards/chosen": 2.8888912200927734, "rewards/margins": 1.140271782875061, "rewards/rejected": 1.7486193180084229, "ronpo/logit": 0.11402717977762222, "ronpo/residual": 0.12274861335754395, "ronpo/residual_abs": 0.22723272442817688, "ronpo/target": -0.008721430785953999, "ronpo/target_abs": 0.06909500062465668, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 5 }, { "drift/chosen_abs": 0.25118276476860046, "drift/rejected_abs": 0.1736268401145935, "epoch": 0.0031783869686134287, "grad_norm": 5.21875, "learning_rate": 7.5e-09, "logits/chosen": -1.9810645580291748, "logits/rejected": -1.9216926097869873, "logps/chosen": -0.2643340528011322, "logps/rejected": -0.2714827060699463, "loss": 0.15867051482200623, "loss/core": 0.14627718925476074, "loss/preference_sft": 0.2700865864753723, "loss/reference_anchor": 0.3270857632160187, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.511749267578125, "rewards/margins": 0.7762027978897095, "rewards/rejected": 1.735546350479126, "ronpo/logit": 0.0776202604174614, "ronpo/residual": 0.0606955885887146, "ronpo/residual_abs": 0.17922377586364746, "ronpo/target": 0.016924677416682243, "ronpo/target_abs": 0.08231900632381439, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 10 }, { "drift/chosen_abs": 0.14721375703811646, "drift/rejected_abs": 0.0799868255853653, "epoch": 0.004767580452920143, "grad_norm": 7.5, "learning_rate": 1.1666666666666667e-08, "logits/chosen": -2.291767120361328, "logits/rejected": -2.2457759380340576, "logps/chosen": -0.2763332426548004, "logps/rejected": -0.28252172470092773, "loss": 0.049369972199201584, "loss/core": 0.04628817364573479, "loss/preference_sft": 0.2778027057647705, "loss/reference_anchor": 0.060271114110946655, "rewards/accuracies": 0.875, "rewards/chosen": 1.4715213775634766, "rewards/margins": 0.6733793020248413, "rewards/rejected": 0.7981420755386353, "ronpo/logit": 0.06733792275190353, "ronpo/residual": 0.09835626929998398, "ronpo/residual_abs": 0.1334967315196991, "ronpo/target": -0.031018350273370743, "ronpo/target_abs": 0.076286181807518, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 15 }, { "drift/chosen_abs": 0.14123335480690002, "drift/rejected_abs": 0.0699237510561943, "epoch": 0.006356773937226857, "grad_norm": 1.1171875, "learning_rate": 1.5833333333333333e-08, "logits/chosen": -2.2912278175354004, "logits/rejected": -2.2170040607452393, "logps/chosen": -0.27552682161331177, "logps/rejected": -0.27909940481185913, "loss": 0.02950289286673069, "loss/core": 0.02670697309076786, "loss/preference_sft": 0.27772125601768494, "loss/reference_anchor": 0.05211133509874344, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 1.4118921756744385, "rewards/margins": 0.7152848839759827, "rewards/rejected": 0.6966072916984558, "ronpo/logit": 0.07152849435806274, "ronpo/residual": 0.06994546204805374, "ronpo/residual_abs": 0.10728304088115692, "ronpo/target": 0.001583026722073555, "ronpo/target_abs": 0.0675927996635437, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 20 }, { "drift/chosen_abs": 0.23433640599250793, "drift/rejected_abs": 0.07729706168174744, "epoch": 0.007945967421533572, "grad_norm": 1.125, "learning_rate": 2e-08, "logits/chosen": -2.353553295135498, "logits/rejected": -2.392822504043579, "logps/chosen": -0.23983506858348846, "logps/rejected": -0.24817518889904022, "loss": 0.30295810103416443, "loss/core": 0.29502326250076294, "loss/preference_sft": 0.2504972517490387, "loss/reference_anchor": 0.20166020095348358, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.342874050140381, "rewards/margins": 1.5709974765777588, "rewards/rejected": 0.7718764543533325, "ronpo/logit": 0.15709975361824036, "ronpo/residual": 0.1602698713541031, "ronpo/residual_abs": 0.20184986293315887, "ronpo/target": -0.0031701307743787766, "ronpo/target_abs": 0.07585608959197998, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 25 }, { "drift/chosen_abs": 0.20710475742816925, "drift/rejected_abs": 0.0870819017291069, "epoch": 0.009535160905840286, "grad_norm": 316.0, "learning_rate": 2.4166666666666668e-08, "logits/chosen": -2.2282848358154297, "logits/rejected": -2.124019145965576, "logps/chosen": -0.28481242060661316, "logps/rejected": -0.28323936462402344, "loss": 0.1839178502559662, "loss/core": 0.17509128153324127, "loss/preference_sft": 0.2906222939491272, "loss/reference_anchor": 0.22312648594379425, "rewards/accuracies": 0.9375, "rewards/chosen": 2.0701568126678467, "rewards/margins": 1.1993378400802612, "rewards/rejected": 0.870819091796875, "ronpo/logit": 0.11993379890918732, "ronpo/residual": 0.10416283458471298, "ronpo/residual_abs": 0.17567749321460724, "ronpo/target": 0.015770956873893738, "ronpo/target_abs": 0.0844925194978714, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 30 }, { "drift/chosen_abs": 0.20917043089866638, "drift/rejected_abs": 0.13052448630332947, "epoch": 0.011124354390147001, "grad_norm": 2.3125, "learning_rate": 2.8333333333333332e-08, "logits/chosen": -2.122420310974121, "logits/rejected": -2.0597376823425293, "logps/chosen": -0.2706679403781891, "logps/rejected": -0.28155890107154846, "loss": 0.04921208694577217, "loss/core": 0.04241333529353142, "loss/preference_sft": 0.2808118462562561, "loss/reference_anchor": 0.16616889834403992, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.091449737548828, "rewards/margins": 0.7882283926010132, "rewards/rejected": 1.303221344947815, "ronpo/logit": 0.07882285118103027, "ronpo/residual": 0.06731168925762177, "ronpo/residual_abs": 0.12287186086177826, "ronpo/target": 0.011511145159602165, "ronpo/target_abs": 0.07380068302154541, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 35 }, { "drift/chosen_abs": 0.13368353247642517, "drift/rejected_abs": 0.10889682918787003, "epoch": 0.012713547874453715, "grad_norm": 430.0, "learning_rate": 3.25e-08, "logits/chosen": -2.080500364303589, "logits/rejected": -2.039989471435547, "logps/chosen": -0.31930357217788696, "logps/rejected": -0.32469144463539124, "loss": 0.22682690620422363, "loss/core": 0.220375657081604, "loss/preference_sft": 0.32273247838020325, "loss/reference_anchor": 0.15204796195030212, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 1.335686445236206, "rewards/margins": 0.24776522815227509, "rewards/rejected": 1.0879215002059937, "ronpo/logit": 0.024776523932814598, "ronpo/residual": 0.01710623875260353, "ronpo/residual_abs": 0.15854685008525848, "ronpo/target": 0.007670276798307896, "ronpo/target_abs": 0.06935645639896393, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 40 }, { "drift/chosen_abs": 0.19959691166877747, "drift/rejected_abs": 0.10330881178379059, "epoch": 0.014302741358760428, "grad_norm": 3.90625, "learning_rate": 3.6666666666666664e-08, "logits/chosen": -2.1476857662200928, "logits/rejected": -2.0756335258483887, "logps/chosen": -0.30488723516464233, "logps/rejected": -0.29676884412765503, "loss": 0.08849599957466125, "loss/core": 0.08234541118144989, "loss/preference_sft": 0.3082302510738373, "loss/reference_anchor": 0.14490821957588196, "rewards/accuracies": 0.875, "rewards/chosen": 1.9952192306518555, "rewards/margins": 0.9640136957168579, "rewards/rejected": 1.0312055349349976, "ronpo/logit": 0.09640135616064072, "ronpo/residual": 0.09527552872896194, "ronpo/residual_abs": 0.14935743808746338, "ronpo/target": 0.0011258311569690704, "ronpo/target_abs": 0.0794779434800148, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 45 }, { "drift/chosen_abs": 0.20762768387794495, "drift/rejected_abs": 0.17166680097579956, "epoch": 0.015891934843067144, "grad_norm": 8.5625, "learning_rate": 4.083333333333333e-08, "logits/chosen": -2.2317585945129395, "logits/rejected": -2.281120538711548, "logps/chosen": -0.38824811577796936, "logps/rejected": -0.37904253602027893, "loss": 0.48683279752731323, "loss/core": 0.4693141579627991, "loss/preference_sft": 0.3856825530529022, "loss/reference_anchor": 0.4619651436805725, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.075303554534912, "rewards/margins": 1.4245798587799072, "rewards/rejected": 0.6507236361503601, "ronpo/logit": 0.1424579918384552, "ronpo/residual": 0.1403721421957016, "ronpo/residual_abs": 0.23278239369392395, "ronpo/target": 0.002085856394842267, "ronpo/target_abs": 0.07827635854482651, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 50 }, { "drift/chosen_abs": 0.2422821968793869, "drift/rejected_abs": 0.11313442885875702, "epoch": 0.01748112832737386, "grad_norm": 1.4140625, "learning_rate": 4.5e-08, "logits/chosen": -2.13415265083313, "logits/rejected": -2.1069016456604004, "logps/chosen": -0.3025519549846649, "logps/rejected": -0.29403483867645264, "loss": 0.2851434350013733, "loss/core": 0.27573922276496887, "loss/preference_sft": 0.29142332077026367, "loss/reference_anchor": 0.2395510971546173, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.42053484916687, "rewards/margins": 1.2919930219650269, "rewards/rejected": 1.1285418272018433, "ronpo/logit": 0.1291993260383606, "ronpo/residual": 0.12877634167671204, "ronpo/residual_abs": 0.18722160160541534, "ronpo/target": 0.00042299646884202957, "ronpo/target_abs": 0.0751943588256836, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 55 }, { "drift/chosen_abs": 0.2859105169773102, "drift/rejected_abs": 0.07585777342319489, "epoch": 0.01907032181168057, "grad_norm": 6.75, "learning_rate": 4.9166666666666664e-08, "logits/chosen": -2.022763252258301, "logits/rejected": -2.0409018993377686, "logps/chosen": -0.2585347890853882, "logps/rejected": -0.26091212034225464, "loss": 0.6952535510063171, "loss/core": 0.6805825233459473, "loss/preference_sft": 0.2590915560722351, "loss/reference_anchor": 0.3932636082172394, "rewards/accuracies": 1.0, "rewards/chosen": 2.8586783409118652, "rewards/margins": 2.1142590045928955, "rewards/rejected": 0.7444195747375488, "ronpo/logit": 0.21142590045928955, "ronpo/residual": 0.22805149853229523, "ronpo/residual_abs": 0.26630499958992004, "ronpo/target": -0.01662559062242508, "ronpo/target_abs": 0.07312311232089996, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 60 }, { "drift/chosen_abs": 0.23467068374156952, "drift/rejected_abs": 0.08460647612810135, "epoch": 0.020659515295987287, "grad_norm": 156.0, "learning_rate": 5.3333333333333334e-08, "logits/chosen": -2.391279458999634, "logits/rejected": -2.461313247680664, "logps/chosen": -0.2505510151386261, "logps/rejected": -0.24410393834114075, "loss": 0.23585395514965057, "loss/core": 0.22895380854606628, "loss/preference_sft": 0.25251513719558716, "loss/reference_anchor": 0.1718951165676117, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.3467066287994385, "rewards/margins": 1.5009348392486572, "rewards/rejected": 0.8457720875740051, "ronpo/logit": 0.15009348094463348, "ronpo/residual": 0.12950626015663147, "ronpo/residual_abs": 0.1809195727109909, "ronpo/target": 0.02058723010122776, "ronpo/target_abs": 0.07401974499225616, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 65 }, { "drift/chosen_abs": 0.2189807891845703, "drift/rejected_abs": 0.11199836432933807, "epoch": 0.022248708780294002, "grad_norm": 3.640625, "learning_rate": 5.75e-08, "logits/chosen": -2.13312029838562, "logits/rejected": -2.1325230598449707, "logps/chosen": -0.2632366716861725, "logps/rejected": -0.26798808574676514, "loss": 0.07575295865535736, "loss/core": 0.07079951465129852, "loss/preference_sft": 0.27200424671173096, "loss/reference_anchor": 0.11432687193155289, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.18943190574646, "rewards/margins": 1.069993495941162, "rewards/rejected": 1.1194385290145874, "ronpo/logit": 0.10699935257434845, "ronpo/residual": 0.09644700586795807, "ronpo/residual_abs": 0.1400878131389618, "ronpo/target": 0.010552345775067806, "ronpo/target_abs": 0.07036933302879333, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 70 }, { "drift/chosen_abs": 0.2812674641609192, "drift/rejected_abs": 0.14779499173164368, "epoch": 0.023837902264600714, "grad_norm": 6.65625, "learning_rate": 6.166666666666666e-08, "logits/chosen": -2.1063144207000732, "logits/rejected": -2.0400238037109375, "logps/chosen": -0.2795808017253876, "logps/rejected": -0.27232033014297485, "loss": 0.6679071187973022, "loss/core": 0.6522617340087891, "loss/preference_sft": 0.27452582120895386, "loss/reference_anchor": 0.4195597171783447, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 2.8126060962677, "rewards/margins": 1.365581750869751, "rewards/rejected": 1.4470244646072388, "ronpo/logit": 0.1365581750869751, "ronpo/residual": 0.14009013772010803, "ronpo/residual_abs": 0.30546045303344727, "ronpo/target": -0.003531957510858774, "ronpo/target_abs": 0.0828089714050293, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 75 }, { "drift/chosen_abs": 0.32651928067207336, "drift/rejected_abs": 0.1381281465291977, "epoch": 0.02542709574890743, "grad_norm": 41.0, "learning_rate": 6.583333333333333e-08, "logits/chosen": -2.0517067909240723, "logits/rejected": -2.019787311553955, "logps/chosen": -0.270771861076355, "logps/rejected": -0.27388402819633484, "loss": 0.371652215719223, "loss/core": 0.35672515630722046, "loss/preference_sft": 0.27029484510421753, "loss/reference_anchor": 0.3994576334953308, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.265192747116089, "rewards/margins": 1.8877226114273071, "rewards/rejected": 1.3774702548980713, "ronpo/logit": 0.18877224624156952, "ronpo/residual": 0.18959882855415344, "ronpo/residual_abs": 0.23960533738136292, "ronpo/target": -0.0008265804499387741, "ronpo/target_abs": 0.08044739812612534, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 80 }, { "drift/chosen_abs": 0.229955792427063, "drift/rejected_abs": 0.07845465838909149, "epoch": 0.027016289233214145, "grad_norm": 1000.0, "learning_rate": 6.999999999999999e-08, "logits/chosen": -2.366101026535034, "logits/rejected": -2.30869722366333, "logps/chosen": -0.29234808683395386, "logps/rejected": -0.26663321256637573, "loss": 0.579170286655426, "loss/core": 0.5666659474372864, "loss/preference_sft": 0.2720109522342682, "loss/reference_anchor": 0.3300662636756897, "rewards/accuracies": 0.875, "rewards/chosen": 2.2987306118011475, "rewards/margins": 1.5194584131240845, "rewards/rejected": 0.779272198677063, "ronpo/logit": 0.15194585919380188, "ronpo/residual": 0.1387050896883011, "ronpo/residual_abs": 0.20297744870185852, "ronpo/target": 0.013240769505500793, "ronpo/target_abs": 0.07829339802265167, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 85 }, { "drift/chosen_abs": 0.1896975040435791, "drift/rejected_abs": 0.16012468934059143, "epoch": 0.028605482717520857, "grad_norm": 1.1875, "learning_rate": 7.416666666666666e-08, "logits/chosen": -2.2618637084960938, "logits/rejected": -2.2726664543151855, "logps/chosen": -0.2831419110298157, "logps/rejected": -0.2784458100795746, "loss": 0.26859554648399353, "loss/core": 0.26028066873550415, "loss/preference_sft": 0.2823284864425659, "loss/reference_anchor": 0.20933446288108826, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.8969752788543701, "rewards/margins": 0.29952192306518555, "rewards/rejected": 1.5974533557891846, "ronpo/logit": 0.029952192679047585, "ronpo/residual": 0.023403458297252655, "ronpo/residual_abs": 0.17928209900856018, "ronpo/target": 0.006548731122165918, "ronpo/target_abs": 0.08282124251127243, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 90 }, { "drift/chosen_abs": 0.22641654312610626, "drift/rejected_abs": 0.10155995935201645, "epoch": 0.030194676201827572, "grad_norm": 3.8125, "learning_rate": 7.499548723985053e-08, "logits/chosen": -2.3383660316467285, "logits/rejected": -2.2575364112854004, "logps/chosen": -0.2721652388572693, "logps/rejected": -0.270842969417572, "loss": 0.3992326259613037, "loss/core": 0.39081233739852905, "loss/preference_sft": 0.2746056616306305, "loss/reference_anchor": 0.21311792731285095, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.2641654014587402, "rewards/margins": 1.2714884281158447, "rewards/rejected": 0.9926770925521851, "ronpo/logit": 0.12714883685112, "ronpo/residual": 0.14186282455921173, "ronpo/residual_abs": 0.22912614047527313, "ronpo/target": -0.014713977463543415, "ronpo/target_abs": 0.08614644408226013, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 95 }, { "drift/chosen_abs": 0.253028929233551, "drift/rejected_abs": 0.10068502277135849, "epoch": 0.03178386968613429, "grad_norm": 10.0, "learning_rate": 7.497715601321609e-08, "logits/chosen": -1.9719501733779907, "logits/rejected": -2.0437729358673096, "logps/chosen": -0.2982824444770813, "logps/rejected": -0.28864842653274536, "loss": 0.24105651676654816, "loss/core": 0.23307080566883087, "loss/preference_sft": 0.28916245698928833, "loss/reference_anchor": 0.19924673438072205, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.5282044410705566, "rewards/margins": 1.5234105587005615, "rewards/rejected": 1.0047937631607056, "ronpo/logit": 0.1523410826921463, "ronpo/residual": 0.1546805202960968, "ronpo/residual_abs": 0.19531986117362976, "ronpo/target": -0.0023394643794745207, "ronpo/target_abs": 0.07932108640670776, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 100 }, { "drift/chosen_abs": 0.3908888101577759, "drift/rejected_abs": 0.12839046120643616, "epoch": 0.033373063170441, "grad_norm": 95.0, "learning_rate": 7.494473116090843e-08, "logits/chosen": -1.92897629737854, "logits/rejected": -1.9361374378204346, "logps/chosen": -0.2795645296573639, "logps/rejected": -0.28167638182640076, "loss": 0.6834398508071899, "loss/core": 0.6608570218086243, "loss/preference_sft": 0.2787085473537445, "loss/reference_anchor": 0.617350697517395, "rewards/accuracies": 0.9375, "rewards/chosen": 3.908715009689331, "rewards/margins": 2.635611057281494, "rewards/rejected": 1.273103952407837, "ronpo/logit": 0.26356109976768494, "ronpo/residual": 0.27055853605270386, "ronpo/residual_abs": 0.3101016879081726, "ronpo/target": -0.00699741393327713, "ronpo/target_abs": 0.08270318806171417, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 105 }, { "drift/chosen_abs": 0.2439744919538498, "drift/rejected_abs": 0.12347511947154999, "epoch": 0.03496225665474772, "grad_norm": 8.0, "learning_rate": 7.489822487658843e-08, "logits/chosen": -2.243443250656128, "logits/rejected": -2.2027456760406494, "logps/chosen": -0.29370397329330444, "logps/rejected": -0.28651857376098633, "loss": 0.12330013513565063, "loss/core": 0.11514942348003387, "loss/preference_sft": 0.29073047637939453, "loss/reference_anchor": 0.20380404591560364, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.438559055328369, "rewards/margins": 1.2159253358840942, "rewards/rejected": 1.2226336002349854, "ronpo/logit": 0.12159254401922226, "ronpo/residual": 0.12451876699924469, "ronpo/residual_abs": 0.18701641261577606, "ronpo/target": -0.002926218556240201, "ronpo/target_abs": 0.08683927357196808, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 110 }, { "drift/chosen_abs": 0.27595922350883484, "drift/rejected_abs": 0.1378968507051468, "epoch": 0.03655145013905443, "grad_norm": 2.1875, "learning_rate": 7.483765464936766e-08, "logits/chosen": -1.8842188119888306, "logits/rejected": -1.7752964496612549, "logps/chosen": -0.2784324586391449, "logps/rejected": -0.2820046842098236, "loss": 0.1508234441280365, "loss/core": 0.14237406849861145, "loss/preference_sft": 0.2840380072593689, "loss/reference_anchor": 0.21300740540027618, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.757049560546875, "rewards/margins": 1.3863294124603271, "rewards/rejected": 1.3707202672958374, "ronpo/logit": 0.13863296806812286, "ronpo/residual": 0.14248371124267578, "ronpo/residual_abs": 0.18661606311798096, "ronpo/target": -0.003850756911560893, "ronpo/target_abs": 0.07908544689416885, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 115 }, { "drift/chosen_abs": 0.25356635451316833, "drift/rejected_abs": 0.11751619726419449, "epoch": 0.03814064362336114, "grad_norm": 1.078125, "learning_rate": 7.476304325723143e-08, "logits/chosen": -2.017167568206787, "logits/rejected": -2.0651519298553467, "logps/chosen": -0.3016659915447235, "logps/rejected": -0.2997520864009857, "loss": 0.2998911142349243, "loss/core": 0.28765398263931274, "loss/preference_sft": 0.3130350410938263, "loss/reference_anchor": 0.31832972168922424, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.534654140472412, "rewards/margins": 1.3620463609695435, "rewards/rejected": 1.1726077795028687, "ronpo/logit": 0.13620463013648987, "ronpo/residual": 0.13591532409191132, "ronpo/residual_abs": 0.20360629260540009, "ronpo/target": 0.0002892996999435127, "ronpo/target_abs": 0.07245339453220367, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 120 }, { "drift/chosen_abs": 0.1270354688167572, "drift/rejected_abs": 0.07065033167600632, "epoch": 0.03972983710766786, "grad_norm": 1.3359375, "learning_rate": 7.467441875847293e-08, "logits/chosen": -2.2720069885253906, "logits/rejected": -2.192866802215576, "logps/chosen": -0.2744554877281189, "logps/rejected": -0.28156858682632446, "loss": 0.04993171617388725, "loss/core": 0.04652109742164612, "loss/preference_sft": 0.2717655599117279, "loss/reference_anchor": 0.07026959955692291, "rewards/accuracies": 0.875, "rewards/chosen": 1.2697632312774658, "rewards/margins": 0.5650413036346436, "rewards/rejected": 0.7047218680381775, "ronpo/logit": 0.05650413781404495, "ronpo/residual": 0.06540760397911072, "ronpo/residual_abs": 0.11058205366134644, "ronpo/target": -0.008903468027710915, "ronpo/target_abs": 0.07374493777751923, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 125 }, { "drift/chosen_abs": 0.3694007992744446, "drift/rejected_abs": 0.15201464295387268, "epoch": 0.04131903059197457, "grad_norm": 16.875, "learning_rate": 7.457181448114169e-08, "logits/chosen": -2.0490753650665283, "logits/rejected": -1.9781653881072998, "logps/chosen": -0.280802845954895, "logps/rejected": -0.30160802602767944, "loss": 0.4936361312866211, "loss/core": 0.4678853154182434, "loss/preference_sft": 0.2995707094669342, "loss/reference_anchor": 0.7057813405990601, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.6940078735351562, "rewards/margins": 2.1743321418762207, "rewards/rejected": 1.5196759700775146, "ronpo/logit": 0.21743318438529968, "ronpo/residual": 0.2153353989124298, "ronpo/residual_abs": 0.2628825604915619, "ronpo/target": 0.0020978045649826527, "ronpo/target_abs": 0.06583398580551147, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 130 }, { "drift/chosen_abs": 0.22835186123847961, "drift/rejected_abs": 0.17439039051532745, "epoch": 0.04290822407628129, "grad_norm": 90.0, "learning_rate": 7.445526901051018e-08, "logits/chosen": -2.157517910003662, "logits/rejected": -2.1436400413513184, "logps/chosen": -0.2549034357070923, "logps/rejected": -0.27489176392555237, "loss": 0.35979098081588745, "loss/core": 0.3489346206188202, "loss/preference_sft": 0.26387128233909607, "loss/reference_anchor": 0.28379374742507935, "rewards/accuracies": 0.875, "rewards/chosen": 2.2820842266082764, "rewards/margins": 0.5401636958122253, "rewards/rejected": 1.7419202327728271, "ronpo/logit": 0.054016370326280594, "ronpo/residual": 0.05434161424636841, "ronpo/residual_abs": 0.2039295881986618, "ronpo/target": -0.0003252413007430732, "ronpo/target_abs": 0.06969951838254929, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 135 }, { "drift/chosen_abs": 0.2118876725435257, "drift/rejected_abs": 0.1539703756570816, "epoch": 0.044497417560588004, "grad_norm": 38.75, "learning_rate": 7.432482617456355e-08, "logits/chosen": -2.0652270317077637, "logits/rejected": -2.0345349311828613, "logps/chosen": -0.28724318742752075, "logps/rejected": -0.28770583868026733, "loss": 0.2550918161869049, "loss/core": 0.2464626580476761, "loss/preference_sft": 0.28313058614730835, "loss/reference_anchor": 0.2182338684797287, "rewards/accuracies": 0.875, "rewards/chosen": 2.117487668991089, "rewards/margins": 0.5793629884719849, "rewards/rejected": 1.538124918937683, "ronpo/logit": 0.057936299592256546, "ronpo/residual": 0.04774229973554611, "ronpo/residual_abs": 0.19685055315494537, "ronpo/target": 0.010194000788033009, "ronpo/target_abs": 0.0766717940568924, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 140 }, { "drift/chosen_abs": 0.3104633092880249, "drift/rejected_abs": 0.13752777874469757, "epoch": 0.04608661104489471, "grad_norm": 213.0, "learning_rate": 7.41805350275177e-08, "logits/chosen": -2.1027872562408447, "logits/rejected": -2.1394100189208984, "logps/chosen": -0.2751466631889343, "logps/rejected": -0.2653554081916809, "loss": 0.20231513679027557, "loss/core": 0.18823571503162384, "loss/preference_sft": 0.2743663191795349, "loss/reference_anchor": 0.37483280897140503, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.104318380355835, "rewards/margins": 1.7324186563491821, "rewards/rejected": 1.3718997240066528, "ronpo/logit": 0.17324186861515045, "ronpo/residual": 0.1822124421596527, "ronpo/residual_abs": 0.214836984872818, "ronpo/target": -0.008970573544502258, "ronpo/target_abs": 0.07618303596973419, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 145 }, { "drift/chosen_abs": 0.40220266580581665, "drift/rejected_abs": 0.11431588977575302, "epoch": 0.04767580452920143, "grad_norm": 2.265625, "learning_rate": 7.402244983137196e-08, "logits/chosen": -2.0275425910949707, "logits/rejected": -2.13136625289917, "logps/chosen": -0.29549291729927063, "logps/rejected": -0.2743282914161682, "loss": 0.994286835193634, "loss/core": 0.9691765904426575, "loss/preference_sft": 0.2976817190647125, "loss/reference_anchor": 0.6876654624938965, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 4.022026062011719, "rewards/margins": 2.8812575340270996, "rewards/rejected": 1.1407687664031982, "ronpo/logit": 0.28812575340270996, "ronpo/residual": 0.2670125663280487, "ronpo/residual_abs": 0.32439857721328735, "ronpo/target": 0.021113192662596703, "ronpo/target_abs": 0.08010004460811615, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 150 }, { "drift/chosen_abs": 0.311504065990448, "drift/rejected_abs": 0.10349141061306, "epoch": 0.04926499801350814, "grad_norm": 4.875, "learning_rate": 7.385063003550334e-08, "logits/chosen": -2.169419765472412, "logits/rejected": -2.1742537021636963, "logps/chosen": -0.2659375071525574, "logps/rejected": -0.26293236017227173, "loss": 0.6533135175704956, "loss/core": 0.6341246366500854, "loss/preference_sft": 0.2658475339412689, "loss/reference_anchor": 0.5216662287712097, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.1136386394500732, "rewards/margins": 2.081437826156616, "rewards/rejected": 1.0322009325027466, "ronpo/logit": 0.20814375579357147, "ronpo/residual": 0.20334522426128387, "ronpo/residual_abs": 0.2493644505739212, "ronpo/target": 0.004798541776835918, "ronpo/target_abs": 0.07737194001674652, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 155 }, { "drift/chosen_abs": 0.20044144988059998, "drift/rejected_abs": 0.06772055476903915, "epoch": 0.05085419149781486, "grad_norm": 844.0, "learning_rate": 7.36651402543102e-08, "logits/chosen": -2.206714153289795, "logits/rejected": -2.2234230041503906, "logps/chosen": -0.3195026218891144, "logps/rejected": -0.27392783761024475, "loss": 0.5297176241874695, "loss/core": 0.517520546913147, "loss/preference_sft": 0.3230019211769104, "loss/reference_anchor": 0.3161885142326355, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 2.0040805339813232, "rewards/margins": 1.3281657695770264, "rewards/rejected": 0.6759150624275208, "ronpo/logit": 0.13281656801700592, "ronpo/residual": 0.12692351639270782, "ronpo/residual_abs": 0.18193206191062927, "ronpo/target": 0.00589305954053998, "ronpo/target_abs": 0.0840735211968422, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 160 }, { "drift/chosen_abs": 0.1306372582912445, "drift/rejected_abs": 0.0512150414288044, "epoch": 0.052443384982121574, "grad_norm": 4.46875, "learning_rate": 7.346605024291327e-08, "logits/chosen": -2.128614902496338, "logits/rejected": -2.0060853958129883, "logps/chosen": -0.2849295735359192, "logps/rejected": -0.2936997413635254, "loss": 0.11120221763849258, "loss/core": 0.10780216753482819, "loss/preference_sft": 0.28842681646347046, "loss/reference_anchor": 0.06830208003520966, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.3055182695388794, "rewards/margins": 0.7997378706932068, "rewards/rejected": 0.5057803392410278, "ronpo/logit": 0.07997378706932068, "ronpo/residual": 0.07414297759532928, "ronpo/residual_abs": 0.13741916418075562, "ronpo/target": 0.005830812267959118, "ronpo/target_abs": 0.07218202203512192, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 165 }, { "drift/chosen_abs": 0.31605249643325806, "drift/rejected_abs": 0.16812530159950256, "epoch": 0.05403257846642829, "grad_norm": 1.0703125, "learning_rate": 7.325343487092362e-08, "logits/chosen": -2.0245869159698486, "logits/rejected": -2.0181970596313477, "logps/chosen": -0.28985458612442017, "logps/rejected": -0.2747141420841217, "loss": 0.4409272074699402, "loss/core": 0.42512696981430054, "loss/preference_sft": 0.2895664572715759, "loss/reference_anchor": 0.42247915267944336, "rewards/accuracies": 0.875, "rewards/chosen": 3.159921407699585, "rewards/margins": 1.4804481267929077, "rewards/rejected": 1.6794731616973877, "ronpo/logit": 0.14804483950138092, "ronpo/residual": 0.14191299676895142, "ronpo/residual_abs": 0.25526586174964905, "ronpo/target": 0.006131820380687714, "ronpo/target_abs": 0.07981385290622711, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 170 }, { "drift/chosen_abs": 0.19817472994327545, "drift/rejected_abs": 0.11151386797428131, "epoch": 0.055621771950735005, "grad_norm": 26.375, "learning_rate": 7.302737409428734e-08, "logits/chosen": -2.2624623775482178, "logits/rejected": -2.1980812549591064, "logps/chosen": -0.29418444633483887, "logps/rejected": -0.30473509430885315, "loss": 0.05296384170651436, "loss/core": 0.0463070385158062, "loss/preference_sft": 0.2961450517177582, "loss/reference_anchor": 0.1605798900127411, "rewards/accuracies": 0.875, "rewards/chosen": 1.9817469120025635, "rewards/margins": 0.8666085004806519, "rewards/rejected": 1.1151385307312012, "ronpo/logit": 0.08666086941957474, "ronpo/residual": 0.09623171389102936, "ronpo/residual_abs": 0.13510657846927643, "ronpo/target": -0.009570842608809471, "ronpo/target_abs": 0.08157099038362503, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 175 }, { "drift/chosen_abs": 0.47317036986351013, "drift/rejected_abs": 0.22457854449748993, "epoch": 0.057210965435041714, "grad_norm": 30.125, "learning_rate": 7.278795292521733e-08, "logits/chosen": -2.0715465545654297, "logits/rejected": -2.0496222972869873, "logps/chosen": -0.28974977135658264, "logps/rejected": -0.28343138098716736, "loss": 0.9135662913322449, "loss/core": 0.8731704950332642, "loss/preference_sft": 0.2855917811393738, "loss/reference_anchor": 1.1256051063537598, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.731703758239746, "rewards/margins": 2.486184597015381, "rewards/rejected": 2.2455201148986816, "ronpo/logit": 0.24861843883991241, "ronpo/residual": 0.26165086030960083, "ronpo/residual_abs": 0.29524511098861694, "ronpo/target": -0.013032438233494759, "ronpo/target_abs": 0.0726519376039505, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 180 }, { "drift/chosen_abs": 0.12218070030212402, "drift/rejected_abs": 0.06443633139133453, "epoch": 0.05880015891934843, "grad_norm": 4.59375, "learning_rate": 7.253526140022365e-08, "logits/chosen": -2.228654384613037, "logits/rejected": -2.33091139793396, "logps/chosen": -0.29517388343811035, "logps/rejected": -0.2922295331954956, "loss": 0.027977192774415016, "loss/core": 0.025672132149338722, "loss/preference_sft": 0.2922584116458893, "loss/reference_anchor": 0.03663304075598717, "rewards/accuracies": 0.9375, "rewards/chosen": 1.2214815616607666, "rewards/margins": 0.5821638107299805, "rewards/rejected": 0.6393176317214966, "ronpo/logit": 0.05821638181805611, "ronpo/residual": 0.06779725104570389, "ronpo/residual_abs": 0.1186339408159256, "ronpo/target": -0.009580868296325207, "ronpo/target_abs": 0.08872409909963608, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 185 }, { "drift/chosen_abs": 0.12466661632061005, "drift/rejected_abs": 0.05561278015375137, "epoch": 0.060389352403655144, "grad_norm": 0.8671875, "learning_rate": 7.226939454625452e-08, "logits/chosen": -2.314230442047119, "logits/rejected": -2.4117889404296875, "logps/chosen": -0.3054582476615906, "logps/rejected": -0.3027587831020355, "loss": 0.05369839817285538, "loss/core": 0.050449974834918976, "loss/preference_sft": 0.3041912019252777, "loss/reference_anchor": 0.0623929388821125, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.245711088180542, "rewards/margins": 0.6905202865600586, "rewards/rejected": 0.5551908612251282, "ronpo/logit": 0.06905203312635422, "ronpo/residual": 0.062486421316862106, "ronpo/residual_abs": 0.13185548782348633, "ronpo/target": 0.00656561553478241, "ronpo/target_abs": 0.08973196148872375, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 190 }, { "drift/chosen_abs": 0.21235451102256775, "drift/rejected_abs": 0.11808476597070694, "epoch": 0.06197854588796186, "grad_norm": 6.40625, "learning_rate": 7.199045234496058e-08, "logits/chosen": -2.054386615753174, "logits/rejected": -2.0310075283050537, "logps/chosen": -0.2783626914024353, "logps/rejected": -0.27244699001312256, "loss": 0.05899541452527046, "loss/core": 0.05395280569791794, "loss/preference_sft": 0.27698707580566406, "loss/reference_anchor": 0.11637584865093231, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.1214401721954346, "rewards/margins": 0.9426848292350769, "rewards/rejected": 1.1787551641464233, "ronpo/logit": 0.09426848590373993, "ronpo/residual": 0.09450878202915192, "ronpo/residual_abs": 0.1405004858970642, "ronpo/target": -0.00024029240012168884, "ronpo/target_abs": 0.07883597910404205, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 195 }, { "drift/chosen_abs": 0.2827044129371643, "drift/rejected_abs": 0.08958513289690018, "epoch": 0.06356773937226858, "grad_norm": 8.875, "learning_rate": 7.169853969509582e-08, "logits/chosen": -2.036879777908325, "logits/rejected": -2.1572928428649902, "logps/chosen": -0.28990262746810913, "logps/rejected": -0.2788967192173004, "loss": 0.4873984456062317, "loss/core": 0.4752472937107086, "loss/preference_sft": 0.28781306743621826, "loss/reference_anchor": 0.3183947503566742, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.8262548446655273, "rewards/margins": 1.9317960739135742, "rewards/rejected": 0.8944588899612427, "ronpo/logit": 0.19317962229251862, "ronpo/residual": 0.19387605786323547, "ronpo/residual_abs": 0.2424454241991043, "ronpo/target": -0.000696455710567534, "ronpo/target_abs": 0.0863131508231163, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 200 }, { "drift/chosen_abs": 0.2943757474422455, "drift/rejected_abs": 0.10825987160205841, "epoch": 0.06515693285657528, "grad_norm": 101.5, "learning_rate": 7.139376637306952e-08, "logits/chosen": -2.000290632247925, "logits/rejected": -2.0813472270965576, "logps/chosen": -0.2815471589565277, "logps/rejected": -0.2869817614555359, "loss": 0.7546241879463196, "loss/core": 0.7346261739730835, "loss/preference_sft": 0.287017822265625, "loss/reference_anchor": 0.5426746606826782, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.9414560794830322, "rewards/margins": 1.859922170639038, "rewards/rejected": 1.0815337896347046, "ronpo/logit": 0.18599221110343933, "ronpo/residual": 0.21781902015209198, "ronpo/residual_abs": 0.2372894585132599, "ronpo/target": -0.031826816499233246, "ronpo/target_abs": 0.06753235310316086, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 205 }, { "drift/chosen_abs": 0.21875524520874023, "drift/rejected_abs": 0.058099307119846344, "epoch": 0.066746126340882, "grad_norm": 3.40625, "learning_rate": 7.107624699166376e-08, "logits/chosen": -2.2508599758148193, "logits/rejected": -2.1692118644714355, "logps/chosen": -0.3082744777202606, "logps/rejected": -0.31492576003074646, "loss": 1.0409514904022217, "loss/core": 1.020789384841919, "loss/preference_sft": 0.3110334277153015, "loss/reference_anchor": 0.5449555516242981, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.1875526905059814, "rewards/margins": 1.6086879968643188, "rewards/rejected": 0.5788646340370178, "ronpo/logit": 0.16086876392364502, "ronpo/residual": 0.16442081332206726, "ronpo/residual_abs": 0.21720726788043976, "ronpo/target": -0.003552014008164406, "ronpo/target_abs": 0.07966691255569458, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 210 }, { "drift/chosen_abs": 0.32707658410072327, "drift/rejected_abs": 0.164951890707016, "epoch": 0.06833531982518871, "grad_norm": 1.0234375, "learning_rate": 7.074610095693229e-08, "logits/chosen": -2.1667943000793457, "logits/rejected": -2.154695510864258, "logps/chosen": -0.26537930965423584, "logps/rejected": -0.2718598246574402, "loss": 0.29404348134994507, "loss/core": 0.27320724725723267, "loss/preference_sft": 0.2671200931072235, "loss/reference_anchor": 0.5686091780662537, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.270766019821167, "rewards/margins": 1.6218563318252563, "rewards/rejected": 1.648909330368042, "ronpo/logit": 0.1621856391429901, "ronpo/residual": 0.16278550028800964, "ronpo/residual_abs": 0.21330192685127258, "ronpo/target": -0.0005998473498038948, "ronpo/target_abs": 0.07392500340938568, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 215 }, { "drift/chosen_abs": 0.11985316127538681, "drift/rejected_abs": 0.08097579330205917, "epoch": 0.06992451330949544, "grad_norm": 0.482421875, "learning_rate": 7.040345242329681e-08, "logits/chosen": -2.214625120162964, "logits/rejected": -2.082509756088257, "logps/chosen": -0.30458956956863403, "logps/rejected": -0.2999945878982544, "loss": 0.06141216307878494, "loss/core": 0.05872996523976326, "loss/preference_sft": 0.302223265171051, "loss/reference_anchor": 0.046411849558353424, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 1.1941266059875488, "rewards/margins": 0.3879542350769043, "rewards/rejected": 0.806172251701355, "ronpo/logit": 0.03879542648792267, "ronpo/residual": 0.04558854550123215, "ronpo/residual_abs": 0.13173437118530273, "ronpo/target": -0.006793119013309479, "ronpo/target_abs": 0.07642895728349686, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 220 }, { "drift/chosen_abs": 0.23159165680408478, "drift/rejected_abs": 0.10313665866851807, "epoch": 0.07151370679380215, "grad_norm": 4.9375, "learning_rate": 7.004843024685753e-08, "logits/chosen": -2.178675651550293, "logits/rejected": -2.195631504058838, "logps/chosen": -0.2871968150138855, "logps/rejected": -0.2853408753871918, "loss": 0.20537880063056946, "loss/core": 0.19829311966896057, "loss/preference_sft": 0.28042349219322205, "loss/reference_anchor": 0.1744067519903183, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.3159165382385254, "rewards/margins": 1.2847979068756104, "rewards/rejected": 1.031118631362915, "ronpo/logit": 0.12847980856895447, "ronpo/residual": 0.11979977041482925, "ronpo/residual_abs": 0.16873392462730408, "ronpo/target": 0.008680026978254318, "ronpo/target_abs": 0.06639182567596436, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 225 }, { "drift/chosen_abs": 0.26562970876693726, "drift/rejected_abs": 0.0964142382144928, "epoch": 0.07310290027810885, "grad_norm": 2.578125, "learning_rate": 6.968116793693563e-08, "logits/chosen": -2.240947723388672, "logits/rejected": -2.258277654647827, "logps/chosen": -0.2617165446281433, "logps/rejected": -0.26082921028137207, "loss": 0.40380316972732544, "loss/core": 0.3928808569908142, "loss/preference_sft": 0.2601296007633209, "loss/reference_anchor": 0.2860526442527771, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.656296730041504, "rewards/margins": 1.6946455240249634, "rewards/rejected": 0.9616514444351196, "ronpo/logit": 0.16946455836296082, "ronpo/residual": 0.16977901756763458, "ronpo/residual_abs": 0.21348360180854797, "ronpo/target": -0.0003144521324429661, "ronpo/target_abs": 0.07685980945825577, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 230 }, { "drift/chosen_abs": 0.46096473932266235, "drift/rejected_abs": 0.2360442876815796, "epoch": 0.07469209376241558, "grad_norm": 63.25, "learning_rate": 6.930180360586597e-08, "logits/chosen": -2.1169967651367188, "logits/rejected": -2.118549108505249, "logps/chosen": -0.28103750944137573, "logps/rejected": -0.28609365224838257, "loss": 1.180558443069458, "loss/core": 1.1377079486846924, "loss/preference_sft": 0.28971725702285767, "loss/reference_anchor": 1.1953245401382446, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 4.6081223487854, "rewards/margins": 2.326866626739502, "rewards/rejected": 2.2812554836273193, "ronpo/logit": 0.23268663883209229, "ronpo/residual": 0.2368365228176117, "ronpo/residual_abs": 0.38683050870895386, "ronpo/target": -0.0041498770006000996, "ronpo/target_abs": 0.06436969339847565, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 235 }, { "drift/chosen_abs": 0.29032057523727417, "drift/rejected_abs": 0.16610702872276306, "epoch": 0.07628128724672228, "grad_norm": 169.0, "learning_rate": 6.89104799170586e-08, "logits/chosen": -2.1400163173675537, "logits/rejected": -2.216646671295166, "logps/chosen": -0.2673208713531494, "logps/rejected": -0.2859800159931183, "loss": 0.206389382481575, "loss/core": 0.18720188736915588, "loss/preference_sft": 0.2810494303703308, "loss/reference_anchor": 0.520110011100769, "rewards/accuracies": 0.9375, "rewards/chosen": 2.9032058715820312, "rewards/margins": 1.2437211275100708, "rewards/rejected": 1.6594845056533813, "ronpo/logit": 0.12437212467193604, "ronpo/residual": 0.10429469496011734, "ronpo/residual_abs": 0.1717267632484436, "ronpo/target": 0.020077422261238098, "ronpo/target_abs": 0.07924573123455048, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 240 }, { "drift/chosen_abs": 0.18758592009544373, "drift/rejected_abs": 0.0732673853635788, "epoch": 0.07787048073102901, "grad_norm": 2.25, "learning_rate": 6.85073440313489e-08, "logits/chosen": -2.178632974624634, "logits/rejected": -2.1970858573913574, "logps/chosen": -0.29963111877441406, "logps/rejected": -0.3051955997943878, "loss": 0.2845226526260376, "loss/core": 0.277618944644928, "loss/preference_sft": 0.3048124611377716, "loss/reference_anchor": 0.1667684018611908, "rewards/accuracies": 0.875, "rewards/chosen": 1.8743228912353516, "rewards/margins": 1.169947624206543, "rewards/rejected": 0.7043755054473877, "ronpo/logit": 0.11699476093053818, "ronpo/residual": 0.12534533441066742, "ronpo/residual_abs": 0.17828436195850372, "ronpo/target": -0.008350581862032413, "ronpo/target_abs": 0.08381186425685883, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 245 }, { "drift/chosen_abs": 0.36205315589904785, "drift/rejected_abs": 0.0857582613825798, "epoch": 0.07945967421533572, "grad_norm": 7.6875, "learning_rate": 6.809254755165645e-08, "logits/chosen": -2.110821485519409, "logits/rejected": -2.0873301029205322, "logps/chosen": -0.30966055393218994, "logps/rejected": -0.3097647726535797, "loss": 1.2090619802474976, "loss/core": 1.1840286254882812, "loss/preference_sft": 0.3095891773700714, "loss/reference_anchor": 0.6842803955078125, "rewards/accuracies": 0.8125, "rewards/chosen": 3.619492769241333, "rewards/margins": 2.763693332672119, "rewards/rejected": 0.8557994961738586, "ronpo/logit": 0.2763693332672119, "ronpo/residual": 0.2691985070705414, "ronpo/residual_abs": 0.32348698377609253, "ronpo/target": 0.007170822471380234, "ronpo/target_abs": 0.07173763960599899, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 250 }, { "drift/chosen_abs": 0.44590863585472107, "drift/rejected_abs": 0.1995648592710495, "epoch": 0.08104886769964244, "grad_norm": 6.875, "learning_rate": 6.766624646597332e-08, "logits/chosen": -2.1040661334991455, "logits/rejected": -2.106905937194824, "logps/chosen": -0.26315388083457947, "logps/rejected": -0.2701566815376282, "loss": 0.7402136325836182, "loss/core": 0.7059299349784851, "loss/preference_sft": 0.27266380190849304, "loss/reference_anchor": 0.9522676467895508, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": 4.455777168273926, "rewards/margins": 2.532688617706299, "rewards/rejected": 1.9230890274047852, "ronpo/logit": 0.25326889753341675, "ronpo/residual": 0.2603941559791565, "ronpo/residual_abs": 0.30658119916915894, "ronpo/target": -0.007125276140868664, "ronpo/target_abs": 0.08376964181661606, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 255 }, { "drift/chosen_abs": 0.33060067892074585, "drift/rejected_abs": 0.0853637307882309, "epoch": 0.08263806118394915, "grad_norm": 880.0, "learning_rate": 6.722860108870342e-08, "logits/chosen": -2.1947333812713623, "logits/rejected": -2.1691174507141113, "logps/chosen": -0.280915766954422, "logps/rejected": -0.28027015924453735, "loss": 0.8541151881217957, "loss/core": 0.8351130485534668, "loss/preference_sft": 0.2944938540458679, "loss/reference_anchor": 0.5134744644165039, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.304910182952881, "rewards/margins": 2.455112934112549, "rewards/rejected": 0.849797248840332, "ronpo/logit": 0.24551133811473846, "ronpo/residual": 0.2461731731891632, "ronpo/residual_abs": 0.2894217371940613, "ronpo/target": -0.000661846250295639, "ronpo/target_abs": 0.07702169567346573, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 260 }, { "drift/chosen_abs": 0.18190790712833405, "drift/rejected_abs": 0.08150045573711395, "epoch": 0.08422725466825585, "grad_norm": 16.0, "learning_rate": 6.677977600037472e-08, "logits/chosen": -2.1558258533477783, "logits/rejected": -2.1550559997558594, "logps/chosen": -0.2850190997123718, "logps/rejected": -0.29064029455184937, "loss": 0.0755200982093811, "loss/core": 0.07134844362735748, "loss/preference_sft": 0.28953614830970764, "loss/reference_anchor": 0.0902366042137146, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.8185592889785767, "rewards/margins": 1.0096482038497925, "rewards/rejected": 0.8089107275009155, "ronpo/logit": 0.10096482932567596, "ronpo/residual": 0.08930448442697525, "ronpo/residual_abs": 0.14850127696990967, "ronpo/target": 0.011660362593829632, "ronpo/target_abs": 0.07749392092227936, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 265 }, { "drift/chosen_abs": 0.29220855236053467, "drift/rejected_abs": 0.16033557057380676, "epoch": 0.08581644815256258, "grad_norm": 2.328125, "learning_rate": 6.631993998574737e-08, "logits/chosen": -2.2243332862854004, "logits/rejected": -2.183389663696289, "logps/chosen": -0.2799707055091858, "logps/rejected": -0.28995639085769653, "loss": 0.3449793756008148, "loss/core": 0.3193924129009247, "loss/preference_sft": 0.28027012944221497, "loss/reference_anchor": 0.7030280828475952, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.920807361602783, "rewards/margins": 1.320322036743164, "rewards/rejected": 1.60048508644104, "ronpo/logit": 0.13203221559524536, "ronpo/residual": 0.11397619545459747, "ronpo/residual_abs": 0.1874753087759018, "ronpo/target": 0.01805601641535759, "ronpo/target_abs": 0.089533731341362, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 270 }, { "drift/chosen_abs": 0.1789226233959198, "drift/rejected_abs": 0.10433357954025269, "epoch": 0.08740564163686929, "grad_norm": 7.40625, "learning_rate": 6.584926597034054e-08, "logits/chosen": -2.2748570442199707, "logits/rejected": -2.3492331504821777, "logps/chosen": -0.27904680371284485, "logps/rejected": -0.2677331864833832, "loss": 0.04550812020897865, "loss/core": 0.04103340208530426, "loss/preference_sft": 0.27703091502189636, "loss/reference_anchor": 0.10014583170413971, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 1.7892261743545532, "rewards/margins": 0.7504833936691284, "rewards/rejected": 1.0387427806854248, "ronpo/logit": 0.07504834234714508, "ronpo/residual": 0.06096462160348892, "ronpo/residual_abs": 0.12806007266044617, "ronpo/target": 0.014083716087043285, "ronpo/target_abs": 0.07628627121448517, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 275 }, { "drift/chosen_abs": 0.1692146509885788, "drift/rejected_abs": 0.11026661098003387, "epoch": 0.08899483512117601, "grad_norm": 50.0, "learning_rate": 6.536793095540228e-08, "logits/chosen": -2.18296480178833, "logits/rejected": -2.1851444244384766, "logps/chosen": -0.2847527265548706, "logps/rejected": -0.29109668731689453, "loss": 0.08967806398868561, "loss/core": 0.08531448990106583, "loss/preference_sft": 0.28928548097610474, "loss/reference_anchor": 0.09574474394321442, "rewards/accuracies": 0.875, "rewards/chosen": 1.6921466588974, "rewards/margins": 0.5903476476669312, "rewards/rejected": 1.1017990112304688, "ronpo/logit": 0.05903477221727371, "ronpo/residual": 0.04740891978144646, "ronpo/residual_abs": 0.1459716111421585, "ronpo/target": 0.011625855229794979, "ronpo/target_abs": 0.067486971616745, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 280 }, { "drift/chosen_abs": 0.4365122318267822, "drift/rejected_abs": 0.14117935299873352, "epoch": 0.09058402860548272, "grad_norm": 9.875, "learning_rate": 6.487611595134655e-08, "logits/chosen": -2.2247390747070312, "logits/rejected": -2.196688175201416, "logps/chosen": -0.28154322504997253, "logps/rejected": -0.2904406487941742, "loss": 1.4670677185058594, "loss/core": 1.4230749607086182, "loss/preference_sft": 0.2900314927101135, "loss/reference_anchor": 1.2279331684112549, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 4.364717960357666, "rewards/margins": 2.953922986984253, "rewards/rejected": 1.4107954502105713, "ronpo/logit": 0.29539233446121216, "ronpo/residual": 0.3029133677482605, "ronpo/residual_abs": 0.355401873588562, "ronpo/target": -0.0075210509821772575, "ronpo/target_abs": 0.07652939856052399, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 285 }, { "drift/chosen_abs": 0.299263060092926, "drift/rejected_abs": 0.13971708714962006, "epoch": 0.09217322208978942, "grad_norm": 84.0, "learning_rate": 6.43740059096826e-08, "logits/chosen": -2.013880968093872, "logits/rejected": -2.087587594985962, "logps/chosen": -0.2723047435283661, "logps/rejected": -0.28252094984054565, "loss": 0.28749948740005493, "loss/core": 0.2771393358707428, "loss/preference_sft": 0.27838370203971863, "loss/reference_anchor": 0.2681671977043152, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.991696357727051, "rewards/margins": 1.5951488018035889, "rewards/rejected": 1.396547555923462, "ronpo/logit": 0.1595148742198944, "ronpo/residual": 0.15534573793411255, "ronpo/residual_abs": 0.23441383242607117, "ronpo/target": 0.004169141408056021, "ronpo/target_abs": 0.08591829240322113, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 290 }, { "drift/chosen_abs": 0.19509822130203247, "drift/rejected_abs": 0.07152483612298965, "epoch": 0.09376241557409615, "grad_norm": 192.0, "learning_rate": 6.386178965346232e-08, "logits/chosen": -2.1909687519073486, "logits/rejected": -2.1839189529418945, "logps/chosen": -0.30111661553382874, "logps/rejected": -0.3057703375816345, "loss": 0.2420537918806076, "loss/core": 0.23458893597126007, "loss/preference_sft": 0.29602617025375366, "loss/reference_anchor": 0.18367911875247955, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.9500179290771484, "rewards/margins": 1.2395662069320679, "rewards/rejected": 0.7104517817497253, "ronpo/logit": 0.12395663559436798, "ronpo/residual": 0.12118228524923325, "ronpo/residual_abs": 0.16567404568195343, "ronpo/target": 0.0027743394020944834, "ronpo/target_abs": 0.06994090974330902, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 295 }, { "drift/chosen_abs": 0.1367836445569992, "drift/rejected_abs": 0.08203138411045074, "epoch": 0.09535160905840286, "grad_norm": 5.65625, "learning_rate": 6.333965980627154e-08, "logits/chosen": -2.4043996334075928, "logits/rejected": -2.4086849689483643, "logps/chosen": -0.29549726843833923, "logps/rejected": -0.2862420082092285, "loss": 0.030557457357645035, "loss/core": 0.027756040915846825, "loss/preference_sft": 0.29091423749923706, "loss/reference_anchor": 0.05094895511865616, "rewards/accuracies": 0.875, "rewards/chosen": 1.3668571710586548, "rewards/margins": 0.5511847734451294, "rewards/rejected": 0.8156723976135254, "ronpo/logit": 0.05511848255991936, "ronpo/residual": 0.05358175188302994, "ronpo/residual_abs": 0.10816502571105957, "ronpo/target": 0.001536733703687787, "ronpo/target_abs": 0.07785020023584366, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 300 }, { "drift/chosen_abs": 0.22175483405590057, "drift/rejected_abs": 0.10230524837970734, "epoch": 0.09694080254270958, "grad_norm": 2.90625, "learning_rate": 6.280781271979229e-08, "logits/chosen": -2.0780434608459473, "logits/rejected": -2.1060025691986084, "logps/chosen": -0.30538249015808105, "logps/rejected": -0.29317212104797363, "loss": 0.18119879066944122, "loss/core": 0.17126640677452087, "loss/preference_sft": 0.3047506511211395, "loss/reference_anchor": 0.25330713391304016, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.2160160541534424, "rewards/margins": 1.1955595016479492, "rewards/rejected": 1.0204565525054932, "ronpo/logit": 0.11955595016479492, "ronpo/residual": 0.12216754257678986, "ronpo/residual_abs": 0.17102210223674774, "ronpo/target": -0.0026116042863577604, "ronpo/target_abs": 0.07656402885913849, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 305 }, { "drift/chosen_abs": 0.229892298579216, "drift/rejected_abs": 0.15156972408294678, "epoch": 0.09852999602701629, "grad_norm": 226.0, "learning_rate": 6.226644839996278e-08, "logits/chosen": -2.0961735248565674, "logits/rejected": -1.9890098571777344, "logps/chosen": -0.30494624376296997, "logps/rejected": -0.29889440536499023, "loss": 0.08074851334095001, "loss/core": 0.06823646277189255, "loss/preference_sft": 0.3070228397846222, "loss/reference_anchor": 0.32678505778312683, "rewards/accuracies": 0.824999988079071, "rewards/chosen": 2.2979280948638916, "rewards/margins": 0.7833226323127747, "rewards/rejected": 1.5146055221557617, "ronpo/logit": 0.07833226025104523, "ronpo/residual": 0.0831359401345253, "ronpo/residual_abs": 0.14295317232608795, "ronpo/target": -0.0048036882653832436, "ronpo/target_abs": 0.0696224644780159, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 310 }, { "drift/chosen_abs": 0.21884290874004364, "drift/rejected_abs": 0.1715756207704544, "epoch": 0.10011918951132301, "grad_norm": 1.96875, "learning_rate": 6.171577043176358e-08, "logits/chosen": -2.206495761871338, "logits/rejected": -2.2399115562438965, "logps/chosen": -0.2883833348751068, "logps/rejected": -0.2834603190422058, "loss": 0.11746679246425629, "loss/core": 0.08842774480581284, "loss/preference_sft": 0.2884564995765686, "loss/reference_anchor": 0.8008415102958679, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.1873815059661865, "rewards/margins": 0.4734341502189636, "rewards/rejected": 1.7139472961425781, "ronpo/logit": 0.047343410551548004, "ronpo/residual": 0.03924637287855148, "ronpo/residual_abs": 0.13970692455768585, "ronpo/target": 0.008097042329609394, "ronpo/target_abs": 0.07850007712841034, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 315 }, { "drift/chosen_abs": 0.30623868107795715, "drift/rejected_abs": 0.15175890922546387, "epoch": 0.10170838299562972, "grad_norm": 5.15625, "learning_rate": 6.115598590265742e-08, "logits/chosen": -2.048631191253662, "logits/rejected": -2.0478978157043457, "logps/chosen": -0.2675008773803711, "logps/rejected": -0.26748839020729065, "loss": 0.41966015100479126, "loss/core": 0.3942663371562958, "loss/preference_sft": 0.26846352219581604, "loss/reference_anchor": 0.6986923217773438, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.06076717376709, "rewards/margins": 1.5459158420562744, "rewards/rejected": 1.5148515701293945, "ronpo/logit": 0.15459157526493073, "ronpo/residual": 0.15290841460227966, "ronpo/residual_abs": 0.209668830037117, "ronpo/target": 0.0016831799875944853, "ronpo/target_abs": 0.0813584178686142, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 320 }, { "drift/chosen_abs": 0.2687004804611206, "drift/rejected_abs": 0.06457342952489853, "epoch": 0.10329757647993643, "grad_norm": 22.5, "learning_rate": 6.058730532471218e-08, "logits/chosen": -2.212750196456909, "logits/rejected": -2.2377688884735107, "logps/chosen": -0.3067273497581482, "logps/rejected": -0.2894396185874939, "loss": 1.0871208906173706, "loss/core": 1.0651031732559204, "loss/preference_sft": 0.3019077479839325, "loss/reference_anchor": 0.5988877415657043, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.686971426010132, "rewards/margins": 2.045879602432251, "rewards/rejected": 0.6410917043685913, "ronpo/logit": 0.20458796620368958, "ronpo/residual": 0.2081504613161087, "ronpo/residual_abs": 0.24282391369342804, "ronpo/target": -0.003562526311725378, "ronpo/target_abs": 0.07360132783651352, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 325 }, { "drift/chosen_abs": 0.25193846225738525, "drift/rejected_abs": 0.1257767379283905, "epoch": 0.10488676996424315, "grad_norm": 15.5, "learning_rate": 6.000994255543592e-08, "logits/chosen": -2.3243191242218018, "logits/rejected": -2.323913812637329, "logps/chosen": -0.265578955411911, "logps/rejected": -0.26127713918685913, "loss": 0.5734900832176208, "loss/core": 0.5604211091995239, "loss/preference_sft": 0.27206915616989136, "loss/reference_anchor": 0.34619513154029846, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.5193843841552734, "rewards/margins": 1.2678606510162354, "rewards/rejected": 1.2515236139297485, "ronpo/logit": 0.12678608298301697, "ronpo/residual": 0.14241120219230652, "ronpo/residual_abs": 0.2712676525115967, "ronpo/target": -0.015625139698386192, "ronpo/target_abs": 0.09194107353687286, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 330 }, { "drift/chosen_abs": 0.4483170509338379, "drift/rejected_abs": 0.2124291956424713, "epoch": 0.10647596344854986, "grad_norm": 13.0625, "learning_rate": 5.942411471735374e-08, "logits/chosen": -1.9917030334472656, "logits/rejected": -1.9397776126861572, "logps/chosen": -0.29203617572784424, "logps/rejected": -0.2925509214401245, "loss": 0.7571202516555786, "loss/core": 0.7059234380722046, "loss/preference_sft": 0.29065781831741333, "loss/reference_anchor": 1.4337010383605957, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 4.481464385986328, "rewards/margins": 2.3606128692626953, "rewards/rejected": 2.120851516723633, "ronpo/logit": 0.23606130480766296, "ronpo/residual": 0.25377950072288513, "ronpo/residual_abs": 0.3121553659439087, "ronpo/target": -0.017718184739351273, "ronpo/target_abs": 0.07344368100166321, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 335 }, { "drift/chosen_abs": 0.45905232429504395, "drift/rejected_abs": 0.16429868340492249, "epoch": 0.10806515693285658, "grad_norm": 82.0, "learning_rate": 5.883004211635694e-08, "logits/chosen": -1.8839290142059326, "logits/rejected": -1.9421613216400146, "logps/chosen": -0.28718405961990356, "logps/rejected": -0.2739580273628235, "loss": 0.9680835008621216, "loss/core": 0.93462735414505, "loss/preference_sft": 0.2868141829967499, "loss/reference_anchor": 0.927208423614502, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 4.588636875152588, "rewards/margins": 2.951829195022583, "rewards/rejected": 1.6368080377578735, "ronpo/logit": 0.29518288373947144, "ronpo/residual": 0.2900192439556122, "ronpo/residual_abs": 0.33183783292770386, "ronpo/target": 0.005163692403584719, "ronpo/target_abs": 0.06677814573049545, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 340 }, { "drift/chosen_abs": 0.2457018792629242, "drift/rejected_abs": 0.12155888229608536, "epoch": 0.10965435041716329, "grad_norm": 63.25, "learning_rate": 5.8227948158854975e-08, "logits/chosen": -2.178396463394165, "logits/rejected": -2.251084566116333, "logps/chosen": -0.27320748567581177, "logps/rejected": -0.27947288751602173, "loss": 0.1250918060541153, "loss/core": 0.1128639206290245, "loss/preference_sft": 0.28871798515319824, "loss/reference_anchor": 0.3204962909221649, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.4553937911987305, "rewards/margins": 1.2922033071517944, "rewards/rejected": 1.1631903648376465, "ronpo/logit": 0.12922033667564392, "ronpo/residual": 0.1374775469303131, "ronpo/residual_abs": 0.1687041074037552, "ronpo/target": -0.008257212117314339, "ronpo/target_abs": 0.06108090281486511, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 345 }, { "drift/chosen_abs": 0.41426244378089905, "drift/rejected_abs": 0.1653062105178833, "epoch": 0.11124354390147001, "grad_norm": 1.578125, "learning_rate": 5.761805926776142e-08, "logits/chosen": -2.268873691558838, "logits/rejected": -2.296691417694092, "logps/chosen": -0.23340287804603577, "logps/rejected": -0.24630527198314667, "loss": 0.9605229496955872, "loss/core": 0.9214657545089722, "loss/preference_sft": 0.233280748128891, "loss/reference_anchor": 1.0925902128219604, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 4.142511367797852, "rewards/margins": 2.4894490242004395, "rewards/rejected": 1.6530622243881226, "ronpo/logit": 0.24894490838050842, "ronpo/residual": 0.24280676245689392, "ronpo/residual_abs": 0.2877592444419861, "ronpo/target": 0.006138127762824297, "ronpo/target_abs": 0.06590522080659866, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 350 }, { "drift/chosen_abs": 0.28039026260375977, "drift/rejected_abs": 0.10265352576971054, "epoch": 0.11283273738577672, "grad_norm": 18.25, "learning_rate": 5.700060479734557e-08, "logits/chosen": -2.1270534992218018, "logits/rejected": -2.02483868598938, "logps/chosen": -0.28407421708106995, "logps/rejected": -0.27686986327171326, "loss": 0.40096959471702576, "loss/core": 0.3891247808933258, "loss/preference_sft": 0.2740500569343567, "loss/reference_anchor": 0.3110194802284241, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.8039023876190186, "rewards/margins": 1.780269980430603, "rewards/rejected": 1.0236326456069946, "ronpo/logit": 0.17802700400352478, "ronpo/residual": 0.16591885685920715, "ronpo/residual_abs": 0.22742481529712677, "ronpo/target": 0.012108135037124157, "ronpo/target_abs": 0.081517793238163, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 355 }, { "drift/chosen_abs": 0.2177037000656128, "drift/rejected_abs": 0.0887526124715805, "epoch": 0.11442193087008343, "grad_norm": 11.0, "learning_rate": 5.6375816946981745e-08, "logits/chosen": -2.2079660892486572, "logits/rejected": -2.2729287147521973, "logps/chosen": -0.26719698309898376, "logps/rejected": -0.25824791193008423, "loss": 0.08846821635961533, "loss/core": 0.08424760401248932, "loss/preference_sft": 0.26704221963882446, "loss/reference_anchor": 0.09388500452041626, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.1766204833984375, "rewards/margins": 1.2892426252365112, "rewards/rejected": 0.8873778581619263, "ronpo/logit": 0.12892426550388336, "ronpo/residual": 0.13182158768177032, "ronpo/residual_abs": 0.17560000717639923, "ronpo/target": -0.002897311467677355, "ronpo/target_abs": 0.07039253413677216, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 360 }, { "drift/chosen_abs": 0.2076272964477539, "drift/rejected_abs": 0.13079994916915894, "epoch": 0.11601112435439015, "grad_norm": 2.203125, "learning_rate": 5.574393067382849e-08, "logits/chosen": -2.094231128692627, "logits/rejected": -2.132718324661255, "logps/chosen": -0.28609710931777954, "logps/rejected": -0.3247900903224945, "loss": 0.05453226715326309, "loss/core": 0.040341801941394806, "loss/preference_sft": 0.29053995013237, "loss/reference_anchor": 0.37638789415359497, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.0743491649627686, "rewards/margins": 0.7902168035507202, "rewards/rejected": 1.284132480621338, "ronpo/logit": 0.07902167737483978, "ronpo/residual": 0.0732402577996254, "ronpo/residual_abs": 0.12813961505889893, "ronpo/target": 0.005781428888440132, "ronpo/target_abs": 0.08649028837680817, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 365 }, { "drift/chosen_abs": 0.31887754797935486, "drift/rejected_abs": 0.2154131680727005, "epoch": 0.11760031783869686, "grad_norm": 13.0625, "learning_rate": 5.51051836044709e-08, "logits/chosen": -2.1323890686035156, "logits/rejected": -2.1665499210357666, "logps/chosen": -0.2781253159046173, "logps/rejected": -0.28154605627059937, "loss": 0.14984655380249023, "loss/core": 0.10850384086370468, "loss/preference_sft": 0.28171804547309875, "loss/reference_anchor": 1.1530487537384033, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.1884636878967285, "rewards/margins": 1.0351537466049194, "rewards/rejected": 2.1533100605010986, "ronpo/logit": 0.1035153716802597, "ronpo/residual": 0.09588007628917694, "ronpo/residual_abs": 0.16027481853961945, "ronpo/target": 0.007635296322405338, "ronpo/target_abs": 0.0714506059885025, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 370 }, { "drift/chosen_abs": 0.21330508589744568, "drift/rejected_abs": 0.09820286929607391, "epoch": 0.11918951132300358, "grad_norm": 19.25, "learning_rate": 5.445981594555891e-08, "logits/chosen": -2.2786364555358887, "logits/rejected": -2.3025929927825928, "logps/chosen": -0.29398036003112793, "logps/rejected": -0.2887040376663208, "loss": 0.14517758786678314, "loss/core": 0.1390768587589264, "loss/preference_sft": 0.2918010950088501, "loss/reference_anchor": 0.14512670040130615, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.132155179977417, "rewards/margins": 1.1507148742675781, "rewards/rejected": 0.9814404249191284, "ronpo/logit": 0.11507149785757065, "ronpo/residual": 0.12740284204483032, "ronpo/residual_abs": 0.17654451727867126, "ronpo/target": -0.01233135536313057, "ronpo/target_abs": 0.08002184331417084, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 375 }, { "drift/chosen_abs": 0.2749318480491638, "drift/rejected_abs": 0.10027341544628143, "epoch": 0.12077870480731029, "grad_norm": 0.65234375, "learning_rate": 5.3808070393475435e-08, "logits/chosen": -2.1572823524475098, "logits/rejected": -2.1735129356384277, "logps/chosen": -0.2892719805240631, "logps/rejected": -0.28280091285705566, "loss": 0.5006850361824036, "loss/core": 0.4873785972595215, "loss/preference_sft": 0.28779616951942444, "loss/reference_anchor": 0.35140353441238403, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.7480342388153076, "rewards/margins": 1.75465989112854, "rewards/rejected": 0.9933745265007019, "ronpo/logit": 0.17546597123146057, "ronpo/residual": 0.16056114435195923, "ronpo/residual_abs": 0.2221776247024536, "ronpo/target": 0.014904849231243134, "ronpo/target_abs": 0.07894332706928253, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 380 }, { "drift/chosen_abs": 0.2522042691707611, "drift/rejected_abs": 0.1486947387456894, "epoch": 0.12236789829161701, "grad_norm": 62.75, "learning_rate": 5.315019204306817e-08, "logits/chosen": -2.2786965370178223, "logits/rejected": -2.2488064765930176, "logps/chosen": -0.27131175994873047, "logps/rejected": -0.26389139890670776, "loss": 0.17779920995235443, "loss/core": 0.16905324161052704, "loss/preference_sft": 0.26512569189071655, "loss/reference_anchor": 0.22337183356285095, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.521192789077759, "rewards/margins": 1.0361357927322388, "rewards/rejected": 1.4850571155548096, "ronpo/logit": 0.10361359268426895, "ronpo/residual": 0.0915413573384285, "ronpo/residual_abs": 0.15346641838550568, "ronpo/target": 0.012072229757905006, "ronpo/target_abs": 0.08567623794078827, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 385 }, { "drift/chosen_abs": 0.2105921506881714, "drift/rejected_abs": 0.12278739362955093, "epoch": 0.12395709177592372, "grad_norm": 3.0625, "learning_rate": 5.248642829547946e-08, "logits/chosen": -2.0767416954040527, "logits/rejected": -2.117583751678467, "logps/chosen": -0.3050832748413086, "logps/rejected": -0.2946804165840149, "loss": 0.08686129003763199, "loss/core": 0.07733297348022461, "loss/preference_sft": 0.2956315875053406, "loss/reference_anchor": 0.2426745593547821, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.1027276515960693, "rewards/margins": 0.8792668581008911, "rewards/rejected": 1.2234607934951782, "ronpo/logit": 0.08792667090892792, "ronpo/residual": 0.10677643120288849, "ronpo/residual_abs": 0.141727477312088, "ronpo/target": -0.018849752843379974, "ronpo/target_abs": 0.07699684798717499, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 390 }, { "drift/chosen_abs": 0.2143975794315338, "drift/rejected_abs": 0.07005457580089569, "epoch": 0.12554628526023043, "grad_norm": 2.703125, "learning_rate": 5.181702876510881e-08, "logits/chosen": -2.1568331718444824, "logits/rejected": -2.2212538719177246, "logps/chosen": -0.3092118203639984, "logps/rejected": -0.2957898676395416, "loss": 0.3654123842716217, "loss/core": 0.3564660847187042, "loss/preference_sft": 0.30225998163223267, "loss/reference_anchor": 0.2253817319869995, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.142920732498169, "rewards/margins": 1.4426730871200562, "rewards/rejected": 0.700247585773468, "ronpo/logit": 0.14426730573177338, "ronpo/residual": 0.14315283298492432, "ronpo/residual_abs": 0.18677830696105957, "ronpo/target": 0.0011144764721393585, "ronpo/target_abs": 0.07045630365610123, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 395 }, { "drift/chosen_abs": 0.19173476099967957, "drift/rejected_abs": 0.08401096612215042, "epoch": 0.12713547874453715, "grad_norm": 9.1875, "learning_rate": 5.114224518574318e-08, "logits/chosen": -2.2294397354125977, "logits/rejected": -2.318349599838257, "logps/chosen": -0.25690674781799316, "logps/rejected": -0.26110321283340454, "loss": 0.07373060286045074, "loss/core": 0.06859154254198074, "loss/preference_sft": 0.2571546137332916, "loss/reference_anchor": 0.12111492455005646, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 1.916455864906311, "rewards/margins": 1.0823928117752075, "rewards/rejected": 0.8340628743171692, "ronpo/logit": 0.10823929309844971, "ronpo/residual": 0.10606695711612701, "ronpo/residual_abs": 0.15262378752231598, "ronpo/target": 0.0021723308600485325, "ronpo/target_abs": 0.07407379150390625, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 400 }, { "drift/chosen_abs": 0.17076550424098969, "drift/rejected_abs": 0.1465177685022354, "epoch": 0.12872467222884387, "grad_norm": 1.5859375, "learning_rate": 5.046233131589012e-08, "logits/chosen": -2.2917768955230713, "logits/rejected": -2.300046443939209, "logps/chosen": -0.29497990012168884, "logps/rejected": -0.2882741093635559, "loss": 0.28003552556037903, "loss/core": 0.27095627784729004, "loss/preference_sft": 0.28535428643226624, "loss/reference_anchor": 0.2308715283870697, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.7055221796035767, "rewards/margins": 0.24134886264801025, "rewards/rejected": 1.464173436164856, "ronpo/logit": 0.024134885519742966, "ronpo/residual": 0.025067275390028954, "ronpo/residual_abs": 0.1801082193851471, "ronpo/target": -0.0009323973208665848, "ronpo/target_abs": 0.07486466318368912, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 405 }, { "drift/chosen_abs": 0.18705379962921143, "drift/rejected_abs": 0.05877493694424629, "epoch": 0.13031386571315057, "grad_norm": 0.94921875, "learning_rate": 4.977754284334966e-08, "logits/chosen": -2.2271358966827393, "logits/rejected": -2.1403744220733643, "logps/chosen": -0.28943097591400146, "logps/rejected": -0.28723686933517456, "loss": 0.34275034070014954, "loss/core": 0.33544498682022095, "loss/preference_sft": 0.2885206937789917, "loss/reference_anchor": 0.1798722892999649, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 1.8687069416046143, "rewards/margins": 1.3043181896209717, "rewards/rejected": 0.5643887519836426, "ronpo/logit": 0.13043180108070374, "ronpo/residual": 0.1327742636203766, "ronpo/residual_abs": 0.1725582629442215, "ronpo/target": -0.0023424329701811075, "ronpo/target_abs": 0.07028114050626755, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 410 }, { "drift/chosen_abs": 0.12771040201187134, "drift/rejected_abs": 0.05937645584344864, "epoch": 0.1319030591974573, "grad_norm": 18.75, "learning_rate": 4.9088137289060525e-08, "logits/chosen": -2.2670741081237793, "logits/rejected": -2.297637701034546, "logps/chosen": -0.2853187322616577, "logps/rejected": -0.27392154932022095, "loss": 0.0417989157140255, "loss/core": 0.03925047814846039, "loss/preference_sft": 0.27748462557792664, "loss/reference_anchor": 0.04506412148475647, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 1.277104139328003, "rewards/margins": 0.6841433644294739, "rewards/rejected": 0.5929605960845947, "ronpo/logit": 0.06841433793306351, "ronpo/residual": 0.06507981568574905, "ronpo/residual_abs": 0.1218712329864502, "ronpo/target": 0.003334520850330591, "ronpo/target_abs": 0.07405976951122284, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 415 }, { "drift/chosen_abs": 0.3893112242221832, "drift/rejected_abs": 0.1252056360244751, "epoch": 0.133492252681764, "grad_norm": 1.9921875, "learning_rate": 4.8394373910257054e-08, "logits/chosen": -2.1123807430267334, "logits/rejected": -2.231128692626953, "logps/chosen": -0.2718810737133026, "logps/rejected": -0.272949755191803, "loss": 0.8291887044906616, "loss/core": 0.8047215342521667, "loss/preference_sft": 0.27599674463272095, "loss/reference_anchor": 0.6714580059051514, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.8931126594543457, "rewards/margins": 2.735173463821411, "rewards/rejected": 1.1579387187957764, "ronpo/logit": 0.27351734042167664, "ronpo/residual": 0.26163390278816223, "ronpo/residual_abs": 0.302009254693985, "ronpo/target": 0.011883469298481941, "ronpo/target_abs": 0.07862479239702225, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 420 }, { "drift/chosen_abs": 0.42474645376205444, "drift/rejected_abs": 0.13992993533611298, "epoch": 0.1350814461660707, "grad_norm": 304.0, "learning_rate": 4.769651360297318e-08, "logits/chosen": -2.1638529300689697, "logits/rejected": -2.1579020023345947, "logps/chosen": -0.29796314239501953, "logps/rejected": -0.29541391134262085, "loss": 1.2007313966751099, "loss/core": 1.1638450622558594, "loss/preference_sft": 0.29007643461227417, "loss/reference_anchor": 1.0248842239379883, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 4.246431350708008, "rewards/margins": 2.849513530731201, "rewards/rejected": 1.3969179391860962, "ronpo/logit": 0.284951388835907, "ronpo/residual": 0.2653922438621521, "ronpo/residual_abs": 0.3415285050868988, "ronpo/target": 0.019559111446142197, "ronpo/target_abs": 0.09040865302085876, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 425 }, { "drift/chosen_abs": 0.18750756978988647, "drift/rejected_abs": 0.09092601388692856, "epoch": 0.13667063965037743, "grad_norm": 11.375, "learning_rate": 4.699481880393007e-08, "logits/chosen": -2.430100679397583, "logits/rejected": -2.416260242462158, "logps/chosen": -0.27946940064430237, "logps/rejected": -0.2753836512565613, "loss": 0.09714354574680328, "loss/core": 0.09125364571809769, "loss/preference_sft": 0.273917555809021, "loss/reference_anchor": 0.1408909112215042, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.8739410638809204, "rewards/margins": 0.9661051630973816, "rewards/rejected": 0.9078359603881836, "ronpo/logit": 0.09661051630973816, "ronpo/residual": 0.090634785592556, "ronpo/residual_abs": 0.14002564549446106, "ronpo/target": 0.005975720938295126, "ronpo/target_abs": 0.06694401055574417, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 430 }, { "drift/chosen_abs": 0.1731262505054474, "drift/rejected_abs": 0.0927744135260582, "epoch": 0.13825983313468415, "grad_norm": 11.875, "learning_rate": 4.628955339184442e-08, "logits/chosen": -2.182917356491089, "logits/rejected": -2.1770148277282715, "logps/chosen": -0.27976903319358826, "logps/rejected": -0.281031996011734, "loss": 0.04321626201272011, "loss/core": 0.03948911279439926, "loss/preference_sft": 0.27680492401123047, "loss/reference_anchor": 0.07880944013595581, "rewards/accuracies": 0.9375, "rewards/chosen": 1.7302322387695312, "rewards/margins": 0.80370032787323, "rewards/rejected": 0.9265319108963013, "ronpo/logit": 0.08037003129720688, "ronpo/residual": 0.06914183497428894, "ronpo/residual_abs": 0.1280720978975296, "ronpo/target": 0.011228211224079132, "ronpo/target_abs": 0.06739668548107147, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 435 }, { "drift/chosen_abs": 0.31899359822273254, "drift/rejected_abs": 0.1331823766231537, "epoch": 0.13984902661899087, "grad_norm": 3.984375, "learning_rate": 4.55809825881945e-08, "logits/chosen": -2.2753958702087402, "logits/rejected": -2.300555467605591, "logps/chosen": -0.2945733666419983, "logps/rejected": -0.29113486409187317, "loss": 0.7341649532318115, "loss/core": 0.7019937634468079, "loss/preference_sft": 0.29861825704574585, "loss/reference_anchor": 0.8893128633499146, "rewards/accuracies": 0.875, "rewards/chosen": 3.1885926723480225, "rewards/margins": 1.86493718624115, "rewards/rejected": 1.323655366897583, "ronpo/logit": 0.18649372458457947, "ronpo/residual": 0.19063273072242737, "ronpo/residual_abs": 0.24889390170574188, "ronpo/target": -0.004139023832976818, "ronpo/target_abs": 0.08812376856803894, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 440 }, { "drift/chosen_abs": 0.19011065363883972, "drift/rejected_abs": 0.16861161589622498, "epoch": 0.14143822010329757, "grad_norm": 286.0, "learning_rate": 4.486937285748113e-08, "logits/chosen": -2.2313015460968018, "logits/rejected": -2.1554415225982666, "logps/chosen": -0.280648797750473, "logps/rejected": -0.27201026678085327, "loss": 0.22713658213615417, "loss/core": 0.21747353672981262, "loss/preference_sft": 0.2815045714378357, "loss/reference_anchor": 0.24793705344200134, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 1.9011064767837524, "rewards/margins": 0.21516799926757812, "rewards/rejected": 1.6859385967254639, "ronpo/logit": 0.021516799926757812, "ronpo/residual": 0.026197385042905807, "ronpo/residual_abs": 0.18634097278118134, "ronpo/target": -0.0046805874444544315, "ronpo/target_abs": 0.0748194083571434, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 445 }, { "drift/chosen_abs": 0.3207959234714508, "drift/rejected_abs": 0.1021362692117691, "epoch": 0.1430274135876043, "grad_norm": 31.25, "learning_rate": 4.415499180702132e-08, "logits/chosen": -2.263270854949951, "logits/rejected": -2.3271467685699463, "logps/chosen": -0.2672632336616516, "logps/rejected": -0.2835923731327057, "loss": 0.7797986268997192, "loss/core": 0.7587357759475708, "loss/preference_sft": 0.2777632474899292, "loss/reference_anchor": 0.5740165114402771, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.2072951793670654, "rewards/margins": 2.1891021728515625, "rewards/rejected": 1.018193006515503, "ronpo/logit": 0.21891024708747864, "ronpo/residual": 0.228356271982193, "ronpo/residual_abs": 0.27203264832496643, "ronpo/target": -0.00944601558148861, "ronpo/target_abs": 0.06918533146381378, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 450 }, { "drift/chosen_abs": 0.2078581303358078, "drift/rejected_abs": 0.09944683313369751, "epoch": 0.144616607071911, "grad_norm": 9.875, "learning_rate": 4.3438108086312104e-08, "logits/chosen": -2.315324306488037, "logits/rejected": -2.3483715057373047, "logps/chosen": -0.2843065857887268, "logps/rejected": -0.2822841703891754, "loss": 0.1076749935746193, "loss/core": 0.10172615200281143, "loss/preference_sft": 0.2944115698337555, "loss/reference_anchor": 0.14052584767341614, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.0785813331604004, "rewards/margins": 1.0842633247375488, "rewards/rejected": 0.994317889213562, "ronpo/logit": 0.10842633247375488, "ronpo/residual": 0.10938017070293427, "ronpo/residual_abs": 0.16080966591835022, "ronpo/target": -0.0009538428857922554, "ronpo/target_abs": 0.07275767624378204, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 455 }, { "drift/chosen_abs": 0.37791651487350464, "drift/rejected_abs": 0.16409462690353394, "epoch": 0.1462058005562177, "grad_norm": 115.5, "learning_rate": 4.271899128600245e-08, "logits/chosen": -1.805584192276001, "logits/rejected": -1.6658241748809814, "logps/chosen": -0.2807144224643707, "logps/rejected": -0.30006617307662964, "loss": 0.5550448298454285, "loss/core": 0.5333111882209778, "loss/preference_sft": 0.29056912660598755, "loss/reference_anchor": 0.5919061303138733, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 3.7732338905334473, "rewards/margins": 2.1390435695648193, "rewards/rejected": 1.634190320968628, "ronpo/logit": 0.21390438079833984, "ronpo/residual": 0.2101598083972931, "ronpo/residual_abs": 0.26126962900161743, "ronpo/target": 0.0037445551715791225, "ronpo/target_abs": 0.07518764585256577, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 460 }, { "drift/chosen_abs": 0.3444373905658722, "drift/rejected_abs": 0.23399516940116882, "epoch": 0.14779499404052443, "grad_norm": 318.0, "learning_rate": 4.199791183651124e-08, "logits/chosen": -2.0957283973693848, "logits/rejected": -2.116079092025757, "logps/chosen": -0.2696365416049957, "logps/rejected": -0.2735532522201538, "loss": 0.8012321591377258, "loss/core": 0.7794309854507446, "loss/preference_sft": 0.277563214302063, "loss/reference_anchor": 0.5951352119445801, "rewards/accuracies": 0.875, "rewards/chosen": 3.4441075325012207, "rewards/margins": 1.1061354875564575, "rewards/rejected": 2.3379721641540527, "ronpo/logit": 0.11061354726552963, "ronpo/residual": 0.099038265645504, "ronpo/residual_abs": 0.29626208543777466, "ronpo/target": 0.011575284413993359, "ronpo/target_abs": 0.06793732196092606, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 465 }, { "drift/chosen_abs": 0.29984086751937866, "drift/rejected_abs": 0.11699891090393066, "epoch": 0.14938418752483115, "grad_norm": 2.640625, "learning_rate": 4.127514090632942e-08, "logits/chosen": -2.075214147567749, "logits/rejected": -2.0434682369232178, "logps/chosen": -0.254089891910553, "logps/rejected": -0.2596934139728546, "loss": 0.7762528657913208, "loss/core": 0.7569972276687622, "loss/preference_sft": 0.2593012750148773, "loss/reference_anchor": 0.5242301821708679, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": 2.996891975402832, "rewards/margins": 1.8279199600219727, "rewards/rejected": 1.1689717769622803, "ronpo/logit": 0.18279200792312622, "ronpo/residual": 0.1851007044315338, "ronpo/residual_abs": 0.25966882705688477, "ronpo/target": -0.0023087128065526485, "ronpo/target_abs": 0.07931704819202423, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 470 }, { "drift/chosen_abs": 0.25055521726608276, "drift/rejected_abs": 0.09638573229312897, "epoch": 0.15097338100913787, "grad_norm": 8.3125, "learning_rate": 4.055095030004469e-08, "logits/chosen": -2.195185661315918, "logits/rejected": -2.1315507888793945, "logps/chosen": -0.30518704652786255, "logps/rejected": -0.29946842789649963, "loss": 0.2174142301082611, "loss/core": 0.21045875549316406, "loss/preference_sft": 0.3060879111289978, "loss/reference_anchor": 0.16811935603618622, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.5054726600646973, "rewards/margins": 1.542541265487671, "rewards/rejected": 0.9629314541816711, "ronpo/logit": 0.15425412356853485, "ronpo/residual": 0.15975168347358704, "ronpo/residual_abs": 0.21089980006217957, "ronpo/target": -0.0054975529201328754, "ronpo/target_abs": 0.07147940248250961, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 475 }, { "drift/chosen_abs": 0.22197389602661133, "drift/rejected_abs": 0.1093837171792984, "epoch": 0.15256257449344457, "grad_norm": 1.5390625, "learning_rate": 3.982561235612682e-08, "logits/chosen": -2.1202902793884277, "logits/rejected": -2.142862319946289, "logps/chosen": -0.2731131911277771, "logps/rejected": -0.2682819962501526, "loss": 0.08932836353778839, "loss/core": 0.08391264826059341, "loss/preference_sft": 0.2691730856895447, "loss/reference_anchor": 0.12781760096549988, "rewards/accuracies": 0.875, "rewards/chosen": 2.2168147563934326, "rewards/margins": 1.1324942111968994, "rewards/rejected": 1.0843205451965332, "ronpo/logit": 0.11324940621852875, "ronpo/residual": 0.13424549996852875, "ronpo/residual_abs": 0.1686258614063263, "ronpo/target": -0.020996104925870895, "ronpo/target_abs": 0.07291201502084732, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 480 }, { "drift/chosen_abs": 0.29734545946121216, "drift/rejected_abs": 0.1604335755109787, "epoch": 0.1541517679777513, "grad_norm": 6.125, "learning_rate": 3.90993998445124e-08, "logits/chosen": -1.9032106399536133, "logits/rejected": -2.025702953338623, "logps/chosen": -0.28302717208862305, "logps/rejected": -0.28014808893203735, "loss": 0.12766507267951965, "loss/core": 0.1155250072479248, "loss/preference_sft": 0.28657975792884827, "loss/reference_anchor": 0.3182009756565094, "rewards/accuracies": 0.875, "rewards/chosen": 2.972499370574951, "rewards/margins": 1.370253086090088, "rewards/rejected": 1.6022460460662842, "ronpo/logit": 0.13702531158924103, "ronpo/residual": 0.1244589239358902, "ronpo/residual_abs": 0.1808883100748062, "ronpo/target": 0.012566402554512024, "ronpo/target_abs": 0.08073397725820541, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 485 }, { "drift/chosen_abs": 0.36853092908859253, "drift/rejected_abs": 0.15083953738212585, "epoch": 0.15574096146205801, "grad_norm": 320.0, "learning_rate": 3.837258586402711e-08, "logits/chosen": -2.1028637886047363, "logits/rejected": -2.0833709239959717, "logps/chosen": -0.2817224860191345, "logps/rejected": -0.2832174301147461, "loss": 0.6819409132003784, "loss/core": 0.6572893857955933, "loss/preference_sft": 0.2822495102882385, "loss/reference_anchor": 0.6761033535003662, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.6840882301330566, "rewards/margins": 2.1773343086242676, "rewards/rejected": 1.50675368309021, "ronpo/logit": 0.2177334576845169, "ronpo/residual": 0.2175024449825287, "ronpo/residual_abs": 0.26726433634757996, "ronpo/target": 0.00023099928512237966, "ronpo/target_abs": 0.07836781442165375, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 490 }, { "drift/chosen_abs": 0.22081363201141357, "drift/rejected_abs": 0.18465587496757507, "epoch": 0.1573301549463647, "grad_norm": 378.0, "learning_rate": 3.7645443739684514e-08, "logits/chosen": -2.126253843307495, "logits/rejected": -2.0972957611083984, "logps/chosen": -0.2862113416194916, "logps/rejected": -0.2739894986152649, "loss": 1.195561170578003, "loss/core": 1.1711348295211792, "loss/preference_sft": 0.2842695713043213, "loss/reference_anchor": 0.6694715619087219, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.2055563926696777, "rewards/margins": 0.3634433150291443, "rewards/rejected": 1.8421132564544678, "ronpo/logit": 0.03634430468082428, "ronpo/residual": 0.03679324686527252, "ronpo/residual_abs": 0.3129720687866211, "ronpo/target": -0.00044894107850268483, "ronpo/target_abs": 0.0857948586344719, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 495 }, { "drift/chosen_abs": 0.2433653324842453, "drift/rejected_abs": 0.14672759175300598, "epoch": 0.15891934843067143, "grad_norm": 3.578125, "learning_rate": 3.6918246919899624e-08, "logits/chosen": -2.231562376022339, "logits/rejected": -2.089289903640747, "logps/chosen": -0.2816031575202942, "logps/rejected": -0.2773313522338867, "loss": 0.23030047118663788, "loss/core": 0.21950602531433105, "loss/preference_sft": 0.27701348066329956, "loss/reference_anchor": 0.28071126341819763, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.4336533546447754, "rewards/margins": 0.9874534606933594, "rewards/rejected": 1.446199893951416, "ronpo/logit": 0.09874534606933594, "ronpo/residual": 0.10226426273584366, "ronpo/residual_abs": 0.20653577148914337, "ronpo/target": -0.0035189155023545027, "ronpo/target_abs": 0.07701299339532852, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 500 }, { "drift/chosen_abs": 0.16075539588928223, "drift/rejected_abs": 0.1068691611289978, "epoch": 0.16050854191497815, "grad_norm": 4.40625, "learning_rate": 3.619126887365622e-08, "logits/chosen": -2.2844841480255127, "logits/rejected": -2.2047648429870605, "logps/chosen": -0.2954888343811035, "logps/rejected": -0.280442476272583, "loss": 0.028839338570833206, "loss/core": 0.025353241711854935, "loss/preference_sft": 0.29189762473106384, "loss/reference_anchor": 0.0704130083322525, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 1.6047475337982178, "rewards/margins": 0.536501944065094, "rewards/rejected": 1.0682456493377686, "ronpo/logit": 0.05365020036697388, "ronpo/residual": 0.03312177211046219, "ronpo/residual_abs": 0.1087198257446289, "ronpo/target": 0.020528428256511688, "ronpo/target_abs": 0.0795794278383255, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 505 }, { "drift/chosen_abs": 0.3148565888404846, "drift/rejected_abs": 0.16070185601711273, "epoch": 0.16209773539928488, "grad_norm": 2.609375, "learning_rate": 3.5464782987666235e-08, "logits/chosen": -1.940596342086792, "logits/rejected": -1.9633086919784546, "logps/chosen": -0.26210013031959534, "logps/rejected": -0.26328200101852417, "loss": 0.426157146692276, "loss/core": 0.40358486771583557, "loss/preference_sft": 0.263058602809906, "loss/reference_anchor": 0.6186167597770691, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.1481029987335205, "rewards/margins": 1.5438060760498047, "rewards/rejected": 1.6042969226837158, "ronpo/logit": 0.15438061952590942, "ronpo/residual": 0.15045881271362305, "ronpo/residual_abs": 0.1978466957807541, "ronpo/target": 0.00392179936170578, "ronpo/target_abs": 0.06807820498943329, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 510 }, { "drift/chosen_abs": 0.2575669288635254, "drift/rejected_abs": 0.15305769443511963, "epoch": 0.16368692888359157, "grad_norm": 5.75, "learning_rate": 3.473906246356032e-08, "logits/chosen": -2.141854763031006, "logits/rejected": -2.2106332778930664, "logps/chosen": -0.2897399663925171, "logps/rejected": -0.27617400884628296, "loss": 0.3367363512516022, "loss/core": 0.32625168561935425, "loss/preference_sft": 0.28051096200942993, "loss/reference_anchor": 0.2715080976486206, "rewards/accuracies": 0.875, "rewards/chosen": 2.574777126312256, "rewards/margins": 1.0462206602096558, "rewards/rejected": 1.528556227684021, "ronpo/logit": 0.10462206602096558, "ronpo/residual": 0.09638858586549759, "ronpo/residual_abs": 0.214156836271286, "ronpo/target": 0.008233482949435711, "ronpo/target_abs": 0.07465226948261261, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 515 }, { "drift/chosen_abs": 0.21658673882484436, "drift/rejected_abs": 0.10446375608444214, "epoch": 0.1652761223678983, "grad_norm": 2.0625, "learning_rate": 3.401438021514779e-08, "logits/chosen": -2.1490402221679688, "logits/rejected": -2.1608524322509766, "logps/chosen": -0.29304903745651245, "logps/rejected": -0.3020794689655304, "loss": 0.2509724795818329, "loss/core": 0.23218867182731628, "loss/preference_sft": 0.2949676513671875, "loss/reference_anchor": 0.5071809887886047, "rewards/accuracies": 0.9375, "rewards/chosen": 2.165867328643799, "rewards/margins": 1.1303069591522217, "rewards/rejected": 1.0355603694915771, "ronpo/logit": 0.11303069442510605, "ronpo/residual": 0.10117007791996002, "ronpo/residual_abs": 0.1607782542705536, "ronpo/target": 0.011860623024404049, "ronpo/target_abs": 0.08329831808805466, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 520 }, { "drift/chosen_abs": 0.2207900583744049, "drift/rejected_abs": 0.06476399302482605, "epoch": 0.16686531585220502, "grad_norm": 7.625, "learning_rate": 3.329100876578495e-08, "logits/chosen": -1.9039376974105835, "logits/rejected": -2.0480172634124756, "logps/chosen": -0.3111041188240051, "logps/rejected": -0.3204375207424164, "loss": 0.27312102913856506, "loss/core": 0.2654520869255066, "loss/preference_sft": 0.32211512327194214, "loss/reference_anchor": 0.18690235912799835, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.2061574459075928, "rewards/margins": 1.5601660013198853, "rewards/rejected": 0.6459915637969971, "ronpo/logit": 0.15601658821105957, "ronpo/residual": 0.16684645414352417, "ronpo/residual_abs": 0.20652273297309875, "ronpo/target": -0.010829855687916279, "ronpo/target_abs": 0.08058717846870422, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 525 }, { "drift/chosen_abs": 0.1540854275226593, "drift/rejected_abs": 0.06486465036869049, "epoch": 0.1684545093365117, "grad_norm": 1.3046875, "learning_rate": 3.256922014589015e-08, "logits/chosen": -2.324816942214966, "logits/rejected": -2.319702625274658, "logps/chosen": -0.26595860719680786, "logps/rejected": -0.26108628511428833, "loss": 0.06081946939229965, "loss/core": 0.05783188343048096, "loss/preference_sft": 0.26668626070022583, "loss/reference_anchor": 0.05869119241833687, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 1.5408542156219482, "rewards/margins": 0.8943354487419128, "rewards/rejected": 0.6465188264846802, "ronpo/logit": 0.08943354338407516, "ronpo/residual": 0.09258811175823212, "ronpo/residual_abs": 0.13264910876750946, "ronpo/target": -0.003154561622068286, "ronpo/target_abs": 0.0733359232544899, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 530 }, { "drift/chosen_abs": 0.3481886386871338, "drift/rejected_abs": 0.14496779441833496, "epoch": 0.17004370282081843, "grad_norm": 2.40625, "learning_rate": 3.184928579064438e-08, "logits/chosen": -2.240485429763794, "logits/rejected": -2.2552919387817383, "logps/chosen": -0.3145575523376465, "logps/rejected": -0.3206997513771057, "loss": 1.516591191291809, "loss/core": 1.477942705154419, "loss/preference_sft": 0.31602850556373596, "loss/reference_anchor": 1.0726332664489746, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.4811675548553467, "rewards/margins": 2.195894241333008, "rewards/rejected": 1.2852733135223389, "ronpo/logit": 0.21958939731121063, "ronpo/residual": 0.23421013355255127, "ronpo/residual_abs": 0.340684711933136, "ronpo/target": -0.014620688743889332, "ronpo/target_abs": 0.08300719410181046, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 535 }, { "drift/chosen_abs": 0.2462986409664154, "drift/rejected_abs": 0.10847475379705429, "epoch": 0.17163289630512515, "grad_norm": 3.234375, "learning_rate": 3.1131476437915473e-08, "logits/chosen": -2.202176570892334, "logits/rejected": -2.2374606132507324, "logps/chosen": -0.27796033024787903, "logps/rejected": -0.2893964350223541, "loss": 0.22307059168815613, "loss/core": 0.20895227789878845, "loss/preference_sft": 0.28652340173721313, "loss/reference_anchor": 0.3747285306453705, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.462252140045166, "rewards/margins": 1.377599835395813, "rewards/rejected": 1.084652304649353, "ronpo/logit": 0.1377600133419037, "ronpo/residual": 0.13737379014492035, "ronpo/residual_abs": 0.19108590483665466, "ronpo/target": 0.0003861982258968055, "ronpo/target_abs": 0.06714820116758347, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 540 }, { "drift/chosen_abs": 0.39844200015068054, "drift/rejected_abs": 0.2459792196750641, "epoch": 0.17322208978943188, "grad_norm": 5.1875, "learning_rate": 3.0416062026444745e-08, "logits/chosen": -1.9908326864242554, "logits/rejected": -2.015047550201416, "logps/chosen": -0.26075753569602966, "logps/rejected": -0.27910637855529785, "loss": 0.45975977182388306, "loss/core": 0.42998823523521423, "loss/preference_sft": 0.2777344584465027, "loss/reference_anchor": 0.8228422999382019, "rewards/accuracies": 0.875, "rewards/chosen": 3.9832959175109863, "rewards/margins": 1.527392029762268, "rewards/rejected": 2.455904006958008, "ronpo/logit": 0.15273919701576233, "ronpo/residual": 0.1590009331703186, "ronpo/residual_abs": 0.25908511877059937, "ronpo/target": -0.006261733826249838, "ronpo/target_abs": 0.08037153631448746, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 545 }, { "drift/chosen_abs": 0.2750501334667206, "drift/rejected_abs": 0.13786940276622772, "epoch": 0.17481128327373857, "grad_norm": 1.1015625, "learning_rate": 2.970331159433403e-08, "logits/chosen": -2.1441848278045654, "logits/rejected": -2.100531578063965, "logps/chosen": -0.25560611486434937, "logps/rejected": -0.26517319679260254, "loss": 0.3065684139728546, "loss/core": 0.2939642667770386, "loss/preference_sft": 0.2637214660644531, "loss/reference_anchor": 0.3337470293045044, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 2.750082492828369, "rewards/margins": 1.3717514276504517, "rewards/rejected": 1.378330945968628, "ronpo/logit": 0.13717512786388397, "ronpo/residual": 0.16072766482830048, "ronpo/residual_abs": 0.20835892856121063, "ronpo/target": -0.023552536964416504, "ronpo/target_abs": 0.08319397270679474, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 550 }, { "drift/chosen_abs": 0.12603361904621124, "drift/rejected_abs": 0.05538574978709221, "epoch": 0.1764004767580453, "grad_norm": 0.87109375, "learning_rate": 2.8993493177871495e-08, "logits/chosen": -2.215672016143799, "logits/rejected": -2.2594668865203857, "logps/chosen": -0.31073468923568726, "logps/rejected": -0.30866894125938416, "loss": 0.06362307071685791, "loss/core": 0.05991923063993454, "loss/preference_sft": 0.3092317581176758, "loss/reference_anchor": 0.07490097731351852, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.2597297430038452, "rewards/margins": 0.7066539525985718, "rewards/rejected": 0.5530757308006287, "ronpo/logit": 0.0706654042005539, "ronpo/residual": 0.05782570317387581, "ronpo/residual_abs": 0.1277032196521759, "ronpo/target": 0.012839692644774914, "ronpo/target_abs": 0.08341912925243378, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 555 }, { "drift/chosen_abs": 0.26530271768569946, "drift/rejected_abs": 0.12235991656780243, "epoch": 0.17798967024235202, "grad_norm": 17.25, "learning_rate": 2.8286873710734132e-08, "logits/chosen": -2.085667133331299, "logits/rejected": -2.0912811756134033, "logps/chosen": -0.27223190665245056, "logps/rejected": -0.2789652347564697, "loss": 0.25534918904304504, "loss/core": 0.24563512206077576, "loss/preference_sft": 0.2783053517341614, "loss/reference_anchor": 0.24971409142017365, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.650958299636841, "rewards/margins": 1.4280980825424194, "rewards/rejected": 1.2228600978851318, "ronpo/logit": 0.14280982315540314, "ronpo/residual": 0.14116224646568298, "ronpo/residual_abs": 0.1777869462966919, "ronpo/target": 0.0016475621378049254, "ronpo/target_abs": 0.06457264721393585, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 560 }, { "drift/chosen_abs": 0.2612389326095581, "drift/rejected_abs": 0.11688123643398285, "epoch": 0.1795788637266587, "grad_norm": 30.75, "learning_rate": 2.7583718923604928e-08, "logits/chosen": -2.042419910430908, "logits/rejected": -2.0682761669158936, "logps/chosen": -0.33960863947868347, "logps/rejected": -0.3336796164512634, "loss": 0.359559029340744, "loss/core": 0.34349602460861206, "loss/preference_sft": 0.34128159284591675, "loss/reference_anchor": 0.4248151183128357, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.6094517707824707, "rewards/margins": 1.4438995122909546, "rewards/rejected": 1.1655519008636475, "ronpo/logit": 0.14438995718955994, "ronpo/residual": 0.13987357914447784, "ronpo/residual_abs": 0.19552505016326904, "ronpo/target": 0.004516408313065767, "ronpo/target_abs": 0.07334297150373459, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 565 }, { "drift/chosen_abs": 0.4135201871395111, "drift/rejected_abs": 0.09967617690563202, "epoch": 0.18116805721096543, "grad_norm": 7.78125, "learning_rate": 2.688429324424236e-08, "logits/chosen": -2.1476290225982666, "logits/rejected": -2.1181139945983887, "logps/chosen": -0.27395740151405334, "logps/rejected": -0.2648341655731201, "loss": 1.3773685693740845, "loss/core": 1.3479124307632446, "loss/preference_sft": 0.27103132009506226, "loss/reference_anchor": 0.8145006895065308, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 4.134523868560791, "rewards/margins": 3.1398844718933105, "rewards/rejected": 0.99463951587677, "ronpo/logit": 0.31398844718933105, "ronpo/residual": 0.30398809909820557, "ronpo/residual_abs": 0.3727145493030548, "ronpo/target": 0.010000362992286682, "ronpo/target_abs": 0.09326197952032089, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 570 }, { "drift/chosen_abs": 0.19183699786663055, "drift/rejected_abs": 0.09668353945016861, "epoch": 0.18275725069527216, "grad_norm": 3.484375, "learning_rate": 2.6188859698040042e-08, "logits/chosen": -2.0081419944763184, "logits/rejected": -2.0951578617095947, "logps/chosen": -0.2836478650569916, "logps/rejected": -0.27917319536209106, "loss": 0.0594443753361702, "loss/core": 0.05546221882104874, "loss/preference_sft": 0.2888682782649994, "loss/reference_anchor": 0.08488909900188446, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.9143584966659546, "rewards/margins": 0.9508224725723267, "rewards/rejected": 0.9635359048843384, "ronpo/logit": 0.09508226811885834, "ronpo/residual": 0.10423971712589264, "ronpo/residual_abs": 0.1455569565296173, "ronpo/target": -0.009157449007034302, "ronpo/target_abs": 0.07435912638902664, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 575 }, { "drift/chosen_abs": 0.3637532591819763, "drift/rejected_abs": 0.16664664447307587, "epoch": 0.18434644417957885, "grad_norm": 1.4453125, "learning_rate": 2.5497679809113563e-08, "logits/chosen": -2.001953125, "logits/rejected": -2.0324103832244873, "logps/chosen": -0.2893816828727722, "logps/rejected": -0.2681158483028412, "loss": 0.548211395740509, "loss/core": 0.5249797105789185, "loss/preference_sft": 0.28467053174972534, "loss/reference_anchor": 0.6352949738502502, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.6368331909179688, "rewards/margins": 1.9708592891693115, "rewards/rejected": 1.6659743785858154, "ronpo/logit": 0.1970859318971634, "ronpo/residual": 0.1857091784477234, "ronpo/residual_abs": 0.2529819905757904, "ronpo/target": 0.011376758106052876, "ronpo/target_abs": 0.06672044098377228, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 580 }, { "drift/chosen_abs": 0.23147885501384735, "drift/rejected_abs": 0.09522049129009247, "epoch": 0.18593563766388557, "grad_norm": 1.8203125, "learning_rate": 2.481101350195196e-08, "logits/chosen": -1.9583909511566162, "logits/rejected": -1.982860803604126, "logps/chosen": -0.28692710399627686, "logps/rejected": -0.2864198684692383, "loss": 0.15366187691688538, "loss/core": 0.14675399661064148, "loss/preference_sft": 0.2956783175468445, "loss/reference_anchor": 0.16780035197734833, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.313133955001831, "rewards/margins": 1.362217903137207, "rewards/rejected": 0.9509159922599792, "ronpo/logit": 0.13622179627418518, "ronpo/residual": 0.13147194683551788, "ronpo/residual_abs": 0.17332948744297028, "ronpo/target": 0.004749844316393137, "ronpo/target_abs": 0.06973059475421906, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 585 }, { "drift/chosen_abs": 0.259533554315567, "drift/rejected_abs": 0.12255913019180298, "epoch": 0.1875248311481923, "grad_norm": 1.3671875, "learning_rate": 2.4129119003670752e-08, "logits/chosen": -2.1177847385406494, "logits/rejected": -2.0852158069610596, "logps/chosen": -0.3071213364601135, "logps/rejected": -0.31466570496559143, "loss": 0.545588493347168, "loss/core": 0.5284724831581116, "loss/preference_sft": 0.315735787153244, "loss/reference_anchor": 0.4574582576751709, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.592897891998291, "rewards/margins": 1.3681689500808716, "rewards/rejected": 1.2247288227081299, "ronpo/logit": 0.13681688904762268, "ronpo/residual": 0.14628082513809204, "ronpo/residual_abs": 0.19750835001468658, "ronpo/target": -0.009463920257985592, "ronpo/target_abs": 0.07016940414905548, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 590 }, { "drift/chosen_abs": 0.23209957778453827, "drift/rejected_abs": 0.11176522821187973, "epoch": 0.18911402463249902, "grad_norm": 5.84375, "learning_rate": 2.3452252746903295e-08, "logits/chosen": -1.974440336227417, "logits/rejected": -1.9321467876434326, "logps/chosen": -0.28932034969329834, "logps/rejected": -0.29376018047332764, "loss": 0.1980138123035431, "loss/core": 0.19030292332172394, "loss/preference_sft": 0.29327264428138733, "loss/reference_anchor": 0.19098426401615143, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.3200583457946777, "rewards/margins": 1.2056306600570679, "rewards/rejected": 1.1144278049468994, "ronpo/logit": 0.12056305259466171, "ronpo/residual": 0.10401593148708344, "ronpo/residual_abs": 0.1900263875722885, "ronpo/target": 0.016547124832868576, "ronpo/target_abs": 0.08112061023712158, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 595 }, { "drift/chosen_abs": 0.2035134732723236, "drift/rejected_abs": 0.1420602947473526, "epoch": 0.1907032181168057, "grad_norm": 147.0, "learning_rate": 2.278066927336698e-08, "logits/chosen": -2.131030797958374, "logits/rejected": -2.104788303375244, "logps/chosen": -0.29952484369277954, "logps/rejected": -0.320384681224823, "loss": 0.45714807510375977, "loss/core": 0.4465022087097168, "loss/preference_sft": 0.3080054819583893, "loss/reference_anchor": 0.2733660936355591, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.0339159965515137, "rewards/margins": 0.6155134439468384, "rewards/rejected": 1.4184025526046753, "ronpo/logit": 0.06155133247375488, "ronpo/residual": 0.06355226784944534, "ronpo/residual_abs": 0.23413407802581787, "ronpo/target": -0.0020009521394968033, "ronpo/target_abs": 0.06787214428186417, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 600 }, { "drift/chosen_abs": 0.34611085057258606, "drift/rejected_abs": 0.15783950686454773, "epoch": 0.19229241160111243, "grad_norm": 1.2890625, "learning_rate": 2.2114621138140448e-08, "logits/chosen": -2.026627540588379, "logits/rejected": -1.951611876487732, "logps/chosen": -0.283507376909256, "logps/rejected": -0.27926164865493774, "loss": 0.4765016436576843, "loss/core": 0.45531606674194336, "loss/preference_sft": 0.2730947434902191, "loss/reference_anchor": 0.5779938101768494, "rewards/accuracies": 0.875, "rewards/chosen": 3.452455520629883, "rewards/margins": 1.884347915649414, "rewards/rejected": 1.5681076049804688, "ronpo/logit": 0.18843480944633484, "ronpo/residual": 0.18703269958496094, "ronpo/residual_abs": 0.2455645054578781, "ronpo/target": 0.0014021346578374505, "ronpo/target_abs": 0.0817280188202858, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 605 }, { "drift/chosen_abs": 0.3367292284965515, "drift/rejected_abs": 0.152937650680542, "epoch": 0.19388160508541916, "grad_norm": 95.5, "learning_rate": 2.145435881468797e-08, "logits/chosen": -2.018470525741577, "logits/rejected": -1.968064308166504, "logps/chosen": -0.3080677092075348, "logps/rejected": -0.29313474893569946, "loss": 0.3543425500392914, "loss/core": 0.3362095355987549, "loss/preference_sft": 0.31406843662261963, "loss/reference_anchor": 0.4866800904273987, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.365661144256592, "rewards/margins": 1.8380382061004639, "rewards/rejected": 1.527622938156128, "ronpo/logit": 0.18380382657051086, "ronpo/residual": 0.18959875404834747, "ronpo/residual_abs": 0.23765261471271515, "ronpo/target": -0.005794919561594725, "ronpo/target_abs": 0.08199281990528107, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 610 }, { "drift/chosen_abs": 0.25560295581817627, "drift/rejected_abs": 0.11357559263706207, "epoch": 0.19547079856972585, "grad_norm": 125.0, "learning_rate": 2.0800130600666648e-08, "logits/chosen": -2.1990818977355957, "logits/rejected": -2.159752368927002, "logps/chosen": -0.28720757365226746, "logps/rejected": -0.28762027621269226, "loss": 0.293748140335083, "loss/core": 0.28089436888694763, "loss/preference_sft": 0.293092280626297, "loss/reference_anchor": 0.33794093132019043, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.5560295581817627, "rewards/margins": 1.4323694705963135, "rewards/rejected": 1.1236598491668701, "ronpo/logit": 0.1432369500398636, "ronpo/residual": 0.14470703899860382, "ronpo/residual_abs": 0.18086595833301544, "ronpo/target": -0.0014700808096677065, "ronpo/target_abs": 0.06537780165672302, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 615 }, { "drift/chosen_abs": 0.22184021770954132, "drift/rejected_abs": 0.1088438630104065, "epoch": 0.19705999205403257, "grad_norm": 13.875, "learning_rate": 2.0152182524551813e-08, "logits/chosen": -2.0863685607910156, "logits/rejected": -2.0217607021331787, "logps/chosen": -0.29196107387542725, "logps/rejected": -0.2876456379890442, "loss": 0.10439170897006989, "loss/core": 0.0983448475599289, "loss/preference_sft": 0.2921106219291687, "loss/reference_anchor": 0.1435559093952179, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.2182328701019287, "rewards/margins": 1.1298673152923584, "rewards/rejected": 1.0883656740188599, "ronpo/logit": 0.1129867285490036, "ronpo/residual": 0.11576826870441437, "ronpo/residual_abs": 0.16257163882255554, "ronpo/target": -0.0027815259527415037, "ronpo/target_abs": 0.06601886451244354, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 620 }, { "drift/chosen_abs": 0.24640531837940216, "drift/rejected_abs": 0.09290675818920135, "epoch": 0.1986491855383393, "grad_norm": 45.25, "learning_rate": 1.951075825311578e-08, "logits/chosen": -2.0369296073913574, "logits/rejected": -2.0407700538635254, "logps/chosen": -0.2959621846675873, "logps/rejected": -0.29841041564941406, "loss": 0.3487691283226013, "loss/core": 0.33975881338119507, "loss/preference_sft": 0.2982815206050873, "loss/reference_anchor": 0.22760829329490662, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.4635446071624756, "rewards/margins": 1.5408732891082764, "rewards/rejected": 0.9226716160774231, "ronpo/logit": 0.15408733487129211, "ronpo/residual": 0.17024736106395721, "ronpo/residual_abs": 0.2152116596698761, "ronpo/target": -0.016160037368535995, "ronpo/target_abs": 0.07987652719020844, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 625 }, { "drift/chosen_abs": 0.23432044684886932, "drift/rejected_abs": 0.08939666301012039, "epoch": 0.20023837902264602, "grad_norm": 2.875, "learning_rate": 1.887609899979465e-08, "logits/chosen": -2.3256447315216064, "logits/rejected": -2.416283130645752, "logps/chosen": -0.2670096158981323, "logps/rejected": -0.2636256814002991, "loss": 0.3040315806865692, "loss/core": 0.29226741194725037, "loss/preference_sft": 0.2660647928714752, "loss/reference_anchor": 0.30951163172721863, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.3428587913513184, "rewards/margins": 1.4511168003082275, "rewards/rejected": 0.891741931438446, "ronpo/logit": 0.14511170983314514, "ronpo/residual": 0.1341066062450409, "ronpo/residual_abs": 0.19294190406799316, "ronpo/target": 0.011005090549588203, "ronpo/target_abs": 0.06993237882852554, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 630 }, { "drift/chosen_abs": 0.2895382344722748, "drift/rejected_abs": 0.10728726536035538, "epoch": 0.2018275725069527, "grad_norm": 16.375, "learning_rate": 1.824844343397797e-08, "logits/chosen": -2.1119954586029053, "logits/rejected": -2.150533676147461, "logps/chosen": -0.2655460238456726, "logps/rejected": -0.2733895182609558, "loss": 0.38659775257110596, "loss/core": 0.37493616342544556, "loss/preference_sft": 0.2679237723350525, "loss/reference_anchor": 0.30639657378196716, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.8941664695739746, "rewards/margins": 1.8319135904312134, "rewards/rejected": 1.0622529983520508, "ronpo/logit": 0.18319137394428253, "ronpo/residual": 0.1691356599330902, "ronpo/residual_abs": 0.2294747531414032, "ronpo/target": 0.014055708423256874, "ronpo/target_abs": 0.07849180698394775, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 635 }, { "drift/chosen_abs": 0.32430991530418396, "drift/rejected_abs": 0.09809484332799911, "epoch": 0.20341676599125944, "grad_norm": 1.703125, "learning_rate": 1.762802759125482e-08, "logits/chosen": -2.017073154449463, "logits/rejected": -2.0082268714904785, "logps/chosen": -0.31356340646743774, "logps/rejected": -0.31727951765060425, "loss": 0.7281572818756104, "loss/core": 0.707539439201355, "loss/preference_sft": 0.3098931312561035, "loss/reference_anchor": 0.5580906867980957, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.2425026893615723, "rewards/margins": 2.297203779220581, "rewards/rejected": 0.9452990293502808, "ronpo/logit": 0.22972038388252258, "ronpo/residual": 0.2376583367586136, "ronpo/residual_abs": 0.27416878938674927, "ronpo/target": -0.007937933318316936, "ronpo/target_abs": 0.07765815407037735, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 640 }, { "drift/chosen_abs": 0.26535215973854065, "drift/rejected_abs": 0.11467768251895905, "epoch": 0.20500595947556616, "grad_norm": 7.9375, "learning_rate": 1.7015084784650487e-08, "logits/chosen": -2.123807430267334, "logits/rejected": -2.129242181777954, "logps/chosen": -0.3087415099143982, "logps/rejected": -0.2945675551891327, "loss": 0.3146616816520691, "loss/core": 0.30256468057632446, "loss/preference_sft": 0.3151400685310364, "loss/reference_anchor": 0.31411463022232056, "rewards/accuracies": 0.9375, "rewards/chosen": 2.6520421504974365, "rewards/margins": 1.5085344314575195, "rewards/rejected": 1.143507957458496, "ronpo/logit": 0.15085344016551971, "ronpo/residual": 0.14317695796489716, "ronpo/residual_abs": 0.20519272983074188, "ronpo/target": 0.007676492445170879, "ronpo/target_abs": 0.08047398179769516, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 645 }, { "drift/chosen_abs": 0.15140828490257263, "drift/rejected_abs": 0.09236256033182144, "epoch": 0.20659515295987285, "grad_norm": 1.84375, "learning_rate": 1.6409845516887003e-08, "logits/chosen": -2.207329273223877, "logits/rejected": -2.135587215423584, "logps/chosen": -0.3012005686759949, "logps/rejected": -0.29473382234573364, "loss": 0.0722493976354599, "loss/core": 0.06875967979431152, "loss/preference_sft": 0.3042813539505005, "loss/reference_anchor": 0.06927807629108429, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 1.5135061740875244, "rewards/margins": 0.6719859838485718, "rewards/rejected": 0.8415201902389526, "ronpo/logit": 0.06719859689474106, "ronpo/residual": 0.0728544145822525, "ronpo/residual_abs": 0.12853005528450012, "ronpo/target": -0.005655817687511444, "ronpo/target_abs": 0.07488958537578583, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 650 }, { "drift/chosen_abs": 0.20458777248859406, "drift/rejected_abs": 0.06509005278348923, "epoch": 0.20818434644417957, "grad_norm": 172.0, "learning_rate": 1.5812537393700506e-08, "logits/chosen": -2.2815442085266113, "logits/rejected": -2.287656784057617, "logps/chosen": -0.28355464339256287, "logps/rejected": -0.29816484451293945, "loss": 0.39056888222694397, "loss/core": 0.3819330930709839, "loss/preference_sft": 0.2903548777103424, "loss/reference_anchor": 0.2177002727985382, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.0455703735351562, "rewards/margins": 1.397626519203186, "rewards/rejected": 0.6479436755180359, "ronpo/logit": 0.13976263999938965, "ronpo/residual": 0.16489243507385254, "ronpo/residual_abs": 0.20165614783763885, "ronpo/target": -0.025129785761237144, "ronpo/target_abs": 0.08682605624198914, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 655 }, { "drift/chosen_abs": 0.17608028650283813, "drift/rejected_abs": 0.07316999137401581, "epoch": 0.2097735399284863, "grad_norm": 8.0, "learning_rate": 1.522338503824803e-08, "logits/chosen": -2.2558116912841797, "logits/rejected": -2.2970988750457764, "logps/chosen": -0.29492825269699097, "logps/rejected": -0.29021143913269043, "loss": 0.12977449595928192, "loss/core": 0.1249893456697464, "loss/preference_sft": 0.29489555954933167, "loss/reference_anchor": 0.10722937434911728, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 1.7590672969818115, "rewards/margins": 1.0307245254516602, "rewards/rejected": 0.7283426523208618, "ronpo/logit": 0.10307244956493378, "ronpo/residual": 0.1016714796423912, "ronpo/residual_abs": 0.14400455355644226, "ronpo/target": 0.0014009776059538126, "ronpo/target_abs": 0.06940744817256927, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 660 }, { "drift/chosen_abs": 0.2352174073457718, "drift/rejected_abs": 0.13656437397003174, "epoch": 0.21136273341279302, "grad_norm": 0.79296875, "learning_rate": 1.464261000663589e-08, "logits/chosen": -2.079434871673584, "logits/rejected": -2.024263858795166, "logps/chosen": -0.2820441424846649, "logps/rejected": -0.2857016921043396, "loss": 0.23443233966827393, "loss/core": 0.2236245572566986, "loss/preference_sft": 0.2865526080131531, "loss/reference_anchor": 0.28013843297958374, "rewards/accuracies": 0.875, "rewards/chosen": 2.349414825439453, "rewards/margins": 0.9838077425956726, "rewards/rejected": 1.3656069040298462, "ronpo/logit": 0.09838077425956726, "ronpo/residual": 0.11156570911407471, "ronpo/residual_abs": 0.17216257750988007, "ronpo/target": -0.013184946961700916, "ronpo/target_abs": 0.07473040372133255, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 665 }, { "drift/chosen_abs": 0.20989990234375, "drift/rejected_abs": 0.07849745452404022, "epoch": 0.2129519268970997, "grad_norm": 5.40625, "learning_rate": 1.40704307046015e-08, "logits/chosen": -2.347775459289551, "logits/rejected": -2.3573763370513916, "logps/chosen": -0.2676052153110504, "logps/rejected": -0.27207422256469727, "loss": 0.21361777186393738, "loss/core": 0.20699867606163025, "loss/preference_sft": 0.26641157269477844, "loss/reference_anchor": 0.16247610747814178, "rewards/accuracies": 0.9375, "rewards/chosen": 2.0982730388641357, "rewards/margins": 1.3146512508392334, "rewards/rejected": 0.7836218476295471, "ronpo/logit": 0.1314651221036911, "ronpo/residual": 0.16295532882213593, "ronpo/residual_abs": 0.18369440734386444, "ronpo/target": -0.03149022161960602, "ronpo/target_abs": 0.07089079916477203, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 670 }, { "drift/chosen_abs": 0.4036272466182709, "drift/rejected_abs": 0.08043356984853745, "epoch": 0.21454112038140644, "grad_norm": 2.21875, "learning_rate": 1.3507062305379941e-08, "logits/chosen": -1.9893615245819092, "logits/rejected": -2.1025524139404297, "logps/chosen": -0.2764974534511566, "logps/rejected": -0.27073296904563904, "loss": 1.4822903871536255, "loss/core": 1.4529348611831665, "loss/preference_sft": 0.27298080921173096, "loss/reference_anchor": 0.8114289045333862, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.034860134124756, "rewards/margins": 3.2362799644470215, "rewards/rejected": 0.7985801100730896, "ronpo/logit": 0.3236280679702759, "ronpo/residual": 0.3155183792114258, "ronpo/residual_abs": 0.37980663776397705, "ronpo/target": 0.008109642192721367, "ronpo/target_abs": 0.08955520391464233, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 675 }, { "drift/chosen_abs": 0.19523149728775024, "drift/rejected_abs": 0.11687342822551727, "epoch": 0.21613031386571316, "grad_norm": 2.484375, "learning_rate": 1.2952716668786109e-08, "logits/chosen": -1.9566532373428345, "logits/rejected": -1.9517863988876343, "logps/chosen": -0.28758880496025085, "logps/rejected": -0.28461915254592896, "loss": 0.066526398062706, "loss/core": 0.05917876213788986, "loss/preference_sft": 0.28697067499160767, "loss/reference_anchor": 0.18123537302017212, "rewards/accuracies": 0.875, "rewards/chosen": 1.950860619544983, "rewards/margins": 0.7826768159866333, "rewards/rejected": 1.1681835651397705, "ronpo/logit": 0.07826767861843109, "ronpo/residual": 0.07248474657535553, "ronpo/residual_abs": 0.13983985781669617, "ronpo/target": 0.0057829380966722965, "ronpo/target_abs": 0.08430159091949463, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 680 }, { "drift/chosen_abs": 0.18238039314746857, "drift/rejected_abs": 0.09428268671035767, "epoch": 0.21771950735001985, "grad_norm": 3.3125, "learning_rate": 1.2407602261542828e-08, "logits/chosen": -1.9722627401351929, "logits/rejected": -2.0617997646331787, "logps/chosen": -0.2924751937389374, "logps/rejected": -0.29996949434280396, "loss": 0.054906345903873444, "loss/core": 0.0511222667992115, "loss/preference_sft": 0.30433911085128784, "loss/reference_anchor": 0.07768283784389496, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.8226171731948853, "rewards/margins": 0.8817318677902222, "rewards/rejected": 0.9408853650093079, "ronpo/logit": 0.08817318081855774, "ronpo/residual": 0.08928640186786652, "ronpo/residual_abs": 0.133590966463089, "ronpo/target": -0.0011132096406072378, "ronpo/target_abs": 0.07680922746658325, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 685 }, { "drift/chosen_abs": 0.45133644342422485, "drift/rejected_abs": 0.18786491453647614, "epoch": 0.21930870083432658, "grad_norm": 434.0, "learning_rate": 1.1871924078885106e-08, "logits/chosen": -1.929914116859436, "logits/rejected": -1.9743080139160156, "logps/chosen": -0.2774960696697235, "logps/rejected": -0.26925212144851685, "loss": 0.8445002436637878, "loss/core": 0.8047998547554016, "loss/preference_sft": 0.2817479968070984, "loss/reference_anchor": 1.1061266660690308, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.513057708740234, "rewards/margins": 2.6355862617492676, "rewards/rejected": 1.8774709701538086, "ronpo/logit": 0.26355868577957153, "ronpo/residual": 0.26055908203125, "ronpo/residual_abs": 0.3207409679889679, "ronpo/target": 0.002999568125233054, "ronpo/target_abs": 0.0797749012708664, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 690 }, { "drift/chosen_abs": 0.1836492121219635, "drift/rejected_abs": 0.10975848138332367, "epoch": 0.2208978943186333, "grad_norm": 12.375, "learning_rate": 1.1345883567469826e-08, "logits/chosen": -2.1611971855163574, "logits/rejected": -2.147216558456421, "logps/chosen": -0.2980535626411438, "logps/rejected": -0.30087488889694214, "loss": 0.11188419908285141, "loss/core": 0.10657135397195816, "loss/preference_sft": 0.2982610762119293, "loss/reference_anchor": 0.1219698041677475, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.8356151580810547, "rewards/margins": 0.7388324737548828, "rewards/rejected": 1.0967828035354614, "ronpo/logit": 0.07388325035572052, "ronpo/residual": 0.06399385631084442, "ronpo/residual_abs": 0.16593526303768158, "ronpo/target": 0.00988937821239233, "ronpo/target_abs": 0.07012668997049332, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 695 }, { "drift/chosen_abs": 0.2231002300977707, "drift/rejected_abs": 0.13982199132442474, "epoch": 0.22248708780294002, "grad_norm": 6.5625, "learning_rate": 1.0829678549619859e-08, "logits/chosen": -2.218724489212036, "logits/rejected": -2.1778321266174316, "logps/chosen": -0.32158035039901733, "logps/rejected": -0.3115960955619812, "loss": 0.43686145544052124, "loss/core": 0.42180243134498596, "loss/preference_sft": 0.31540364027023315, "loss/reference_anchor": 0.39871746301651, "rewards/accuracies": 0.875, "rewards/chosen": 2.2305080890655518, "rewards/margins": 0.8341946601867676, "rewards/rejected": 1.3963135480880737, "ronpo/logit": 0.08341946452856064, "ronpo/residual": 0.08280835300683975, "ronpo/residual_abs": 0.19750329852104187, "ronpo/target": 0.0006111121037974954, "ronpo/target_abs": 0.0668053850531578, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 700 }, { "drift/chosen_abs": 0.1843855381011963, "drift/rejected_abs": 0.08389295637607574, "epoch": 0.22407628128724671, "grad_norm": 3.328125, "learning_rate": 1.0323503148931167e-08, "logits/chosen": -2.261746644973755, "logits/rejected": -2.3672595024108887, "logps/chosen": -0.26710596680641174, "logps/rejected": -0.27074798941612244, "loss": 0.08147259801626205, "loss/core": 0.07696676254272461, "loss/preference_sft": 0.27835264801979065, "loss/reference_anchor": 0.10090293735265732, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 1.8432375192642212, "rewards/margins": 1.0078269243240356, "rewards/rejected": 0.8354104161262512, "ronpo/logit": 0.10078269243240356, "ronpo/residual": 0.10459145158529282, "ronpo/residual_abs": 0.15122047066688538, "ronpo/target": -0.0038087614811956882, "ronpo/target_abs": 0.08357212692499161, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 705 }, { "drift/chosen_abs": 0.2887698709964752, "drift/rejected_abs": 0.10348562896251678, "epoch": 0.22566547477155344, "grad_norm": 107.5, "learning_rate": 9.827547717270884e-09, "logits/chosen": -2.164956569671631, "logits/rejected": -2.177293300628662, "logps/chosen": -0.29969245195388794, "logps/rejected": -0.29687872529029846, "loss": 0.6983797550201416, "loss/core": 0.6833909749984741, "loss/preference_sft": 0.2986063063144684, "loss/reference_anchor": 0.3983899652957916, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 2.8871381282806396, "rewards/margins": 1.9205951690673828, "rewards/rejected": 0.9665428400039673, "ronpo/logit": 0.19205954670906067, "ronpo/residual": 0.18855328857898712, "ronpo/residual_abs": 0.2506033778190613, "ronpo/target": 0.003506250213831663, "ronpo/target_abs": 0.06901577860116959, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 710 }, { "drift/chosen_abs": 0.1665627658367157, "drift/rejected_abs": 0.0922744944691658, "epoch": 0.22725466825586016, "grad_norm": 1.1171875, "learning_rate": 9.34199876319378e-09, "logits/chosen": -2.0484366416931152, "logits/rejected": -1.9690176248550415, "logps/chosen": -0.3025513291358948, "logps/rejected": -0.30018630623817444, "loss": 0.07143449038267136, "loss/core": 0.06748920679092407, "loss/preference_sft": 0.3050025403499603, "loss/reference_anchor": 0.08222213387489319, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.6645787954330444, "rewards/margins": 0.7441354990005493, "rewards/rejected": 0.9204433560371399, "ronpo/logit": 0.07441355288028717, "ronpo/residual": 0.06392879039049149, "ronpo/residual_abs": 0.13627013564109802, "ronpo/target": 0.010484759695827961, "ronpo/target_abs": 0.0739731639623642, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 715 }, { "drift/chosen_abs": 0.3448890745639801, "drift/rejected_abs": 0.14354726672172546, "epoch": 0.22884386174016685, "grad_norm": 482.0, "learning_rate": 8.867038881803992e-09, "logits/chosen": -1.9485433101654053, "logits/rejected": -2.0286898612976074, "logps/chosen": -0.2920309901237488, "logps/rejected": -0.2847922146320343, "loss": 0.5865887403488159, "loss/core": 0.5658643841743469, "loss/preference_sft": 0.2796561121940613, "loss/reference_anchor": 0.5641579031944275, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.4488911628723145, "rewards/margins": 2.0442280769348145, "rewards/rejected": 1.4046627283096313, "ronpo/logit": 0.20442280173301697, "ronpo/residual": 0.18873873353004456, "ronpo/residual_abs": 0.24582239985466003, "ronpo/target": 0.015684079378843307, "ronpo/target_abs": 0.07453683018684387, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 720 }, { "drift/chosen_abs": 0.2959468960762024, "drift/rejected_abs": 0.06901375949382782, "epoch": 0.23043305522447358, "grad_norm": 4.0625, "learning_rate": 8.402846686088465e-09, "logits/chosen": -2.08479642868042, "logits/rejected": -2.0257925987243652, "logps/chosen": -0.29016560316085815, "logps/rejected": -0.2911625802516937, "loss": 1.0842726230621338, "loss/core": 1.0630663633346558, "loss/preference_sft": 0.2913043797016144, "loss/reference_anchor": 0.5767573118209839, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.959277629852295, "rewards/margins": 2.2727534770965576, "rewards/rejected": 0.6865240931510925, "ronpo/logit": 0.22727534174919128, "ronpo/residual": 0.20963330566883087, "ronpo/residual_abs": 0.27996546030044556, "ronpo/target": 0.017642047256231308, "ronpo/target_abs": 0.08470751345157623, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 725 }, { "drift/chosen_abs": 0.31818053126335144, "drift/rejected_abs": 0.09311111271381378, "epoch": 0.2320222487087803, "grad_norm": 2656.0, "learning_rate": 7.949596739747928e-09, "logits/chosen": -2.1799933910369873, "logits/rejected": -2.214536190032959, "logps/chosen": -0.2960435748100281, "logps/rejected": -0.2987004816532135, "loss": 1.0148961544036865, "loss/core": 0.9916189908981323, "loss/preference_sft": 0.30723002552986145, "loss/reference_anchor": 0.6343376040458679, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.18082332611084, "rewards/margins": 2.2561564445495605, "rewards/rejected": 0.9246670007705688, "ronpo/logit": 0.22561565041542053, "ronpo/residual": 0.211147278547287, "ronpo/residual_abs": 0.27076631784439087, "ronpo/target": 0.014468356966972351, "ronpo/target_abs": 0.07857581973075867, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 730 }, { "drift/chosen_abs": 0.41159167885780334, "drift/rejected_abs": 0.15270794928073883, "epoch": 0.23361144219308702, "grad_norm": 9.9375, "learning_rate": 7.507459491550653e-09, "logits/chosen": -1.872210144996643, "logits/rejected": -1.9234001636505127, "logps/chosen": -0.31087052822113037, "logps/rejected": -0.28266483545303345, "loss": 1.28376042842865, "loss/core": 1.2525687217712402, "loss/preference_sft": 0.30879443883895874, "loss/reference_anchor": 0.8603096008300781, "rewards/accuracies": 0.875, "rewards/chosen": 4.114415645599365, "rewards/margins": 2.59476900100708, "rewards/rejected": 1.5196462869644165, "ronpo/logit": 0.259476900100708, "ronpo/residual": 0.2532740831375122, "ronpo/residual_abs": 0.36283624172210693, "ronpo/target": 0.006202817894518375, "ronpo/target_abs": 0.09357239305973053, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 735 }, { "drift/chosen_abs": 0.40805894136428833, "drift/rejected_abs": 0.17579728364944458, "epoch": 0.23520063567739372, "grad_norm": 6.1875, "learning_rate": 7.076601211233599e-09, "logits/chosen": -2.061128854751587, "logits/rejected": -2.0528006553649902, "logps/chosen": -0.27439576387405396, "logps/rejected": -0.2714352607727051, "loss": 0.8380556106567383, "loss/core": 0.8007208704948425, "loss/preference_sft": 0.2815498411655426, "loss/reference_anchor": 1.0385510921478271, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 4.079874515533447, "rewards/margins": 2.3227553367614746, "rewards/rejected": 1.7571195363998413, "ronpo/logit": 0.23227553069591522, "ronpo/residual": 0.21789321303367615, "ronpo/residual_abs": 0.29552918672561646, "ronpo/target": 0.01438230276107788, "ronpo/target_abs": 0.08076776564121246, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 740 }, { "drift/chosen_abs": 0.2793598175048828, "drift/rejected_abs": 0.1420426368713379, "epoch": 0.23678982916170044, "grad_norm": 4.1875, "learning_rate": 6.6571839269751625e-09, "logits/chosen": -1.9656693935394287, "logits/rejected": -1.9407997131347656, "logps/chosen": -0.310263454914093, "logps/rejected": -0.2949405312538147, "loss": 0.4082837998867035, "loss/core": 0.39522331953048706, "loss/preference_sft": 0.302360862493515, "loss/reference_anchor": 0.34292274713516235, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.7910044193267822, "rewards/margins": 1.4027537107467651, "rewards/rejected": 1.3882505893707275, "ronpo/logit": 0.14027537405490875, "ronpo/residual": 0.14826272428035736, "ronpo/residual_abs": 0.2111869603395462, "ronpo/target": -0.007987345568835735, "ronpo/target_abs": 0.06692041456699371, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 745 }, { "drift/chosen_abs": 0.4917718470096588, "drift/rejected_abs": 0.09017399698495865, "epoch": 0.23837902264600716, "grad_norm": 25.875, "learning_rate": 6.249365364463068e-09, "logits/chosen": -1.8745949268341064, "logits/rejected": -2.0258686542510986, "logps/chosen": -0.32771915197372437, "logps/rejected": -0.3020108938217163, "loss": 1.669013261795044, "loss/core": 1.633209228515625, "loss/preference_sft": 0.3086247444152832, "loss/reference_anchor": 0.9921077489852905, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.917025566101074, "rewards/margins": 4.016125679016113, "rewards/rejected": 0.9008998870849609, "ronpo/logit": 0.4016125798225403, "ronpo/residual": 0.39909854531288147, "ronpo/residual_abs": 0.4497567117214203, "ronpo/target": 0.002514071064069867, "ronpo/target_abs": 0.08284027129411697, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 750 }, { "drift/chosen_abs": 0.5578740835189819, "drift/rejected_abs": 0.25196394324302673, "epoch": 0.23996821613031386, "grad_norm": 176.0, "learning_rate": 5.853298887580098e-09, "logits/chosen": -1.7803856134414673, "logits/rejected": -1.7086734771728516, "logps/chosen": -0.26622751355171204, "logps/rejected": -0.2793108820915222, "loss": 1.8586667776107788, "loss/core": 1.813750982284546, "loss/preference_sft": 0.258129745721817, "loss/reference_anchor": 1.257493019104004, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 5.57759952545166, "rewards/margins": 3.06099009513855, "rewards/rejected": 2.5166096687316895, "ronpo/logit": 0.3060990273952484, "ronpo/residual": 0.29286080598831177, "ronpo/residual_abs": 0.49150973558425903, "ronpo/target": 0.013238211162388325, "ronpo/target_abs": 0.07374797761440277, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 755 }, { "drift/chosen_abs": 0.2570843994617462, "drift/rejected_abs": 0.10265134274959564, "epoch": 0.24155740961462058, "grad_norm": 1.515625, "learning_rate": 5.4691334407302935e-09, "logits/chosen": -2.2019875049591064, "logits/rejected": -2.2530570030212402, "logps/chosen": -0.26051491498947144, "logps/rejected": -0.2609294652938843, "loss": 0.4085308611392975, "loss/core": 0.3968563675880432, "loss/preference_sft": 0.27111831307411194, "loss/reference_anchor": 0.3064435124397278, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.5702319145202637, "rewards/margins": 1.548225998878479, "rewards/rejected": 1.0220061540603638, "ronpo/logit": 0.15482258796691895, "ronpo/residual": 0.14783695340156555, "ronpo/residual_abs": 0.20230190455913544, "ronpo/target": 0.0069856420159339905, "ronpo/target_abs": 0.0757329910993576, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 760 }, { "drift/chosen_abs": 0.16770626604557037, "drift/rejected_abs": 0.09157443791627884, "epoch": 0.2431466030989273, "grad_norm": 1.203125, "learning_rate": 5.097013492826982e-09, "logits/chosen": -2.3989851474761963, "logits/rejected": -2.3377420902252197, "logps/chosen": -0.2859249711036682, "logps/rejected": -0.28967946767807007, "loss": 0.04619941860437393, "loss/core": 0.041453104466199875, "loss/preference_sft": 0.28654763102531433, "loss/reference_anchor": 0.10695407539606094, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.6764370203018188, "rewards/margins": 0.7609391212463379, "rewards/rejected": 0.9154976606369019, "ronpo/logit": 0.07609390467405319, "ronpo/residual": 0.09639062732458115, "ronpo/residual_abs": 0.12980125844478607, "ronpo/target": -0.020296704024076462, "ronpo/target_abs": 0.06380324065685272, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 765 }, { "drift/chosen_abs": 0.33088406920433044, "drift/rejected_abs": 0.10067299753427505, "epoch": 0.24473579658323402, "grad_norm": 49.5, "learning_rate": 4.737078982963998e-09, "logits/chosen": -1.9620716571807861, "logits/rejected": -2.093608856201172, "logps/chosen": -0.2551668882369995, "logps/rejected": -0.27200862765312195, "loss": 0.6154618263244629, "loss/core": 0.6015649437904358, "loss/preference_sft": 0.26770323514938354, "loss/reference_anchor": 0.3702848255634308, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.3076298236846924, "rewards/margins": 2.3778507709503174, "rewards/rejected": 0.929779052734375, "ronpo/logit": 0.23778507113456726, "ronpo/residual": 0.2486831247806549, "ronpo/residual_abs": 0.29293423891067505, "ronpo/target": -0.010898038744926453, "ronpo/target_abs": 0.08667393028736115, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 770 }, { "drift/chosen_abs": 0.34422239661216736, "drift/rejected_abs": 0.14332148432731628, "epoch": 0.24632499006754072, "grad_norm": 1.125, "learning_rate": 4.389465267790238e-09, "logits/chosen": -1.8292220830917358, "logits/rejected": -1.8172327280044556, "logps/chosen": -0.2949088513851166, "logps/rejected": -0.2962006628513336, "loss": 0.5463058948516846, "loss/core": 0.5306318402290344, "loss/preference_sft": 0.29769080877304077, "loss/reference_anchor": 0.41806188225746155, "rewards/accuracies": 0.875, "rewards/chosen": 3.440873384475708, "rewards/margins": 2.0447137355804443, "rewards/rejected": 1.3961597681045532, "ronpo/logit": 0.2044713944196701, "ronpo/residual": 0.21325626969337463, "ronpo/residual_abs": 0.2596508264541626, "ronpo/target": -0.008784900419414043, "ronpo/target_abs": 0.08347609639167786, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 775 }, { "drift/chosen_abs": 0.3502070903778076, "drift/rejected_abs": 0.15574514865875244, "epoch": 0.24791418355184744, "grad_norm": 4.03125, "learning_rate": 4.05430307060764e-09, "logits/chosen": -2.158703565597534, "logits/rejected": -2.1818182468414307, "logps/chosen": -0.2700011134147644, "logps/rejected": -0.2787485718727112, "loss": 0.3507295846939087, "loss/core": 0.33437737822532654, "loss/preference_sft": 0.27247267961502075, "loss/reference_anchor": 0.43995794653892517, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 3.501782178878784, "rewards/margins": 1.9450876712799072, "rewards/rejected": 1.556694507598877, "ronpo/logit": 0.19450877606868744, "ronpo/residual": 0.18948352336883545, "ronpo/residual_abs": 0.27817171812057495, "ronpo/target": 0.0050252461805939674, "ronpo/target_abs": 0.0843028575181961, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 780 }, { "drift/chosen_abs": 0.47840428352355957, "drift/rejected_abs": 0.1084831953048706, "epoch": 0.24950337703615416, "grad_norm": 4.3125, "learning_rate": 3.731718432211417e-09, "logits/chosen": -2.1047401428222656, "logits/rejected": -2.21636962890625, "logps/chosen": -0.2704011797904968, "logps/rejected": -0.26196351647377014, "loss": 1.810654878616333, "loss/core": 1.7768656015396118, "loss/preference_sft": 0.26972612738609314, "loss/reference_anchor": 0.9384282827377319, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.778249263763428, "rewards/margins": 3.708155870437622, "rewards/rejected": 1.0700935125350952, "ronpo/logit": 0.37081560492515564, "ronpo/residual": 0.3735385537147522, "ronpo/residual_abs": 0.4183095097541809, "ronpo/target": -0.0027229245752096176, "ronpo/target_abs": 0.08420609682798386, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 785 }, { "drift/chosen_abs": 0.3598479926586151, "drift/rejected_abs": 0.12107064574956894, "epoch": 0.25109257052046086, "grad_norm": 191.0, "learning_rate": 3.4218326634913406e-09, "logits/chosen": -2.0067882537841797, "logits/rejected": -2.0267016887664795, "logps/chosen": -0.31197962164878845, "logps/rejected": -0.29012903571128845, "loss": 0.4827456474304199, "loss/core": 0.46548813581466675, "loss/preference_sft": 0.307406485080719, "loss/reference_anchor": 0.46233057975769043, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.597403049468994, "rewards/margins": 2.416919231414795, "rewards/rejected": 1.1804835796356201, "ronpo/logit": 0.2416919767856598, "ronpo/residual": 0.2469751387834549, "ronpo/residual_abs": 0.3075826168060303, "ronpo/target": -0.005283153150230646, "ronpo/target_abs": 0.07869095355272293, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 790 }, { "drift/chosen_abs": 0.24024005234241486, "drift/rejected_abs": 0.11211339384317398, "epoch": 0.2526817640047676, "grad_norm": 1.140625, "learning_rate": 3.1247622998116148e-09, "logits/chosen": -2.133969783782959, "logits/rejected": -2.1671390533447266, "logps/chosen": -0.25962501764297485, "logps/rejected": -0.25368732213974, "loss": 0.12422094494104385, "loss/core": 0.11874745786190033, "loss/preference_sft": 0.2524915635585785, "loss/reference_anchor": 0.13113582134246826, "rewards/accuracies": 0.9375, "rewards/chosen": 2.401646375656128, "rewards/margins": 1.2813524007797241, "rewards/rejected": 1.1202940940856934, "ronpo/logit": 0.12813523411750793, "ronpo/residual": 0.1344965100288391, "ronpo/residual_abs": 0.19210845232009888, "ronpo/target": -0.006361273117363453, "ronpo/target_abs": 0.07786816358566284, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 795 }, { "drift/chosen_abs": 0.2830193042755127, "drift/rejected_abs": 0.15909279882907867, "epoch": 0.2542709574890743, "grad_norm": 16.75, "learning_rate": 2.840619057186745e-09, "logits/chosen": -2.169642448425293, "logits/rejected": -2.2404894828796387, "logps/chosen": -0.2703879475593567, "logps/rejected": -0.2701282203197479, "loss": 0.2124510258436203, "loss/core": 0.1838868260383606, "loss/preference_sft": 0.2592373192310333, "loss/reference_anchor": 0.7901961803436279, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.8296594619750977, "rewards/margins": 1.2395825386047363, "rewards/rejected": 1.5900771617889404, "ronpo/logit": 0.12395823001861572, "ronpo/residual": 0.1275547742843628, "ronpo/residual_abs": 0.1707366406917572, "ronpo/target": -0.003596528898924589, "ronpo/target_abs": 0.06492110341787338, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 800 }, { "drift/chosen_abs": 0.3190115988254547, "drift/rejected_abs": 0.13805243372917175, "epoch": 0.255860150973381, "grad_norm": 217.0, "learning_rate": 2.5695097902696582e-09, "logits/chosen": -2.2752726078033447, "logits/rejected": -2.237405300140381, "logps/chosen": -0.30355507135391235, "logps/rejected": -0.3101564049720764, "loss": 0.8964834213256836, "loss/core": 0.8716424703598022, "loss/preference_sft": 0.2993467450141907, "loss/reference_anchor": 0.6798084378242493, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.1900362968444824, "rewards/margins": 1.8883402347564697, "rewards/rejected": 1.3016960620880127, "ronpo/logit": 0.1888340413570404, "ronpo/residual": 0.1919974535703659, "ronpo/residual_abs": 0.29645854234695435, "ronpo/target": -0.0031634189654141665, "ronpo/target_abs": 0.07282833755016327, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 805 }, { "drift/chosen_abs": 0.21134933829307556, "drift/rejected_abs": 0.11000971496105194, "epoch": 0.25744934445768775, "grad_norm": 7.40625, "learning_rate": 2.311536452168106e-09, "logits/chosen": -2.0637238025665283, "logits/rejected": -2.114058494567871, "logps/chosen": -0.28332144021987915, "logps/rejected": -0.28769245743751526, "loss": 0.04527657851576805, "loss/core": 0.04074571654200554, "loss/preference_sft": 0.2843453884124756, "loss/reference_anchor": 0.10101868957281113, "rewards/accuracies": 0.875, "rewards/chosen": 2.1122398376464844, "rewards/margins": 1.0128422975540161, "rewards/rejected": 1.0993976593017578, "ronpo/logit": 0.10128422826528549, "ronpo/residual": 0.07714606821537018, "ronpo/residual_abs": 0.1348331868648529, "ronpo/target": 0.02413817122578621, "ronpo/target_abs": 0.07104022800922394, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 810 }, { "drift/chosen_abs": 0.2593992352485657, "drift/rejected_abs": 0.16390137374401093, "epoch": 0.25903853794199444, "grad_norm": 12.125, "learning_rate": 2.0667960561041974e-09, "logits/chosen": -2.107893228530884, "logits/rejected": -2.0309271812438965, "logps/chosen": -0.2672129273414612, "logps/rejected": -0.2773855924606323, "loss": 0.2365700900554657, "loss/core": 0.22861385345458984, "loss/preference_sft": 0.2656956911087036, "loss/reference_anchor": 0.20075194537639618, "rewards/accuracies": 0.8374999761581421, "rewards/chosen": 2.593992233276367, "rewards/margins": 0.9726476669311523, "rewards/rejected": 1.6213443279266357, "ronpo/logit": 0.09726477414369583, "ronpo/residual": 0.08035314828157425, "ronpo/residual_abs": 0.22539310157299042, "ronpo/target": 0.016911622136831284, "ronpo/target_abs": 0.08380316197872162, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 815 }, { "drift/chosen_abs": 0.1997155249118805, "drift/rejected_abs": 0.1135120838880539, "epoch": 0.26062773142630113, "grad_norm": 1.828125, "learning_rate": 1.8353806389317424e-09, "logits/chosen": -2.0197744369506836, "logits/rejected": -1.9759705066680908, "logps/chosen": -0.2759111225605011, "logps/rejected": -0.28906577825546265, "loss": 0.06284098327159882, "loss/core": 0.056352388113737106, "loss/preference_sft": 0.27423539757728577, "loss/reference_anchor": 0.15796485543251038, "rewards/accuracies": 0.9375, "rewards/chosen": 1.993607521057129, "rewards/margins": 0.8597732782363892, "rewards/rejected": 1.1338340044021606, "ronpo/logit": 0.08597733825445175, "ronpo/residual": 0.08307936042547226, "ronpo/residual_abs": 0.14249305427074432, "ronpo/target": 0.0028979722410440445, "ronpo/target_abs": 0.07825162261724472, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 820 }, { "drift/chosen_abs": 0.2867492735385895, "drift/rejected_abs": 0.18776382505893707, "epoch": 0.2622169249106079, "grad_norm": 67.5, "learning_rate": 1.6173772265249478e-09, "logits/chosen": -2.2137134075164795, "logits/rejected": -2.161581516265869, "logps/chosen": -0.2998419404029846, "logps/rejected": -0.3022862374782562, "loss": 0.3233335018157959, "loss/core": 0.29298847913742065, "loss/preference_sft": 0.3009437024593353, "loss/reference_anchor": 0.8369055986404419, "rewards/accuracies": 0.875, "rewards/chosen": 2.8670103549957275, "rewards/margins": 0.9923250079154968, "rewards/rejected": 1.8746850490570068, "ronpo/logit": 0.0992325022816658, "ronpo/residual": 0.09889347851276398, "ronpo/residual_abs": 0.20365075767040253, "ronpo/target": 0.00033902880386449397, "ronpo/target_abs": 0.07141275703907013, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 825 }, { "drift/chosen_abs": 0.18498516082763672, "drift/rejected_abs": 0.11326082795858383, "epoch": 0.2638061183949146, "grad_norm": 35.5, "learning_rate": 1.4128678010515385e-09, "logits/chosen": -2.2336018085479736, "logits/rejected": -2.234981060028076, "logps/chosen": -0.2774164378643036, "logps/rejected": -0.2730526328086853, "loss": 0.13492383062839508, "loss/core": 0.12916125357151031, "loss/preference_sft": 0.27852898836135864, "loss/reference_anchor": 0.13679227232933044, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.8494479656219482, "rewards/margins": 0.7175174951553345, "rewards/rejected": 1.1319307088851929, "ronpo/logit": 0.07175175100564957, "ronpo/residual": 0.05915927141904831, "ronpo/residual_abs": 0.17424622178077698, "ronpo/target": 0.012592477723956108, "ronpo/target_abs": 0.0779508426785469, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 830 }, { "drift/chosen_abs": 0.4061504006385803, "drift/rejected_abs": 0.16050119698047638, "epoch": 0.2653953118792213, "grad_norm": 17.25, "learning_rate": 1.2219292701426076e-09, "logits/chosen": -2.190483808517456, "logits/rejected": -2.155860424041748, "logps/chosen": -0.24286535382270813, "logps/rejected": -0.24661651253700256, "loss": 1.0600229501724243, "loss/core": 1.0351521968841553, "loss/preference_sft": 0.24154765903949738, "loss/reference_anchor": 0.686432957649231, "rewards/accuracies": 0.824999988079071, "rewards/chosen": 4.060586452484131, "rewards/margins": 2.4555745124816895, "rewards/rejected": 1.6050119400024414, "ronpo/logit": 0.24555747210979462, "ronpo/residual": 0.233054518699646, "ronpo/residual_abs": 0.3451952636241913, "ronpo/target": 0.012502968311309814, "ronpo/target_abs": 0.07277142256498337, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 835 }, { "drift/chosen_abs": 0.28720027208328247, "drift/rejected_abs": 0.13702043890953064, "epoch": 0.266984505363528, "grad_norm": 1.0546875, "learning_rate": 1.0446334379708085e-09, "logits/chosen": -2.3651621341705322, "logits/rejected": -2.4263358116149902, "logps/chosen": -0.26630455255508423, "logps/rejected": -0.2645034193992615, "loss": 0.3639865517616272, "loss/core": 0.34910276532173157, "loss/preference_sft": 0.2732158899307251, "loss/reference_anchor": 0.39793118834495544, "rewards/accuracies": 0.9375, "rewards/chosen": 2.8720028400421143, "rewards/margins": 1.5028787851333618, "rewards/rejected": 1.369124174118042, "ronpo/logit": 0.15028786659240723, "ronpo/residual": 0.15997345745563507, "ronpo/residual_abs": 0.20985810458660126, "ronpo/target": -0.009685601107776165, "ronpo/target_abs": 0.08229205757379532, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 840 }, { "drift/chosen_abs": 0.29841169714927673, "drift/rejected_abs": 0.18531523644924164, "epoch": 0.2685736988478347, "grad_norm": 6.65625, "learning_rate": 8.810469782477639e-10, "logits/chosen": -2.1420059204101562, "logits/rejected": -2.135344982147217, "logps/chosen": -0.24753496050834656, "logps/rejected": -0.23413236439228058, "loss": 0.0860786885023117, "loss/core": 0.07448365539312363, "loss/preference_sft": 0.24143639206886292, "loss/reference_anchor": 0.30714288353919983, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 2.9827346801757812, "rewards/margins": 1.1305488348007202, "rewards/rejected": 1.8521859645843506, "ronpo/logit": 0.11305488646030426, "ronpo/residual": 0.10625892877578735, "ronpo/residual_abs": 0.16595976054668427, "ronpo/target": 0.006795956287533045, "ronpo/target_abs": 0.08061005175113678, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 845 }, { "drift/chosen_abs": 0.41207045316696167, "drift/rejected_abs": 0.1471131145954132, "epoch": 0.2701628923321414, "grad_norm": 82.5, "learning_rate": 7.31231409150776e-10, "logits/chosen": -1.953844428062439, "logits/rejected": -1.8874391317367554, "logps/chosen": -0.2713695466518402, "logps/rejected": -0.2831560969352722, "loss": 0.9032095670700073, "loss/core": 0.8764148950576782, "loss/preference_sft": 0.2667026221752167, "loss/reference_anchor": 0.7388928532600403, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 4.118669509887695, "rewards/margins": 2.6498663425445557, "rewards/rejected": 1.4688036441802979, "ronpo/logit": 0.26498663425445557, "ronpo/residual": 0.2674812972545624, "ronpo/residual_abs": 0.32319194078445435, "ronpo/target": -0.0024946490302681923, "ronpo/target_abs": 0.07483317703008652, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 850 }, { "drift/chosen_abs": 0.20735082030296326, "drift/rejected_abs": 0.07366625219583511, "epoch": 0.27175208581644816, "grad_norm": 2.21875, "learning_rate": 5.952430701883781e-10, "logits/chosen": -2.3780345916748047, "logits/rejected": -2.365642547607422, "logps/chosen": -0.27689987421035767, "logps/rejected": -0.2746134400367737, "loss": 0.4126233160495758, "loss/core": 0.40223345160484314, "loss/preference_sft": 0.27305346727371216, "loss/reference_anchor": 0.2695460915565491, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.073309898376465, "rewards/margins": 1.336647391319275, "rewards/rejected": 0.7366625070571899, "ronpo/logit": 0.13366474211215973, "ronpo/residual": 0.1408972442150116, "ronpo/residual_abs": 0.18301846086978912, "ronpo/target": -0.0072325123474001884, "ronpo/target_abs": 0.07463272660970688, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 855 }, { "drift/chosen_abs": 0.1881171017885208, "drift/rejected_abs": 0.10389295965433121, "epoch": 0.27334127930075486, "grad_norm": 24.875, "learning_rate": 4.731331010133288e-10, "logits/chosen": -2.2028229236602783, "logits/rejected": -2.12843656539917, "logps/chosen": -0.2806995213031769, "logps/rejected": -0.2797695994377136, "loss": 0.1201997771859169, "loss/core": 0.11440330743789673, "loss/preference_sft": 0.28371673822402954, "loss/reference_anchor": 0.13724195957183838, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 1.8811709880828857, "rewards/margins": 0.8511055111885071, "rewards/rejected": 1.0300655364990234, "ronpo/logit": 0.08511054515838623, "ronpo/residual": 0.07745916396379471, "ronpo/residual_abs": 0.1381129026412964, "ronpo/target": 0.007651382125914097, "ronpo/target_abs": 0.06799162924289703, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 860 }, { "drift/chosen_abs": 0.24889037013053894, "drift/rejected_abs": 0.12972140312194824, "epoch": 0.2749304727850616, "grad_norm": 0.75, "learning_rate": 3.649474221911114e-10, "logits/chosen": -2.090186595916748, "logits/rejected": -2.0353407859802246, "logps/chosen": -0.27941226959228516, "logps/rejected": -0.28891220688819885, "loss": 0.11919602006673813, "loss/core": 0.11207473278045654, "loss/preference_sft": 0.2893761694431305, "loss/reference_anchor": 0.1745276153087616, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.488442897796631, "rewards/margins": 1.1927058696746826, "rewards/rejected": 1.2957369089126587, "ronpo/logit": 0.11927058547735214, "ronpo/residual": 0.11880163848400116, "ronpo/residual_abs": 0.18198655545711517, "ronpo/target": 0.0004689468187279999, "ronpo/target_abs": 0.07780329138040543, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 865 }, { "drift/chosen_abs": 0.46323925256729126, "drift/rejected_abs": 0.21974530816078186, "epoch": 0.2765196662693683, "grad_norm": 532.0, "learning_rate": 2.7072671793106074e-10, "logits/chosen": -2.0998735427856445, "logits/rejected": -2.087890625, "logps/chosen": -0.26934343576431274, "logps/rejected": -0.27645033597946167, "loss": 0.7379623651504517, "loss/core": 0.7018780708312988, "loss/preference_sft": 0.2766755223274231, "loss/reference_anchor": 1.0033100843429565, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.631134986877441, "rewards/margins": 2.4353508949279785, "rewards/rejected": 2.195784091949463, "ronpo/logit": 0.24353507161140442, "ronpo/residual": 0.2610352635383606, "ronpo/residual_abs": 0.31035056710243225, "ronpo/target": -0.017500165849924088, "ronpo/target_abs": 0.0813521146774292, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 870 }, { "drift/chosen_abs": 0.3059462904930115, "drift/rejected_abs": 0.15434163808822632, "epoch": 0.278108859753675, "grad_norm": 5.0, "learning_rate": 1.9050642078670547e-10, "logits/chosen": -1.9450798034667969, "logits/rejected": -1.9134384393692017, "logps/chosen": -0.2710108757019043, "logps/rejected": -0.2715742588043213, "loss": 0.22777685523033142, "loss/core": 0.21646137535572052, "loss/preference_sft": 0.27517765760421753, "loss/reference_anchor": 0.2957829236984253, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.0580132007598877, "rewards/margins": 1.5153281688690186, "rewards/rejected": 1.5426852703094482, "ronpo/logit": 0.15153281390666962, "ronpo/residual": 0.1544264703989029, "ronpo/residual_abs": 0.21350125968456268, "ronpo/target": -0.002893655328080058, "ronpo/target_abs": 0.08072160184383392, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 875 }, { "drift/chosen_abs": 0.24425086379051208, "drift/rejected_abs": 0.0907929316163063, "epoch": 0.27969805323798175, "grad_norm": 5.90625, "learning_rate": 1.2431669833102288e-10, "logits/chosen": -2.1156883239746094, "logits/rejected": -2.1574745178222656, "logps/chosen": -0.27831846475601196, "logps/rejected": -0.29058295488357544, "loss": 0.42100924253463745, "loss/core": 0.41138726472854614, "loss/preference_sft": 0.2927462160587311, "loss/reference_anchor": 0.2456396371126175, "rewards/accuracies": 0.9375, "rewards/chosen": 2.4425082206726074, "rewards/margins": 1.5455248355865479, "rewards/rejected": 0.8969835042953491, "ronpo/logit": 0.15455248951911926, "ronpo/residual": 0.1457471400499344, "ronpo/residual_abs": 0.20269238948822021, "ronpo/target": 0.008805358782410622, "ronpo/target_abs": 0.08658818900585175, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 880 }, { "drift/chosen_abs": 0.26528865098953247, "drift/rejected_abs": 0.1037355437874794, "epoch": 0.28128724672228844, "grad_norm": 5.625, "learning_rate": 7.218244181167594e-11, "logits/chosen": -2.1619465351104736, "logits/rejected": -2.223400592803955, "logps/chosen": -0.29743996262550354, "logps/rejected": -0.2958257794380188, "loss": 0.3847818076610565, "loss/core": 0.36538928747177124, "loss/preference_sft": 0.2988830506801605, "loss/reference_anchor": 0.5241828560829163, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.6526153087615967, "rewards/margins": 1.6183156967163086, "rewards/rejected": 1.034299612045288, "ronpo/logit": 0.1618315875530243, "ronpo/residual": 0.1641121506690979, "ronpo/residual_abs": 0.20270681381225586, "ronpo/target": -0.0022805710323154926, "ronpo/target_abs": 0.07088293880224228, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 885 }, { "drift/chosen_abs": 0.2995665669441223, "drift/rejected_abs": 0.12155304849147797, "epoch": 0.28287644020659514, "grad_norm": 3.765625, "learning_rate": 3.4123256790405397e-11, "logits/chosen": -2.307978868484497, "logits/rejected": -2.2648723125457764, "logps/chosen": -0.2815799415111542, "logps/rejected": -0.2809089720249176, "loss": 0.4028624892234802, "loss/core": 0.3849177956581116, "loss/preference_sft": 0.286651074886322, "loss/reference_anchor": 0.48404186964035034, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.99299693107605, "rewards/margins": 1.7814449071884155, "rewards/rejected": 1.2115522623062134, "ronpo/logit": 0.17814448475837708, "ronpo/residual": 0.18465931713581085, "ronpo/residual_abs": 0.22240400314331055, "ronpo/target": -0.006514822132885456, "ronpo/target_abs": 0.07123463600873947, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 890 }, { "drift/chosen_abs": 0.30803316831588745, "drift/rejected_abs": 0.12433400005102158, "epoch": 0.2844656336909019, "grad_norm": 11.5, "learning_rate": 1.0153455770185725e-11, "logits/chosen": -2.0798468589782715, "logits/rejected": -2.068073272705078, "logps/chosen": -0.2840914726257324, "logps/rejected": -0.2947894334793091, "loss": 0.3525415360927582, "loss/core": 0.33794763684272766, "loss/preference_sft": 0.29777130484580994, "loss/reference_anchor": 0.38719049096107483, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 3.0781116485595703, "rewards/margins": 1.9379209280014038, "rewards/rejected": 1.1401903629302979, "ronpo/logit": 0.19379207491874695, "ronpo/residual": 0.1745019257068634, "ronpo/residual_abs": 0.2425132542848587, "ronpo/target": 0.019290190190076828, "ronpo/target_abs": 0.08941071480512619, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 895 }, { "drift/chosen_abs": 0.16016218066215515, "drift/rejected_abs": 0.08440212905406952, "epoch": 0.2860548271752086, "grad_norm": 15.3125, "learning_rate": 2.820528128971711e-13, "logits/chosen": -2.104757308959961, "logits/rejected": -2.0879600048065186, "logps/chosen": -0.31782156229019165, "logps/rejected": -0.3181937634944916, "loss": 0.14113518595695496, "loss/core": 0.13576209545135498, "loss/preference_sft": 0.3204381763935089, "loss/reference_anchor": 0.12147295475006104, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.6012519598007202, "rewards/margins": 0.888868510723114, "rewards/rejected": 0.7123832106590271, "ronpo/logit": 0.08888684213161469, "ronpo/residual": 0.08423057198524475, "ronpo/residual_abs": 0.14461413025856018, "ronpo/target": 0.004656284116208553, "ronpo/target_abs": 0.07319948077201843, "ronpo/weight": 1.0, "ronpo/weight_max": 1.0, "step": 900 }, { "epoch": 0.2860548271752086, "step": 900, "total_flos": 0.0, "train_loss": 0.4220078429910872, "train_runtime": 7042.2326, "train_samples_per_second": 2.045, "train_steps_per_second": 0.128 } ], "logging_steps": 5, "max_steps": 900, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 900, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }