{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 3.0, "eval_steps": 500, "global_step": 2316, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.025906735751295335, "grad_norm": 12.56043815612793, "kl": 11.764090538024902, "learning_rate": 1.8999999999999998e-07, "logits/chosen": -36900768.744186044, "logits/rejected": -37248082.242519684, "logps/chosen": -498.79932170542634, "logps/rejected": -360.7293553149606, "loss": 0.5315, "loss/base_kto": 3.9338440895080566, "loss/total_with_kl": 4.252355575561523, "metrics/advantage_var": 191.29795837402344, "regularization/advantage_var": 191.29795837402344, "regularization/kl": 0.31851109862327576, "rewards/chosen": 0.1798099251680596, "rewards/margins": 0.05079575828352467, "rewards/rejected": 0.12901416688453493, "step": 20 }, { "epoch": 0.05181347150259067, "grad_norm": 17.16169548034668, "kl": 10.632026672363281, "learning_rate": 3.8999999999999997e-07, "logits/chosen": -35866617.73700306, "logits/rejected": -37734197.162939295, "logps/chosen": -485.1208906727829, "logps/rejected": -375.26163138977637, "loss": 0.5304, "loss/base_kto": 3.914513111114502, "loss/total_with_kl": 4.243302345275879, "metrics/advantage_var": 197.47079467773438, "regularization/advantage_var": 197.47079467773438, "regularization/kl": 0.3287888765335083, "rewards/chosen": 0.1405060094430906, "rewards/margins": 0.06401507097003811, "rewards/rejected": 0.0764909384730525, "step": 40 }, { "epoch": 0.07772020725388601, "grad_norm": 28.62113380432129, "kl": 26.664426803588867, "learning_rate": 5.9e-07, "logits/chosen": -37339925.13761468, "logits/rejected": -38068243.629392974, "logps/chosen": -465.74904434250766, "logps/rejected": -350.9490315495208, "loss": 0.5139, "loss/base_kto": 3.793339252471924, "loss/total_with_kl": 4.11139440536499, "metrics/advantage_var": 191.02430725097656, "regularization/advantage_var": 191.02430725097656, "regularization/kl": 0.3180554509162903, "rewards/chosen": 0.48558973026567276, "rewards/margins": 0.1880954574303343, "rewards/rejected": 0.29749427283533847, "step": 60 }, { "epoch": 0.10362694300518134, "grad_norm": 29.357450485229492, "kl": 11.40846061706543, "learning_rate": 7.9e-07, "logits/chosen": -37080985.754916795, "logits/rejected": -38277980.22617125, "logps/chosen": -501.6247163388805, "logps/rejected": -382.78412762520196, "loss": 0.5205, "loss/base_kto": 3.7919883728027344, "loss/total_with_kl": 4.163951873779297, "metrics/advantage_var": 223.40139770507812, "regularization/advantage_var": 223.40139770507812, "regularization/kl": 0.3719632923603058, "rewards/chosen": 0.22859849799959814, "rewards/margins": 0.20049083792479408, "rewards/rejected": 0.028107660074804055, "step": 80 }, { "epoch": 0.12953367875647667, "grad_norm": 18.8193359375, "kl": 21.178363800048828, "learning_rate": 9.9e-07, "logits/chosen": -36232604.85079365, "logits/rejected": -38612396.50461538, "logps/chosen": -493.13412698412696, "logps/rejected": -358.96997596153847, "loss": 0.5237, "loss/base_kto": 3.816561222076416, "loss/total_with_kl": 4.189939022064209, "metrics/advantage_var": 224.25108337402344, "regularization/advantage_var": 224.25108337402344, "regularization/kl": 0.3733780086040497, "rewards/chosen": 0.36637655591207835, "rewards/margins": 0.16838343879969855, "rewards/rejected": 0.1979931171123798, "step": 100 }, { "epoch": 0.15544041450777202, "grad_norm": 12.725244522094727, "kl": 25.881696701049805, "learning_rate": 9.998186234298189e-07, "logits/chosen": -36246479.83004552, "logits/rejected": -38376083.58132045, "logps/chosen": -479.6000569044006, "logps/rejected": -351.1389895330113, "loss": 0.511, "loss/base_kto": 3.7016937732696533, "loss/total_with_kl": 4.088140487670898, "metrics/advantage_var": 232.1000518798828, "regularization/advantage_var": 232.1000518798828, "regularization/kl": 0.3864465653896332, "rewards/chosen": 0.4568905171928348, "rewards/margins": 0.2578212345259543, "rewards/rejected": 0.19906928266688054, "step": 120 }, { "epoch": 0.18134715025906736, "grad_norm": 131.57684326171875, "kl": 29.29970359802246, "learning_rate": 9.992359552107714e-07, "logits/chosen": -36191890.977917984, "logits/rejected": -38352350.7120743, "logps/chosen": -487.19962539432174, "logps/rejected": -384.77087364551085, "loss": 0.5272, "loss/base_kto": 3.760270833969116, "loss/total_with_kl": 4.2179059982299805, "metrics/advantage_var": 274.8559265136719, "regularization/advantage_var": 274.8559265136719, "regularization/kl": 0.4576350748538971, "rewards/chosen": 0.43132303039358244, "rewards/margins": 0.1845467920790749, "rewards/rejected": 0.24677623831450754, "step": 140 }, { "epoch": 0.20725388601036268, "grad_norm": 32.52603530883789, "kl": 5.176639080047607, "learning_rate": 9.982519612796161e-07, "logits/chosen": -34978281.22570016, "logits/rejected": -37123606.062407136, "logps/chosen": -492.46643327841844, "logps/rejected": -384.6894502228826, "loss": 0.521, "loss/base_kto": 3.815791368484497, "loss/total_with_kl": 4.168036937713623, "metrics/advantage_var": 211.55894470214844, "regularization/advantage_var": 211.55894470214844, "regularization/kl": 0.35224565863609314, "rewards/chosen": -0.028169433995958608, "rewards/margins": 0.1667565855904316, "rewards/rejected": -0.19492601958639022, "step": 160 }, { "epoch": 0.23316062176165803, "grad_norm": 15.889501571655273, "kl": 2.1030290126800537, "learning_rate": 9.968674326492213e-07, "logits/chosen": -37257417.549206346, "logits/rejected": -38514832.935384616, "logps/chosen": -508.6459821428571, "logps/rejected": -379.45134615384615, "loss": 0.5209, "loss/base_kto": 3.7495439052581787, "loss/total_with_kl": 4.16711950302124, "metrics/advantage_var": 250.7963409423828, "regularization/advantage_var": 250.7963409423828, "regularization/kl": 0.4175758957862854, "rewards/chosen": -0.1384769742451017, "rewards/margins": 0.2594093782939608, "rewards/rejected": -0.3978863525390625, "step": 180 }, { "epoch": 0.25906735751295334, "grad_norm": 17.3436279296875, "kl": 3.039137601852417, "learning_rate": 9.950834823142198e-07, "logits/chosen": -37118635.78947368, "logits/rejected": -38403294.47619048, "logps/chosen": -487.4664884868421, "logps/rejected": -393.27464657738096, "loss": 0.5172, "loss/base_kto": 3.731966495513916, "loss/total_with_kl": 4.137502193450928, "metrics/advantage_var": 243.5651397705078, "regularization/advantage_var": 243.5651397705078, "regularization/kl": 0.4055359959602356, "rewards/chosen": -0.0446096376368874, "rewards/margins": 0.23779303672020896, "rewards/rejected": -0.2824026743570964, "step": 200 }, { "epoch": 0.2849740932642487, "grad_norm": 12.679957389831543, "kl": 3.0125131607055664, "learning_rate": 9.929015443562942e-07, "logits/chosen": -35068480.81012658, "logits/rejected": -36613856.39506173, "logps/chosen": -497.21004746835445, "logps/rejected": -390.15875771604937, "loss": 0.5194, "loss/base_kto": 3.726527452468872, "loss/total_with_kl": 4.15496301651001, "metrics/advantage_var": 257.31890869140625, "regularization/advantage_var": 257.31890869140625, "regularization/kl": 0.42843589186668396, "rewards/chosen": -0.045610340335701084, "rewards/margins": 0.2788971492807871, "rewards/rejected": -0.32450748961648823, "step": 220 }, { "epoch": 0.31088082901554404, "grad_norm": 16.49644660949707, "kl": 14.773193359375, "learning_rate": 9.90323372791347e-07, "logits/chosen": -36608568.88888889, "logits/rejected": -38326196.40268456, "logps/chosen": -503.44590643274853, "logps/rejected": -353.9297661493289, "loss": 0.5124, "loss/base_kto": 3.6660022735595703, "loss/total_with_kl": 4.099441051483154, "metrics/advantage_var": 260.3235168457031, "regularization/advantage_var": 260.3235168457031, "regularization/kl": 0.43343862891197205, "rewards/chosen": 0.3166549638000845, "rewards/margins": 0.2954046038360338, "rewards/rejected": 0.021250359964050704, "step": 240 }, { "epoch": 0.33678756476683935, "grad_norm": 18.86716651916504, "kl": 24.02515411376953, "learning_rate": 9.87351040159484e-07, "logits/chosen": -36005391.802469134, "logits/rejected": -37409529.51898734, "logps/chosen": -482.8971354166667, "logps/rejected": -348.525761471519, "loss": 0.5237, "loss/base_kto": 3.7093505859375, "loss/total_with_kl": 4.189932823181152, "metrics/advantage_var": 288.637451171875, "regularization/advantage_var": 288.637451171875, "regularization/kl": 0.4805813431739807, "rewards/chosen": 0.49618826972113717, "rewards/margins": 0.26080010544231, "rewards/rejected": 0.23538816427882714, "step": 260 }, { "epoch": 0.3626943005181347, "grad_norm": 18.295459747314453, "kl": 10.224114418029785, "learning_rate": 9.839869358589396e-07, "logits/chosen": -36287873.87928222, "logits/rejected": -38906473.16341829, "logps/chosen": -488.47705954323004, "logps/rejected": -365.48046289355324, "loss": 0.5156, "loss/base_kto": 3.6996846199035645, "loss/total_with_kl": 4.1250152587890625, "metrics/advantage_var": 255.45396423339844, "regularization/advantage_var": 255.45396423339844, "regularization/kl": 0.42533078789711, "rewards/chosen": 0.17645793871233942, "rewards/margins": 0.25755616065849524, "rewards/rejected": -0.08109822194615583, "step": 280 }, { "epoch": 0.38860103626943004, "grad_norm": 14.254644393920898, "kl": 8.755472183227539, "learning_rate": 9.80233764225289e-07, "logits/chosen": -35747625.6, "logits/rejected": -37146771.2, "logps/chosen": -495.8552734375, "logps/rejected": -380.7497802734375, "loss": 0.5209, "loss/base_kto": 3.6856186389923096, "loss/total_with_kl": 4.167328834533691, "metrics/advantage_var": 289.31536865234375, "regularization/advantage_var": 289.31536865234375, "regularization/kl": 0.4817100465297699, "rewards/chosen": 0.1729241967201233, "rewards/margins": 0.2861413359642029, "rewards/rejected": -0.11321713924407958, "step": 300 }, { "epoch": 0.41450777202072536, "grad_norm": 14.803353309631348, "kl": 9.75937557220459, "learning_rate": 9.760945423574868e-07, "logits/chosen": -36956386.86494689, "logits/rejected": -37225166.94363929, "logps/chosen": -457.41099203338393, "logps/rejected": -376.5743508454106, "loss": 0.5136, "loss/base_kto": 3.7342541217803955, "loss/total_with_kl": 4.108650207519531, "metrics/advantage_var": 224.862548828125, "regularization/advantage_var": 224.862548828125, "regularization/kl": 0.3743961453437805, "rewards/chosen": 0.15819099206302162, "rewards/margins": 0.2578718204585027, "rewards/rejected": -0.09968082839548108, "step": 320 }, { "epoch": 0.44041450777202074, "grad_norm": 13.023077964782715, "kl": 11.907632827758789, "learning_rate": 9.71572597692482e-07, "logits/chosen": -37419965.00934579, "logits/rejected": -38185871.64890282, "logps/chosen": -473.6628699376947, "logps/rejected": -384.80721003134795, "loss": 0.5177, "loss/base_kto": 3.687344789505005, "loss/total_with_kl": 4.141505241394043, "metrics/advantage_var": 272.76885986328125, "regularization/advantage_var": 272.76885986328125, "regularization/kl": 0.4541601240634918, "rewards/chosen": 0.24721781412760416, "rewards/margins": 0.3008286606680014, "rewards/rejected": -0.05361084654039724, "step": 340 }, { "epoch": 0.46632124352331605, "grad_norm": 15.86916732788086, "kl": 13.91089153289795, "learning_rate": 9.666715653303588e-07, "logits/chosen": -37737871.72628305, "logits/rejected": -38219968.90423862, "logps/chosen": -491.63258164852255, "logps/rejected": -380.27624607535324, "loss": 0.5199, "loss/base_kto": 3.6812775135040283, "loss/total_with_kl": 4.1595377922058105, "metrics/advantage_var": 287.2435302734375, "regularization/advantage_var": 287.2435302734375, "regularization/kl": 0.47826048731803894, "rewards/chosen": 0.3224105479001628, "rewards/margins": 0.3177741920913598, "rewards/rejected": 0.004636355808803013, "step": 360 }, { "epoch": 0.49222797927461137, "grad_norm": 15.11502456665039, "kl": 14.820399284362793, "learning_rate": 9.613953851121528e-07, "logits/chosen": -37378483.2, "logits/rejected": -38810985.6, "logps/chosen": -499.506591796875, "logps/rejected": -369.033740234375, "loss": 0.5161, "loss/base_kto": 3.703874349594116, "loss/total_with_kl": 4.129049301147461, "metrics/advantage_var": 255.36026000976562, "regularization/advantage_var": 255.36026000976562, "regularization/kl": 0.4251748025417328, "rewards/chosen": 0.26812138557434084, "rewards/margins": 0.25979379862546925, "rewards/rejected": 0.008327586948871613, "step": 380 }, { "epoch": 0.5181347150259067, "grad_norm": 12.838387489318848, "kl": 13.0317964553833, "learning_rate": 9.557482984526924e-07, "logits/chosen": -36799193.784911714, "logits/rejected": -38216184.98630137, "logps/chosen": -494.7509028892456, "logps/rejected": -375.41412195585997, "loss": 0.5134, "loss/base_kto": 3.679405689239502, "loss/total_with_kl": 4.107245922088623, "metrics/advantage_var": 256.9610290527344, "regularization/advantage_var": 256.9610290527344, "regularization/kl": 0.42784014344215393, "rewards/chosen": 0.17315232964235555, "rewards/margins": 0.2958745082945804, "rewards/rejected": -0.12272217865222484, "step": 400 }, { "epoch": 0.5440414507772021, "grad_norm": 14.595232963562012, "kl": 11.939623832702637, "learning_rate": 9.497348449310107e-07, "logits/chosen": -36126301.769854136, "logits/rejected": -36686945.30316742, "logps/chosen": -482.40903565640195, "logps/rejected": -374.1887254901961, "loss": 0.5189, "loss/base_kto": 3.674595594406128, "loss/total_with_kl": 4.15156888961792, "metrics/advantage_var": 286.4701232910156, "regularization/advantage_var": 286.4701232910156, "regularization/kl": 0.4769727289676666, "rewards/chosen": 0.20014795337349323, "rewards/margins": 0.30965150234228445, "rewards/rejected": -0.10950354896879125, "step": 420 }, { "epoch": 0.5699481865284974, "grad_norm": 19.584970474243164, "kl": 4.382344722747803, "learning_rate": 9.433598586410701e-07, "logits/chosen": -36109483.20504732, "logits/rejected": -37060931.368421055, "logps/chosen": -497.10917783911674, "logps/rejected": -363.86595394736844, "loss": 0.5239, "loss/base_kto": 3.7143712043762207, "loss/total_with_kl": 4.19157600402832, "metrics/advantage_var": 286.6097106933594, "regularization/advantage_var": 286.6097106933594, "regularization/kl": 0.4772051274776459, "rewards/chosen": -0.04339043650341335, "rewards/margins": 0.27319334454704136, "rewards/rejected": -0.3165837810504547, "step": 440 }, { "epoch": 0.5958549222797928, "grad_norm": 15.339016914367676, "kl": 11.145998001098633, "learning_rate": 9.366284643057313e-07, "logits/chosen": -36344467.986710966, "logits/rejected": -37687371.51622419, "logps/chosen": -476.24133098006644, "logps/rejected": -372.29374078171094, "loss": 0.5207, "loss/base_kto": 3.6438357830047607, "loss/total_with_kl": 4.165812015533447, "metrics/advantage_var": 313.4992370605469, "regularization/advantage_var": 313.4992370605469, "regularization/kl": 0.5219761729240417, "rewards/chosen": 0.19266740665879362, "rewards/margins": 0.33707507847185825, "rewards/rejected": -0.14440767181306463, "step": 460 }, { "epoch": 0.6217616580310881, "grad_norm": 26.200326919555664, "kl": 3.715789794921875, "learning_rate": 9.295460731570917e-07, "logits/chosen": -35384174.60763359, "logits/rejected": -35745798.5536, "logps/chosen": -458.30143129770994, "logps/rejected": -371.50825, "loss": 0.5302, "loss/base_kto": 3.7983169555664062, "loss/total_with_kl": 4.2412495613098145, "metrics/advantage_var": 266.0255126953125, "regularization/advantage_var": 266.0255126953125, "regularization/kl": 0.44293245673179626, "rewards/chosen": -0.08463416063148557, "rewards/margins": 0.21929518507163945, "rewards/rejected": -0.303929345703125, "step": 480 }, { "epoch": 0.6476683937823834, "grad_norm": 17.56812858581543, "kl": 18.693069458007812, "learning_rate": 9.221183785865056e-07, "logits/chosen": -35942031.67751938, "logits/rejected": -35166940.11968504, "logps/chosen": -473.8987403100775, "logps/rejected": -373.3375246062992, "loss": 0.524, "loss/base_kto": 3.657953977584839, "loss/total_with_kl": 4.19184684753418, "metrics/advantage_var": 320.6563720703125, "regularization/advantage_var": 320.6563720703125, "regularization/kl": 0.533892810344696, "rewards/chosen": 0.3846660288729409, "rewards/margins": 0.3189899659961569, "rewards/rejected": 0.06567606287678396, "step": 500 }, { "epoch": 0.6735751295336787, "grad_norm": 10.936003684997559, "kl": 13.084571838378906, "learning_rate": 9.143513515677817e-07, "logits/chosen": -34986207.94928685, "logits/rejected": -35362913.82434515, "logps/chosen": -449.13287440570525, "logps/rejected": -334.2512278505393, "loss": 0.502, "loss/base_kto": 3.5975258350372314, "loss/total_with_kl": 4.015763759613037, "metrics/advantage_var": 251.19371032714844, "regularization/advantage_var": 251.19371032714844, "regularization/kl": 0.41823750734329224, "rewards/chosen": 0.27444801391021195, "rewards/margins": 0.3917814010999389, "rewards/rejected": -0.11733338718972698, "step": 520 }, { "epoch": 0.6994818652849741, "grad_norm": 19.385421752929688, "kl": 8.8245210647583, "learning_rate": 9.062512358572373e-07, "logits/chosen": -35239906.602870815, "logits/rejected": -36412386.20520674, "logps/chosen": -468.02641547049444, "logps/rejected": -387.21604613323126, "loss": 0.5238, "loss/base_kto": 3.7126381397247314, "loss/total_with_kl": 4.190079689025879, "metrics/advantage_var": 286.7515563964844, "regularization/advantage_var": 286.7515563964844, "regularization/kl": 0.47744131088256836, "rewards/chosen": 0.07935350287283817, "rewards/margins": 0.29322699951992376, "rewards/rejected": -0.21387349664708558, "step": 540 }, { "epoch": 0.7253886010362695, "grad_norm": 14.663601875305176, "kl": 7.2484130859375, "learning_rate": 8.978245429744691e-07, "logits/chosen": -35352050.87179487, "logits/rejected": -36537226.165316045, "logps/chosen": -494.62877073906486, "logps/rejected": -374.1108438006483, "loss": 0.5194, "loss/base_kto": 3.6950180530548096, "loss/total_with_kl": 4.15555477142334, "metrics/advantage_var": 276.598388671875, "regularization/advantage_var": 276.598388671875, "regularization/kl": 0.46053630113601685, "rewards/chosen": 0.09303807277305454, "rewards/margins": 0.28864844065776, "rewards/rejected": -0.19561036788470548, "step": 560 }, { "epoch": 0.7512953367875648, "grad_norm": 16.714582443237305, "kl": 17.105743408203125, "learning_rate": 8.890780469678738e-07, "logits/chosen": -38186708.36802413, "logits/rejected": -37993178.243111834, "logps/chosen": -506.79520173453994, "logps/rejected": -381.8746454619125, "loss": 0.5168, "loss/base_kto": 3.6692006587982178, "loss/total_with_kl": 4.13415002822876, "metrics/advantage_var": 279.248779296875, "regularization/advantage_var": 279.248779296875, "regularization/kl": 0.46494922041893005, "rewards/chosen": 0.39226553846448436, "rewards/margins": 0.2876216039147477, "rewards/rejected": 0.10464393454973663, "step": 580 }, { "epoch": 0.7772020725388601, "grad_norm": 18.708269119262695, "kl": 14.641237258911133, "learning_rate": 8.800187789691269e-07, "logits/chosen": -36220029.58490566, "logits/rejected": -37642640.69565217, "logps/chosen": -494.0291863207547, "logps/rejected": -355.74058618012424, "loss": 0.5168, "loss/base_kto": 3.6611649990081787, "loss/total_with_kl": 4.134397029876709, "metrics/advantage_var": 284.2236022949219, "regularization/advantage_var": 284.2236022949219, "regularization/kl": 0.4732322692871094, "rewards/chosen": 0.26615013266509435, "rewards/margins": 0.30125557729483465, "rewards/rejected": -0.03510544462974027, "step": 600 }, { "epoch": 0.8031088082901554, "grad_norm": 26.249277114868164, "kl": 13.373329162597656, "learning_rate": 8.706540215409981e-07, "logits/chosen": -35497366.974358976, "logits/rejected": -36902468.682926826, "logps/chosen": -465.55398637820514, "logps/rejected": -383.5540205792683, "loss": 0.5152, "loss/base_kto": 3.695056200027466, "loss/total_with_kl": 4.1215715408325195, "metrics/advantage_var": 256.1650390625, "regularization/advantage_var": 256.1650390625, "regularization/kl": 0.4265148341655731, "rewards/chosen": 0.13917385003505608, "rewards/margins": 0.2562612607525318, "rewards/rejected": -0.1170874107174757, "step": 620 }, { "epoch": 0.8290155440414507, "grad_norm": 12.480875015258789, "kl": 14.748756408691406, "learning_rate": 8.609913028230462e-07, "logits/chosen": -37256670.94173228, "logits/rejected": -38256262.151937984, "logps/chosen": -482.16225393700785, "logps/rejected": -385.0124515503876, "loss": 0.5089, "loss/base_kto": 3.6076576709747314, "loss/total_with_kl": 4.0711669921875, "metrics/advantage_var": 278.38385009765625, "regularization/advantage_var": 278.38385009765625, "regularization/kl": 0.46350908279418945, "rewards/chosen": 0.29258932278850885, "rewards/margins": 0.34477712559858636, "rewards/rejected": -0.05218780281007752, "step": 640 }, { "epoch": 0.8549222797927462, "grad_norm": 17.288246154785156, "kl": 12.5535888671875, "learning_rate": 8.510383904798994e-07, "logits/chosen": -38125598.98335855, "logits/rejected": -38174147.6187399, "logps/chosen": -479.7106183812405, "logps/rejected": -382.77867528271406, "loss": 0.5207, "loss/base_kto": 3.720900058746338, "loss/total_with_kl": 4.1653361320495605, "metrics/advantage_var": 266.92828369140625, "regularization/advantage_var": 266.92828369140625, "regularization/kl": 0.44443559646606445, "rewards/chosen": 0.24331503448255484, "rewards/margins": 0.24921536201406436, "rewards/rejected": -0.0059003275315095225, "step": 660 }, { "epoch": 0.8808290155440415, "grad_norm": 27.970935821533203, "kl": 19.024961471557617, "learning_rate": 8.408032854569865e-07, "logits/chosen": -37159446.84958678, "logits/rejected": -37783288.03555556, "logps/chosen": -461.6587809917355, "logps/rejected": -370.68409722222225, "loss": 0.5233, "loss/base_kto": 3.680556535720825, "loss/total_with_kl": 4.186591625213623, "metrics/advantage_var": 303.924560546875, "regularization/advantage_var": 303.924560546875, "regularization/kl": 0.5060344338417053, "rewards/chosen": 0.34186113846203514, "rewards/margins": 0.24035254154724928, "rewards/rejected": 0.10150859691478588, "step": 680 }, { "epoch": 0.9067357512953368, "grad_norm": 12.256890296936035, "kl": 11.472315788269043, "learning_rate": 8.302942155487377e-07, "logits/chosen": -36766501.35791091, "logits/rejected": -38025525.316375196, "logps/chosen": -486.63148041474653, "logps/rejected": -384.1498161764706, "loss": 0.5195, "loss/base_kto": 3.6514556407928467, "loss/total_with_kl": 4.156064510345459, "metrics/advantage_var": 303.0688171386719, "regularization/advantage_var": 303.0688171386719, "regularization/kl": 0.5046095252037048, "rewards/chosen": 0.2328626688358055, "rewards/margins": 0.34065466044109416, "rewards/rejected": -0.10779199160528866, "step": 700 }, { "epoch": 0.9326424870466321, "grad_norm": 14.921036720275879, "kl": 14.676401138305664, "learning_rate": 8.195196287844278e-07, "logits/chosen": -37102108.44444445, "logits/rejected": -37644236.151898734, "logps/chosen": -505.00626929012344, "logps/rejected": -377.8958168512658, "loss": 0.5201, "loss/base_kto": 3.6620795726776123, "loss/total_with_kl": 4.160859107971191, "metrics/advantage_var": 299.5671691894531, "regularization/advantage_var": 299.5671691894531, "regularization/kl": 0.498779296875, "rewards/chosen": 0.1833903701217086, "rewards/margins": 0.2806555231132811, "rewards/rejected": -0.09726515299157251, "step": 720 }, { "epoch": 0.9585492227979274, "grad_norm": 16.012065887451172, "kl": 18.5018310546875, "learning_rate": 8.08488186636975e-07, "logits/chosen": -35699431.541401275, "logits/rejected": -38184702.42944785, "logps/chosen": -472.077826433121, "logps/rejected": -359.90203220858893, "loss": 0.5079, "loss/base_kto": 3.6436784267425537, "loss/total_with_kl": 4.0634050369262695, "metrics/advantage_var": 252.0883026123047, "regularization/advantage_var": 252.0883026123047, "regularization/kl": 0.41972705721855164, "rewards/chosen": 0.2600595753663664, "rewards/margins": 0.2945126339017024, "rewards/rejected": -0.034453058535335986, "step": 740 }, { "epoch": 0.9844559585492227, "grad_norm": 17.976825714111328, "kl": 20.56618309020996, "learning_rate": 7.972087570601576e-07, "logits/chosen": -36451590.717201166, "logits/rejected": -37006042.93602694, "logps/chosen": -479.79282069970844, "logps/rejected": -400.649647516835, "loss": 0.512, "loss/base_kto": 3.646868944168091, "loss/total_with_kl": 4.095690727233887, "metrics/advantage_var": 269.56268310546875, "regularization/advantage_var": 269.56268310546875, "regularization/kl": 0.4488218426704407, "rewards/chosen": 0.38110556199321244, "rewards/margins": 0.2800363395975435, "rewards/rejected": 0.10106922239566893, "step": 760 }, { "epoch": 1.0103626943005182, "grad_norm": 14.430757522583008, "kl": 28.526700973510742, "learning_rate": 7.856904073598458e-07, "logits/chosen": -36917991.58153846, "logits/rejected": -37864881.73248408, "logps/chosen": -476.4955769230769, "logps/rejected": -368.49400378184714, "loss": 0.5152, "loss/base_kto": 3.432401418685913, "loss/total_with_kl": 4.121790885925293, "metrics/advantage_var": 414.0479431152344, "regularization/advantage_var": 414.0479431152344, "regularization/kl": 0.6893897652626038, "rewards/chosen": 0.655761483999399, "rewards/margins": 0.5595810541155758, "rewards/rejected": 0.09618042988382328, "step": 780 }, { "epoch": 1.0362694300518134, "grad_norm": 16.027830123901367, "kl": 22.440448760986328, "learning_rate": 7.739423969049761e-07, "logits/chosen": -36057283.66878981, "logits/rejected": -36913073.47239264, "logps/chosen": -463.7797571656051, "logps/rejected": -357.4802051380368, "loss": 0.5414, "loss/base_kto": 3.18515944480896, "loss/total_with_kl": 4.3311591148376465, "metrics/advantage_var": 688.2881469726562, "regularization/advantage_var": 688.2881469726562, "regularization/kl": 1.145999789237976, "rewards/chosen": 0.7192742414535231, "rewards/margins": 0.8778118311527359, "rewards/rejected": -0.15853758969921275, "step": 800 }, { "epoch": 1.0621761658031088, "grad_norm": 24.44966697692871, "kl": 22.706125259399414, "learning_rate": 7.619741696841322e-07, "logits/chosen": -36837197.20634921, "logits/rejected": -38595253.16923077, "logps/chosen": -477.5025297619048, "logps/rejected": -400.7340384615385, "loss": 0.7072, "loss/base_kto": 3.1124749183654785, "loss/total_with_kl": 5.657690048217773, "metrics/advantage_var": 1528.6578369140625, "regularization/advantage_var": 1528.6578369140625, "regularization/kl": 2.545215368270874, "rewards/chosen": 0.7123586987692212, "rewards/margins": 1.0149622237842453, "rewards/rejected": -0.302603525015024, "step": 820 }, { "epoch": 1.0880829015544042, "grad_norm": 11.723468780517578, "kl": 14.782014846801758, "learning_rate": 7.497953467137135e-07, "logits/chosen": -36865033.54037267, "logits/rejected": -37690387.320754714, "logps/chosen": -483.75296001552795, "logps/rejected": -400.18204599056605, "loss": 0.521, "loss/base_kto": 3.120527982711792, "loss/total_with_kl": 4.167871952056885, "metrics/advantage_var": 629.0353393554688, "regularization/advantage_var": 629.0353393554688, "regularization/kl": 1.0473438501358032, "rewards/chosen": 0.6224194639217779, "rewards/margins": 0.9727715292900464, "rewards/rejected": -0.35035206536826846, "step": 840 }, { "epoch": 1.1139896373056994, "grad_norm": 47.848121643066406, "kl": 17.3000545501709, "learning_rate": 7.37415718303793e-07, "logits/chosen": -37203643.43581616, "logits/rejected": -37336807.14946071, "logps/chosen": -504.6797246434231, "logps/rejected": -365.1705989984592, "loss": 0.5173, "loss/base_kto": 3.144667148590088, "loss/total_with_kl": 4.138281345367432, "metrics/advantage_var": 596.765380859375, "regularization/advantage_var": 596.765380859375, "regularization/kl": 0.9936143755912781, "rewards/chosen": 0.6061912089255646, "rewards/margins": 0.9172739989300185, "rewards/rejected": -0.311082790004454, "step": 860 }, { "epoch": 1.1398963730569949, "grad_norm": 17.364564895629883, "kl": 16.685171127319336, "learning_rate": 7.248452361878855e-07, "logits/chosen": -34775951.975346684, "logits/rejected": -36733768.62123613, "logps/chosen": -485.69597457627117, "logps/rejected": -381.38975832012676, "loss": 0.5397, "loss/base_kto": 3.1717493534088135, "loss/total_with_kl": 4.317812442779541, "metrics/advantage_var": 688.326171875, "regularization/advantage_var": 688.326171875, "regularization/kl": 1.1460630893707275, "rewards/chosen": 0.5993905324597939, "rewards/margins": 0.8719567338406007, "rewards/rejected": -0.2725662013808067, "step": 880 }, { "epoch": 1.16580310880829, "grad_norm": 22.1566162109375, "kl": 14.815750122070312, "learning_rate": 7.120940055229497e-07, "logits/chosen": -36743694.989214174, "logits/rejected": -37517130.24405705, "logps/chosen": -480.1051617873652, "logps/rejected": -381.6002872424723, "loss": 0.5198, "loss/base_kto": 3.1873226165771484, "loss/total_with_kl": 4.158039093017578, "metrics/advantage_var": 583.0128784179688, "regularization/advantage_var": 583.0128784179688, "regularization/kl": 0.9707164764404297, "rewards/chosen": 0.5157440608749037, "rewards/margins": 0.8582328772223433, "rewards/rejected": -0.3424888163474396, "step": 900 }, { "epoch": 1.1917098445595855, "grad_norm": 111.60948944091797, "kl": 15.490511894226074, "learning_rate": 6.991722767660556e-07, "logits/chosen": -34866865.56765163, "logits/rejected": -37142437.97802198, "logps/chosen": -467.9593701399689, "logps/rejected": -369.319613422292, "loss": 0.534, "loss/base_kto": 3.219972610473633, "loss/total_with_kl": 4.2719950675964355, "metrics/advantage_var": 631.8453369140625, "regularization/advantage_var": 631.8453369140625, "regularization/kl": 1.0520225763320923, "rewards/chosen": 0.5430646217073289, "rewards/margins": 0.815589756310449, "rewards/rejected": -0.2725251346031201, "step": 920 }, { "epoch": 1.2176165803108807, "grad_norm": 19.880008697509766, "kl": 19.41802406311035, "learning_rate": 6.860904374342499e-07, "logits/chosen": -36163899.5624037, "logits/rejected": -37782344.62123613, "logps/chosen": -513.7564522342064, "logps/rejected": -379.47432151347067, "loss": 0.532, "loss/base_kto": 3.1227097511291504, "loss/total_with_kl": 4.255713939666748, "metrics/advantage_var": 680.4828491210938, "regularization/advantage_var": 680.4828491210938, "regularization/kl": 1.1330039501190186, "rewards/chosen": 0.6724691758354199, "rewards/margins": 0.9759685941152461, "rewards/rejected": -0.3034994182798262, "step": 940 }, { "epoch": 1.2435233160621761, "grad_norm": 29.675657272338867, "kl": 19.0522518157959, "learning_rate": 6.7285900375424e-07, "logits/chosen": -35016233.555895865, "logits/rejected": -36021631.79585327, "logps/chosen": -505.2218606431853, "logps/rejected": -373.53162380382776, "loss": 0.5315, "loss/base_kto": 3.199005365371704, "loss/total_with_kl": 4.2517476081848145, "metrics/advantage_var": 632.2777709960938, "regularization/advantage_var": 632.2777709960938, "regularization/kl": 1.0527423620224, "rewards/chosen": 0.6828806761760624, "rewards/margins": 0.8645740498607402, "rewards/rejected": -0.18169337368467778, "step": 960 }, { "epoch": 1.2694300518134716, "grad_norm": 15.505671501159668, "kl": 17.601240158081055, "learning_rate": 6.594886122086123e-07, "logits/chosen": -35504928.0, "logits/rejected": -36978988.8, "logps/chosen": -460.32314453125, "logps/rejected": -373.08994140625, "loss": 0.5234, "loss/base_kto": 3.1828761100769043, "loss/total_with_kl": 4.187551975250244, "metrics/advantage_var": 603.4091186523438, "regularization/advantage_var": 603.4091186523438, "regularization/kl": 1.0046762228012085, "rewards/chosen": 0.6137538433074952, "rewards/margins": 0.8548269987106324, "rewards/rejected": -0.2410731554031372, "step": 980 }, { "epoch": 1.2953367875647668, "grad_norm": 20.777111053466797, "kl": 16.595083236694336, "learning_rate": 6.459900109853766e-07, "logits/chosen": -35664453.81818182, "logits/rejected": -37547754.409638554, "logps/chosen": -468.5825892857143, "logps/rejected": -367.95336031626505, "loss": 0.5056, "loss/base_kto": 3.174976110458374, "loss/total_with_kl": 4.044863700866699, "metrics/advantage_var": 522.4550170898438, "regularization/advantage_var": 522.4550170898438, "regularization/kl": 0.86988765001297, "rewards/chosen": 0.5709245855158026, "rewards/margins": 0.8491906190101315, "rewards/rejected": -0.2782660334943289, "step": 1000 }, { "epoch": 1.3212435233160622, "grad_norm": 21.215492248535156, "kl": 15.830972671508789, "learning_rate": 6.323740513377171e-07, "logits/chosen": -34005008.46302251, "logits/rejected": -36186712.705167174, "logps/chosen": -475.83832395498393, "logps/rejected": -367.00113981762917, "loss": 0.5125, "loss/base_kto": 3.14848256111145, "loss/total_with_kl": 4.100399971008301, "metrics/advantage_var": 571.72216796875, "regularization/advantage_var": 571.72216796875, "regularization/kl": 0.9519174695014954, "rewards/chosen": 0.5289721718938404, "rewards/margins": 0.866860292411745, "rewards/rejected": -0.3378881205179046, "step": 1020 }, { "epoch": 1.3471502590673574, "grad_norm": 28.345291137695312, "kl": 18.425092697143555, "learning_rate": 6.186516788608865e-07, "logits/chosen": -36062101.30015552, "logits/rejected": -36489894.379905805, "logps/chosen": -490.91417185069986, "logps/rejected": -398.04466738618527, "loss": 0.5135, "loss/base_kto": 3.1371660232543945, "loss/total_with_kl": 4.108198642730713, "metrics/advantage_var": 583.2025756835938, "regularization/advantage_var": 583.2025756835938, "regularization/kl": 0.9710323214530945, "rewards/chosen": 0.6313026629932932, "rewards/margins": 0.9115406483617194, "rewards/rejected": -0.2802379853684262, "step": 1040 }, { "epoch": 1.3730569948186528, "grad_norm": 12.197693824768066, "kl": 16.49967384338379, "learning_rate": 6.048339246932652e-07, "logits/chosen": -34652210.7244582, "logits/rejected": -36630841.337539434, "logps/chosen": -485.7217492260062, "logps/rejected": -377.4125098580442, "loss": 0.5388, "loss/base_kto": 3.1774182319641113, "loss/total_with_kl": 4.310180187225342, "metrics/advantage_var": 680.337158203125, "regularization/advantage_var": 680.337158203125, "regularization/kl": 1.1327613592147827, "rewards/chosen": 0.6493429178054857, "rewards/margins": 0.8816621208904374, "rewards/rejected": -0.2323192030849517, "step": 1060 }, { "epoch": 1.3989637305699483, "grad_norm": 12.110732078552246, "kl": 15.026209831237793, "learning_rate": 5.909318966486494e-07, "logits/chosen": -35231660.929797195, "logits/rejected": -35182101.633802816, "logps/chosen": -473.28207878315135, "logps/rejected": -377.75750684663535, "loss": 0.4983, "loss/base_kto": 3.1897284984588623, "loss/total_with_kl": 3.9866340160369873, "metrics/advantage_var": 478.62188720703125, "regularization/advantage_var": 478.62188720703125, "regularization/kl": 0.796905517578125, "rewards/chosen": 0.5617964293767063, "rewards/margins": 0.8690470955100078, "rewards/rejected": -0.30725066613330154, "step": 1080 }, { "epoch": 1.4248704663212435, "grad_norm": 15.362382888793945, "kl": 18.008405685424805, "learning_rate": 5.769567702869052e-07, "logits/chosen": -33535168.882082697, "logits/rejected": -35901397.537480064, "logps/chosen": -457.31537136294025, "logps/rejected": -375.2753688197767, "loss": 0.507, "loss/base_kto": 3.1821868419647217, "loss/total_with_kl": 4.056239604949951, "metrics/advantage_var": 524.9563598632812, "regularization/advantage_var": 524.9563598632812, "regularization/kl": 0.8740522265434265, "rewards/chosen": 0.6425474672186064, "rewards/margins": 0.8705690676285447, "rewards/rejected": -0.22802160040993819, "step": 1100 }, { "epoch": 1.450777202072539, "grad_norm": 13.028266906738281, "kl": 20.347318649291992, "learning_rate": 5.629197799301614e-07, "logits/chosen": -35240293.58730159, "logits/rejected": -36347024.935384616, "logps/chosen": -470.25982142857146, "logps/rejected": -367.3821394230769, "loss": 0.5179, "loss/base_kto": 3.178182601928711, "loss/total_with_kl": 4.143165111541748, "metrics/advantage_var": 579.5691528320312, "regularization/advantage_var": 579.5691528320312, "regularization/kl": 0.9649826288223267, "rewards/chosen": 0.6228441995287698, "rewards/margins": 0.8269883692628444, "rewards/rejected": -0.2041441697340745, "step": 1120 }, { "epoch": 1.4766839378238341, "grad_norm": 13.994969367980957, "kl": 12.647303581237793, "learning_rate": 5.488322096317633e-07, "logits/chosen": -34107188.14088821, "logits/rejected": -34854216.267942585, "logps/chosen": -469.74813361408883, "logps/rejected": -374.46511164274324, "loss": 0.5208, "loss/base_kto": 3.205705404281616, "loss/total_with_kl": 4.166758060455322, "metrics/advantage_var": 577.208740234375, "regularization/advantage_var": 577.208740234375, "regularization/kl": 0.9610525369644165, "rewards/chosen": 0.5281509469515697, "rewards/margins": 0.8834324056587068, "rewards/rejected": -0.3552814587071372, "step": 1140 }, { "epoch": 1.5025906735751295, "grad_norm": 21.57211685180664, "kl": 23.30204963684082, "learning_rate": 5.347053841052518e-07, "logits/chosen": -36361083.974842764, "logits/rejected": -36832236.91925466, "logps/chosen": -484.1163522012579, "logps/rejected": -376.38327348602485, "loss": 0.5196, "loss/base_kto": 3.1640663146972656, "loss/total_with_kl": 4.1570353507995605, "metrics/advantage_var": 596.3777465820312, "regularization/advantage_var": 596.3777465820312, "regularization/kl": 0.9929689764976501, "rewards/chosen": 0.7034506168005601, "rewards/margins": 0.8748559470083375, "rewards/rejected": -0.17140533020777732, "step": 1160 }, { "epoch": 1.528497409326425, "grad_norm": 14.304961204528809, "kl": 11.102791786193848, "learning_rate": 5.205506596206531e-07, "logits/chosen": -36154552.980645165, "logits/rejected": -37462292.16969697, "logps/chosen": -498.20650201612904, "logps/rejected": -384.77845643939395, "loss": 0.5262, "loss/base_kto": 3.193181037902832, "loss/total_with_kl": 4.209733486175537, "metrics/advantage_var": 610.5421142578125, "regularization/advantage_var": 610.5421142578125, "regularization/kl": 1.0165525674819946, "rewards/chosen": 0.3935492730909778, "rewards/margins": 0.8816277199127107, "rewards/rejected": -0.48807844682173296, "step": 1180 }, { "epoch": 1.5544041450777202, "grad_norm": 18.025127410888672, "kl": 14.871289253234863, "learning_rate": 5.063794148754032e-07, "logits/chosen": -34359508.73798449, "logits/rejected": -36444150.32440945, "logps/chosen": -480.859253875969, "logps/rejected": -355.9941929133858, "loss": 0.5052, "loss/base_kto": 3.2017250061035156, "loss/total_with_kl": 4.041450500488281, "metrics/advantage_var": 504.3397521972656, "regularization/advantage_var": 504.3397521972656, "regularization/kl": 0.8397256731987, "rewards/chosen": 0.5066620109617248, "rewards/margins": 0.8373793470607036, "rewards/rejected": -0.33071733609897885, "step": 1200 }, { "epoch": 1.5803108808290154, "grad_norm": 12.405550003051758, "kl": 19.35162925720215, "learning_rate": 4.922030418472422e-07, "logits/chosen": -34966591.209876545, "logits/rejected": -36841151.18987342, "logps/chosen": -498.6701388888889, "logps/rejected": -370.04966871044303, "loss": 0.5206, "loss/base_kto": 3.1798698902130127, "loss/total_with_kl": 4.164729118347168, "metrics/advantage_var": 591.5068969726562, "regularization/advantage_var": 591.5068969726562, "regularization/kl": 0.9848589301109314, "rewards/chosen": 0.6469076651114004, "rewards/margins": 0.8770311125779007, "rewards/rejected": -0.2301234474665002, "step": 1220 }, { "epoch": 1.6062176165803108, "grad_norm": 11.384881019592285, "kl": 15.16826057434082, "learning_rate": 4.780329366364336e-07, "logits/chosen": -33603148.523882896, "logits/rejected": -35084193.87638669, "logps/chosen": -460.21398305084745, "logps/rejected": -354.7185023771791, "loss": 0.507, "loss/base_kto": 3.2701079845428467, "loss/total_with_kl": 4.056195259094238, "metrics/advantage_var": 472.1244201660156, "regularization/advantage_var": 472.1244201660156, "regularization/kl": 0.786087155342102, "rewards/chosen": 0.5577042260779805, "rewards/margins": 0.7868489194435461, "rewards/rejected": -0.22914469336556556, "step": 1240 }, { "epoch": 1.6321243523316062, "grad_norm": 21.23419189453125, "kl": 10.736686706542969, "learning_rate": 4.638804903046684e-07, "logits/chosen": -35548129.684210524, "logits/rejected": -37481152.0, "logps/chosen": -502.76932565789474, "logps/rejected": -379.7354445684524, "loss": 0.544, "loss/base_kto": 3.194667100906372, "loss/total_with_kl": 4.352288246154785, "metrics/advantage_var": 695.2678833007812, "regularization/advantage_var": 695.2678833007812, "regularization/kl": 1.157620906829834, "rewards/chosen": 0.4874106959292763, "rewards/margins": 0.878785374768097, "rewards/rejected": -0.3913746788388207, "step": 1260 }, { "epoch": 1.6580310880829017, "grad_norm": 12.550798416137695, "kl": 15.807379722595215, "learning_rate": 4.497570797180217e-07, "logits/chosen": -34019533.1155624, "logits/rejected": -35015321.35657686, "logps/chosen": -489.33407164869027, "logps/rejected": -366.04796454041207, "loss": 0.5074, "loss/base_kto": 3.1485722064971924, "loss/total_with_kl": 4.059228420257568, "metrics/advantage_var": 546.9407348632812, "regularization/advantage_var": 546.9407348632812, "regularization/kl": 0.9106562733650208, "rewards/chosen": 0.5557488910222458, "rewards/margins": 0.8697980812956214, "rewards/rejected": -0.3140491902733756, "step": 1280 }, { "epoch": 1.6839378238341969, "grad_norm": 13.203516960144043, "kl": 12.83787727355957, "learning_rate": 4.3567405840131853e-07, "logits/chosen": -34067463.91344668, "logits/rejected": -34785558.24328594, "logps/chosen": -470.4313176197836, "logps/rejected": -361.5815560821485, "loss": 0.5118, "loss/base_kto": 3.2267396450042725, "loss/total_with_kl": 4.094069957733154, "metrics/advantage_var": 520.9192504882812, "regularization/advantage_var": 520.9192504882812, "regularization/kl": 0.8673305511474609, "rewards/chosen": 0.5369890970645769, "rewards/margins": 0.8186976321128342, "rewards/rejected": -0.2817085350482573, "step": 1300 }, { "epoch": 1.709844559585492, "grad_norm": 17.600608825683594, "kl": 21.662384033203125, "learning_rate": 4.2164274741126506e-07, "logits/chosen": -34932157.28257687, "logits/rejected": -34804480.42881072, "logps/chosen": -470.1298041727672, "logps/rejected": -360.8416561976549, "loss": 0.5017, "loss/base_kto": 3.214611530303955, "loss/total_with_kl": 4.013759136199951, "metrics/advantage_var": 479.9686584472656, "regularization/advantage_var": 479.9686584472656, "regularization/kl": 0.7991477251052856, "rewards/chosen": 0.6904898290229228, "rewards/margins": 0.8291842343789673, "rewards/rejected": -0.13869440535604455, "step": 1320 }, { "epoch": 1.7357512953367875, "grad_norm": 12.323432922363281, "kl": 21.623647689819336, "learning_rate": 4.0767442623567856e-07, "logits/chosen": -35091959.427701674, "logits/rejected": -37777434.298555374, "logps/chosen": -465.1884512937595, "logps/rejected": -363.67345505617976, "loss": 0.509, "loss/base_kto": 3.236440420150757, "loss/total_with_kl": 4.072193145751953, "metrics/advantage_var": 501.95361328125, "regularization/advantage_var": 501.95361328125, "regularization/kl": 0.8357527852058411, "rewards/chosen": 0.670259658604452, "rewards/margins": 0.8174756717214263, "rewards/rejected": -0.14721601311697433, "step": 1340 }, { "epoch": 1.761658031088083, "grad_norm": 17.97987937927246, "kl": 24.90900230407715, "learning_rate": 3.937803237261357e-07, "logits/chosen": -35157740.895522386, "logits/rejected": -36166132.249180324, "logps/chosen": -463.6839552238806, "logps/rejected": -394.0257172131148, "loss": 0.5186, "loss/base_kto": 3.186596393585205, "loss/total_with_kl": 4.149149417877197, "metrics/advantage_var": 578.1098022460938, "regularization/advantage_var": 578.1098022460938, "regularization/kl": 0.9625527262687683, "rewards/chosen": 0.752466457993237, "rewards/margins": 0.814937616380468, "rewards/rejected": -0.062471158387231045, "step": 1360 }, { "epoch": 1.7875647668393784, "grad_norm": 15.260648727416992, "kl": 23.88718032836914, "learning_rate": 3.7997160907132456e-07, "logits/chosen": -35047574.01273885, "logits/rejected": -35695019.19018405, "logps/chosen": -453.6136544585987, "logps/rejected": -390.8337327453988, "loss": 0.5149, "loss/base_kto": 3.2465310096740723, "loss/total_with_kl": 4.119444847106934, "metrics/advantage_var": 524.2727661132812, "regularization/advantage_var": 524.2727661132812, "regularization/kl": 0.8729141354560852, "rewards/chosen": 0.6648632462616939, "rewards/margins": 0.8069072200323035, "rewards/rejected": -0.14204397377060965, "step": 1380 }, { "epoch": 1.8134715025906736, "grad_norm": 40.36940002441406, "kl": 24.01422882080078, "learning_rate": 3.662593828183601e-07, "logits/chosen": -33963539.482496195, "logits/rejected": -34700914.23434992, "logps/chosen": -498.3009893455099, "logps/rejected": -375.87713182182983, "loss": 0.5174, "loss/base_kto": 3.266451358795166, "loss/total_with_kl": 4.139065742492676, "metrics/advantage_var": 524.0924072265625, "regularization/advantage_var": 524.0924072265625, "regularization/kl": 0.8726138472557068, "rewards/chosen": 0.629130231372479, "rewards/margins": 0.7368029089406973, "rewards/rejected": -0.1076726775682183, "step": 1400 }, { "epoch": 1.8393782383419688, "grad_norm": 13.748312950134277, "kl": 18.63722801208496, "learning_rate": 3.5265466794927725e-07, "logits/chosen": -33118719.17552335, "logits/rejected": -34697340.30955994, "logps/chosen": -478.16928341384863, "logps/rejected": -366.45023235963583, "loss": 0.4882, "loss/base_kto": 3.1683220863342285, "loss/total_with_kl": 3.9054718017578125, "metrics/advantage_var": 442.7323913574219, "regularization/advantage_var": 442.7323913574219, "regularization/kl": 0.7371494174003601, "rewards/chosen": 0.6168487106544384, "rewards/margins": 0.8708910625696271, "rewards/rejected": -0.25404235191518876, "step": 1420 }, { "epoch": 1.8652849740932642, "grad_norm": 45.252323150634766, "kl": 14.570375442504883, "learning_rate": 3.3916840101987887e-07, "logits/chosen": -34964176.768992245, "logits/rejected": -37252874.88503937, "logps/chosen": -503.100484496124, "logps/rejected": -387.47586122047244, "loss": 0.5094, "loss/base_kto": 3.2020890712738037, "loss/total_with_kl": 4.074826717376709, "metrics/advantage_var": 524.1666870117188, "regularization/advantage_var": 524.1666870117188, "regularization/kl": 0.8727375268936157, "rewards/chosen": 0.5829836823219476, "rewards/margins": 0.8559707957339187, "rewards/rejected": -0.27298711341197096, "step": 1440 }, { "epoch": 1.8911917098445596, "grad_norm": 12.377616882324219, "kl": 21.388090133666992, "learning_rate": 3.258114233680583e-07, "logits/chosen": -32697256.038897894, "logits/rejected": -35308440.518853694, "logps/chosen": -467.6978322528363, "logps/rejected": -364.8227281297134, "loss": 0.5111, "loss/base_kto": 3.175271987915039, "loss/total_with_kl": 4.088471412658691, "metrics/advantage_var": 548.4679565429688, "regularization/advantage_var": 548.4679565429688, "regularization/kl": 0.913199245929718, "rewards/chosen": 0.6209790988718091, "rewards/margins": 0.7975338521019775, "rewards/rejected": -0.17655475323016828, "step": 1460 }, { "epoch": 1.917098445595855, "grad_norm": 17.96770477294922, "kl": 14.73278522491455, "learning_rate": 3.1259447239866796e-07, "logits/chosen": -33914628.43464567, "logits/rejected": -35950239.553488374, "logps/chosen": -454.064468503937, "logps/rejected": -363.4874515503876, "loss": 0.4983, "loss/base_kto": 3.246981143951416, "loss/total_with_kl": 3.9862403869628906, "metrics/advantage_var": 443.9997253417969, "regularization/advantage_var": 443.9997253417969, "regularization/kl": 0.7392594218254089, "rewards/chosen": 0.5303596556656004, "rewards/margins": 0.7817680773816251, "rewards/rejected": -0.25140842171602473, "step": 1480 }, { "epoch": 1.9430051813471503, "grad_norm": 19.688234329223633, "kl": 12.038764953613281, "learning_rate": 2.9952817295193435e-07, "logits/chosen": -35928193.21135647, "logits/rejected": -36824149.59752322, "logps/chosen": -509.84966482649844, "logps/rejected": -374.5051518962848, "loss": 0.5101, "loss/base_kto": 3.1669700145721436, "loss/total_with_kl": 4.080795764923096, "metrics/advantage_var": 548.8440551757812, "regularization/advantage_var": 548.8440551757812, "regularization/kl": 0.9138253331184387, "rewards/chosen": 0.4717784123465842, "rewards/margins": 0.9075726403482289, "rewards/rejected": -0.43579422800164475, "step": 1500 }, { "epoch": 1.9689119170984455, "grad_norm": 19.103435516357422, "kl": 12.448521614074707, "learning_rate": 2.866230287623651e-07, "logits/chosen": -35412740.08945687, "logits/rejected": -36733310.04281346, "logps/chosen": -493.96350838658145, "logps/rejected": -365.94973241590213, "loss": 0.5006, "loss/base_kto": 3.174381971359253, "loss/total_with_kl": 4.0048346519470215, "metrics/advantage_var": 498.7702331542969, "regularization/advantage_var": 498.7702331542969, "regularization/kl": 0.8304523825645447, "rewards/chosen": 0.482789060939996, "rewards/margins": 0.8583273648868615, "rewards/rejected": -0.37553830394686544, "step": 1520 }, { "epoch": 1.994818652849741, "grad_norm": 22.631084442138672, "kl": 11.49282455444336, "learning_rate": 2.738894140150075e-07, "logits/chosen": -36232848.74959613, "logits/rejected": -35711846.63237519, "logps/chosen": -485.0666397415186, "logps/rejected": -403.40260968229956, "loss": 0.5091, "loss/base_kto": 3.2113113403320312, "loss/total_with_kl": 4.073005199432373, "metrics/advantage_var": 517.5335083007812, "regularization/advantage_var": 517.5335083007812, "regularization/kl": 0.8616933226585388, "rewards/chosen": 0.4211052569511056, "rewards/margins": 0.8206276314606082, "rewards/rejected": -0.3995223745095026, "step": 1540 }, { "epoch": 2.0207253886010363, "grad_norm": 17.78579330444336, "kl": 11.901885032653809, "learning_rate": 2.6133756500585156e-07, "logits/chosen": -35878359.0708899, "logits/rejected": -35850949.30731707, "logps/chosen": -473.42317119155354, "logps/rejected": -403.6257113821138, "loss": 0.4735, "loss/base_kto": 3.16479229927063, "loss/total_with_kl": 3.7882308959960938, "metrics/advantage_var": 374.4375, "regularization/advantage_var": 374.4375, "regularization/kl": 0.6234384775161743, "rewards/chosen": 0.508323703955741, "rewards/margins": 0.8497739191170723, "rewards/rejected": -0.3414502151613313, "step": 1560 }, { "epoch": 2.0466321243523318, "grad_norm": 14.26706600189209, "kl": 15.9487943649292, "learning_rate": 2.489775719130767e-07, "logits/chosen": -35277747.48489666, "logits/rejected": -35760879.877112135, "logps/chosen": -474.0095886327504, "logps/rejected": -370.0624039938556, "loss": 0.4641, "loss/base_kto": 3.1360933780670166, "loss/total_with_kl": 3.713010549545288, "metrics/advantage_var": 346.4967956542969, "regularization/advantage_var": 346.4967956542969, "regularization/kl": 0.5769171714782715, "rewards/chosen": 0.6138253265041236, "rewards/margins": 0.8728191052309622, "rewards/rejected": -0.25899377872683854, "step": 1580 }, { "epoch": 2.0725388601036268, "grad_norm": 10.086207389831543, "kl": 10.952147483825684, "learning_rate": 2.3681937068576303e-07, "logits/chosen": -33841814.78317152, "logits/rejected": -35682746.39274924, "logps/chosen": -481.45009101941747, "logps/rejected": -366.0855126510574, "loss": 0.4652, "loss/base_kto": 3.165739059448242, "loss/total_with_kl": 3.7217299938201904, "metrics/advantage_var": 333.9284362792969, "regularization/advantage_var": 333.9284362792969, "regularization/kl": 0.5559908747673035, "rewards/chosen": 0.5068062594021794, "rewards/margins": 0.8481513866577142, "rewards/rejected": -0.34134512725553484, "step": 1600 }, { "epoch": 2.098445595854922, "grad_norm": 12.579350471496582, "kl": 10.133259773254395, "learning_rate": 2.2487273505658e-07, "logits/chosen": -33835605.333333336, "logits/rejected": -35693168.3902439, "logps/chosen": -469.95678084935895, "logps/rejected": -368.1609422637195, "loss": 0.4622, "loss/base_kto": 3.183089256286621, "loss/total_with_kl": 3.697826385498047, "metrics/advantage_var": 309.1514587402344, "regularization/advantage_var": 309.1514587402344, "regularization/kl": 0.5147371888160706, "rewards/chosen": 0.4313532022329477, "rewards/margins": 0.8092067120893811, "rewards/rejected": -0.3778535098564334, "step": 1620 }, { "epoch": 2.1243523316062176, "grad_norm": 10.727472305297852, "kl": 8.098077774047852, "learning_rate": 2.131472686848817e-07, "logits/chosen": -33968363.32919255, "logits/rejected": -34895025.10691824, "logps/chosen": -510.988887810559, "logps/rejected": -376.80380306603774, "loss": 0.4683, "loss/base_kto": 3.1759884357452393, "loss/total_with_kl": 3.7461278438568115, "metrics/advantage_var": 342.4258728027344, "regularization/advantage_var": 342.4258728027344, "regularization/kl": 0.5701390504837036, "rewards/chosen": 0.4529393356038917, "rewards/margins": 0.8565572904578374, "rewards/rejected": -0.4036179548539456, "step": 1640 }, { "epoch": 2.150259067357513, "grad_norm": 14.88467025756836, "kl": 11.241103172302246, "learning_rate": 2.016523974365188e-07, "logits/chosen": -33316190.60030166, "logits/rejected": -34900694.92382496, "logps/chosen": -469.4418834841629, "logps/rejected": -370.39341065640195, "loss": 0.462, "loss/base_kto": 3.16162109375, "loss/total_with_kl": 3.6957168579101562, "metrics/advantage_var": 320.778076171875, "regularization/advantage_var": 320.778076171875, "regularization/kl": 0.5340954661369324, "rewards/chosen": 0.5090312698847568, "rewards/margins": 0.8521236828488218, "rewards/rejected": -0.343092412964065, "step": 1660 }, { "epoch": 2.1761658031088085, "grad_norm": 25.685148239135742, "kl": 16.984634399414062, "learning_rate": 1.9039736180657372e-07, "logits/chosen": -33908816.125195615, "logits/rejected": -35897938.271450855, "logps/chosen": -493.41867175273865, "logps/rejected": -405.77610666926677, "loss": 0.4663, "loss/base_kto": 3.0821049213409424, "loss/total_with_kl": 3.730085849761963, "metrics/advantage_var": 389.1777648925781, "regularization/advantage_var": 389.1777648925781, "regularization/kl": 0.6479809880256653, "rewards/chosen": 0.6098842471604803, "rewards/margins": 0.9221466052085132, "rewards/rejected": -0.3122623580480329, "step": 1680 }, { "epoch": 2.2020725388601035, "grad_norm": 17.96446990966797, "kl": 12.134880065917969, "learning_rate": 1.7939120949111498e-07, "logits/chosen": -34742538.71627907, "logits/rejected": -35549590.37480315, "logps/chosen": -456.46143410852716, "logps/rejected": -401.46407480314963, "loss": 0.4669, "loss/base_kto": 3.198035955429077, "loss/total_with_kl": 3.735119342803955, "metrics/advantage_var": 322.5724792480469, "regularization/advantage_var": 322.5724792480469, "regularization/kl": 0.5370831489562988, "rewards/chosen": 0.49280371850775195, "rewards/margins": 0.8004921056801806, "rewards/rejected": -0.30768838717242863, "step": 1700 }, { "epoch": 2.227979274611399, "grad_norm": 11.199472427368164, "kl": 20.47431755065918, "learning_rate": 1.6864278811393523e-07, "logits/chosen": -34062554.96855346, "logits/rejected": -35988575.403726704, "logps/chosen": -484.41627358490564, "logps/rejected": -367.4661296583851, "loss": 0.4608, "loss/base_kto": 3.1300294399261475, "loss/total_with_kl": 3.6866562366485596, "metrics/advantage_var": 334.3104553222656, "regularization/advantage_var": 334.3104553222656, "regularization/kl": 0.5566269755363464, "rewards/chosen": 0.6354286145864043, "rewards/margins": 0.8338579828839101, "rewards/rejected": -0.19842936829750582, "step": 1720 }, { "epoch": 2.2538860103626943, "grad_norm": 12.289140701293945, "kl": 15.893994331359863, "learning_rate": 1.5816073811412584e-07, "logits/chosen": -34612028.87559055, "logits/rejected": -34521475.37364341, "logps/chosen": -476.50994094488186, "logps/rejected": -376.8841569767442, "loss": 0.4724, "loss/base_kto": 3.167349338531494, "loss/total_with_kl": 3.779182195663452, "metrics/advantage_var": 367.46722412109375, "regularization/advantage_var": 367.46722412109375, "regularization/kl": 0.6118329167366028, "rewards/chosen": 0.582288558839813, "rewards/margins": 0.8153732803680955, "rewards/rejected": -0.23308472152828247, "step": 1740 }, { "epoch": 2.2797927461139897, "grad_norm": 11.060856819152832, "kl": 16.986530303955078, "learning_rate": 1.4795348580020207e-07, "logits/chosen": -35904337.32707355, "logits/rejected": -35920271.37597504, "logps/chosen": -478.1259780907668, "logps/rejected": -371.7021255850234, "loss": 0.4674, "loss/base_kto": 3.1196110248565674, "loss/total_with_kl": 3.7393479347229004, "metrics/advantage_var": 372.21417236328125, "regularization/advantage_var": 372.21417236328125, "regularization/kl": 0.6197366118431091, "rewards/chosen": 0.6117885153804773, "rewards/margins": 0.878207305677011, "rewards/rejected": -0.26641879029653376, "step": 1760 }, { "epoch": 2.305699481865285, "grad_norm": 8.367744445800781, "kl": 17.850460052490234, "learning_rate": 1.3802923657636355e-07, "logits/chosen": -34448627.95751138, "logits/rejected": -35959338.04830918, "logps/chosen": -467.9199070561457, "logps/rejected": -360.73362016908214, "loss": 0.4673, "loss/base_kto": 3.1459972858428955, "loss/total_with_kl": 3.738208055496216, "metrics/advantage_var": 355.6819763183594, "regularization/advantage_var": 355.6819763183594, "regularization/kl": 0.5922104716300964, "rewards/chosen": 0.6343060552802542, "rewards/margins": 0.84190760559288, "rewards/rejected": -0.2076015503126258, "step": 1780 }, { "epoch": 2.33160621761658, "grad_norm": 14.395444869995117, "kl": 19.718313217163086, "learning_rate": 1.28395968346336e-07, "logits/chosen": -35564519.26570048, "logits/rejected": -37770583.40515933, "logps/chosen": -494.33031400966183, "logps/rejected": -385.8139226100152, "loss": 0.4693, "loss/base_kto": 3.1141934394836426, "loss/total_with_kl": 3.7542195320129395, "metrics/advantage_var": 384.3999938964844, "regularization/advantage_var": 384.3999938964844, "regularization/kl": 0.6400260329246521, "rewards/chosen": 0.6306573004730274, "rewards/margins": 0.8881337366422943, "rewards/rejected": -0.2574764361692669, "step": 1800 }, { "epoch": 2.3575129533678756, "grad_norm": 14.89570426940918, "kl": 12.683195114135742, "learning_rate": 1.1906142510009415e-07, "logits/chosen": -34974333.58490566, "logits/rejected": -36554720.19875777, "logps/chosen": -474.2201257861635, "logps/rejected": -371.3301387810559, "loss": 0.4688, "loss/base_kto": 3.1612491607666016, "loss/total_with_kl": 3.75059175491333, "metrics/advantage_var": 353.959228515625, "regularization/advantage_var": 353.959228515625, "regularization/kl": 0.5893421173095703, "rewards/chosen": 0.5535465456404776, "rewards/margins": 0.8291624036355523, "rewards/rejected": -0.2756158579950747, "step": 1820 }, { "epoch": 2.383419689119171, "grad_norm": 14.248080253601074, "kl": 17.36821937561035, "learning_rate": 1.1003311068862547e-07, "logits/chosen": -36378132.09419152, "logits/rejected": -36636722.96111975, "logps/chosen": -488.78973704866564, "logps/rejected": -388.22278382581646, "loss": 0.4662, "loss/base_kto": 3.1176862716674805, "loss/total_with_kl": 3.72993540763855, "metrics/advantage_var": 367.71728515625, "regularization/advantage_var": 367.71728515625, "regularization/kl": 0.6122493147850037, "rewards/chosen": 0.6243669399099784, "rewards/margins": 0.8787355573222548, "rewards/rejected": -0.25436861741227645, "step": 1840 }, { "epoch": 2.4093264248704664, "grad_norm": 12.850327491760254, "kl": 16.18716049194336, "learning_rate": 1.0131828279173588e-07, "logits/chosen": -34844874.27160494, "logits/rejected": -36962329.92405064, "logps/chosen": -471.06240354938274, "logps/rejected": -345.65758504746833, "loss": 0.4568, "loss/base_kto": 3.0813486576080322, "loss/total_with_kl": 3.6540393829345703, "metrics/advantage_var": 343.9584045410156, "regularization/advantage_var": 343.9584045410156, "regularization/kl": 0.5726906657218933, "rewards/chosen": 0.6342455075110918, "rewards/margins": 0.9026876372981023, "rewards/rejected": -0.2684421297870105, "step": 1860 }, { "epoch": 2.4352331606217614, "grad_norm": 13.700258255004883, "kl": 19.546560287475586, "learning_rate": 9.292394708374596e-08, "logits/chosen": -34399608.28915662, "logits/rejected": -36052626.28571428, "logps/chosen": -482.5048004518072, "logps/rejected": -387.1890726461039, "loss": 0.4657, "loss/base_kto": 3.0845305919647217, "loss/total_with_kl": 3.7255101203918457, "metrics/advantage_var": 384.9725036621094, "regularization/advantage_var": 384.9725036621094, "regularization/kl": 0.6409791111946106, "rewards/chosen": 0.7085653557834855, "rewards/margins": 0.9112956336995391, "rewards/rejected": -0.20273027791605366, "step": 1880 }, { "epoch": 2.461139896373057, "grad_norm": 11.574172973632812, "kl": 20.4260196685791, "learning_rate": 8.48568516017727e-08, "logits/chosen": -34066046.00623053, "logits/rejected": -35182123.335423194, "logps/chosen": -476.22322819314644, "logps/rejected": -360.31279388714734, "loss": 0.4615, "loss/base_kto": 3.1729626655578613, "loss/total_with_kl": 3.691901922225952, "metrics/advantage_var": 311.6752624511719, "regularization/advantage_var": 311.6752624511719, "regularization/kl": 0.5189393162727356, "rewards/chosen": 0.6363134651540596, "rewards/margins": 0.8237576677326133, "rewards/rejected": -0.1874442025785536, "step": 1900 }, { "epoch": 2.4870466321243523, "grad_norm": 11.468894004821777, "kl": 21.85495376586914, "learning_rate": 7.71234813211169e-08, "logits/chosen": -34670083.30322581, "logits/rejected": -36920363.442424245, "logps/chosen": -492.3671875, "logps/rejected": -373.9569602272727, "loss": 0.4618, "loss/base_kto": 3.12001895904541, "loss/total_with_kl": 3.6945621967315674, "metrics/advantage_var": 345.07086181640625, "regularization/advantage_var": 345.07086181640625, "regularization/kl": 0.5745429396629333, "rewards/chosen": 0.6417865384009577, "rewards/margins": 0.8691116310634222, "rewards/rejected": -0.22732509266246448, "step": 1920 }, { "epoch": 2.5129533678756477, "grad_norm": 12.923368453979492, "kl": 14.473068237304688, "learning_rate": 6.973005294212353e-08, "logits/chosen": -34439092.497695856, "logits/rejected": -36047349.418124005, "logps/chosen": -490.56451612903226, "logps/rejected": -385.7020319952305, "loss": 0.4654, "loss/base_kto": 3.133051633834839, "loss/total_with_kl": 3.722947835922241, "metrics/advantage_var": 354.2922058105469, "regularization/advantage_var": 354.2922058105469, "regularization/kl": 0.5898965001106262, "rewards/chosen": 0.5951190251176075, "rewards/margins": 0.8729170940390105, "rewards/rejected": -0.277798068921403, "step": 1940 }, { "epoch": 2.538860103626943, "grad_norm": 11.678525924682617, "kl": 20.182462692260742, "learning_rate": 6.268250989270102e-08, "logits/chosen": -33799696.66976744, "logits/rejected": -37651702.72755905, "logps/chosen": -492.54588178294574, "logps/rejected": -350.1196850393701, "loss": 0.4618, "loss/base_kto": 3.1142308712005615, "loss/total_with_kl": 3.6947124004364014, "metrics/advantage_var": 348.6377258300781, "regularization/advantage_var": 348.6377258300781, "regularization/kl": 0.5804818272590637, "rewards/chosen": 0.6593832326489826, "rewards/margins": 0.8821094808419575, "rewards/rejected": -0.2227262481929749, "step": 1960 }, { "epoch": 2.5647668393782386, "grad_norm": 12.077221870422363, "kl": 19.26957130432129, "learning_rate": 5.598651755051975e-08, "logits/chosen": -33815259.19224556, "logits/rejected": -36040109.11951589, "logps/chosen": -460.8996365105008, "logps/rejected": -371.63178422844175, "loss": 0.4616, "loss/base_kto": 3.1193220615386963, "loss/total_with_kl": 3.6931564807891846, "metrics/advantage_var": 344.6451721191406, "regularization/advantage_var": 344.6451721191406, "regularization/kl": 0.573834240436554, "rewards/chosen": 0.6056515596602383, "rewards/margins": 0.8724790953616495, "rewards/rejected": -0.2668275357014112, "step": 1980 }, { "epoch": 2.5906735751295336, "grad_norm": 9.597661018371582, "kl": 17.388717651367188, "learning_rate": 4.964745868872933e-08, "logits/chosen": -34505823.327327326, "logits/rejected": -35755911.921824105, "logps/chosen": -474.30808933933935, "logps/rejected": -387.89418770358304, "loss": 0.4602, "loss/base_kto": 3.1345348358154297, "loss/total_with_kl": 3.681957721710205, "metrics/advantage_var": 328.78240966796875, "regularization/advantage_var": 328.78240966796875, "regularization/kl": 0.5474227070808411, "rewards/chosen": 0.6305892531936234, "rewards/margins": 0.8648866421215677, "rewards/rejected": -0.23429738892794433, "step": 2000 }, { "epoch": 2.616580310880829, "grad_norm": 12.27588176727295, "kl": 17.52077865600586, "learning_rate": 4.3670429148855493e-08, "logits/chosen": -35681623.56792144, "logits/rejected": -36064839.174887896, "logps/chosen": -487.5958981178396, "logps/rejected": -368.54386210762334, "loss": 0.4634, "loss/base_kto": 3.1000630855560303, "loss/total_with_kl": 3.7071731090545654, "metrics/advantage_var": 364.6307067871094, "regularization/advantage_var": 364.6307067871094, "regularization/kl": 0.6071101427078247, "rewards/chosen": 0.6279889245665405, "rewards/margins": 0.8973022179323384, "rewards/rejected": -0.26931329336579785, "step": 2020 }, { "epoch": 2.6424870466321244, "grad_norm": 13.711464881896973, "kl": 18.362272262573242, "learning_rate": 3.806023374435663e-08, "logits/chosen": -35292941.59635812, "logits/rejected": -35814583.03381643, "logps/chosen": -483.1783004552352, "logps/rejected": -385.9627616747182, "loss": 0.4625, "loss/base_kto": 3.1211092472076416, "loss/total_with_kl": 3.7002124786376953, "metrics/advantage_var": 347.8098449707031, "regularization/advantage_var": 347.8098449707031, "regularization/kl": 0.579103410243988, "rewards/chosen": 0.6044007273834645, "rewards/margins": 0.8793050614102987, "rewards/rejected": -0.27490433402683423, "step": 2040 }, { "epoch": 2.66839378238342, "grad_norm": 10.548821449279785, "kl": 19.548810958862305, "learning_rate": 3.282138239813037e-08, "logits/chosen": -34212640.14883721, "logits/rejected": -35593936.83149606, "logps/chosen": -481.8520833333333, "logps/rejected": -374.74562007874016, "loss": 0.4607, "loss/base_kto": 3.057556629180908, "loss/total_with_kl": 3.685317277908325, "metrics/advantage_var": 377.0332946777344, "regularization/advantage_var": 377.0332946777344, "regularization/kl": 0.6277604103088379, "rewards/chosen": 0.6706285580184108, "rewards/margins": 0.9070045259302587, "rewards/rejected": -0.23637596791184792, "step": 2060 }, { "epoch": 2.694300518134715, "grad_norm": 7.823846817016602, "kl": 19.428497314453125, "learning_rate": 2.795808651707793e-08, "logits/chosen": -36310088.33908948, "logits/rejected": -37475453.810264386, "logps/chosen": -484.6997154631083, "logps/rejected": -369.9785672628305, "loss": 0.4609, "loss/base_kto": 3.1207172870635986, "loss/total_with_kl": 3.687396287918091, "metrics/advantage_var": 340.3479919433594, "regularization/advantage_var": 340.3479919433594, "regularization/kl": 0.5666794776916504, "rewards/chosen": 0.6668884181527178, "rewards/margins": 0.8729672396938944, "rewards/rejected": -0.20607882154117663, "step": 2080 }, { "epoch": 2.7202072538860103, "grad_norm": 8.664216995239258, "kl": 17.887357711791992, "learning_rate": 2.3474255606639293e-08, "logits/chosen": -33848489.07165109, "logits/rejected": -36312625.75548589, "logps/chosen": -459.8963200934579, "logps/rejected": -375.4141849529781, "loss": 0.4623, "loss/base_kto": 3.104853630065918, "loss/total_with_kl": 3.6987578868865967, "metrics/advantage_var": 356.699462890625, "regularization/advantage_var": 356.699462890625, "regularization/kl": 0.5939046144485474, "rewards/chosen": 0.6817976811964563, "rewards/margins": 0.8805802402953249, "rewards/rejected": -0.19878255909886852, "step": 2100 }, { "epoch": 2.7461139896373057, "grad_norm": 12.456644058227539, "kl": 19.308988571166992, "learning_rate": 1.9373494128020195e-08, "logits/chosen": -34445936.918429, "logits/rejected": -35370073.475728154, "logps/chosen": -476.6726774924471, "logps/rejected": -351.433277710356, "loss": 0.4636, "loss/base_kto": 3.1250016689300537, "loss/total_with_kl": 3.708803653717041, "metrics/advantage_var": 350.63189697265625, "regularization/advantage_var": 350.63189697265625, "regularization/kl": 0.5838021636009216, "rewards/chosen": 0.6458656795075293, "rewards/margins": 0.8593024852079363, "rewards/rejected": -0.21343680570040707, "step": 2120 }, { "epoch": 2.772020725388601, "grad_norm": 13.652802467346191, "kl": 19.432348251342773, "learning_rate": 1.5659098600638798e-08, "logits/chosen": -34813539.1232, "logits/rejected": -35671157.2519084, "logps/chosen": -477.397, "logps/rejected": -384.62273377862596, "loss": 0.4592, "loss/base_kto": 3.1075828075408936, "loss/total_with_kl": 3.673489809036255, "metrics/advantage_var": 339.88409423828125, "regularization/advantage_var": 339.88409423828125, "regularization/kl": 0.5659070014953613, "rewards/chosen": 0.66147333984375, "rewards/margins": 0.8755134292253102, "rewards/rejected": -0.2140400893815601, "step": 2140 }, { "epoch": 2.7979274611398965, "grad_norm": 12.318232536315918, "kl": 20.481626510620117, "learning_rate": 1.2334054952120143e-08, "logits/chosen": -36536352.768, "logits/rejected": -36707848.59847328, "logps/chosen": -503.8237, "logps/rejected": -384.4250238549618, "loss": 0.4766, "loss/base_kto": 3.09419846534729, "loss/total_with_kl": 3.8124756813049316, "metrics/advantage_var": 431.39794921875, "regularization/advantage_var": 431.39794921875, "regularization/kl": 0.7182775735855103, "rewards/chosen": 0.67184501953125, "rewards/margins": 0.9049889963397543, "rewards/rejected": -0.2331439768085043, "step": 2160 }, { "epoch": 2.823834196891192, "grad_norm": 14.077258110046387, "kl": 20.919355392456055, "learning_rate": 9.401036117969108e-09, "logits/chosen": -35301458.71405493, "logits/rejected": -36259285.397882, "logps/chosen": -479.1207592891761, "logps/rejected": -378.84084247352496, "loss": 0.4554, "loss/base_kto": 3.1235177516937256, "loss/total_with_kl": 3.64288330078125, "metrics/advantage_var": 311.9313659667969, "regularization/advantage_var": 311.9313659667969, "regularization/kl": 0.5193657279014587, "rewards/chosen": 0.6403917056870961, "rewards/margins": 0.8508738658773264, "rewards/rejected": -0.21048216019023025, "step": 2180 }, { "epoch": 2.849740932642487, "grad_norm": 10.713716506958008, "kl": 17.51523780822754, "learning_rate": 6.8623998928517555e-09, "logits/chosen": -35305854.627871364, "logits/rejected": -35604692.31259968, "logps/chosen": -488.16653905053596, "logps/rejected": -394.6655452551834, "loss": 0.4643, "loss/base_kto": 3.073132038116455, "loss/total_with_kl": 3.714580535888672, "metrics/advantage_var": 385.254638671875, "regularization/advantage_var": 385.254638671875, "regularization/kl": 0.641448974609375, "rewards/chosen": 0.6772497390349828, "rewards/margins": 0.9285689550437547, "rewards/rejected": -0.25131921600877194, "step": 2200 }, { "epoch": 2.8756476683937824, "grad_norm": 9.375003814697266, "kl": 19.961950302124023, "learning_rate": 4.72018703521132e-09, "logits/chosen": -34793932.55502392, "logits/rejected": -34593166.3093415, "logps/chosen": -466.6762360446571, "logps/rejected": -366.905915007657, "loss": 0.4479, "loss/base_kto": 3.0823726654052734, "loss/total_with_kl": 3.5829293727874756, "metrics/advantage_var": 300.634521484375, "regularization/advantage_var": 300.634521484375, "regularization/kl": 0.500556468963623, "rewards/chosen": 0.6577253569826556, "rewards/margins": 0.8845860226004771, "rewards/rejected": -0.2268606656178216, "step": 2220 }, { "epoch": 2.901554404145078, "grad_norm": 8.692347526550293, "kl": 16.773324966430664, "learning_rate": 2.976119626744267e-09, "logits/chosen": -36092797.67272727, "logits/rejected": -35963418.425806455, "logps/chosen": -482.5468276515152, "logps/rejected": -370.9232610887097, "loss": 0.4619, "loss/base_kto": 3.1150739192962646, "loss/total_with_kl": 3.6949822902679443, "metrics/advantage_var": 348.2931823730469, "regularization/advantage_var": 348.2931823730469, "regularization/kl": 0.5799080729484558, "rewards/chosen": 0.6352346709280303, "rewards/margins": 0.8743967960540384, "rewards/rejected": -0.23916212512600807, "step": 2240 }, { "epoch": 2.927461139896373, "grad_norm": 16.19487762451172, "kl": 16.275922775268555, "learning_rate": 1.631599688052765e-09, "logits/chosen": -34692276.327044025, "logits/rejected": -34984737.39130435, "logps/chosen": -477.4905169025157, "logps/rejected": -376.2428425854037, "loss": 0.4632, "loss/base_kto": 3.1360573768615723, "loss/total_with_kl": 3.705369710922241, "metrics/advantage_var": 341.9292297363281, "regularization/advantage_var": 341.9292297363281, "regularization/kl": 0.5693122148513794, "rewards/chosen": 0.6395522303551248, "rewards/margins": 0.8553214116955024, "rewards/rejected": -0.21576918134037754, "step": 2260 }, { "epoch": 2.9533678756476682, "grad_norm": 13.71744441986084, "kl": 17.854293823242188, "learning_rate": 6.877080515894085e-10, "logits/chosen": -34928216.27586207, "logits/rejected": -35329256.872274145, "logps/chosen": -484.6248040752351, "logps/rejected": -353.38101148753896, "loss": 0.4618, "loss/base_kto": 3.1671254634857178, "loss/total_with_kl": 3.6941890716552734, "metrics/advantage_var": 316.5549011230469, "regularization/advantage_var": 316.5549011230469, "regularization/kl": 0.5270638465881348, "rewards/chosen": 0.6265961937022433, "rewards/margins": 0.8139161542518563, "rewards/rejected": -0.18731996054961303, "step": 2280 }, { "epoch": 2.9792746113989637, "grad_norm": 10.524263381958008, "kl": 17.332983016967773, "learning_rate": 1.4520349279739663e-10, "logits/chosen": -34855024.88188976, "logits/rejected": -36169356.50232558, "logps/chosen": -484.57942913385824, "logps/rejected": -380.99890988372096, "loss": 0.4618, "loss/base_kto": 3.0859153270721436, "loss/total_with_kl": 3.6943321228027344, "metrics/advantage_var": 365.4156188964844, "regularization/advantage_var": 365.4156188964844, "regularization/kl": 0.6084169745445251, "rewards/chosen": 0.6552281657541831, "rewards/margins": 0.9204792426225212, "rewards/rejected": -0.26525107686833815, "step": 2300 }, { "epoch": 3.0, "step": 2316, "total_flos": 9.977114557203087e+17, "train_loss": 0.5011991713331153, "train_runtime": 10985.9282, "train_samples_per_second": 13.492, "train_steps_per_second": 0.211 } ], "logging_steps": 20, "max_steps": 2316, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 9.977114557203087e+17, "train_batch_size": 8, "trial_name": null, "trial_params": null }