{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 3.0, "eval_steps": 500, "global_step": 2316, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.025906735751295335, "grad_norm": 21.30620574951172, "kl": 16.385543823242188, "learning_rate": 1.8999999999999998e-07, "logits/chosen": -37237433.32515337, "logits/rejected": -38183799.031847134, "logps/chosen": -458.3118769171779, "logps/rejected": -380.0686703821656, "loss": 0.5989, "loss/base_kto": 3.887625217437744, "metrics/advantage_var": 191.3936004638672, "regularization/lipschitz_norm": 919.4276733398438, "regularization/mmd": 0.13125856220722198, "regularization/rkhs_norm": 156.2601776123047, "regularization/w1": 0.7723193168640137, "rewards/chosen": 0.2465540412013516, "rewards/margins": 0.08952261140033399, "rewards/rejected": 0.1570314298010176, "step": 20 }, { "epoch": 0.05181347150259067, "grad_norm": 23.404104232788086, "kl": 9.088505744934082, "learning_rate": 3.8999999999999997e-07, "logits/chosen": -36791531.810725555, "logits/rejected": -37880429.374613, "logps/chosen": -479.285636829653, "logps/rejected": -363.20839783281735, "loss": 0.604, "loss/base_kto": 3.9080753326416016, "metrics/advantage_var": 237.0071258544922, "regularization/lipschitz_norm": 941.30517578125, "regularization/mmd": 0.1333685964345932, "regularization/rkhs_norm": 158.7721405029297, "regularization/w1": 0.790696382522583, "rewards/chosen": 0.1017112852271047, "rewards/margins": 0.09241618113830667, "rewards/rejected": 0.009295104088798026, "step": 40 }, { "epoch": 0.07772020725388601, "grad_norm": 21.467105865478516, "kl": 11.46056079864502, "learning_rate": 5.9e-07, "logits/chosen": -35823341.92941176, "logits/rejected": -37332244.48, "logps/chosen": -463.5670955882353, "logps/rejected": -382.3347395833333, "loss": 0.6039, "loss/base_kto": 3.8919856548309326, "metrics/advantage_var": 192.63877868652344, "regularization/lipschitz_norm": 956.9761962890625, "regularization/mmd": 0.1353202611207962, "regularization/rkhs_norm": 161.09556579589844, "regularization/w1": 0.8038601279258728, "rewards/chosen": 0.18667184044333066, "rewards/margins": 0.10561023562562233, "rewards/rejected": 0.08106160481770833, "step": 60 }, { "epoch": 0.10362694300518134, "grad_norm": 22.6914005279541, "kl": 8.611804008483887, "learning_rate": 7.9e-07, "logits/chosen": -36030081.83732057, "logits/rejected": -37339907.52833078, "logps/chosen": -487.0245215311005, "logps/rejected": -385.3145817381317, "loss": 0.5973, "loss/base_kto": 3.8769938945770264, "metrics/advantage_var": 182.3870849609375, "regularization/lipschitz_norm": 915.8214721679688, "regularization/mmd": 0.13175193965435028, "regularization/rkhs_norm": 156.84756469726562, "regularization/w1": 0.7692900896072388, "rewards/chosen": 0.08962506502629088, "rewards/margins": 0.10581642927372834, "rewards/rejected": -0.01619136424743746, "step": 80 }, { "epoch": 0.12953367875647667, "grad_norm": 23.67812728881836, "kl": 16.48870086669922, "learning_rate": 9.9e-07, "logits/chosen": -37817934.88821752, "logits/rejected": -37920091.96116505, "logps/chosen": -480.29966012084594, "logps/rejected": -369.67157160194176, "loss": 0.5939, "loss/base_kto": 3.8202621936798096, "metrics/advantage_var": 204.16806030273438, "regularization/lipschitz_norm": 944.6422729492188, "regularization/mmd": 0.1374884694814682, "regularization/rkhs_norm": 163.6767578125, "regularization/w1": 0.7934995889663696, "rewards/chosen": 0.3411060805767324, "rewards/margins": 0.14616479136016308, "rewards/rejected": 0.19494128921656934, "step": 100 }, { "epoch": 0.15544041450777202, "grad_norm": 21.86313819885254, "kl": 13.873870849609375, "learning_rate": 9.998186234298189e-07, "logits/chosen": -36919010.6749226, "logits/rejected": -36583236.643533126, "logps/chosen": -469.7824109907121, "logps/rejected": -343.7594390772871, "loss": 0.5917, "loss/base_kto": 3.8518707752227783, "metrics/advantage_var": 185.32675170898438, "regularization/lipschitz_norm": 897.4934692382812, "regularization/mmd": 0.12806062400341034, "regularization/rkhs_norm": 152.45315551757812, "regularization/w1": 0.7538945078849792, "rewards/chosen": 0.2243381417572683, "rewards/margins": 0.15287296539454773, "rewards/rejected": 0.07146517636272057, "step": 120 }, { "epoch": 0.18134715025906736, "grad_norm": 24.354427337646484, "kl": 5.70906400680542, "learning_rate": 9.992359552107714e-07, "logits/chosen": -36384878.3518225, "logits/rejected": -37398903.5192604, "logps/chosen": -474.1553090332805, "logps/rejected": -379.6500385208012, "loss": 0.5914, "loss/base_kto": 3.8447353839874268, "metrics/advantage_var": 179.997314453125, "regularization/lipschitz_norm": 901.4913940429688, "regularization/mmd": 0.1292407363653183, "regularization/rkhs_norm": 153.85801696777344, "regularization/w1": 0.7572528123855591, "rewards/chosen": 0.08524320091574014, "rewards/margins": 0.14999670540582105, "rewards/rejected": -0.0647535044900809, "step": 140 }, { "epoch": 0.20725388601036268, "grad_norm": 24.62518310546875, "kl": 7.673250675201416, "learning_rate": 9.982519612796161e-07, "logits/chosen": -36190016.81012658, "logits/rejected": -37570857.086419754, "logps/chosen": -490.16307357594934, "logps/rejected": -368.49573206018516, "loss": 0.5996, "loss/base_kto": 3.8244552612304688, "metrics/advantage_var": 207.548583984375, "regularization/lipschitz_norm": 994.6609497070312, "regularization/mmd": 0.1368967741727829, "regularization/rkhs_norm": 162.97235107421875, "regularization/w1": 0.8355152010917664, "rewards/chosen": 0.1482733955866174, "rewards/margins": 0.17031480321885648, "rewards/rejected": -0.022041407632239073, "step": 160 }, { "epoch": 0.23316062176165803, "grad_norm": 18.731962203979492, "kl": 7.896986484527588, "learning_rate": 9.968674326492213e-07, "logits/chosen": -36152657.01265823, "logits/rejected": -38387958.518518515, "logps/chosen": -520.5198773734177, "logps/rejected": -395.7608748070988, "loss": 0.6026, "loss/base_kto": 3.7690110206604004, "metrics/advantage_var": 246.31382751464844, "regularization/lipschitz_norm": 1073.66796875, "regularization/mmd": 0.14970853924751282, "regularization/rkhs_norm": 178.22445678710938, "regularization/w1": 0.901881217956543, "rewards/chosen": 0.08006769494165349, "rewards/margins": 0.2120290465011394, "rewards/rejected": -0.1319613515594859, "step": 180 }, { "epoch": 0.25906735751295334, "grad_norm": 21.32183265686035, "kl": 9.421449661254883, "learning_rate": 9.950834823142198e-07, "logits/chosen": -36843970.03821656, "logits/rejected": -37945268.61349693, "logps/chosen": -519.3416600318471, "logps/rejected": -361.66645897239266, "loss": 0.5974, "loss/base_kto": 3.828326463699341, "metrics/advantage_var": 211.9000701904297, "regularization/lipschitz_norm": 965.6625366210938, "regularization/mmd": 0.13956843316555023, "regularization/rkhs_norm": 166.1528778076172, "regularization/w1": 0.8111564517021179, "rewards/chosen": 0.10966109014620447, "rewards/margins": 0.16354339063426104, "rewards/rejected": -0.05388230048805658, "step": 200 }, { "epoch": 0.2849740932642487, "grad_norm": 22.96819496154785, "kl": 3.105022430419922, "learning_rate": 9.929015443562942e-07, "logits/chosen": -37329087.18987342, "logits/rejected": -37404877.43209877, "logps/chosen": -473.15644778481015, "logps/rejected": -414.8044945987654, "loss": 0.5992, "loss/base_kto": 3.8584632873535156, "metrics/advantage_var": 207.459716796875, "regularization/lipschitz_norm": 950.4669189453125, "regularization/mmd": 0.1365087777376175, "regularization/rkhs_norm": 162.5104522705078, "regularization/w1": 0.7983922362327576, "rewards/chosen": -0.08360711834098719, "rewards/margins": 0.13516261447573252, "rewards/rejected": -0.21876973281671971, "step": 220 }, { "epoch": 0.31088082901554404, "grad_norm": 28.522031784057617, "kl": 11.790318489074707, "learning_rate": 9.90323372791347e-07, "logits/chosen": -35376852.45151033, "logits/rejected": -34778815.115207374, "logps/chosen": -492.73400238473766, "logps/rejected": -388.49961597542244, "loss": 0.5992, "loss/base_kto": 3.7751381397247314, "metrics/advantage_var": 243.459228515625, "regularization/lipschitz_norm": 1035.931396484375, "regularization/mmd": 0.14857906103134155, "regularization/rkhs_norm": 176.8798370361328, "regularization/w1": 0.870182454586029, "rewards/chosen": 0.1879951464921378, "rewards/margins": 0.21998687950992596, "rewards/rejected": -0.03199173301778814, "step": 240 }, { "epoch": 0.33678756476683935, "grad_norm": 26.046768188476562, "kl": 16.253116607666016, "learning_rate": 9.87351040159484e-07, "logits/chosen": -36735619.83233533, "logits/rejected": -37659105.88235294, "logps/chosen": -476.267870508982, "logps/rejected": -351.4927236519608, "loss": 0.6088, "loss/base_kto": 3.823444366455078, "metrics/advantage_var": 260.6925964355469, "regularization/lipschitz_norm": 1064.2943115234375, "regularization/mmd": 0.15288196504116058, "regularization/rkhs_norm": 182.00230407714844, "regularization/w1": 0.8940073251724243, "rewards/chosen": 0.2839615873234001, "rewards/margins": 0.1419578485211572, "rewards/rejected": 0.1420037388022429, "step": 260 }, { "epoch": 0.3626943005181347, "grad_norm": 22.29317855834961, "kl": 18.77423667907715, "learning_rate": 9.839869358589396e-07, "logits/chosen": -37005181.67272727, "logits/rejected": -37470330.21935484, "logps/chosen": -485.25710227272725, "logps/rejected": -359.2376008064516, "loss": 0.5959, "loss/base_kto": 3.746065616607666, "metrics/advantage_var": 290.56085205078125, "regularization/lipschitz_norm": 1033.027587890625, "regularization/mmd": 0.15310192108154297, "regularization/rkhs_norm": 182.26419067382812, "regularization/w1": 0.8677431344985962, "rewards/chosen": 0.3154003952488755, "rewards/margins": 0.2193312069770761, "rewards/rejected": 0.09606918827179939, "step": 280 }, { "epoch": 0.38860103626943004, "grad_norm": 17.760087966918945, "kl": 5.830892086029053, "learning_rate": 9.80233764225289e-07, "logits/chosen": -36442131.91572123, "logits/rejected": -36848749.65912519, "logps/chosen": -496.06624797406806, "logps/rejected": -370.4114347662142, "loss": 0.5979, "loss/base_kto": 3.778766632080078, "metrics/advantage_var": 234.72276306152344, "regularization/lipschitz_norm": 1019.4176025390625, "regularization/mmd": 0.14803357422351837, "regularization/rkhs_norm": 176.23045349121094, "regularization/w1": 0.8563107848167419, "rewards/chosen": 0.028018224954218688, "rewards/margins": 0.2139656834429477, "rewards/rejected": -0.185947458488729, "step": 300 }, { "epoch": 0.41450777202072536, "grad_norm": 27.020835876464844, "kl": 12.562323570251465, "learning_rate": 9.760945423574868e-07, "logits/chosen": -36587044.864, "logits/rejected": -37103853.63053435, "logps/chosen": -466.8468, "logps/rejected": -394.52194656488547, "loss": 0.605, "loss/base_kto": 3.8280041217803955, "metrics/advantage_var": 261.6440124511719, "regularization/lipschitz_norm": 1030.7325439453125, "regularization/mmd": 0.14636702835559845, "regularization/rkhs_norm": 174.2464599609375, "regularization/w1": 0.8658153414726257, "rewards/chosen": 0.2149145263671875, "rewards/margins": 0.15796180671517174, "rewards/rejected": 0.056952719652015746, "step": 320 }, { "epoch": 0.44041450777202074, "grad_norm": 19.868404388427734, "kl": 14.786349296569824, "learning_rate": 9.71572597692482e-07, "logits/chosen": -37221587.699846864, "logits/rejected": -37740988.22328549, "logps/chosen": -465.51124617151606, "logps/rejected": -361.75343899521533, "loss": 0.5978, "loss/base_kto": 3.7788236141204834, "metrics/advantage_var": 236.5540008544922, "regularization/lipschitz_norm": 1020.3147583007812, "regularization/mmd": 0.1464555859565735, "regularization/rkhs_norm": 174.35189819335938, "regularization/w1": 0.857064425945282, "rewards/chosen": 0.2893020139171372, "rewards/margins": 0.19903800230897084, "rewards/rejected": 0.09026401160816637, "step": 340 }, { "epoch": 0.46632124352331605, "grad_norm": 29.049692153930664, "kl": 4.6120924949646, "learning_rate": 9.666715653303588e-07, "logits/chosen": -36731300.72665535, "logits/rejected": -37294133.348769896, "logps/chosen": -508.23418930390494, "logps/rejected": -369.49638205499275, "loss": 0.5931, "loss/base_kto": 3.7556424140930176, "metrics/advantage_var": 233.97671508789062, "regularization/lipschitz_norm": 1001.9880981445312, "regularization/mmd": 0.14747083187103271, "regularization/rkhs_norm": 175.56053161621094, "regularization/w1": 0.841670036315918, "rewards/chosen": -0.03298977073060636, "rewards/margins": 0.21002983012243723, "rewards/rejected": -0.24301960085304358, "step": 360 }, { "epoch": 0.49222797927461137, "grad_norm": 16.215953826904297, "kl": 5.461564540863037, "learning_rate": 9.613953851121528e-07, "logits/chosen": -36202803.869281046, "logits/rejected": -36440959.233532935, "logps/chosen": -453.9362745098039, "logps/rejected": -358.4580838323353, "loss": 0.597, "loss/base_kto": 3.7826297283172607, "metrics/advantage_var": 239.4388427734375, "regularization/lipschitz_norm": 1004.7897338867188, "regularization/mmd": 0.14958620071411133, "regularization/rkhs_norm": 178.0788116455078, "regularization/w1": 0.8440233469009399, "rewards/chosen": -0.11870555005042381, "rewards/margins": 0.1800786945097722, "rewards/rejected": -0.29878424456019603, "step": 380 }, { "epoch": 0.5181347150259067, "grad_norm": 24.50790786743164, "kl": 1.323299527168274, "learning_rate": 9.557482984526924e-07, "logits/chosen": -35323897.456869006, "logits/rejected": -36453892.69724771, "logps/chosen": -481.3480431309904, "logps/rejected": -379.06751720183485, "loss": 0.5986, "loss/base_kto": 3.7839763164520264, "metrics/advantage_var": 236.5087890625, "regularization/lipschitz_norm": 1018.1357421875, "regularization/mmd": 0.14926938712596893, "regularization/rkhs_norm": 177.70164489746094, "regularization/w1": 0.8552339673042297, "rewards/chosen": -0.2617264281446561, "rewards/margins": 0.21538011534671914, "rewards/rejected": -0.4771065434913752, "step": 400 }, { "epoch": 0.5440414507772021, "grad_norm": 20.045339584350586, "kl": 4.559103488922119, "learning_rate": 9.497348449310107e-07, "logits/chosen": -36375742.54765507, "logits/rejected": -36945344.310177706, "logps/chosen": -485.43792549167927, "logps/rejected": -375.23434975767367, "loss": 0.591, "loss/base_kto": 3.755572557449341, "metrics/advantage_var": 222.8901824951172, "regularization/lipschitz_norm": 987.20556640625, "regularization/mmd": 0.143472820520401, "regularization/rkhs_norm": 170.8009796142578, "regularization/w1": 0.8292526602745056, "rewards/chosen": 0.08200028227605545, "rewards/margins": 0.25845988282018534, "rewards/rejected": -0.1764596005441299, "step": 420 }, { "epoch": 0.5699481865284974, "grad_norm": 18.75247573852539, "kl": 4.5143818855285645, "learning_rate": 9.433598586410701e-07, "logits/chosen": -35564661.12418301, "logits/rejected": -36188647.473053895, "logps/chosen": -487.8157169117647, "logps/rejected": -380.7473100673653, "loss": 0.6056, "loss/base_kto": 3.810009002685547, "metrics/advantage_var": 253.3105926513672, "regularization/lipschitz_norm": 1051.658447265625, "regularization/mmd": 0.15111172199249268, "regularization/rkhs_norm": 179.89492797851562, "regularization/w1": 0.8833931088447571, "rewards/chosen": -0.10365574655969159, "rewards/margins": 0.147400673315402, "rewards/rejected": -0.2510564198750936, "step": 440 }, { "epoch": 0.5958549222797928, "grad_norm": 32.510154724121094, "kl": 7.408751010894775, "learning_rate": 9.366284643057313e-07, "logits/chosen": -35925008.15141956, "logits/rejected": -37290427.839009285, "logps/chosen": -495.5834976340694, "logps/rejected": -358.63201431888547, "loss": 0.5926, "loss/base_kto": 3.683509111404419, "metrics/advantage_var": 265.0987854003906, "regularization/lipschitz_norm": 1074.9356689453125, "regularization/mmd": 0.15402238070964813, "regularization/rkhs_norm": 183.35997009277344, "regularization/w1": 0.9029459357261658, "rewards/chosen": 0.139697872125764, "rewards/margins": 0.2905832503698865, "rewards/rejected": -0.1508853782441225, "step": 460 }, { "epoch": 0.6217616580310881, "grad_norm": 20.378629684448242, "kl": 4.379853248596191, "learning_rate": 9.295460731570917e-07, "logits/chosen": -36566377.50544324, "logits/rejected": -36899237.17425432, "logps/chosen": -467.98833592534993, "logps/rejected": -379.43597919937207, "loss": 0.6013, "loss/base_kto": 3.7656960487365723, "metrics/advantage_var": 267.1982421875, "regularization/lipschitz_norm": 1059.279296875, "regularization/mmd": 0.15457189083099365, "regularization/rkhs_norm": 184.01417541503906, "regularization/w1": 0.8897945284843445, "rewards/chosen": -8.799796156341885e-05, "rewards/margins": 0.22673953067609356, "rewards/rejected": -0.22682752863765698, "step": 480 }, { "epoch": 0.6476683937823834, "grad_norm": 21.59541130065918, "kl": 8.324943542480469, "learning_rate": 9.221183785865056e-07, "logits/chosen": -36415978.73482428, "logits/rejected": -38221222.75229358, "logps/chosen": -507.32438099041536, "logps/rejected": -362.50936544342505, "loss": 0.5947, "loss/base_kto": 3.7547378540039062, "metrics/advantage_var": 231.9820098876953, "regularization/lipschitz_norm": 1020.3601684570312, "regularization/mmd": 0.14552414417266846, "regularization/rkhs_norm": 173.24302673339844, "regularization/w1": 0.8571025729179382, "rewards/chosen": 0.04323362161557134, "rewards/margins": 0.19958254709817838, "rewards/rejected": -0.15634892548260704, "step": 500 }, { "epoch": 0.6735751295336787, "grad_norm": 20.982234954833984, "kl": 5.086883068084717, "learning_rate": 9.143513515677817e-07, "logits/chosen": -34935692.41335453, "logits/rejected": -35675260.26420891, "logps/chosen": -494.2798589030207, "logps/rejected": -365.8594950076805, "loss": 0.6067, "loss/base_kto": 3.7751259803771973, "metrics/advantage_var": 269.0093688964844, "regularization/lipschitz_norm": 1094.8719482421875, "regularization/mmd": 0.15867407619953156, "regularization/rkhs_norm": 188.897705078125, "regularization/w1": 0.9196924567222595, "rewards/chosen": -0.06926192388246473, "rewards/margins": 0.22349587511667124, "rewards/rejected": -0.29275779899913595, "step": 520 }, { "epoch": 0.6994818652849741, "grad_norm": 20.61083984375, "kl": 5.628817558288574, "learning_rate": 9.062512358572373e-07, "logits/chosen": -37833826.99207607, "logits/rejected": -38360889.19568567, "logps/chosen": -504.9411152931854, "logps/rejected": -371.56091101694915, "loss": 0.5907, "loss/base_kto": 3.678312063217163, "metrics/advantage_var": 269.90863037109375, "regularization/lipschitz_norm": 1062.9803466796875, "regularization/mmd": 0.15461526811122894, "regularization/rkhs_norm": 184.0657958984375, "regularization/w1": 0.8929035067558289, "rewards/chosen": 0.06762640903188763, "rewards/margins": 0.3164247500041767, "rewards/rejected": -0.2487983409722891, "step": 540 }, { "epoch": 0.7253886010362695, "grad_norm": 18.963178634643555, "kl": 6.945270538330078, "learning_rate": 8.978245429744691e-07, "logits/chosen": -35717953.41324921, "logits/rejected": -37219115.591331266, "logps/chosen": -472.8264491324921, "logps/rejected": -363.4963960913313, "loss": 0.5972, "loss/base_kto": 3.7133948802948, "metrics/advantage_var": 272.83770751953125, "regularization/lipschitz_norm": 1079.7265625, "regularization/mmd": 0.1568920910358429, "regularization/rkhs_norm": 186.77630615234375, "regularization/w1": 0.9069703221321106, "rewards/chosen": 0.03154011554898524, "rewards/margins": 0.2772439869541468, "rewards/rejected": -0.24570387140516156, "step": 560 }, { "epoch": 0.7512953367875648, "grad_norm": 21.301929473876953, "kl": 7.833394527435303, "learning_rate": 8.890780469678738e-07, "logits/chosen": -36862122.398744114, "logits/rejected": -37882766.93001555, "logps/chosen": -471.8631279434851, "logps/rejected": -384.9840347978227, "loss": 0.6027, "loss/base_kto": 3.8068490028381348, "metrics/advantage_var": 243.0074920654297, "regularization/lipschitz_norm": 1030.75439453125, "regularization/mmd": 0.14884667098522186, "regularization/rkhs_norm": 177.1984100341797, "regularization/w1": 0.8658336997032166, "rewards/chosen": 0.08991812761387608, "rewards/margins": 0.19279751094427314, "rewards/rejected": -0.10287938333039706, "step": 580 }, { "epoch": 0.7772020725388601, "grad_norm": 16.994449615478516, "kl": 6.220043659210205, "learning_rate": 8.800187789691269e-07, "logits/chosen": -36866568.02507837, "logits/rejected": -38472190.40498442, "logps/chosen": -503.31612460815046, "logps/rejected": -382.5021417445483, "loss": 0.5947, "loss/base_kto": 3.7638473510742188, "metrics/advantage_var": 235.528564453125, "regularization/lipschitz_norm": 1007.619140625, "regularization/mmd": 0.1473793238401413, "regularization/rkhs_norm": 175.45156860351562, "regularization/w1": 0.8463999629020691, "rewards/chosen": 0.01837258204397363, "rewards/margins": 0.22212782707445697, "rewards/rejected": -0.20375524503048334, "step": 600 }, { "epoch": 0.8031088082901554, "grad_norm": 22.941701889038086, "kl": 9.876753807067871, "learning_rate": 8.706540215409981e-07, "logits/chosen": -36347578.469135806, "logits/rejected": -37466653.164556965, "logps/chosen": -512.2447916666666, "logps/rejected": -355.0952828322785, "loss": 0.6013, "loss/base_kto": 3.7260215282440186, "metrics/advantage_var": 275.5116882324219, "regularization/lipschitz_norm": 1103.078369140625, "regularization/mmd": 0.15810923278331757, "regularization/rkhs_norm": 188.2252655029297, "regularization/w1": 0.9265857934951782, "rewards/chosen": 0.10814003885528188, "rewards/margins": 0.25599663047832255, "rewards/rejected": -0.14785659162304068, "step": 620 }, { "epoch": 0.8290155440414507, "grad_norm": 22.692665100097656, "kl": 12.700552940368652, "learning_rate": 8.609913028230462e-07, "logits/chosen": -35791884.3003003, "logits/rejected": -36865420.92508143, "logps/chosen": -476.85923423423424, "logps/rejected": -370.5443556596091, "loss": 0.592, "loss/base_kto": 3.660593032836914, "metrics/advantage_var": 277.190673828125, "regularization/lipschitz_norm": 1092.2738037109375, "regularization/mmd": 0.15797871351242065, "regularization/rkhs_norm": 188.0699005126953, "regularization/w1": 0.9175099730491638, "rewards/chosen": 0.2714547738656625, "rewards/margins": 0.2995656942778069, "rewards/rejected": -0.028110920412144365, "step": 640 }, { "epoch": 0.8549222797927462, "grad_norm": 16.765060424804688, "kl": 7.800302982330322, "learning_rate": 8.510383904798994e-07, "logits/chosen": -36281006.71810089, "logits/rejected": -36525944.81848185, "logps/chosen": -473.14706973293767, "logps/rejected": -382.9906662541254, "loss": 0.607, "loss/base_kto": 3.772947072982788, "metrics/advantage_var": 256.4257507324219, "regularization/lipschitz_norm": 1104.7586669921875, "regularization/mmd": 0.1550176590681076, "regularization/rkhs_norm": 184.5448455810547, "regularization/w1": 0.9279972314834595, "rewards/chosen": 0.14004827889915036, "rewards/margins": 0.2160759381988938, "rewards/rejected": -0.07602765929974345, "step": 660 }, { "epoch": 0.8808290155440415, "grad_norm": 19.784156799316406, "kl": 18.611282348632812, "learning_rate": 8.408032854569865e-07, "logits/chosen": -36242914.96119403, "logits/rejected": -36432563.619672135, "logps/chosen": -487.7375932835821, "logps/rejected": -374.94795081967214, "loss": 0.5943, "loss/base_kto": 3.7035744190216064, "metrics/advantage_var": 255.98374938964844, "regularization/lipschitz_norm": 1066.238525390625, "regularization/mmd": 0.15486612915992737, "regularization/rkhs_norm": 184.3644256591797, "regularization/w1": 0.8956403732299805, "rewards/chosen": 0.3499934182238223, "rewards/margins": 0.24982007462791106, "rewards/rejected": 0.10017334359591125, "step": 680 }, { "epoch": 0.9067357512953368, "grad_norm": 21.27347183227539, "kl": 16.115873336791992, "learning_rate": 8.302942155487377e-07, "logits/chosen": -36517990.72100314, "logits/rejected": -37601879.7258567, "logps/chosen": -504.5242456896552, "logps/rejected": -397.8655081775701, "loss": 0.5956, "loss/base_kto": 3.7432262897491455, "metrics/advantage_var": 252.15293884277344, "regularization/lipschitz_norm": 1038.50537109375, "regularization/mmd": 0.14949001371860504, "regularization/rkhs_norm": 177.96429443359375, "regularization/w1": 0.8723444938659668, "rewards/chosen": 0.2303431878642976, "rewards/margins": 0.22849981562628105, "rewards/rejected": 0.0018433722380165741, "step": 700 }, { "epoch": 0.9326424870466321, "grad_norm": 20.591951370239258, "kl": 14.438664436340332, "learning_rate": 8.195196287844278e-07, "logits/chosen": -35569708.24691358, "logits/rejected": -37203047.69620253, "logps/chosen": -489.99455054012344, "logps/rejected": -360.44986155063293, "loss": 0.5936, "loss/base_kto": 3.7016303539276123, "metrics/advantage_var": 259.8722839355469, "regularization/lipschitz_norm": 1064.8228759765625, "regularization/mmd": 0.15258057415485382, "regularization/rkhs_norm": 181.64353942871094, "regularization/w1": 0.8944511413574219, "rewards/chosen": 0.22748824696481965, "rewards/margins": 0.2628883539018901, "rewards/rejected": -0.035400106937070436, "step": 720 }, { "epoch": 0.9585492227979274, "grad_norm": 21.718122482299805, "kl": 10.604936599731445, "learning_rate": 8.08488186636975e-07, "logits/chosen": -36374055.384615384, "logits/rejected": -35905230.048780486, "logps/chosen": -485.7256610576923, "logps/rejected": -383.9867568597561, "loss": 0.6052, "loss/base_kto": 3.7834489345550537, "metrics/advantage_var": 255.31553649902344, "regularization/lipschitz_norm": 1078.7470703125, "regularization/mmd": 0.15181118249893188, "regularization/rkhs_norm": 180.72760009765625, "regularization/w1": 0.9061475992202759, "rewards/chosen": 0.08294287706032777, "rewards/margins": 0.19643550548350686, "rewards/rejected": -0.11349262842317907, "step": 740 }, { "epoch": 0.9844559585492227, "grad_norm": 17.114286422729492, "kl": 28.3470516204834, "learning_rate": 7.972087570601576e-07, "logits/chosen": -35719567.97650514, "logits/rejected": -36538672.29382304, "logps/chosen": -486.05892070484583, "logps/rejected": -366.78078046744577, "loss": 0.5914, "loss/base_kto": 3.6441636085510254, "metrics/advantage_var": 277.0114440917969, "regularization/lipschitz_norm": 1102.2738037109375, "regularization/mmd": 0.16112275421619415, "regularization/rkhs_norm": 191.81280517578125, "regularization/w1": 0.9259099960327148, "rewards/chosen": 0.5076863965273953, "rewards/margins": 0.30991189473322667, "rewards/rejected": 0.19777450179416867, "step": 760 }, { "epoch": 1.0103626943005182, "grad_norm": 20.1845760345459, "kl": 22.826278686523438, "learning_rate": 7.856904073598458e-07, "logits/chosen": -36458379.15689382, "logits/rejected": -36623621.14374034, "logps/chosen": -469.39659270998413, "logps/rejected": -360.1577714451314, "loss": 0.6075, "loss/base_kto": 3.614739179611206, "metrics/advantage_var": 394.32861328125, "regularization/lipschitz_norm": 1266.2174072265625, "regularization/mmd": 0.18139131367206573, "regularization/rkhs_norm": 215.94204711914062, "regularization/w1": 1.0636225938796997, "rewards/chosen": 0.42696843698927794, "rewards/margins": 0.3763576632748085, "rewards/rejected": 0.050610773714469426, "step": 780 }, { "epoch": 1.0362694300518134, "grad_norm": 30.14720916748047, "kl": 15.291643142700195, "learning_rate": 7.739423969049761e-07, "logits/chosen": -36967712.24883359, "logits/rejected": -38327731.642072216, "logps/chosen": -482.3858864696734, "logps/rejected": -378.95947802197804, "loss": 0.6284, "loss/base_kto": 3.3046669960021973, "metrics/advantage_var": 679.3814697265625, "regularization/lipschitz_norm": 1763.589111328125, "regularization/mmd": 0.24101614952087402, "regularization/rkhs_norm": 286.9239807128906, "regularization/w1": 1.481414794921875, "rewards/chosen": 0.5338768847808126, "rewards/margins": 0.7598774205870823, "rewards/rejected": -0.22600053580626964, "step": 800 }, { "epoch": 1.0621761658031088, "grad_norm": 19.852436065673828, "kl": 14.40466022491455, "learning_rate": 7.619741696841322e-07, "logits/chosen": -36692998.32098766, "logits/rejected": -38535135.594936706, "logps/chosen": -493.6412037037037, "logps/rejected": -356.8260977056962, "loss": 0.6061, "loss/base_kto": 3.3381354808807373, "metrics/advantage_var": 522.1862182617188, "regularization/lipschitz_norm": 1542.626953125, "regularization/mmd": 0.21481800079345703, "regularization/rkhs_norm": 255.7357177734375, "regularization/w1": 1.2958067655563354, "rewards/chosen": 0.44550949850200133, "rewards/margins": 0.7030229552087606, "rewards/rejected": -0.2575134567067593, "step": 820 }, { "epoch": 1.0880829015544042, "grad_norm": 17.26534080505371, "kl": 12.90103816986084, "learning_rate": 7.497953467137135e-07, "logits/chosen": -36583257.6, "logits/rejected": -37187059.2, "logps/chosen": -471.836767578125, "logps/rejected": -396.18720703125, "loss": 0.6197, "loss/base_kto": 3.31327223777771, "metrics/advantage_var": 666.0192260742188, "regularization/lipschitz_norm": 1676.6455078125, "regularization/mmd": 0.23618967831134796, "regularization/rkhs_norm": 281.1781921386719, "regularization/w1": 1.4083822965621948, "rewards/chosen": 0.5340013980865479, "rewards/margins": 0.7987452745437622, "rewards/rejected": -0.26474387645721437, "step": 840 }, { "epoch": 1.1139896373056994, "grad_norm": 23.737882614135742, "kl": 12.132939338684082, "learning_rate": 7.37415718303793e-07, "logits/chosen": -35278184.65408805, "logits/rejected": -37071305.9378882, "logps/chosen": -499.165634827044, "logps/rejected": -377.6488014363354, "loss": 0.6043, "loss/base_kto": 3.3615880012512207, "metrics/advantage_var": 558.8425903320312, "regularization/lipschitz_norm": 1498.6766357421875, "regularization/mmd": 0.2139439582824707, "regularization/rkhs_norm": 254.69522094726562, "regularization/w1": 1.2588883638381958, "rewards/chosen": 0.44297598592890136, "rewards/margins": 0.7101170082260675, "rewards/rejected": -0.26714102229716613, "step": 860 }, { "epoch": 1.1398963730569949, "grad_norm": 15.236355781555176, "kl": 17.843761444091797, "learning_rate": 7.248452361878855e-07, "logits/chosen": -35046467.43032159, "logits/rejected": -36572099.57256778, "logps/chosen": -445.42783307810106, "logps/rejected": -381.4280801435407, "loss": 0.5917, "loss/base_kto": 3.2963523864746094, "metrics/advantage_var": 551.6969604492188, "regularization/lipschitz_norm": 1460.2900390625, "regularization/mmd": 0.21053610742092133, "regularization/rkhs_norm": 250.6382293701172, "regularization/w1": 1.2266435623168945, "rewards/chosen": 0.5309551057917783, "rewards/margins": 0.7822974655141044, "rewards/rejected": -0.25134235972232605, "step": 880 }, { "epoch": 1.16580310880829, "grad_norm": 16.517234802246094, "kl": 17.103546142578125, "learning_rate": 7.120940055229497e-07, "logits/chosen": -35180473.26844584, "logits/rejected": -36107501.28771384, "logps/chosen": -471.614010989011, "logps/rejected": -369.73233378693624, "loss": 0.6009, "loss/base_kto": 3.380141019821167, "metrics/advantage_var": 526.4321899414062, "regularization/lipschitz_norm": 1447.9232177734375, "regularization/mmd": 0.2106965035200119, "regularization/rkhs_norm": 250.82920837402344, "regularization/w1": 1.2162554264068604, "rewards/chosen": 0.498858537179896, "rewards/margins": 0.674133043507049, "rewards/rejected": -0.175274506327153, "step": 900 }, { "epoch": 1.1917098445595855, "grad_norm": 13.176434516906738, "kl": 15.887646675109863, "learning_rate": 6.991722767660556e-07, "logits/chosen": -33349247.42771982, "logits/rejected": -35284178.18062397, "logps/chosen": -476.1371087928465, "logps/rejected": -368.3392857142857, "loss": 0.6098, "loss/base_kto": 3.3418033123016357, "metrics/advantage_var": 591.3095092773438, "regularization/lipschitz_norm": 1566.5013427734375, "regularization/mmd": 0.2209499329328537, "regularization/rkhs_norm": 263.0356750488281, "regularization/w1": 1.3158612251281738, "rewards/chosen": 0.5200721069881706, "rewards/margins": 0.7304523229383708, "rewards/rejected": -0.21038021595020012, "step": 920 }, { "epoch": 1.2176165803108807, "grad_norm": 18.833049774169922, "kl": 20.219152450561523, "learning_rate": 6.860904374342499e-07, "logits/chosen": -35351840.66468843, "logits/rejected": -36238461.04290429, "logps/chosen": -492.69621661721067, "logps/rejected": -365.50549195544556, "loss": 0.6237, "loss/base_kto": 3.3600921630859375, "metrics/advantage_var": 596.228759765625, "regularization/lipschitz_norm": 1668.950439453125, "regularization/mmd": 0.22760716080665588, "regularization/rkhs_norm": 270.9609069824219, "regularization/w1": 1.4019182920455933, "rewards/chosen": 0.5726751219978672, "rewards/margins": 0.7042602966591321, "rewards/rejected": -0.13158517466126496, "step": 940 }, { "epoch": 1.2435233160621761, "grad_norm": 17.778493881225586, "kl": 18.271499633789062, "learning_rate": 6.7285900375424e-07, "logits/chosen": -35268249.27388535, "logits/rejected": -35581867.19018405, "logps/chosen": -494.1021098726115, "logps/rejected": -388.5136359279141, "loss": 0.6139, "loss/base_kto": 3.3075549602508545, "metrics/advantage_var": 640.4003295898438, "regularization/lipschitz_norm": 1638.5855712890625, "regularization/mmd": 0.22699318826198578, "regularization/rkhs_norm": 270.22998046875, "regularization/w1": 1.3764116764068604, "rewards/chosen": 0.5335830396907345, "rewards/margins": 0.774583373137401, "rewards/rejected": -0.24100033344666652, "step": 960 }, { "epoch": 1.2694300518134716, "grad_norm": 20.075347900390625, "kl": 12.981180191040039, "learning_rate": 6.594886122086123e-07, "logits/chosen": -35015986.58682635, "logits/rejected": -37582911.58169935, "logps/chosen": -487.8549775449102, "logps/rejected": -356.66048815359477, "loss": 0.6053, "loss/base_kto": 3.288564443588257, "metrics/advantage_var": 623.2282104492188, "regularization/lipschitz_norm": 1576.2935791015625, "regularization/mmd": 0.22978658974170685, "regularization/rkhs_norm": 273.55548095703125, "regularization/w1": 1.3240865468978882, "rewards/chosen": 0.5048243357035929, "rewards/margins": 0.8054399330981497, "rewards/rejected": -0.3006155973945568, "step": 980 }, { "epoch": 1.2953367875647668, "grad_norm": 16.932571411132812, "kl": 31.370588302612305, "learning_rate": 6.459900109853766e-07, "logits/chosen": -34778587.372907154, "logits/rejected": -36209751.113964684, "logps/chosen": -489.5570776255708, "logps/rejected": -363.12103731942216, "loss": 0.5876, "loss/base_kto": 3.262110948562622, "metrics/advantage_var": 507.5049743652344, "regularization/lipschitz_norm": 1457.1304931640625, "regularization/mmd": 0.21449874341487885, "regularization/rkhs_norm": 255.3556671142578, "regularization/w1": 1.2239896059036255, "rewards/chosen": 0.7377876270125808, "rewards/margins": 0.7224614929294131, "rewards/rejected": 0.015326134083167698, "step": 1000 }, { "epoch": 1.3212435233160622, "grad_norm": 19.6783504486084, "kl": 25.338829040527344, "learning_rate": 6.323740513377171e-07, "logits/chosen": -35122894.568288855, "logits/rejected": -34815159.141524106, "logps/chosen": -472.63417386185245, "logps/rejected": -354.4738287325039, "loss": 0.589, "loss/base_kto": 3.291515350341797, "metrics/advantage_var": 518.9993286132812, "regularization/lipschitz_norm": 1441.677978515625, "regularization/mmd": 0.2093811333179474, "regularization/rkhs_norm": 249.2632598876953, "regularization/w1": 1.2110095024108887, "rewards/chosen": 0.6328515452530171, "rewards/margins": 0.7417727683254073, "rewards/rejected": -0.10892122307239016, "step": 1020 }, { "epoch": 1.3471502590673574, "grad_norm": 14.299549102783203, "kl": 13.822909355163574, "learning_rate": 6.186516788608865e-07, "logits/chosen": -34826474.057142854, "logits/rejected": -35530600.763076924, "logps/chosen": -480.8300595238095, "logps/rejected": -373.80310096153846, "loss": 0.5948, "loss/base_kto": 3.2905819416046143, "metrics/advantage_var": 525.7487182617188, "regularization/lipschitz_norm": 1492.8621826171875, "regularization/mmd": 0.21371899545192719, "regularization/rkhs_norm": 254.42739868164062, "regularization/w1": 1.2540040016174316, "rewards/chosen": 0.4819301060267857, "rewards/margins": 0.7838381354363411, "rewards/rejected": -0.3019080294095553, "step": 1040 }, { "epoch": 1.3730569948186528, "grad_norm": 21.68499183654785, "kl": 11.745469093322754, "learning_rate": 6.048339246932652e-07, "logits/chosen": -36805399.497584544, "logits/rejected": -36395620.2245827, "logps/chosen": -521.2514090177134, "logps/rejected": -378.34476953717757, "loss": 0.6245, "loss/base_kto": 3.3373310565948486, "metrics/advantage_var": 717.9779663085938, "regularization/lipschitz_norm": 1694.2879638671875, "regularization/mmd": 0.2353643923997879, "regularization/rkhs_norm": 280.1957092285156, "regularization/w1": 1.4232019186019897, "rewards/chosen": 0.3695133473370194, "rewards/margins": 0.7401284020334107, "rewards/rejected": -0.37061505469639133, "step": 1060 }, { "epoch": 1.3989637305699483, "grad_norm": 21.23410987854004, "kl": 8.063403129577637, "learning_rate": 5.909318966486494e-07, "logits/chosen": -34965694.433931485, "logits/rejected": -36071813.94902549, "logps/chosen": -462.24525897226755, "logps/rejected": -379.5708395802099, "loss": 0.6046, "loss/base_kto": 3.344383955001831, "metrics/advantage_var": 544.3243408203125, "regularization/lipschitz_norm": 1522.578369140625, "regularization/mmd": 0.21384473145008087, "regularization/rkhs_norm": 254.57705688476562, "regularization/w1": 1.2789658308029175, "rewards/chosen": 0.28101740925774876, "rewards/margins": 0.719642853615648, "rewards/rejected": -0.4386254443578992, "step": 1080 }, { "epoch": 1.4248704663212435, "grad_norm": 21.19676971435547, "kl": 10.65280532836914, "learning_rate": 5.769567702869052e-07, "logits/chosen": -34922690.43037975, "logits/rejected": -36481046.12345679, "logps/chosen": -491.1184731012658, "logps/rejected": -386.28317901234567, "loss": 0.6083, "loss/base_kto": 3.267338514328003, "metrics/advantage_var": 670.4760131835938, "regularization/lipschitz_norm": 1626.25390625, "regularization/mmd": 0.23278503119945526, "regularization/rkhs_norm": 277.12506103515625, "regularization/w1": 1.3660532236099243, "rewards/chosen": 0.42179600196548656, "rewards/margins": 0.803499508246833, "rewards/rejected": -0.38170350628134647, "step": 1100 }, { "epoch": 1.450777202072539, "grad_norm": 17.94518280029297, "kl": 7.550856113433838, "learning_rate": 5.629197799301614e-07, "logits/chosen": -34210965.97165354, "logits/rejected": -35365899.1131783, "logps/chosen": -454.8527559055118, "logps/rejected": -370.1827277131783, "loss": 0.6042, "loss/base_kto": 3.37127685546875, "metrics/advantage_var": 532.8030395507812, "regularization/lipschitz_norm": 1485.170166015625, "regularization/mmd": 0.2144092172384262, "regularization/rkhs_norm": 255.2490692138672, "regularization/w1": 1.2475429773330688, "rewards/chosen": 0.2536710724117249, "rewards/margins": 0.6749354369647772, "rewards/rejected": -0.42126436455305233, "step": 1120 }, { "epoch": 1.4766839378238341, "grad_norm": 21.139720916748047, "kl": 12.362833976745605, "learning_rate": 5.488322096317633e-07, "logits/chosen": -33882437.02927581, "logits/rejected": -34784679.55625991, "logps/chosen": -472.1962153312789, "logps/rejected": -360.87633716323296, "loss": 0.6008, "loss/base_kto": 3.286090850830078, "metrics/advantage_var": 561.7142944335938, "regularization/lipschitz_norm": 1551.6668701171875, "regularization/mmd": 0.21712155640125275, "regularization/rkhs_norm": 258.47802734375, "regularization/w1": 1.3034000396728516, "rewards/chosen": 0.4976035472240948, "rewards/margins": 0.7835909780444692, "rewards/rejected": -0.2859874308203744, "step": 1140 }, { "epoch": 1.5025906735751295, "grad_norm": 16.639875411987305, "kl": 9.712878227233887, "learning_rate": 5.347053841052518e-07, "logits/chosen": -35478420.123176664, "logits/rejected": -36061906.823529415, "logps/chosen": -482.0229943273906, "logps/rejected": -369.93636877828055, "loss": 0.5927, "loss/base_kto": 3.3484184741973877, "metrics/advantage_var": 488.9422912597656, "regularization/lipschitz_norm": 1415.404541015625, "regularization/mmd": 0.2046360969543457, "regularization/rkhs_norm": 243.61441040039062, "regularization/w1": 1.1889398097991943, "rewards/chosen": 0.33435288588463585, "rewards/margins": 0.6939580105178725, "rewards/rejected": -0.3596051246332367, "step": 1160 }, { "epoch": 1.528497409326425, "grad_norm": 24.6983642578125, "kl": 9.777175903320312, "learning_rate": 5.205506596206531e-07, "logits/chosen": -34844887.245749615, "logits/rejected": -35976772.751974724, "logps/chosen": -485.0933636012365, "logps/rejected": -366.4594687993681, "loss": 0.6151, "loss/base_kto": 3.35014271736145, "metrics/advantage_var": 736.5438842773438, "regularization/lipschitz_norm": 1605.5198974609375, "regularization/mmd": 0.2221064418554306, "regularization/rkhs_norm": 264.4124450683594, "regularization/w1": 1.3486367464065552, "rewards/chosen": 0.39864683077545404, "rewards/margins": 0.6804745434747776, "rewards/rejected": -0.28182771269932366, "step": 1180 }, { "epoch": 1.5544041450777202, "grad_norm": 19.533191680908203, "kl": 9.757631301879883, "learning_rate": 5.063794148754032e-07, "logits/chosen": -35173026.18633541, "logits/rejected": -37269793.81132075, "logps/chosen": -474.92294254658384, "logps/rejected": -391.2468307783019, "loss": 0.6054, "loss/base_kto": 3.274069309234619, "metrics/advantage_var": 546.0035400390625, "regularization/lipschitz_norm": 1608.5828857421875, "regularization/mmd": 0.21795068681240082, "regularization/rkhs_norm": 259.465087890625, "regularization/w1": 1.3512096405029297, "rewards/chosen": 0.445318423442959, "rewards/margins": 0.785911821435171, "rewards/rejected": -0.34059339799221205, "step": 1200 }, { "epoch": 1.5803108808290154, "grad_norm": 20.48908233642578, "kl": 11.801558494567871, "learning_rate": 4.922030418472422e-07, "logits/chosen": -35335841.77198697, "logits/rejected": -35890348.2042042, "logps/chosen": -486.5277890879479, "logps/rejected": -372.42149962462463, "loss": 0.6001, "loss/base_kto": 3.3054816722869873, "metrics/advantage_var": 570.5050659179688, "regularization/lipschitz_norm": 1523.2435302734375, "regularization/mmd": 0.21592283248901367, "regularization/rkhs_norm": 257.051025390625, "regularization/w1": 1.2795246839523315, "rewards/chosen": 0.4257883575141236, "rewards/margins": 0.7725356556545492, "rewards/rejected": -0.3467472981404256, "step": 1220 }, { "epoch": 1.6062176165803108, "grad_norm": 19.19207191467285, "kl": 13.553595542907715, "learning_rate": 4.780329366364336e-07, "logits/chosen": -34889640.18376723, "logits/rejected": -36536169.74800638, "logps/chosen": -473.4591787901991, "logps/rejected": -378.58951355661884, "loss": 0.6004, "loss/base_kto": 3.3230807781219482, "metrics/advantage_var": 545.9921875, "regularization/lipschitz_norm": 1504.0831298828125, "regularization/mmd": 0.21675610542297363, "regularization/rkhs_norm": 258.04296875, "regularization/w1": 1.2634297609329224, "rewards/chosen": 0.4697913364100785, "rewards/margins": 0.7035303831791674, "rewards/rejected": -0.2337390467690889, "step": 1240 }, { "epoch": 1.6321243523316062, "grad_norm": 15.541764259338379, "kl": 12.717080116271973, "learning_rate": 4.638804903046684e-07, "logits/chosen": -35423642.24605678, "logits/rejected": -37378539.39318886, "logps/chosen": -505.2036179022082, "logps/rejected": -369.9245839783282, "loss": 0.6017, "loss/base_kto": 3.327488660812378, "metrics/advantage_var": 538.381591796875, "regularization/lipschitz_norm": 1517.1551513671875, "regularization/mmd": 0.2117990255355835, "regularization/rkhs_norm": 252.14170837402344, "regularization/w1": 1.2744102478027344, "rewards/chosen": 0.42909361060109424, "rewards/margins": 0.7177926396319089, "rewards/rejected": -0.2886990290308146, "step": 1260 }, { "epoch": 1.6580310880829017, "grad_norm": 18.025327682495117, "kl": 12.646842956542969, "learning_rate": 4.497570797180217e-07, "logits/chosen": -35896735.90153846, "logits/rejected": -35636058.20952381, "logps/chosen": -471.96846153846155, "logps/rejected": -390.17410714285717, "loss": 0.604, "loss/base_kto": 3.312110185623169, "metrics/advantage_var": 576.8339233398438, "regularization/lipschitz_norm": 1548.7454833984375, "regularization/mmd": 0.2193342000246048, "regularization/rkhs_norm": 261.1121520996094, "regularization/w1": 1.3009463548660278, "rewards/chosen": 0.4091322678786058, "rewards/margins": 0.7262386783138737, "rewards/rejected": -0.3171064104352679, "step": 1280 }, { "epoch": 1.6839378238341969, "grad_norm": 17.990734100341797, "kl": 14.885505676269531, "learning_rate": 4.3567405840131853e-07, "logits/chosen": -35742065.609467454, "logits/rejected": -35703214.62251656, "logps/chosen": -480.8216068786982, "logps/rejected": -376.9612737996689, "loss": 0.6055, "loss/base_kto": 3.234381914138794, "metrics/advantage_var": 622.4717407226562, "regularization/lipschitz_norm": 1642.2193603515625, "regularization/mmd": 0.23044441640377045, "regularization/rkhs_norm": 274.3385925292969, "regularization/w1": 1.3794642686843872, "rewards/chosen": 0.586639133430797, "rewards/margins": 0.8144280712331433, "rewards/rejected": -0.22778893780234633, "step": 1300 }, { "epoch": 1.709844559585492, "grad_norm": 19.09687042236328, "kl": 13.606826782226562, "learning_rate": 4.2164274741126506e-07, "logits/chosen": -36210587.771615006, "logits/rejected": -36060106.266866565, "logps/chosen": -481.3544045676998, "logps/rejected": -383.96675880809596, "loss": 0.6081, "loss/base_kto": 3.22845721244812, "metrics/advantage_var": 648.509521484375, "regularization/lipschitz_norm": 1670.5927734375, "regularization/mmd": 0.23286108672618866, "regularization/rkhs_norm": 277.215576171875, "regularization/w1": 1.403298020362854, "rewards/chosen": 0.5233538629180261, "rewards/margins": 0.8296589089423771, "rewards/rejected": -0.3063050460243511, "step": 1320 }, { "epoch": 1.7357512953367875, "grad_norm": 21.471250534057617, "kl": 9.653098106384277, "learning_rate": 4.0767442623567856e-07, "logits/chosen": -34026022.88607595, "logits/rejected": -35710957.03703704, "logps/chosen": -485.3389042721519, "logps/rejected": -362.10927854938274, "loss": 0.6016, "loss/base_kto": 3.334829092025757, "metrics/advantage_var": 608.798828125, "regularization/lipschitz_norm": 1510.160888671875, "regularization/mmd": 0.20944009721279144, "regularization/rkhs_norm": 249.3334503173828, "regularization/w1": 1.2685352563858032, "rewards/chosen": 0.2437453209599362, "rewards/margins": 0.6843596209695813, "rewards/rejected": -0.4406143000096451, "step": 1340 }, { "epoch": 1.761658031088083, "grad_norm": 22.113142013549805, "kl": 11.807127952575684, "learning_rate": 3.937803237261357e-07, "logits/chosen": -35004644.9689441, "logits/rejected": -35935972.628930815, "logps/chosen": -472.87990100931677, "logps/rejected": -383.3904530267296, "loss": 0.6044, "loss/base_kto": 3.2573845386505127, "metrics/advantage_var": 629.4389038085938, "regularization/lipschitz_norm": 1610.2572021484375, "regularization/mmd": 0.22549353539943695, "regularization/rkhs_norm": 268.4446716308594, "regularization/w1": 1.3526160717010498, "rewards/chosen": 0.4835328285738548, "rewards/margins": 0.8426700457776308, "rewards/rejected": -0.35913721720377606, "step": 1360 }, { "epoch": 1.7875647668393784, "grad_norm": 17.822072982788086, "kl": 12.865683555603027, "learning_rate": 3.7997160907132456e-07, "logits/chosen": -34269693.625966, "logits/rejected": -35976850.401263826, "logps/chosen": -500.7642967542504, "logps/rejected": -386.6487954186414, "loss": 0.6125, "loss/base_kto": 3.3427696228027344, "metrics/advantage_var": 572.8134155273438, "regularization/lipschitz_norm": 1589.7178955078125, "regularization/mmd": 0.22191600501537323, "regularization/rkhs_norm": 264.1857604980469, "regularization/w1": 1.3353630304336548, "rewards/chosen": 0.4387526829054772, "rewards/margins": 0.7057367516690534, "rewards/rejected": -0.2669840687635762, "step": 1380 }, { "epoch": 1.8134715025906736, "grad_norm": 12.618535041809082, "kl": 12.74691104888916, "learning_rate": 3.662593828183601e-07, "logits/chosen": -34078279.49919743, "logits/rejected": -34741355.543378994, "logps/chosen": -492.42676565008026, "logps/rejected": -354.0341990106545, "loss": 0.5933, "loss/base_kto": 3.249702215194702, "metrics/advantage_var": 520.4142456054688, "regularization/lipschitz_norm": 1528.2381591796875, "regularization/mmd": 0.21315963566303253, "regularization/rkhs_norm": 253.761474609375, "regularization/w1": 1.2837201356887817, "rewards/chosen": 0.511796537984049, "rewards/margins": 0.7991691677390313, "rewards/rejected": -0.2873726297549824, "step": 1400 }, { "epoch": 1.8393782383419688, "grad_norm": 20.017213821411133, "kl": 7.575507640838623, "learning_rate": 3.5265466794927725e-07, "logits/chosen": -34369425.6481775, "logits/rejected": -35838734.59476117, "logps/chosen": -482.71711568938196, "logps/rejected": -385.2330026964561, "loss": 0.6088, "loss/base_kto": 3.3300721645355225, "metrics/advantage_var": 551.4451904296875, "regularization/lipschitz_norm": 1572.4085693359375, "regularization/mmd": 0.21954384446144104, "regularization/rkhs_norm": 261.3617248535156, "regularization/w1": 1.320823311805725, "rewards/chosen": 0.377362476475832, "rewards/margins": 0.7479974753322457, "rewards/rejected": -0.3706349988564137, "step": 1420 }, { "epoch": 1.8652849740932642, "grad_norm": 13.82175350189209, "kl": 8.901938438415527, "learning_rate": 3.3916840101987887e-07, "logits/chosen": -33453443.062200956, "logits/rejected": -34858838.64012251, "logps/chosen": -480.2731259968102, "logps/rejected": -383.6234207503828, "loss": 0.5934, "loss/base_kto": 3.3000335693359375, "metrics/advantage_var": 538.3826293945312, "regularization/lipschitz_norm": 1468.3470458984375, "regularization/mmd": 0.21402302384376526, "regularization/rkhs_norm": 254.78929138183594, "regularization/w1": 1.2334115505218506, "rewards/chosen": 0.40801843767911433, "rewards/margins": 0.7755517256738262, "rewards/rejected": -0.3675332879947119, "step": 1440 }, { "epoch": 1.8911917098445596, "grad_norm": 16.749231338500977, "kl": 6.636883735656738, "learning_rate": 3.258114233680583e-07, "logits/chosen": -34588749.6492891, "logits/rejected": -34832643.561051, "logps/chosen": -456.658570300158, "logps/rejected": -403.6516132148377, "loss": 0.6128, "loss/base_kto": 3.294590711593628, "metrics/advantage_var": 709.7509155273438, "regularization/lipschitz_norm": 1638.9635009765625, "regularization/mmd": 0.23090210556983948, "regularization/rkhs_norm": 274.8834533691406, "regularization/w1": 1.3767293691635132, "rewards/chosen": 0.3069215265307193, "rewards/margins": 0.7958374666872108, "rewards/rejected": -0.4889159401564915, "step": 1460 }, { "epoch": 1.917098445595855, "grad_norm": 21.665218353271484, "kl": 8.190171241760254, "learning_rate": 3.1259447239866796e-07, "logits/chosen": -34391653.085072234, "logits/rejected": -34643344.56012177, "logps/chosen": -498.69166332263245, "logps/rejected": -374.8472460045662, "loss": 0.605, "loss/base_kto": 3.238154649734497, "metrics/advantage_var": 579.7916259765625, "regularization/lipschitz_norm": 1639.0191650390625, "regularization/mmd": 0.22491280734539032, "regularization/rkhs_norm": 267.7533264160156, "regularization/w1": 1.3767759799957275, "rewards/chosen": 0.34365690424201195, "rewards/margins": 0.8382388961025047, "rewards/rejected": -0.4945819918604928, "step": 1480 }, { "epoch": 1.9430051813471503, "grad_norm": 12.00883960723877, "kl": 8.099600791931152, "learning_rate": 2.9952817295193435e-07, "logits/chosen": -34364068.256651014, "logits/rejected": -35125107.41965678, "logps/chosen": -481.04171557120503, "logps/rejected": -391.90459243369736, "loss": 0.5997, "loss/base_kto": 3.3013312816619873, "metrics/advantage_var": 561.3448486328125, "regularization/lipschitz_norm": 1522.6522216796875, "regularization/mmd": 0.21698199212551117, "regularization/rkhs_norm": 258.3118591308594, "regularization/w1": 1.2790278196334839, "rewards/chosen": 0.3362527289114461, "rewards/margins": 0.7826477697716743, "rewards/rejected": -0.44639504086022813, "step": 1500 }, { "epoch": 1.9689119170984455, "grad_norm": 16.841245651245117, "kl": 9.618056297302246, "learning_rate": 2.866230287623651e-07, "logits/chosen": -35000446.41975309, "logits/rejected": -36844343.088607594, "logps/chosen": -492.89515817901236, "logps/rejected": -367.14542128164555, "loss": 0.6085, "loss/base_kto": 3.3525917530059814, "metrics/advantage_var": 560.1251831054688, "regularization/lipschitz_norm": 1544.7421875, "regularization/mmd": 0.21756787598133087, "regularization/rkhs_norm": 259.0093688964844, "regularization/w1": 1.2975834608078003, "rewards/chosen": 0.34516786645959924, "rewards/margins": 0.7210477094833581, "rewards/rejected": -0.3758798430237589, "step": 1520 }, { "epoch": 1.994818652849741, "grad_norm": 21.1080379486084, "kl": 11.35669231414795, "learning_rate": 2.738894140150075e-07, "logits/chosen": -33410666.9010989, "logits/rejected": -34769791.00466563, "logps/chosen": -498.6289246467818, "logps/rejected": -381.9893565318818, "loss": 0.6024, "loss/base_kto": 3.252659559249878, "metrics/advantage_var": 662.2719116210938, "regularization/lipschitz_norm": 1594.5869140625, "regularization/mmd": 0.2272685021162033, "regularization/rkhs_norm": 270.5577697753906, "regularization/w1": 1.339453101158142, "rewards/chosen": 0.5052544370738079, "rewards/margins": 0.8519850267550506, "rewards/rejected": -0.3467305896812427, "step": 1540 }, { "epoch": 2.0207253886010363, "grad_norm": 15.84586238861084, "kl": 10.274452209472656, "learning_rate": 2.6133756500585156e-07, "logits/chosen": -33943520.87537994, "logits/rejected": -35674970.838709675, "logps/chosen": -489.6505509118541, "logps/rejected": -350.32694052419356, "loss": 0.563, "loss/base_kto": 3.2420356273651123, "metrics/advantage_var": 389.9266052246094, "regularization/lipschitz_norm": 1279.0777587890625, "regularization/mmd": 0.18764524161815643, "regularization/rkhs_norm": 223.3871612548828, "regularization/w1": 1.074425220489502, "rewards/chosen": 0.4061503308884641, "rewards/margins": 0.8001105576242251, "rewards/rejected": -0.3939602267357611, "step": 1560 }, { "epoch": 2.0466321243523318, "grad_norm": 14.673681259155273, "kl": 13.122614860534668, "learning_rate": 2.489775719130767e-07, "logits/chosen": -33392299.20760697, "logits/rejected": -34548346.280431435, "logps/chosen": -464.9544374009509, "logps/rejected": -392.4648979198767, "loss": 0.5569, "loss/base_kto": 3.1842801570892334, "metrics/advantage_var": 402.1956481933594, "regularization/lipschitz_norm": 1288.0323486328125, "regularization/mmd": 0.1890570968389511, "regularization/rkhs_norm": 225.0679931640625, "regularization/w1": 1.0819472074508667, "rewards/chosen": 0.5139663140104992, "rewards/margins": 0.839497931161886, "rewards/rejected": -0.32553161715138673, "step": 1580 }, { "epoch": 2.0725388601036268, "grad_norm": 15.556137084960938, "kl": 10.652277946472168, "learning_rate": 2.3681937068576303e-07, "logits/chosen": -34607248.77761414, "logits/rejected": -35966676.12645591, "logps/chosen": -488.3823177466863, "logps/rejected": -364.26169925124793, "loss": 0.5582, "loss/base_kto": 3.1443049907684326, "metrics/advantage_var": 442.4376525878906, "regularization/lipschitz_norm": 1332.1800537109375, "regularization/mmd": 0.20206427574157715, "regularization/rkhs_norm": 240.5526885986328, "regularization/w1": 1.119031310081482, "rewards/chosen": 0.5578369679963642, "rewards/margins": 0.9110108316659643, "rewards/rejected": -0.35317386366960013, "step": 1600 }, { "epoch": 2.098445595854922, "grad_norm": 18.974214553833008, "kl": 14.737631797790527, "learning_rate": 2.2487273505658e-07, "logits/chosen": -35975270.702064894, "logits/rejected": -36277033.674418606, "logps/chosen": -497.10550331858406, "logps/rejected": -396.29347487541526, "loss": 0.5696, "loss/base_kto": 3.211864471435547, "metrics/advantage_var": 431.98687744140625, "regularization/lipschitz_norm": 1365.83544921875, "regularization/mmd": 0.19773311913013458, "regularization/rkhs_norm": 235.39659118652344, "regularization/w1": 1.1473017930984497, "rewards/chosen": 0.5352629264899059, "rewards/margins": 0.8344073291808061, "rewards/rejected": -0.29914440269090015, "step": 1620 }, { "epoch": 2.1243523316062176, "grad_norm": 17.793354034423828, "kl": 15.390420913696289, "learning_rate": 2.131472686848817e-07, "logits/chosen": -34687612.121212125, "logits/rejected": -36155078.19354839, "logps/chosen": -484.9957386363636, "logps/rejected": -385.4767137096774, "loss": 0.5633, "loss/base_kto": 3.226102828979492, "metrics/advantage_var": 418.45245361328125, "regularization/lipschitz_norm": 1294.6143798828125, "regularization/mmd": 0.19300396740436554, "regularization/rkhs_norm": 229.7666015625, "regularization/w1": 1.087476134300232, "rewards/chosen": 0.5520561911843039, "rewards/margins": 0.7991140505435529, "rewards/rejected": -0.24705785935924898, "step": 1640 }, { "epoch": 2.150259067357513, "grad_norm": 18.184839248657227, "kl": 17.659698486328125, "learning_rate": 2.016523974365188e-07, "logits/chosen": -34934539.32743363, "logits/rejected": -35940869.10299003, "logps/chosen": -495.3005162241888, "logps/rejected": -388.21631021594686, "loss": 0.5557, "loss/base_kto": 3.187596559524536, "metrics/advantage_var": 409.62841796875, "regularization/lipschitz_norm": 1267.803466796875, "regularization/mmd": 0.19272783398628235, "regularization/rkhs_norm": 229.43789672851562, "regularization/w1": 1.0649548768997192, "rewards/chosen": 0.6390756252592644, "rewards/margins": 0.835071146171004, "rewards/rejected": -0.19599552091173952, "step": 1660 }, { "epoch": 2.1761658031088085, "grad_norm": 15.002678871154785, "kl": 22.223430633544922, "learning_rate": 1.9039736180657372e-07, "logits/chosen": -34828452.36597111, "logits/rejected": -35678684.93150685, "logps/chosen": -474.8338683788122, "logps/rejected": -376.26436453576866, "loss": 0.5523, "loss/base_kto": 3.159904956817627, "metrics/advantage_var": 433.2723083496094, "regularization/lipschitz_norm": 1267.8875732421875, "regularization/mmd": 0.19317255914211273, "regularization/rkhs_norm": 229.96731567382812, "regularization/w1": 1.0650255680084229, "rewards/chosen": 0.6723762622423004, "rewards/margins": 0.8744663586797052, "rewards/rejected": -0.20209009643740486, "step": 1680 }, { "epoch": 2.2020725388601035, "grad_norm": 12.862616539001465, "kl": 14.726593017578125, "learning_rate": 1.7939120949111498e-07, "logits/chosen": -34499999.65163297, "logits/rejected": -34946869.45054945, "logps/chosen": -487.72657465007774, "logps/rejected": -349.1521291208791, "loss": 0.5632, "loss/base_kto": 3.1749675273895264, "metrics/advantage_var": 461.6251525878906, "regularization/lipschitz_norm": 1347.8260498046875, "regularization/mmd": 0.19879385828971863, "regularization/rkhs_norm": 236.6593780517578, "regularization/w1": 1.1321738958358765, "rewards/chosen": 0.5931191926425204, "rewards/margins": 0.8781586901497564, "rewards/rejected": -0.28503949750723606, "step": 1700 }, { "epoch": 2.227979274611399, "grad_norm": 13.166388511657715, "kl": 17.485754013061523, "learning_rate": 1.6864278811393523e-07, "logits/chosen": -34458167.439490445, "logits/rejected": -35242247.85276074, "logps/chosen": -442.3670382165605, "logps/rejected": -391.0656154141104, "loss": 0.5546, "loss/base_kto": 3.207674503326416, "metrics/advantage_var": 432.341064453125, "regularization/lipschitz_norm": 1239.1695556640625, "regularization/mmd": 0.18834565579891205, "regularization/rkhs_norm": 224.2209930419922, "regularization/w1": 1.0409023761749268, "rewards/chosen": 0.5516619348222283, "rewards/margins": 0.8118083274572695, "rewards/rejected": -0.26014639263504125, "step": 1720 }, { "epoch": 2.2538860103626943, "grad_norm": 15.19034194946289, "kl": 10.276593208312988, "learning_rate": 1.5816073811412584e-07, "logits/chosen": -34619947.60883281, "logits/rejected": -35284015.554179564, "logps/chosen": -468.8888505520505, "logps/rejected": -375.39357585139317, "loss": 0.5575, "loss/base_kto": 3.203594923019409, "metrics/advantage_var": 395.5413513183594, "regularization/lipschitz_norm": 1271.7320556640625, "regularization/mmd": 0.18793752789497375, "regularization/rkhs_norm": 223.7351531982422, "regularization/w1": 1.0682549476623535, "rewards/chosen": 0.45097327006728116, "rewards/margins": 0.8379109340874533, "rewards/rejected": -0.3869376640201722, "step": 1740 }, { "epoch": 2.2797927461139897, "grad_norm": 13.577218055725098, "kl": 11.351633071899414, "learning_rate": 1.4795348580020207e-07, "logits/chosen": -34957907.96141479, "logits/rejected": -36514862.68693009, "logps/chosen": -495.5143689710611, "logps/rejected": -358.3359612462006, "loss": 0.5599, "loss/base_kto": 3.219320774078369, "metrics/advantage_var": 407.5980224609375, "regularization/lipschitz_norm": 1273.7225341796875, "regularization/mmd": 0.18991895020008087, "regularization/rkhs_norm": 226.09402465820312, "regularization/w1": 1.0699270963668823, "rewards/chosen": 0.4134593117275422, "rewards/margins": 0.8221395869460073, "rewards/rejected": -0.40868027521846506, "step": 1760 }, { "epoch": 2.305699481865285, "grad_norm": 17.006799697875977, "kl": 13.061903953552246, "learning_rate": 1.3802923657636355e-07, "logits/chosen": -35450206.77112135, "logits/rejected": -36295626.27662957, "logps/chosen": -459.8193644393241, "logps/rejected": -370.8439487281399, "loss": 0.55, "loss/base_kto": 3.189333438873291, "metrics/advantage_var": 382.8031311035156, "regularization/lipschitz_norm": 1222.7635498046875, "regularization/mmd": 0.18362049758434296, "regularization/rkhs_norm": 218.59580993652344, "regularization/w1": 1.0271214246749878, "rewards/chosen": 0.490606211297523, "rewards/margins": 0.8540748120573476, "rewards/rejected": -0.3634686007598246, "step": 1780 }, { "epoch": 2.33160621761658, "grad_norm": 17.623613357543945, "kl": 14.34494686126709, "learning_rate": 1.28395968346336e-07, "logits/chosen": -34216982.47021943, "logits/rejected": -34302519.82554517, "logps/chosen": -480.9950039184953, "logps/rejected": -356.5640332943925, "loss": 0.5585, "loss/base_kto": 3.2039711475372314, "metrics/advantage_var": 440.133544921875, "regularization/lipschitz_norm": 1274.4517822265625, "regularization/mmd": 0.19310761988162994, "regularization/rkhs_norm": 229.8900146484375, "regularization/w1": 1.0705394744873047, "rewards/chosen": 0.5658861118424275, "rewards/margins": 0.8475557523092305, "rewards/rejected": -0.281669640466803, "step": 1800 }, { "epoch": 2.3575129533678756, "grad_norm": 18.474836349487305, "kl": 11.145959854125977, "learning_rate": 1.1906142510009415e-07, "logits/chosen": -33727795.67334361, "logits/rejected": -36828096.70998415, "logps/chosen": -513.0128563174114, "logps/rejected": -379.7720384310618, "loss": 0.5523, "loss/base_kto": 3.1854116916656494, "metrics/advantage_var": 385.63336181640625, "regularization/lipschitz_norm": 1243.1856689453125, "regularization/mmd": 0.18834401667118073, "regularization/rkhs_norm": 224.21910095214844, "regularization/w1": 1.0442759990692139, "rewards/chosen": 0.539585765962057, "rewards/margins": 0.837411381266076, "rewards/rejected": -0.2978256153040189, "step": 1820 }, { "epoch": 2.383419689119171, "grad_norm": 16.720338821411133, "kl": 11.830164909362793, "learning_rate": 1.1003311068862547e-07, "logits/chosen": -34697306.55782313, "logits/rejected": -35999335.58381503, "logps/chosen": -488.01615646258506, "logps/rejected": -391.93781611271675, "loss": 0.5543, "loss/base_kto": 3.1345040798187256, "metrics/advantage_var": 429.4234924316406, "regularization/lipschitz_norm": 1317.3709716796875, "regularization/mmd": 0.19330470263957977, "regularization/rkhs_norm": 230.12466430664062, "regularization/w1": 1.1065915822982788, "rewards/chosen": 0.4993897522387861, "rewards/margins": 0.8876647223209755, "rewards/rejected": -0.3882749700821893, "step": 1840 }, { "epoch": 2.4093264248704664, "grad_norm": 17.21449851989746, "kl": 8.917579650878906, "learning_rate": 1.0131828279173588e-07, "logits/chosen": -34767588.5568, "logits/rejected": -35621611.28549618, "logps/chosen": -501.3483, "logps/rejected": -380.98365935114504, "loss": 0.5667, "loss/base_kto": 3.2373528480529785, "metrics/advantage_var": 441.436767578125, "regularization/lipschitz_norm": 1307.6292724609375, "regularization/mmd": 0.1978851854801178, "regularization/rkhs_norm": 235.5775909423828, "regularization/w1": 1.098408579826355, "rewards/chosen": 0.37561064453125, "rewards/margins": 0.8065592426795085, "rewards/rejected": -0.4309485981482586, "step": 1860 }, { "epoch": 2.4352331606217614, "grad_norm": 13.06633472442627, "kl": 8.588080406188965, "learning_rate": 9.292394708374596e-08, "logits/chosen": -34330590.69918699, "logits/rejected": -35925837.37744361, "logps/chosen": -493.2400406504065, "logps/rejected": -349.3700187969925, "loss": 0.5537, "loss/base_kto": 3.1561896800994873, "metrics/advantage_var": 422.1860046386719, "regularization/lipschitz_norm": 1285.82568359375, "regularization/mmd": 0.19351409375667572, "regularization/rkhs_norm": 230.37393188476562, "regularization/w1": 1.0800936222076416, "rewards/chosen": 0.4347711826727642, "rewards/margins": 0.886212346617783, "rewards/rejected": -0.4514411639450188, "step": 1880 }, { "epoch": 2.461139896373057, "grad_norm": 13.171441078186035, "kl": 8.947189331054688, "learning_rate": 8.48568516017727e-08, "logits/chosen": -34306215.24807396, "logits/rejected": -36405744.58320127, "logps/chosen": -476.3931047765794, "logps/rejected": -364.52929377971475, "loss": 0.553, "loss/base_kto": 3.2210700511932373, "metrics/advantage_var": 362.1111145019531, "regularization/lipschitz_norm": 1216.04833984375, "regularization/mmd": 0.18131333589553833, "regularization/rkhs_norm": 215.84922790527344, "regularization/w1": 1.021480679512024, "rewards/chosen": 0.4055867481672525, "rewards/margins": 0.8161228477054359, "rewards/rejected": -0.41053609953818343, "step": 1900 }, { "epoch": 2.4870466321243523, "grad_norm": 14.532591819763184, "kl": 8.963698387145996, "learning_rate": 7.71234813211169e-08, "logits/chosen": -33834569.571847506, "logits/rejected": -34866480.80267558, "logps/chosen": -479.7925219941349, "logps/rejected": -356.2524561036789, "loss": 0.553, "loss/base_kto": 3.202587127685547, "metrics/advantage_var": 395.7680358886719, "regularization/lipschitz_norm": 1235.2418212890625, "regularization/mmd": 0.18388526141643524, "regularization/rkhs_norm": 218.9110107421875, "regularization/w1": 1.0376030206680298, "rewards/chosen": 0.4610707571080004, "rewards/margins": 0.85213384765886, "rewards/rejected": -0.39106309055085964, "step": 1920 }, { "epoch": 2.5129533678756477, "grad_norm": 16.621479034423828, "kl": 13.260581016540527, "learning_rate": 6.973005294212353e-08, "logits/chosen": -33507639.51807229, "logits/rejected": -34799286.85714286, "logps/chosen": -488.0190135542169, "logps/rejected": -379.2658532873377, "loss": 0.5604, "loss/base_kto": 3.1591362953186035, "metrics/advantage_var": 432.0214538574219, "regularization/lipschitz_norm": 1340.5941162109375, "regularization/mmd": 0.19805912673473358, "regularization/rkhs_norm": 235.78466796875, "regularization/w1": 1.1260989904403687, "rewards/chosen": 0.5622578816241529, "rewards/margins": 0.8873437569000001, "rewards/rejected": -0.3250858752758472, "step": 1940 }, { "epoch": 2.538860103626943, "grad_norm": 15.701424598693848, "kl": 14.250332832336426, "learning_rate": 6.268250989270102e-08, "logits/chosen": -35499924.21052632, "logits/rejected": -35915858.28571428, "logps/chosen": -473.46273643092104, "logps/rejected": -372.7054501488095, "loss": 0.5429, "loss/base_kto": 3.1128509044647217, "metrics/advantage_var": 417.1114196777344, "regularization/lipschitz_norm": 1239.255615234375, "regularization/mmd": 0.1897449940443039, "regularization/rkhs_norm": 225.8868865966797, "regularization/w1": 1.0409746170043945, "rewards/chosen": 0.5564124960648386, "rewards/margins": 0.9140466090133017, "rewards/rejected": -0.3576341129484631, "step": 1960 }, { "epoch": 2.5647668393782386, "grad_norm": 15.832449913024902, "kl": 14.343947410583496, "learning_rate": 5.598651755051975e-08, "logits/chosen": -34636168.50695518, "logits/rejected": -35802016.55608215, "logps/chosen": -473.04337326120555, "logps/rejected": -390.03134873617694, "loss": 0.5578, "loss/base_kto": 3.1775963306427, "metrics/advantage_var": 383.69873046875, "regularization/lipschitz_norm": 1305.6473388671875, "regularization/mmd": 0.18771111965179443, "regularization/rkhs_norm": 223.4656219482422, "regularization/w1": 1.0967435836791992, "rewards/chosen": 0.5248484781020576, "rewards/margins": 0.8445120577204422, "rewards/rejected": -0.31966357961838465, "step": 1980 }, { "epoch": 2.5906735751295336, "grad_norm": 16.625328063964844, "kl": 14.722457885742188, "learning_rate": 4.964745868872933e-08, "logits/chosen": -34853143.27272727, "logits/rejected": -35681442.69158878, "logps/chosen": -484.8646159874608, "logps/rejected": -366.8214563862928, "loss": 0.5522, "loss/base_kto": 3.22148060798645, "metrics/advantage_var": 375.38916015625, "regularization/lipschitz_norm": 1205.2469482421875, "regularization/mmd": 0.1837475746870041, "regularization/rkhs_norm": 218.74710083007812, "regularization/w1": 1.0124075412750244, "rewards/chosen": 0.5078481835631368, "rewards/margins": 0.8058288991157312, "rewards/rejected": -0.2979807155525944, "step": 2000 }, { "epoch": 2.616580310880829, "grad_norm": 15.182455062866211, "kl": 13.751710891723633, "learning_rate": 4.3670429148855493e-08, "logits/chosen": -34762249.721518986, "logits/rejected": -36049483.85185185, "logps/chosen": -489.95352056962025, "logps/rejected": -373.3676456404321, "loss": 0.5569, "loss/base_kto": 3.205108404159546, "metrics/advantage_var": 401.9645690917969, "regularization/lipschitz_norm": 1263.1134033203125, "regularization/mmd": 0.18922996520996094, "regularization/rkhs_norm": 225.27378845214844, "regularization/w1": 1.061015248298645, "rewards/chosen": 0.4993312690831438, "rewards/margins": 0.8156143655850004, "rewards/rejected": -0.31628309650185665, "step": 2020 }, { "epoch": 2.6424870466321244, "grad_norm": 14.551630973815918, "kl": 12.297151565551758, "learning_rate": 3.806023374435663e-08, "logits/chosen": -35766256.12403101, "logits/rejected": -35776399.11811024, "logps/chosen": -490.0437015503876, "logps/rejected": -368.00113188976377, "loss": 0.558, "loss/base_kto": 3.2429046630859375, "metrics/advantage_var": 408.36981201171875, "regularization/lipschitz_norm": 1229.8167724609375, "regularization/mmd": 0.18843434751033783, "regularization/rkhs_norm": 224.3266143798828, "regularization/w1": 1.0330461263656616, "rewards/chosen": 0.5068759652071221, "rewards/margins": 0.8017690527594105, "rewards/rejected": -0.2948930875522884, "step": 2040 }, { "epoch": 2.66839378238342, "grad_norm": 15.871299743652344, "kl": 13.784205436706543, "learning_rate": 3.282138239813037e-08, "logits/chosen": -32945674.6496, "logits/rejected": -35192819.49312977, "logps/chosen": -451.0255, "logps/rejected": -388.55708492366415, "loss": 0.5493, "loss/base_kto": 3.1717541217803955, "metrics/advantage_var": 378.71173095703125, "regularization/lipschitz_norm": 1236.295166015625, "regularization/mmd": 0.18411408364772797, "regularization/rkhs_norm": 219.18345642089844, "regularization/w1": 1.0384880304336548, "rewards/chosen": 0.5057248046875, "rewards/margins": 0.8527784839426289, "rewards/rejected": -0.3470536792551288, "step": 2060 }, { "epoch": 2.694300518134715, "grad_norm": 18.033416748046875, "kl": 13.068841934204102, "learning_rate": 2.795808651707793e-08, "logits/chosen": -34780834.025316454, "logits/rejected": -36113515.45679013, "logps/chosen": -458.52037183544303, "logps/rejected": -364.9637345679012, "loss": 0.5534, "loss/base_kto": 3.164045572280884, "metrics/advantage_var": 413.14263916015625, "regularization/lipschitz_norm": 1274.2684326171875, "regularization/mmd": 0.19258885085582733, "regularization/rkhs_norm": 229.2724609375, "regularization/w1": 1.0703853368759155, "rewards/chosen": 0.5223610793487935, "rewards/margins": 0.877933485505357, "rewards/rejected": -0.3555724061565635, "step": 2080 }, { "epoch": 2.7202072538860103, "grad_norm": 18.45494842529297, "kl": 13.948599815368652, "learning_rate": 2.3474255606639293e-08, "logits/chosen": -34890844.25225225, "logits/rejected": -36268308.84690554, "logps/chosen": -501.97475600600603, "logps/rejected": -373.98310260586317, "loss": 0.5615, "loss/base_kto": 3.147346258163452, "metrics/advantage_var": 466.57666015625, "regularization/lipschitz_norm": 1360.3629150390625, "regularization/mmd": 0.20164580643177032, "regularization/rkhs_norm": 240.05455017089844, "regularization/w1": 1.1427048444747925, "rewards/chosen": 0.5498876256627722, "rewards/margins": 0.8893085034238675, "rewards/rejected": -0.33942087776109525, "step": 2100 }, { "epoch": 2.7461139896373057, "grad_norm": 14.303071022033691, "kl": 12.555839538574219, "learning_rate": 1.9373494128020195e-08, "logits/chosen": -33900531.27950311, "logits/rejected": -34763418.56603774, "logps/chosen": -474.8220108695652, "logps/rejected": -383.9449685534591, "loss": 0.5536, "loss/base_kto": 3.195784568786621, "metrics/advantage_var": 405.81805419921875, "regularization/lipschitz_norm": 1243.735595703125, "regularization/mmd": 0.18844656646251678, "regularization/rkhs_norm": 224.34117126464844, "regularization/w1": 1.0447378158569336, "rewards/chosen": 0.528636482191382, "rewards/margins": 0.8448140422898982, "rewards/rejected": -0.3161775600985161, "step": 2120 }, { "epoch": 2.772020725388601, "grad_norm": 17.929088592529297, "kl": 13.852238655090332, "learning_rate": 1.5659098600638798e-08, "logits/chosen": -34382224.55520505, "logits/rejected": -34066194.229102165, "logps/chosen": -477.84118690851733, "logps/rejected": -374.51456075851394, "loss": 0.562, "loss/base_kto": 3.195948839187622, "metrics/advantage_var": 431.516845703125, "regularization/lipschitz_norm": 1316.0577392578125, "regularization/mmd": 0.19486133754253387, "regularization/rkhs_norm": 231.97781372070312, "regularization/w1": 1.1054884195327759, "rewards/chosen": 0.5255168349208892, "rewards/margins": 0.8452415644765683, "rewards/rejected": -0.31972472955567915, "step": 2140 }, { "epoch": 2.7979274611398965, "grad_norm": 17.644804000854492, "kl": 13.09256649017334, "learning_rate": 1.2334054952120143e-08, "logits/chosen": -34767201.215189874, "logits/rejected": -36606381.82716049, "logps/chosen": -478.00820806962025, "logps/rejected": -386.5558690200617, "loss": 0.5583, "loss/base_kto": 3.164849042892456, "metrics/advantage_var": 428.4366149902344, "regularization/lipschitz_norm": 1319.111572265625, "regularization/mmd": 0.1931963413953781, "regularization/rkhs_norm": 229.9956512451172, "regularization/w1": 1.1080538034439087, "rewards/chosen": 0.5406829254536689, "rewards/margins": 0.8741082589241429, "rewards/rejected": -0.33342533347047404, "step": 2160 }, { "epoch": 2.823834196891192, "grad_norm": 18.839326858520508, "kl": 14.727572441101074, "learning_rate": 9.401036117969108e-09, "logits/chosen": -34945595.6108453, "logits/rejected": -36285121.6661562, "logps/chosen": -476.87764154704945, "logps/rejected": -378.3577239663093, "loss": 0.5696, "loss/base_kto": 3.226821184158325, "metrics/advantage_var": 441.74688720703125, "regularization/lipschitz_norm": 1350.0693359375, "regularization/mmd": 0.1956876814365387, "regularization/rkhs_norm": 232.96153259277344, "regularization/w1": 1.1340582370758057, "rewards/chosen": 0.471017791894064, "rewards/margins": 0.797263367078908, "rewards/rejected": -0.326245575184844, "step": 2180 }, { "epoch": 2.849740932642487, "grad_norm": 17.33658218383789, "kl": 13.067753791809082, "learning_rate": 6.8623998928517555e-09, "logits/chosen": -36326757.06298003, "logits/rejected": -36277031.47853736, "logps/chosen": -482.0709485407066, "logps/rejected": -391.3971333465819, "loss": 0.5539, "loss/base_kto": 3.179420232772827, "metrics/advantage_var": 400.2893981933594, "regularization/lipschitz_norm": 1266.2547607421875, "regularization/mmd": 0.18828484416007996, "regularization/rkhs_norm": 224.1486053466797, "regularization/w1": 1.0636539459228516, "rewards/chosen": 0.5448522142917147, "rewards/margins": 0.8471926990564081, "rewards/rejected": -0.30234048476469344, "step": 2200 }, { "epoch": 2.8756476683937824, "grad_norm": 15.574108123779297, "kl": 11.259978294372559, "learning_rate": 4.72018703521132e-09, "logits/chosen": -36309418.40729483, "logits/rejected": -35371538.10932476, "logps/chosen": -503.0312974924012, "logps/rejected": -383.38972065916397, "loss": 0.5635, "loss/base_kto": 3.139202833175659, "metrics/advantage_var": 470.6390686035156, "regularization/lipschitz_norm": 1390.9105224609375, "regularization/mmd": 0.2007589340209961, "regularization/rkhs_norm": 238.9987335205078, "regularization/w1": 1.1683648824691772, "rewards/chosen": 0.5424724915107333, "rewards/margins": 0.9181714519897828, "rewards/rejected": -0.37569896047904944, "step": 2220 }, { "epoch": 2.901554404145078, "grad_norm": 11.174159049987793, "kl": 12.35366439819336, "learning_rate": 2.976119626744267e-09, "logits/chosen": -34988064.2519685, "logits/rejected": -35116671.80155039, "logps/chosen": -463.37052165354334, "logps/rejected": -388.6034399224806, "loss": 0.5506, "loss/base_kto": 3.1795132160186768, "metrics/advantage_var": 378.6888427734375, "regularization/lipschitz_norm": 1238.9239501953125, "regularization/mmd": 0.18431107699871063, "regularization/rkhs_norm": 219.41796875, "regularization/w1": 1.0406960248947144, "rewards/chosen": 0.47429660586860234, "rewards/margins": 0.8348589998850753, "rewards/rejected": -0.3605623940164729, "step": 2240 }, { "epoch": 2.927461139896373, "grad_norm": 22.472700119018555, "kl": 13.033785820007324, "learning_rate": 1.631599688052765e-09, "logits/chosen": -35498326.998373985, "logits/rejected": -37163365.245112784, "logps/chosen": -489.7123983739837, "logps/rejected": -376.54320958646616, "loss": 0.5607, "loss/base_kto": 3.1383798122406006, "metrics/advantage_var": 430.47833251953125, "regularization/lipschitz_norm": 1368.0810546875, "regularization/mmd": 0.19792315363883972, "regularization/rkhs_norm": 235.622802734375, "regularization/w1": 1.1491881608963013, "rewards/chosen": 0.5409264045033029, "rewards/margins": 0.9076404001858687, "rewards/rejected": -0.3667139956825658, "step": 2260 }, { "epoch": 2.9533678756476682, "grad_norm": 20.603042602539062, "kl": 13.964932441711426, "learning_rate": 6.877080515894085e-10, "logits/chosen": -33562544.12480499, "logits/rejected": -35524364.41940532, "logps/chosen": -480.97723283931356, "logps/rejected": -365.04161776212834, "loss": 0.5519, "loss/base_kto": 3.180474042892456, "metrics/advantage_var": 403.8537292480469, "regularization/lipschitz_norm": 1242.5142822265625, "regularization/mmd": 0.19066281616687775, "regularization/rkhs_norm": 226.9795379638672, "regularization/w1": 1.0437120199203491, "rewards/chosen": 0.5153949041262432, "rewards/margins": 0.8494487197500207, "rewards/rejected": -0.3340538156237774, "step": 2280 }, { "epoch": 2.9792746113989637, "grad_norm": 14.573284149169922, "kl": 13.512062072753906, "learning_rate": 1.4520349279739663e-10, "logits/chosen": -34238379.6133122, "logits/rejected": -35476383.753466874, "logps/chosen": -480.55923137876385, "logps/rejected": -369.87403697996916, "loss": 0.5594, "loss/base_kto": 3.1754276752471924, "metrics/advantage_var": 431.06494140625, "regularization/lipschitz_norm": 1316.7503662109375, "regularization/mmd": 0.19363752007484436, "regularization/rkhs_norm": 230.5208740234375, "regularization/w1": 1.1060702800750732, "rewards/chosen": 0.5244395018757428, "rewards/margins": 0.8500996314169842, "rewards/rejected": -0.32566012954124135, "step": 2300 }, { "epoch": 3.0, "step": 2316, "total_flos": 9.978042558275912e+17, "train_loss": 0.5867222310759654, "train_runtime": 11075.4163, "train_samples_per_second": 13.383, "train_steps_per_second": 0.209 } ], "logging_steps": 20, "max_steps": 2316, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 9.978042558275912e+17, "train_batch_size": 8, "trial_name": null, "trial_params": null }