{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 300, "global_step": 927, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.010794764539198488, "grad_norm": 24.147397994995117, "learning_rate": 4.8387096774193546e-08, "logits/chosen": -1.4572813510894775, "logits/rejected": -1.5238107442855835, "logps/chosen": -328.8013610839844, "logps/rejected": -253.98362731933594, "loss": 0.6928, "metrics/advantage_var": 0.2102208137512207, "rewards/accuracies": 0.40937501192092896, "rewards/chosen": 0.0015822252025827765, "rewards/margins": 0.0017194593092426658, "rewards/rejected": -0.0001372337428620085, "step": 10 }, { "epoch": 0.021589529078396976, "grad_norm": 25.48040199279785, "learning_rate": 1.0215053763440861e-07, "logits/chosen": -1.4601737260818481, "logits/rejected": -1.495823860168457, "logps/chosen": -329.26373291015625, "logps/rejected": -269.0686950683594, "loss": 0.6933, "metrics/advantage_var": 0.26926887035369873, "rewards/accuracies": 0.5140625238418579, "rewards/chosen": 0.0026884928811341524, "rewards/margins": 0.0011197883868589997, "rewards/rejected": 0.0015687048435211182, "stability/repetition_rate_mean": 0.8257812261581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.434472680091858, "step": 20 }, { "epoch": 0.03238429361759547, "grad_norm": 19.910463333129883, "learning_rate": 1.5591397849462365e-07, "logits/chosen": -1.4658056497573853, "logits/rejected": -1.5197511911392212, "logps/chosen": -316.17425537109375, "logps/rejected": -256.8540954589844, "loss": 0.6925, "metrics/advantage_var": 0.24051065742969513, "rewards/accuracies": 0.5140625238418579, "rewards/chosen": 0.004414338618516922, "rewards/margins": 0.0025372428353875875, "rewards/rejected": 0.0018770955502986908, "stability/repetition_rate_mean": 0.8824218511581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.364843726158142, "step": 30 }, { "epoch": 0.04317905815679395, "grad_norm": 23.751585006713867, "learning_rate": 2.0967741935483871e-07, "logits/chosen": -1.4785693883895874, "logits/rejected": -1.5407034158706665, "logps/chosen": -329.3841247558594, "logps/rejected": -258.57672119140625, "loss": 0.6913, "metrics/advantage_var": 0.2763865292072296, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.005997015163302422, "rewards/margins": 0.005208877846598625, "rewards/rejected": 0.0007881380734033883, "stability/repetition_rate_mean": 0.8031250238418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.0068359375, "step": 40 }, { "epoch": 0.05397382269599244, "grad_norm": 20.962650299072266, "learning_rate": 2.6344086021505376e-07, "logits/chosen": -1.438301682472229, "logits/rejected": -1.503927230834961, "logps/chosen": -313.481689453125, "logps/rejected": -258.3254089355469, "loss": 0.6885, "metrics/advantage_var": 0.281412810087204, "rewards/accuracies": 0.546875, "rewards/chosen": 0.01573883555829525, "rewards/margins": 0.010696313343942165, "rewards/rejected": 0.005042524076998234, "stability/repetition_rate_mean": 0.8675781488418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.07568359375, "step": 50 }, { "epoch": 0.06476858723519094, "grad_norm": 20.89899444580078, "learning_rate": 3.172043010752688e-07, "logits/chosen": -1.4596624374389648, "logits/rejected": -1.501914620399475, "logps/chosen": -318.685302734375, "logps/rejected": -265.6797180175781, "loss": 0.6846, "metrics/advantage_var": 0.43760356307029724, "rewards/accuracies": 0.604687511920929, "rewards/chosen": 0.02920054830610752, "rewards/margins": 0.019200993701815605, "rewards/rejected": 0.009999553672969341, "stability/repetition_rate_mean": 0.7318359613418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.2527344226837158, "step": 60 }, { "epoch": 0.07556335177438941, "grad_norm": 22.085494995117188, "learning_rate": 3.7096774193548384e-07, "logits/chosen": -1.5026830434799194, "logits/rejected": -1.5432065725326538, "logps/chosen": -339.0155334472656, "logps/rejected": -273.65655517578125, "loss": 0.6787, "metrics/advantage_var": 0.8573139309883118, "rewards/accuracies": 0.609375, "rewards/chosen": 0.06427477300167084, "rewards/margins": 0.03314407169818878, "rewards/rejected": 0.03113069012761116, "stability/repetition_rate_mean": 0.8822265863418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.3330078125, "step": 70 }, { "epoch": 0.0863581163135879, "grad_norm": 19.948606491088867, "learning_rate": 4.247311827956989e-07, "logits/chosen": -1.4859027862548828, "logits/rejected": -1.5006340742111206, "logps/chosen": -325.33526611328125, "logps/rejected": -254.3518524169922, "loss": 0.6616, "metrics/advantage_var": 1.5368133783340454, "rewards/accuracies": 0.643750011920929, "rewards/chosen": 0.1002170592546463, "rewards/margins": 0.07156896591186523, "rewards/rejected": 0.028648091480135918, "stability/repetition_rate_mean": 0.8583984375, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.825048804283142, "step": 80 }, { "epoch": 0.0971528808527864, "grad_norm": 16.032991409301758, "learning_rate": 4.78494623655914e-07, "logits/chosen": -1.4689861536026, "logits/rejected": -1.4905648231506348, "logps/chosen": -295.8207702636719, "logps/rejected": -248.85952758789062, "loss": 0.6497, "metrics/advantage_var": 3.12138295173645, "rewards/accuracies": 0.6953125, "rewards/chosen": 0.14759615063667297, "rewards/margins": 0.10207539796829224, "rewards/rejected": 0.04552074521780014, "stability/repetition_rate_mean": 0.8740234375, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.663671851158142, "step": 90 }, { "epoch": 0.10794764539198488, "grad_norm": 19.689159393310547, "learning_rate": 4.999361498869529e-07, "logits/chosen": -1.4919625520706177, "logits/rejected": -1.5506508350372314, "logps/chosen": -314.435791015625, "logps/rejected": -249.51695251464844, "loss": 0.6413, "metrics/advantage_var": 5.264100551605225, "rewards/accuracies": 0.6890625357627869, "rewards/chosen": 0.19397814571857452, "rewards/margins": 0.12933655083179474, "rewards/rejected": 0.06464159488677979, "stability/repetition_rate_mean": 0.9400390386581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.4281249046325684, "step": 100 }, { "epoch": 0.11874240993118337, "grad_norm": 17.759952545166016, "learning_rate": 4.995460728562402e-07, "logits/chosen": -1.4327154159545898, "logits/rejected": -1.5077978372573853, "logps/chosen": -319.29156494140625, "logps/rejected": -253.4399871826172, "loss": 0.6214, "metrics/advantage_var": 8.01412582397461, "rewards/accuracies": 0.714062511920929, "rewards/chosen": 0.24892178177833557, "rewards/margins": 0.18393009901046753, "rewards/rejected": 0.06499166041612625, "stability/repetition_rate_mean": 0.809374988079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.246875047683716, "step": 110 }, { "epoch": 0.12953717447038188, "grad_norm": 20.968534469604492, "learning_rate": 4.988019438437758e-07, "logits/chosen": -1.4564826488494873, "logits/rejected": -1.5484386682510376, "logps/chosen": -330.7845764160156, "logps/rejected": -248.1776885986328, "loss": 0.6032, "metrics/advantage_var": 14.087158203125, "rewards/accuracies": 0.739062488079071, "rewards/chosen": 0.3554464876651764, "rewards/margins": 0.2473072111606598, "rewards/rejected": 0.1081392914056778, "stability/repetition_rate_mean": 0.7470703125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.070214867591858, "step": 120 }, { "epoch": 0.14033193900958035, "grad_norm": 19.773218154907227, "learning_rate": 4.977048186079155e-07, "logits/chosen": -1.4577782154083252, "logits/rejected": -1.508943796157837, "logps/chosen": -317.5703125, "logps/rejected": -253.7754364013672, "loss": 0.5951, "metrics/advantage_var": 16.9871883392334, "rewards/accuracies": 0.721875011920929, "rewards/chosen": 0.40177080035209656, "rewards/margins": 0.28298041224479675, "rewards/rejected": 0.11879038065671921, "stability/repetition_rate_mean": 0.773242175579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.4833984375, "step": 130 }, { "epoch": 0.15112670354877883, "grad_norm": 13.235716819763184, "learning_rate": 4.962562537324176e-07, "logits/chosen": -1.4774430990219116, "logits/rejected": -1.512833833694458, "logps/chosen": -343.68548583984375, "logps/rejected": -272.2403869628906, "loss": 0.6036, "metrics/advantage_var": 24.688919067382812, "rewards/accuracies": 0.6890625357627869, "rewards/chosen": 0.42533954977989197, "rewards/margins": 0.2894915044307709, "rewards/rejected": 0.1358480602502823, "stability/repetition_rate_mean": 0.8072265386581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.019140601158142, "step": 140 }, { "epoch": 0.16192146808797733, "grad_norm": 19.26375961303711, "learning_rate": 4.944583044179871e-07, "logits/chosen": -1.4802706241607666, "logits/rejected": -1.5120983123779297, "logps/chosen": -320.7796936035156, "logps/rejected": -268.3492126464844, "loss": 0.6074, "metrics/advantage_var": 21.159940719604492, "rewards/accuracies": 0.6968750357627869, "rewards/chosen": 0.412137508392334, "rewards/margins": 0.2721858024597168, "rewards/rejected": 0.13995173573493958, "stability/repetition_rate_mean": 0.807421863079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.5673828125, "step": 150 }, { "epoch": 0.1727162326271758, "grad_norm": 24.66869354248047, "learning_rate": 4.923135215663896e-07, "logits/chosen": -1.479994773864746, "logits/rejected": -1.5399045944213867, "logps/chosen": -317.5917663574219, "logps/rejected": -243.84231567382812, "loss": 0.5807, "metrics/advantage_var": 26.341684341430664, "rewards/accuracies": 0.7484375238418579, "rewards/chosen": 0.46426326036453247, "rewards/margins": 0.3650646209716797, "rewards/rejected": 0.09919857233762741, "stability/repetition_rate_mean": 0.789257824420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.4421875476837158, "step": 160 }, { "epoch": 0.1835109971663743, "grad_norm": 22.35947036743164, "learning_rate": 4.89824948161273e-07, "logits/chosen": -1.4407405853271484, "logits/rejected": -1.5115913152694702, "logps/chosen": -308.2352600097656, "logps/rejected": -256.36358642578125, "loss": 0.5752, "metrics/advantage_var": 30.53228759765625, "rewards/accuracies": 0.706250011920929, "rewards/chosen": 0.47893524169921875, "rewards/margins": 0.39258310198783875, "rewards/rejected": 0.08635219186544418, "stability/repetition_rate_mean": 0.7845703363418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.5763671398162842, "step": 170 }, { "epoch": 0.1943057617055728, "grad_norm": 14.936376571655273, "learning_rate": 4.8699611495083e-07, "logits/chosen": -1.504378318786621, "logits/rejected": -1.5372484922409058, "logps/chosen": -317.1926574707031, "logps/rejected": -261.8941650390625, "loss": 0.5744, "metrics/advantage_var": 36.89189910888672, "rewards/accuracies": 0.7109375, "rewards/chosen": 0.5145285725593567, "rewards/margins": 0.40907564759254456, "rewards/rejected": 0.10545291751623154, "stability/repetition_rate_mean": 0.7490234375, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.5693359375, "step": 180 }, { "epoch": 0.2051005262447713, "grad_norm": 20.977313995361328, "learning_rate": 4.838310354384302e-07, "logits/chosen": -1.4963879585266113, "logits/rejected": -1.5239351987838745, "logps/chosen": -307.32684326171875, "logps/rejected": -263.76373291015625, "loss": 0.5629, "metrics/advantage_var": 36.270076751708984, "rewards/accuracies": 0.707812488079071, "rewards/chosen": 0.5523476600646973, "rewards/margins": 0.4342571198940277, "rewards/rejected": 0.11809053272008896, "stability/repetition_rate_mean": 0.850781261920929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.1296875476837158, "step": 190 }, { "epoch": 0.21589529078396977, "grad_norm": 13.247014045715332, "learning_rate": 4.803342001883246e-07, "logits/chosen": -1.5078216791152954, "logits/rejected": -1.541197419166565, "logps/chosen": -318.7608337402344, "logps/rejected": -267.5777282714844, "loss": 0.5524, "metrics/advantage_var": 35.416255950927734, "rewards/accuracies": 0.745312511920929, "rewards/chosen": 0.556470513343811, "rewards/margins": 0.4586350917816162, "rewards/rejected": 0.09783537685871124, "stability/repetition_rate_mean": 0.796679675579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.4445312023162842, "step": 200 }, { "epoch": 0.22669005532316827, "grad_norm": 16.3123722076416, "learning_rate": 4.7651057045450515e-07, "logits/chosen": -1.4529699087142944, "logits/rejected": -1.5274174213409424, "logps/chosen": -326.5751647949219, "logps/rejected": -245.41970825195312, "loss": 0.5515, "metrics/advantage_var": 36.527748107910156, "rewards/accuracies": 0.7484375238418579, "rewards/chosen": 0.5690556764602661, "rewards/margins": 0.4599929749965668, "rewards/rejected": 0.10906264930963516, "stability/repetition_rate_mean": 0.81640625, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.3254883289337158, "step": 210 }, { "epoch": 0.23748481986236675, "grad_norm": 21.77730369567871, "learning_rate": 4.72365571141757e-07, "logits/chosen": -1.4564087390899658, "logits/rejected": -1.4975181818008423, "logps/chosen": -331.7127380371094, "logps/rejected": -264.9964294433594, "loss": 0.5548, "metrics/advantage_var": 43.936622619628906, "rewards/accuracies": 0.746874988079071, "rewards/chosen": 0.5794717669487, "rewards/margins": 0.4975171983242035, "rewards/rejected": 0.08195456117391586, "stability/repetition_rate_mean": 0.7367187738418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.763671875, "step": 220 }, { "epoch": 0.24827958440156525, "grad_norm": 18.723806381225586, "learning_rate": 4.6790508310889007e-07, "logits/chosen": -1.4929759502410889, "logits/rejected": -1.5188223123550415, "logps/chosen": -311.7873229980469, "logps/rejected": -257.8661193847656, "loss": 0.5575, "metrics/advantage_var": 45.32672882080078, "rewards/accuracies": 0.7359375357627869, "rewards/chosen": 0.5666539072990417, "rewards/margins": 0.48853203654289246, "rewards/rejected": 0.07812191545963287, "stability/repetition_rate_mean": 0.8841797113418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.4400391578674316, "step": 230 }, { "epoch": 0.25907434894076375, "grad_norm": 12.753899574279785, "learning_rate": 4.6313543482507056e-07, "logits/chosen": -1.4668247699737549, "logits/rejected": -1.50505530834198, "logps/chosen": -319.4496154785156, "logps/rejected": -261.7314453125, "loss": 0.5496, "metrics/advantage_var": 46.35799789428711, "rewards/accuracies": 0.7281250357627869, "rewards/chosen": 0.5762147307395935, "rewards/margins": 0.5230244398117065, "rewards/rejected": 0.05319035053253174, "stability/repetition_rate_mean": 0.802539050579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.0635743141174316, "step": 240 }, { "epoch": 0.26986911347996223, "grad_norm": 17.50574493408203, "learning_rate": 4.580633933910901e-07, "logits/chosen": -1.436219334602356, "logits/rejected": -1.46418035030365, "logps/chosen": -305.2557678222656, "logps/rejected": -253.59902954101562, "loss": 0.5612, "metrics/advantage_var": 47.63227081298828, "rewards/accuracies": 0.723437488079071, "rewards/chosen": 0.5559987425804138, "rewards/margins": 0.4999403655529022, "rewards/rejected": 0.0560583770275116, "stability/repetition_rate_mean": 0.811718761920929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.45166015625, "step": 250 }, { "epoch": 0.2806638780191607, "grad_norm": 15.86179256439209, "learning_rate": 4.526961549383108e-07, "logits/chosen": -1.4230726957321167, "logits/rejected": -1.4305095672607422, "logps/chosen": -325.4045104980469, "logps/rejected": -272.9486389160156, "loss": 0.5574, "metrics/advantage_var": 50.00086212158203, "rewards/accuracies": 0.7203125357627869, "rewards/chosen": 0.6053926348686218, "rewards/margins": 0.5237736105918884, "rewards/rejected": 0.08161897957324982, "stability/repetition_rate_mean": 0.813281238079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.894628882408142, "step": 260 }, { "epoch": 0.2914586425583592, "grad_norm": 15.887772560119629, "learning_rate": 4.470413344189098e-07, "logits/chosen": -1.4393119812011719, "logits/rejected": -1.467215895652771, "logps/chosen": -323.3600158691406, "logps/rejected": -270.2887878417969, "loss": 0.5356, "metrics/advantage_var": 55.44666290283203, "rewards/accuracies": 0.7437500357627869, "rewards/chosen": 0.6630539894104004, "rewards/margins": 0.5985837578773499, "rewards/rejected": 0.06447024643421173, "stability/repetition_rate_mean": 0.828906238079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.030859351158142, "step": 270 }, { "epoch": 0.30225340709755766, "grad_norm": 15.110116958618164, "learning_rate": 4.4110695480190597e-07, "logits/chosen": -1.4905976057052612, "logits/rejected": -1.5187627077102661, "logps/chosen": -330.3580627441406, "logps/rejected": -267.19561767578125, "loss": 0.5221, "metrics/advantage_var": 60.70363235473633, "rewards/accuracies": 0.7671875357627869, "rewards/chosen": 0.7377834916114807, "rewards/margins": 0.6288355588912964, "rewards/rejected": 0.10894794762134552, "stability/repetition_rate_mean": 0.8851562738418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.202734351158142, "step": 280 }, { "epoch": 0.3130481716367562, "grad_norm": 15.053842544555664, "learning_rate": 4.3490143569030017e-07, "logits/chosen": -1.4678796529769897, "logits/rejected": -1.5179208517074585, "logps/chosen": -320.11309814453125, "logps/rejected": -252.4005584716797, "loss": 0.5299, "metrics/advantage_var": 55.2855110168457, "rewards/accuracies": 0.746874988079071, "rewards/chosen": 0.6983198523521423, "rewards/margins": 0.5931033492088318, "rewards/rejected": 0.10521648079156876, "stability/repetition_rate_mean": 0.8099609613418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.6193358898162842, "step": 290 }, { "epoch": 0.32384293617595467, "grad_norm": 21.189123153686523, "learning_rate": 4.284335813754769e-07, "logits/chosen": -1.4394444227218628, "logits/rejected": -1.4822733402252197, "logps/chosen": -321.5018005371094, "logps/rejected": -276.88409423828125, "loss": 0.5031, "metrics/advantage_var": 60.6578483581543, "rewards/accuracies": 0.770312488079071, "rewards/chosen": 0.6781904101371765, "rewards/margins": 0.6781806349754333, "rewards/rejected": 9.752810001373291e-06, "stability/repetition_rate_mean": 0.7289062738418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.1236329078674316, "step": 300 }, { "epoch": 0.32384293617595467, "eval_logits/chosen": -1.4329925775527954, "eval_logits/rejected": -1.4661825895309448, "eval_logps/chosen": -319.7928161621094, "eval_logps/rejected": -254.64100646972656, "eval_loss": 0.513998806476593, "eval_metrics/advantage_var": 65.03108215332031, "eval_rewards/accuracies": 0.7623737454414368, "eval_rewards/chosen": 0.7029823660850525, "eval_rewards/margins": 0.680192232131958, "eval_rewards/rejected": 0.022790152579545975, "eval_runtime": 272.245, "eval_samples_per_second": 14.524, "eval_stability/repetition_rate_mean": 0.9048827886581421, "eval_stability/response_length_mean": 512.0, "eval_stability/response_length_std": 0.0, "eval_stability/response_length_var": 0.0, "eval_stability/token_entropy_mean": 1.9105956554412842, "eval_steps_per_second": 1.818, "step": 300 }, { "epoch": 0.33463770071515314, "grad_norm": 20.66907501220703, "learning_rate": 4.217125683458161e-07, "logits/chosen": -1.4233049154281616, "logits/rejected": -1.452370285987854, "logps/chosen": -315.331298828125, "logps/rejected": -258.9646911621094, "loss": 0.534, "metrics/advantage_var": 62.50199508666992, "rewards/accuracies": 0.75, "rewards/chosen": 0.6554938554763794, "rewards/margins": 0.6286452412605286, "rewards/rejected": 0.0268485676497221, "stability/repetition_rate_mean": 0.799023449420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 0.908203125, "step": 310 }, { "epoch": 0.3454324652543516, "grad_norm": 15.366827011108398, "learning_rate": 4.1474793226723825e-07, "logits/chosen": -1.448975682258606, "logits/rejected": -1.4607841968536377, "logps/chosen": -327.7336120605469, "logps/rejected": -262.0153503417969, "loss": 0.5117, "metrics/advantage_var": 68.2557373046875, "rewards/accuracies": 0.760937511920929, "rewards/chosen": 0.7362013459205627, "rewards/margins": 0.6878542304039001, "rewards/rejected": 0.048347149044275284, "stability/repetition_rate_mean": 0.9072265625, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.083886742591858, "step": 320 }, { "epoch": 0.35622722979355015, "grad_norm": 19.474281311035156, "learning_rate": 4.0754955445415396e-07, "logits/chosen": -1.447295069694519, "logits/rejected": -1.47311270236969, "logps/chosen": -330.9326477050781, "logps/rejected": -256.48638916015625, "loss": 0.4903, "metrics/advantage_var": 77.84224700927734, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": 0.7997223734855652, "rewards/margins": 0.7868985533714294, "rewards/rejected": 0.012823845259845257, "stability/repetition_rate_mean": 0.955273449420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.7205078601837158, "step": 330 }, { "epoch": 0.3670219943327486, "grad_norm": 13.608744621276855, "learning_rate": 4.001276478500126e-07, "logits/chosen": -1.4461034536361694, "logits/rejected": -1.4699530601501465, "logps/chosen": -316.4836120605469, "logps/rejected": -258.7716064453125, "loss": 0.4981, "metrics/advantage_var": 72.29796600341797, "rewards/accuracies": 0.770312488079071, "rewards/chosen": 0.7650503516197205, "rewards/margins": 0.738609790802002, "rewards/rejected": 0.026440534740686417, "stability/repetition_rate_mean": 0.7232421636581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.8572266101837158, "step": 340 }, { "epoch": 0.3778167588719471, "grad_norm": 13.565873146057129, "learning_rate": 3.9249274253734164e-07, "logits/chosen": -1.4219696521759033, "logits/rejected": -1.4259979724884033, "logps/chosen": -314.6985778808594, "logps/rejected": -269.089599609375, "loss": 0.5247, "metrics/advantage_var": 80.2893295288086, "rewards/accuracies": 0.7359375357627869, "rewards/chosen": 0.7249447107315063, "rewards/margins": 0.712753176689148, "rewards/rejected": 0.012191593647003174, "stability/repetition_rate_mean": 0.76953125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.2517578601837158, "step": 350 }, { "epoch": 0.3886115234111456, "grad_norm": 13.095726013183594, "learning_rate": 3.846556707978337e-07, "logits/chosen": -1.427175760269165, "logits/rejected": -1.461297869682312, "logps/chosen": -334.6982727050781, "logps/rejected": -252.6654815673828, "loss": 0.5044, "metrics/advantage_var": 77.54660034179688, "rewards/accuracies": 0.7515625357627869, "rewards/chosen": 0.8103561401367188, "rewards/margins": 0.7708074450492859, "rewards/rejected": 0.039548713713884354, "stability/repetition_rate_mean": 0.889843761920929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.2664062976837158, "step": 360 }, { "epoch": 0.3994062879503441, "grad_norm": 14.872525215148926, "learning_rate": 3.766275517436779e-07, "logits/chosen": -1.3873792886734009, "logits/rejected": -1.3985546827316284, "logps/chosen": -319.5567321777344, "logps/rejected": -254.91746520996094, "loss": 0.4983, "metrics/advantage_var": 75.8227767944336, "rewards/accuracies": 0.729687511920929, "rewards/chosen": 0.7894806861877441, "rewards/margins": 0.7587611079216003, "rewards/rejected": 0.03071962669491768, "stability/repetition_rate_mean": 0.777148425579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.752539038658142, "step": 370 }, { "epoch": 0.4102010524895426, "grad_norm": 14.203469276428223, "learning_rate": 3.684197755419419e-07, "logits/chosen": -1.4125322103500366, "logits/rejected": -1.417395830154419, "logps/chosen": -306.7962341308594, "logps/rejected": -258.84234619140625, "loss": 0.5001, "metrics/advantage_var": 81.72932434082031, "rewards/accuracies": 0.7593750357627869, "rewards/chosen": 0.7916366457939148, "rewards/margins": 0.7797738909721375, "rewards/rejected": 0.01186272781342268, "stability/repetition_rate_mean": 0.7945312261581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.844921827316284, "step": 380 }, { "epoch": 0.42099581702874106, "grad_norm": 14.88988971710205, "learning_rate": 3.60043987254384e-07, "logits/chosen": -1.425724983215332, "logits/rejected": -1.4372109174728394, "logps/chosen": -316.85101318359375, "logps/rejected": -264.2350769042969, "loss": 0.4952, "metrics/advantage_var": 92.4764404296875, "rewards/accuracies": 0.765625, "rewards/chosen": 0.8086903691291809, "rewards/margins": 0.8516300320625305, "rewards/rejected": -0.04293971136212349, "stability/repetition_rate_mean": 0.74609375, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.869531273841858, "step": 390 }, { "epoch": 0.43179058156793954, "grad_norm": 13.483888626098633, "learning_rate": 3.5151207031562633e-07, "logits/chosen": -1.3795498609542847, "logits/rejected": -1.3812373876571655, "logps/chosen": -310.2779235839844, "logps/rejected": -268.259033203125, "loss": 0.5137, "metrics/advantage_var": 87.57612609863281, "rewards/accuracies": 0.776562511920929, "rewards/chosen": 0.7608602046966553, "rewards/margins": 0.7845363020896912, "rewards/rejected": -0.023676123470067978, "stability/repetition_rate_mean": 0.7679687738418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.6525390148162842, "step": 400 }, { "epoch": 0.442585346107138, "grad_norm": 15.127874374389648, "learning_rate": 3.4283612967312687e-07, "logits/chosen": -1.3496493101119995, "logits/rejected": -1.3831802606582642, "logps/chosen": -313.1993103027344, "logps/rejected": -250.6515655517578, "loss": 0.5069, "metrics/advantage_var": 93.4025650024414, "rewards/accuracies": 0.760937511920929, "rewards/chosen": 0.7829875349998474, "rewards/margins": 0.8043760657310486, "rewards/rejected": -0.021388515830039978, "stability/repetition_rate_mean": 0.854296863079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.6925780773162842, "step": 410 }, { "epoch": 0.45338011064633654, "grad_norm": 13.931750297546387, "learning_rate": 3.34028474612874e-07, "logits/chosen": -1.3803904056549072, "logits/rejected": -1.394149661064148, "logps/chosen": -299.86065673828125, "logps/rejected": -261.5897521972656, "loss": 0.5122, "metrics/advantage_var": 100.02517700195312, "rewards/accuracies": 0.737500011920929, "rewards/chosen": 0.7716613411903381, "rewards/margins": 0.8435766100883484, "rewards/rejected": -0.07191526889801025, "stability/repetition_rate_mean": 0.822460949420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.1460938453674316, "step": 420 }, { "epoch": 0.464174875185535, "grad_norm": 15.035143852233887, "learning_rate": 3.2510160129516775e-07, "logits/chosen": -1.3800678253173828, "logits/rejected": -1.3930643796920776, "logps/chosen": -319.5307922363281, "logps/rejected": -265.67938232421875, "loss": 0.4937, "metrics/advantage_var": 104.3858413696289, "rewards/accuracies": 0.768750011920929, "rewards/chosen": 0.8203991055488586, "rewards/margins": 0.8988785147666931, "rewards/rejected": -0.07847942411899567, "stability/repetition_rate_mean": 0.78125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.808789014816284, "step": 430 }, { "epoch": 0.4749696397247335, "grad_norm": 14.138893127441406, "learning_rate": 3.1606817502526736e-07, "logits/chosen": -1.3767389059066772, "logits/rejected": -1.4001781940460205, "logps/chosen": -326.2301330566406, "logps/rejected": -256.16412353515625, "loss": 0.4891, "metrics/advantage_var": 99.69866943359375, "rewards/accuracies": 0.792187511920929, "rewards/chosen": 0.8151880502700806, "rewards/margins": 0.8713592886924744, "rewards/rejected": -0.05617132410407066, "stability/repetition_rate_mean": 0.8707031011581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.164990186691284, "step": 440 }, { "epoch": 0.48576440426393197, "grad_norm": 11.794190406799316, "learning_rate": 3.069410122840585e-07, "logits/chosen": -1.372667670249939, "logits/rejected": -1.4003467559814453, "logps/chosen": -318.3290100097656, "logps/rejected": -268.3873596191406, "loss": 0.4817, "metrics/advantage_var": 106.1712417602539, "rewards/accuracies": 0.7890625, "rewards/chosen": 0.8885055780410767, "rewards/margins": 0.924716591835022, "rewards/rejected": -0.03621084615588188, "stability/repetition_rate_mean": 0.8236328363418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.7429687976837158, "step": 450 }, { "epoch": 0.4965591688031305, "grad_norm": 12.17612361907959, "learning_rate": 2.9773306254423513e-07, "logits/chosen": -1.3681141138076782, "logits/rejected": -1.3875712156295776, "logps/chosen": -294.7362976074219, "logps/rejected": -240.1558074951172, "loss": 0.4832, "metrics/advantage_var": 90.08233642578125, "rewards/accuracies": 0.7718750238418579, "rewards/chosen": 0.853641152381897, "rewards/margins": 0.8702511191368103, "rewards/rejected": -0.016610044986009598, "stability/repetition_rate_mean": 0.8646484613418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.7371094226837158, "step": 460 }, { "epoch": 0.5073539333423289, "grad_norm": 16.81270980834961, "learning_rate": 2.884573898977941e-07, "logits/chosen": -1.388026237487793, "logits/rejected": -1.3998411893844604, "logps/chosen": -308.7089538574219, "logps/rejected": -268.0931091308594, "loss": 0.4801, "metrics/advantage_var": 102.28583526611328, "rewards/accuracies": 0.7593750357627869, "rewards/chosen": 0.8505555391311646, "rewards/margins": 0.8920961618423462, "rewards/rejected": -0.04154070094227791, "stability/repetition_rate_mean": 0.8687499761581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.244531273841858, "step": 470 }, { "epoch": 0.5181486978815275, "grad_norm": 17.988006591796875, "learning_rate": 2.791271545209101e-07, "logits/chosen": -1.3824865818023682, "logits/rejected": -1.4162545204162598, "logps/chosen": -294.66412353515625, "logps/rejected": -248.6920928955078, "loss": 0.4827, "metrics/advantage_var": 95.26998138427734, "rewards/accuracies": 0.7890625, "rewards/chosen": 0.8823606371879578, "rewards/margins": 0.8613284230232239, "rewards/rejected": 0.021032243967056274, "stability/repetition_rate_mean": 0.8519531488418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.1624999046325684, "step": 480 }, { "epoch": 0.528943462420726, "grad_norm": 15.620615005493164, "learning_rate": 2.697555940024887e-07, "logits/chosen": -1.3622970581054688, "logits/rejected": -1.3937467336654663, "logps/chosen": -303.913330078125, "logps/rejected": -242.33706665039062, "loss": 0.4647, "metrics/advantage_var": 105.72547912597656, "rewards/accuracies": 0.792187511920929, "rewards/chosen": 0.9351348280906677, "rewards/margins": 0.9704492688179016, "rewards/rejected": -0.035314515233039856, "stability/repetition_rate_mean": 0.8125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.9038817882537842, "step": 490 }, { "epoch": 0.5397382269599245, "grad_norm": 28.97076416015625, "learning_rate": 2.603560045628857e-07, "logits/chosen": -1.3485071659088135, "logits/rejected": -1.3485997915267944, "logps/chosen": -314.1101989746094, "logps/rejected": -261.1115417480469, "loss": 0.482, "metrics/advantage_var": 112.3708267211914, "rewards/accuracies": 0.7828125357627869, "rewards/chosen": 0.9495781064033508, "rewards/margins": 0.9663735628128052, "rewards/rejected": -0.016795417293906212, "stability/repetition_rate_mean": 0.7855468988418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.1546874046325684, "step": 500 }, { "epoch": 0.5505329914991229, "grad_norm": 16.771753311157227, "learning_rate": 2.509417221894427e-07, "logits/chosen": -1.3849689960479736, "logits/rejected": -1.3808684349060059, "logps/chosen": -307.50494384765625, "logps/rejected": -269.51959228515625, "loss": 0.4861, "metrics/advantage_var": 120.5189208984375, "rewards/accuracies": 0.796875, "rewards/chosen": 0.9477736353874207, "rewards/margins": 0.9401771426200867, "rewards/rejected": 0.007596464361995459, "stability/repetition_rate_mean": 0.8619140386581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.330175757408142, "step": 510 }, { "epoch": 0.5613277560383214, "grad_norm": 16.037240982055664, "learning_rate": 2.4152610371560093e-07, "logits/chosen": -1.4082396030426025, "logits/rejected": -1.4073408842086792, "logps/chosen": -308.14276123046875, "logps/rejected": -257.72283935546875, "loss": 0.4896, "metrics/advantage_var": 103.19306945800781, "rewards/accuracies": 0.765625, "rewards/chosen": 0.9386131167411804, "rewards/margins": 0.8744432330131531, "rewards/rejected": 0.06416996568441391, "stability/repetition_rate_mean": 0.856640636920929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.715429663658142, "step": 520 }, { "epoch": 0.5721225205775199, "grad_norm": 20.248550415039062, "learning_rate": 2.321225078704399e-07, "logits/chosen": -1.4284799098968506, "logits/rejected": -1.4444469213485718, "logps/chosen": -317.8136291503906, "logps/rejected": -260.39678955078125, "loss": 0.4874, "metrics/advantage_var": 103.1753921508789, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": 1.0152686834335327, "rewards/margins": 0.8757182955741882, "rewards/rejected": 0.1395503580570221, "stability/repetition_rate_mean": 0.7701171636581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.365820288658142, "step": 530 }, { "epoch": 0.5829172851167184, "grad_norm": 17.81089210510254, "learning_rate": 2.2274427632552503e-07, "logits/chosen": -1.4013619422912598, "logits/rejected": -1.405015468597412, "logps/chosen": -302.94537353515625, "logps/rejected": -262.5538635253906, "loss": 0.4931, "metrics/advantage_var": 99.6611557006836, "rewards/accuracies": 0.7734375, "rewards/chosen": 0.9857921600341797, "rewards/margins": 0.8826519250869751, "rewards/rejected": 0.1031401976943016, "stability/repetition_rate_mean": 0.8062499761581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.7644531726837158, "step": 540 }, { "epoch": 0.5937120496559168, "grad_norm": 15.281890869140625, "learning_rate": 2.134047147659583e-07, "logits/chosen": -1.3965346813201904, "logits/rejected": -1.3893383741378784, "logps/chosen": -309.70013427734375, "logps/rejected": -264.7682189941406, "loss": 0.4744, "metrics/advantage_var": 104.08345794677734, "rewards/accuracies": 0.7671875357627869, "rewards/chosen": 1.062072992324829, "rewards/margins": 0.9215413928031921, "rewards/rejected": 0.140531525015831, "stability/repetition_rate_mean": 0.8472656011581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.9962890148162842, "step": 550 }, { "epoch": 0.6045068141951153, "grad_norm": 19.41432762145996, "learning_rate": 2.0411707401248403e-07, "logits/chosen": -1.4111377000808716, "logits/rejected": -1.4183025360107422, "logps/chosen": -310.02789306640625, "logps/rejected": -263.0741882324219, "loss": 0.5088, "metrics/advantage_var": 106.92705535888672, "rewards/accuracies": 0.7328125238418579, "rewards/chosen": 1.0473458766937256, "rewards/margins": 0.841526210308075, "rewards/rejected": 0.2058195173740387, "stability/repetition_rate_mean": 0.784960925579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.4849610328674316, "step": 560 }, { "epoch": 0.6153015787343139, "grad_norm": 15.733675956726074, "learning_rate": 1.9489453122143603e-07, "logits/chosen": -1.4051156044006348, "logits/rejected": -1.4276247024536133, "logps/chosen": -312.1006164550781, "logps/rejected": -254.21788024902344, "loss": 0.4437, "metrics/advantage_var": 103.9989013671875, "rewards/accuracies": 0.793749988079071, "rewards/chosen": 1.1061866283416748, "rewards/margins": 0.9930472373962402, "rewards/rejected": 0.11313938349485397, "stability/repetition_rate_mean": 0.828906238079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 0.817578136920929, "step": 570 }, { "epoch": 0.6260963432735124, "grad_norm": 17.128223419189453, "learning_rate": 1.8575017118919928e-07, "logits/chosen": -1.4090861082077026, "logits/rejected": -1.4279987812042236, "logps/chosen": -319.3863830566406, "logps/rejected": -252.8662109375, "loss": 0.5064, "metrics/advantage_var": 100.0505142211914, "rewards/accuracies": 0.746874988079071, "rewards/chosen": 0.9926168322563171, "rewards/margins": 0.8279793858528137, "rewards/rejected": 0.16463755071163177, "stability/repetition_rate_mean": 0.7662109136581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.6736328601837158, "step": 580 }, { "epoch": 0.6368911078127109, "grad_norm": 10.917092323303223, "learning_rate": 1.7669696778770938e-07, "logits/chosen": -1.4007326364517212, "logits/rejected": -1.4263814687728882, "logps/chosen": -316.9762268066406, "logps/rejected": -245.96548461914062, "loss": 0.4494, "metrics/advantage_var": 108.5511245727539, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 1.0922644138336182, "rewards/margins": 1.006994605064392, "rewards/rejected": 0.08526992052793503, "stability/repetition_rate_mean": 0.8466796875, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.351171851158142, "step": 590 }, { "epoch": 0.6476858723519093, "grad_norm": 12.8851318359375, "learning_rate": 1.6774776555733028e-07, "logits/chosen": -1.382874846458435, "logits/rejected": -1.3788057565689087, "logps/chosen": -317.29132080078125, "logps/rejected": -270.30157470703125, "loss": 0.479, "metrics/advantage_var": 106.93010711669922, "rewards/accuracies": 0.768750011920929, "rewards/chosen": 1.0857884883880615, "rewards/margins": 0.9378126263618469, "rewards/rejected": 0.14797575771808624, "stability/repetition_rate_mean": 0.8042968511581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.7417969703674316, "step": 600 }, { "epoch": 0.6476858723519093, "eval_logits/chosen": -1.3723105192184448, "eval_logits/rejected": -1.3827242851257324, "eval_logps/chosen": -316.3534240722656, "eval_logps/rejected": -254.0533447265625, "eval_loss": 0.47359293699264526, "eval_metrics/advantage_var": 112.27565002441406, "eval_rewards/accuracies": 0.7823232412338257, "eval_rewards/chosen": 1.046919584274292, "eval_rewards/margins": 0.9653631448745728, "eval_rewards/rejected": 0.08155640214681625, "eval_runtime": 271.831, "eval_samples_per_second": 14.546, "eval_stability/repetition_rate_mean": 0.8873046636581421, "eval_stability/response_length_mean": 512.0, "eval_stability/response_length_std": 0.0, "eval_stability/response_length_var": 0.0, "eval_stability/token_entropy_mean": 2.07421875, "eval_steps_per_second": 1.821, "step": 600 }, { "epoch": 0.6584806368911078, "grad_norm": 17.53192710876465, "learning_rate": 1.5891526148322593e-07, "logits/chosen": -1.3868850469589233, "logits/rejected": -1.386324167251587, "logps/chosen": -337.97906494140625, "logps/rejected": -273.7126159667969, "loss": 0.4827, "metrics/advantage_var": 108.11744689941406, "rewards/accuracies": 0.785937488079071, "rewards/chosen": 1.0058691501617432, "rewards/margins": 0.8874250650405884, "rewards/rejected": 0.11844401806592941, "stability/repetition_rate_mean": 0.744921863079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.9982421398162842, "step": 610 }, { "epoch": 0.6692754014303063, "grad_norm": 12.167177200317383, "learning_rate": 1.5021198698108036e-07, "logits/chosen": -1.37397038936615, "logits/rejected": -1.378446102142334, "logps/chosen": -317.61688232421875, "logps/rejected": -255.9473114013672, "loss": 0.494, "metrics/advantage_var": 114.5400390625, "rewards/accuracies": 0.7734375, "rewards/chosen": 1.0009812116622925, "rewards/margins": 0.914712131023407, "rewards/rejected": 0.0862690731883049, "stability/repetition_rate_mean": 0.8783203363418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.2964844703674316, "step": 620 }, { "epoch": 0.6800701659695048, "grad_norm": 17.123615264892578, "learning_rate": 1.416502901177251e-07, "logits/chosen": -1.3341145515441895, "logits/rejected": -1.3377490043640137, "logps/chosen": -311.6447448730469, "logps/rejected": -254.44578552246094, "loss": 0.4832, "metrics/advantage_var": 98.87296295166016, "rewards/accuracies": 0.7640625238418579, "rewards/chosen": 0.9517079591751099, "rewards/margins": 0.8569992184638977, "rewards/rejected": 0.09470874816179276, "stability/repetition_rate_mean": 0.8720703125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.377539038658142, "step": 630 }, { "epoch": 0.6908649305087032, "grad_norm": 12.504801750183105, "learning_rate": 1.3324231809189983e-07, "logits/chosen": -1.3613817691802979, "logits/rejected": -1.3694230318069458, "logps/chosen": -319.6551818847656, "logps/rejected": -258.7291564941406, "loss": 0.4615, "metrics/advantage_var": 120.3536376953125, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 1.0257498025894165, "rewards/margins": 1.0272339582443237, "rewards/rejected": -0.0014841229422017932, "stability/repetition_rate_mean": 0.7818359136581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.7732422351837158, "step": 640 }, { "epoch": 0.7016596950479018, "grad_norm": 14.149001121520996, "learning_rate": 1.2500000000000005e-07, "logits/chosen": -1.322688341140747, "logits/rejected": -1.3375654220581055, "logps/chosen": -292.4341735839844, "logps/rejected": -252.3300323486328, "loss": 0.4443, "metrics/advantage_var": 113.12126922607422, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.9646196365356445, "rewards/margins": 1.0090590715408325, "rewards/rejected": -0.04443943500518799, "stability/repetition_rate_mean": 0.850390613079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.5888671875, "step": 650 }, { "epoch": 0.7124544595871003, "grad_norm": 14.957926750183105, "learning_rate": 1.1693502991126608e-07, "logits/chosen": -1.3551677465438843, "logits/rejected": -1.361363410949707, "logps/chosen": -333.7082214355469, "logps/rejected": -250.5390625, "loss": 0.461, "metrics/advantage_var": 120.99027252197266, "rewards/accuracies": 0.7890625, "rewards/chosen": 1.0033434629440308, "rewards/margins": 1.0358381271362305, "rewards/rejected": -0.03249470144510269, "stability/repetition_rate_mean": 0.775585949420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.3021483421325684, "step": 660 }, { "epoch": 0.7232492241262988, "grad_norm": 19.125764846801758, "learning_rate": 1.0905885027642483e-07, "logits/chosen": -1.3491506576538086, "logits/rejected": -1.3566621541976929, "logps/chosen": -295.5746765136719, "logps/rejected": -237.2896728515625, "loss": 0.477, "metrics/advantage_var": 120.28877258300781, "rewards/accuracies": 0.78125, "rewards/chosen": 0.9816705584526062, "rewards/margins": 0.9865358471870422, "rewards/rejected": -0.0048652635887265205, "stability/repetition_rate_mean": 0.785937488079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.6103515625, "step": 670 }, { "epoch": 0.7340439886654972, "grad_norm": 15.960895538330078, "learning_rate": 1.0138263569332267e-07, "logits/chosen": -1.372681975364685, "logits/rejected": -1.3638715744018555, "logps/chosen": -314.92364501953125, "logps/rejected": -262.690673828125, "loss": 0.4736, "metrics/advantage_var": 117.98992156982422, "rewards/accuracies": 0.7796875238418579, "rewards/chosen": 1.0467315912246704, "rewards/margins": 0.9876863360404968, "rewards/rejected": 0.059045251458883286, "stability/repetition_rate_mean": 0.7417968511581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.3708984851837158, "step": 680 }, { "epoch": 0.7448387532046957, "grad_norm": 13.511040687561035, "learning_rate": 9.391727705258502e-08, "logits/chosen": -1.3565599918365479, "logits/rejected": -1.3600114583969116, "logps/chosen": -320.1853942871094, "logps/rejected": -271.3731384277344, "loss": 0.4613, "metrics/advantage_var": 122.61201477050781, "rewards/accuracies": 0.785937488079071, "rewards/chosen": 1.0267409086227417, "rewards/margins": 1.0403625965118408, "rewards/rejected": -0.01362178660929203, "stability/repetition_rate_mean": 0.8603515625, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.596289038658142, "step": 690 }, { "epoch": 0.7556335177438942, "grad_norm": 10.739056587219238, "learning_rate": 8.667336608579487e-08, "logits/chosen": -1.3605684041976929, "logits/rejected": -1.3892539739608765, "logps/chosen": -318.6557312011719, "logps/rejected": -261.914794921875, "loss": 0.4775, "metrics/advantage_var": 106.7211685180664, "rewards/accuracies": 0.801562488079071, "rewards/chosen": 0.9467880129814148, "rewards/margins": 0.9349954724311829, "rewards/rejected": 0.011792674660682678, "stability/repetition_rate_mean": 0.8111327886581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.692675828933716, "step": 700 }, { "epoch": 0.7664282822830927, "grad_norm": 15.086265563964844, "learning_rate": 7.96611803381127e-08, "logits/chosen": -1.3547624349594116, "logits/rejected": -1.3488959074020386, "logps/chosen": -303.3160095214844, "logps/rejected": -258.77264404296875, "loss": 0.4735, "metrics/advantage_var": 115.3608627319336, "rewards/accuracies": 0.796875, "rewards/chosen": 0.9548945426940918, "rewards/margins": 0.9802401661872864, "rewards/rejected": -0.02534562349319458, "stability/repetition_rate_mean": 0.840039074420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.209765672683716, "step": 710 }, { "epoch": 0.7772230468222912, "grad_norm": 16.009767532348633, "learning_rate": 7.28906685866599e-08, "logits/chosen": -1.3702967166900635, "logits/rejected": -1.3795828819274902, "logps/chosen": -314.6918640136719, "logps/rejected": -246.10391235351562, "loss": 0.4632, "metrics/advantage_var": 128.08262634277344, "rewards/accuracies": 0.770312488079071, "rewards/chosen": 1.0151904821395874, "rewards/margins": 1.0434449911117554, "rewards/rejected": -0.02825441025197506, "stability/repetition_rate_mean": 0.8203125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 0.8656250238418579, "step": 720 }, { "epoch": 0.7880178113614896, "grad_norm": 17.335712432861328, "learning_rate": 6.637143672535281e-08, "logits/chosen": -1.3211309909820557, "logits/rejected": -1.3452644348144531, "logps/chosen": -312.05279541015625, "logps/rejected": -256.790283203125, "loss": 0.4905, "metrics/advantage_var": 118.5810317993164, "rewards/accuracies": 0.78125, "rewards/chosen": 1.0083658695220947, "rewards/margins": 0.9445858001708984, "rewards/rejected": 0.06377997249364853, "stability/repetition_rate_mean": 0.7437499761581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.055468797683716, "step": 730 }, { "epoch": 0.7988125759006882, "grad_norm": 10.28477668762207, "learning_rate": 6.01127341362138e-08, "logits/chosen": -1.339066505432129, "logits/rejected": -1.343850016593933, "logps/chosen": -314.6363220214844, "logps/rejected": -265.0621032714844, "loss": 0.4786, "metrics/advantage_var": 117.64568328857422, "rewards/accuracies": 0.7828125357627869, "rewards/chosen": 0.9195186495780945, "rewards/margins": 0.9779030680656433, "rewards/rejected": -0.05838441848754883, "stability/repetition_rate_mean": 0.775195300579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.513671875, "step": 740 }, { "epoch": 0.8096073404398867, "grad_norm": 19.894166946411133, "learning_rate": 5.412344056649526e-08, "logits/chosen": -1.3286861181259155, "logits/rejected": -1.3161166906356812, "logps/chosen": -323.9461364746094, "logps/rejected": -263.8832702636719, "loss": 0.4702, "metrics/advantage_var": 130.6468048095703, "rewards/accuracies": 0.78125, "rewards/chosen": 1.004286766052246, "rewards/margins": 1.0564974546432495, "rewards/rejected": -0.05221066623926163, "stability/repetition_rate_mean": 0.825390636920929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.7333984375, "step": 750 }, { "epoch": 0.8204021049790852, "grad_norm": 17.493206024169922, "learning_rate": 4.841205353023714e-08, "logits/chosen": -1.316092848777771, "logits/rejected": -1.3103103637695312, "logps/chosen": -308.4322814941406, "logps/rejected": -251.6645965576172, "loss": 0.4974, "metrics/advantage_var": 116.90795135498047, "rewards/accuracies": 0.7718750238418579, "rewards/chosen": 0.985223114490509, "rewards/margins": 0.9163877367973328, "rewards/rejected": 0.0688353106379509, "stability/repetition_rate_mean": 0.779101550579071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.017578125, "step": 760 }, { "epoch": 0.8311968695182836, "grad_norm": 19.531171798706055, "learning_rate": 4.298667625212904e-08, "logits/chosen": -1.3329784870147705, "logits/rejected": -1.3454676866531372, "logps/chosen": -312.3675842285156, "logps/rejected": -260.13189697265625, "loss": 0.4839, "metrics/advantage_var": 117.96380615234375, "rewards/accuracies": 0.776562511920929, "rewards/chosen": 0.9914433360099792, "rewards/margins": 0.9715946316719055, "rewards/rejected": 0.01984873227775097, "stability/repetition_rate_mean": 0.8814452886581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.833984375, "step": 770 }, { "epoch": 0.8419916340574821, "grad_norm": 13.06292724609375, "learning_rate": 3.785500617078424e-08, "logits/chosen": -1.3594963550567627, "logits/rejected": -1.3589779138565063, "logps/chosen": -339.4462585449219, "logps/rejected": -283.0810546875, "loss": 0.451, "metrics/advantage_var": 135.08409118652344, "rewards/accuracies": 0.8109375238418579, "rewards/chosen": 1.0986813306808472, "rewards/margins": 1.1291017532348633, "rewards/rejected": -0.03042042814195156, "stability/repetition_rate_mean": 0.795703113079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.575585961341858, "step": 780 }, { "epoch": 0.8527863985966806, "grad_norm": 12.330864906311035, "learning_rate": 3.3024324017737554e-08, "logits/chosen": -1.341722846031189, "logits/rejected": -1.332808256149292, "logps/chosen": -314.4818115234375, "logps/rejected": -266.3319396972656, "loss": 0.4454, "metrics/advantage_var": 123.73223876953125, "rewards/accuracies": 0.809374988079071, "rewards/chosen": 1.0409488677978516, "rewards/margins": 1.057476282119751, "rewards/rejected": -0.016527360305190086, "stability/repetition_rate_mean": 0.8095703125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.439843773841858, "step": 790 }, { "epoch": 0.8635811631358791, "grad_norm": 15.701562881469727, "learning_rate": 2.850148348765921e-08, "logits/chosen": -1.3520551919937134, "logits/rejected": -1.355186939239502, "logps/chosen": -304.0136413574219, "logps/rejected": -258.58209228515625, "loss": 0.4918, "metrics/advantage_var": 115.4345703125, "rewards/accuracies": 0.770312488079071, "rewards/chosen": 0.9924200177192688, "rewards/margins": 0.9020625948905945, "rewards/rejected": 0.09035740047693253, "stability/repetition_rate_mean": 0.9134765863418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.61376953125, "step": 800 }, { "epoch": 0.8743759276750775, "grad_norm": 9.2330322265625, "learning_rate": 2.4292901514442327e-08, "logits/chosen": -1.3454469442367554, "logits/rejected": -1.3386812210083008, "logps/chosen": -321.5726013183594, "logps/rejected": -258.95526123046875, "loss": 0.4636, "metrics/advantage_var": 119.41340637207031, "rewards/accuracies": 0.7875000238418579, "rewards/chosen": 1.0107624530792236, "rewards/margins": 0.9988824725151062, "rewards/rejected": 0.011880074627697468, "stability/repetition_rate_mean": 0.8667968511581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.4040038585662842, "step": 810 }, { "epoch": 0.885170692214276, "grad_norm": 10.787253379821777, "learning_rate": 2.0404549166959718e-08, "logits/chosen": -1.300979495048523, "logits/rejected": -1.3161647319793701, "logps/chosen": -308.5863037109375, "logps/rejected": -254.77041625976562, "loss": 0.4683, "metrics/advantage_var": 121.4715347290039, "rewards/accuracies": 0.7796875238418579, "rewards/chosen": 1.0170072317123413, "rewards/margins": 0.9836593866348267, "rewards/rejected": 0.03334783762693405, "stability/repetition_rate_mean": 0.9224609136581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.8826172351837158, "step": 820 }, { "epoch": 0.8959654567534746, "grad_norm": 13.196191787719727, "learning_rate": 1.6841943177406976e-08, "logits/chosen": -1.3147996664047241, "logits/rejected": -1.3255729675292969, "logps/chosen": -319.1125793457031, "logps/rejected": -274.5699462890625, "loss": 0.4925, "metrics/advantage_var": 126.98355865478516, "rewards/accuracies": 0.7828125357627869, "rewards/chosen": 0.9902805685997009, "rewards/margins": 1.020569086074829, "rewards/rejected": -0.030288374051451683, "stability/repetition_rate_mean": 0.8330078125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.1563477516174316, "step": 830 }, { "epoch": 0.9067602212926731, "grad_norm": 11.895269393920898, "learning_rate": 1.3610138114250519e-08, "logits/chosen": -1.327235221862793, "logits/rejected": -1.343729853630066, "logps/chosen": -327.689697265625, "logps/rejected": -261.86480712890625, "loss": 0.4735, "metrics/advantage_var": 126.14949035644531, "rewards/accuracies": 0.765625, "rewards/chosen": 1.0660076141357422, "rewards/margins": 1.0003070831298828, "rewards/rejected": 0.06570061296224594, "stability/repetition_rate_mean": 0.8158203363418579, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.4929687976837158, "step": 840 }, { "epoch": 0.9175549858318716, "grad_norm": 15.15986156463623, "learning_rate": 1.0713719210886928e-08, "logits/chosen": -1.3229340314865112, "logits/rejected": -1.3207443952560425, "logps/chosen": -324.7861328125, "logps/rejected": -261.70849609375, "loss": 0.4859, "metrics/advantage_var": 124.1085433959961, "rewards/accuracies": 0.776562511920929, "rewards/chosen": 1.053748369216919, "rewards/margins": 0.9940328598022461, "rewards/rejected": 0.05971544608473778, "stability/repetition_rate_mean": 0.7544921636581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.3533203601837158, "step": 850 }, { "epoch": 0.92834975037107, "grad_norm": 13.380340576171875, "learning_rate": 8.156795860187027e-09, "logits/chosen": -1.348737359046936, "logits/rejected": -1.3422983884811401, "logps/chosen": -336.0633850097656, "logps/rejected": -262.0978088378906, "loss": 0.4306, "metrics/advantage_var": 125.40618133544922, "rewards/accuracies": 0.8109375238418579, "rewards/chosen": 1.0677509307861328, "rewards/margins": 1.1280078887939453, "rewards/rejected": -0.06025683507323265, "stability/repetition_rate_mean": 0.805468738079071, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.556640625, "step": 860 }, { "epoch": 0.9391445149102685, "grad_norm": 15.88653564453125, "learning_rate": 5.942995784154692e-09, "logits/chosen": -1.3338909149169922, "logits/rejected": -1.3578466176986694, "logps/chosen": -307.34771728515625, "logps/rejected": -260.82586669921875, "loss": 0.4598, "metrics/advantage_var": 123.701904296875, "rewards/accuracies": 0.7953125238418579, "rewards/chosen": 1.0478211641311646, "rewards/margins": 1.0490620136260986, "rewards/rejected": -0.0012408018810674548, "stability/repetition_rate_mean": 0.864453136920929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.4718749523162842, "step": 870 }, { "epoch": 0.949939279449467, "grad_norm": 14.681868553161621, "learning_rate": 4.075459886973082e-09, "logits/chosen": -1.322386622428894, "logits/rejected": -1.3134368658065796, "logps/chosen": -288.3506774902344, "logps/rejected": -259.43109130859375, "loss": 0.4837, "metrics/advantage_var": 120.60890197753906, "rewards/accuracies": 0.7750000357627869, "rewards/chosen": 0.9852436184883118, "rewards/margins": 0.9725152850151062, "rewards/rejected": 0.012728266417980194, "stability/repetition_rate_mean": 0.8818359375, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.583984375, "step": 880 }, { "epoch": 0.9607340439886655, "grad_norm": 16.591888427734375, "learning_rate": 2.556837798739886e-09, "logits/chosen": -1.3387622833251953, "logits/rejected": -1.34397554397583, "logps/chosen": -314.2873229980469, "logps/rejected": -257.3379821777344, "loss": 0.4647, "metrics/advantage_var": 120.80268859863281, "rewards/accuracies": 0.8109375238418579, "rewards/chosen": 0.9938045740127563, "rewards/margins": 0.9979391098022461, "rewards/rejected": -0.0041345953941345215, "stability/repetition_rate_mean": 0.8443359136581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.6566405296325684, "step": 890 }, { "epoch": 0.9715288085278639, "grad_norm": 13.192713737487793, "learning_rate": 1.3892841162143899e-09, "logits/chosen": -1.3134535551071167, "logits/rejected": -1.3294223546981812, "logps/chosen": -304.279541015625, "logps/rejected": -257.1634216308594, "loss": 0.455, "metrics/advantage_var": 119.05987548828125, "rewards/accuracies": 0.785937488079071, "rewards/chosen": 0.9897605776786804, "rewards/margins": 1.0290673971176147, "rewards/rejected": -0.039306994527578354, "stability/repetition_rate_mean": 0.7369140386581421, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.3954589366912842, "step": 900 }, { "epoch": 0.9715288085278639, "eval_logits/chosen": -1.3294035196304321, "eval_logits/rejected": -1.3341506719589233, "eval_logps/chosen": -316.4003601074219, "eval_logps/rejected": -254.81178283691406, "eval_loss": 0.46732690930366516, "eval_metrics/advantage_var": 125.62010955810547, "eval_rewards/accuracies": 0.7828283309936523, "eval_rewards/chosen": 1.0422265529632568, "eval_rewards/margins": 1.036514401435852, "eval_rewards/rejected": 0.005712195765227079, "eval_runtime": 269.7729, "eval_samples_per_second": 14.657, "eval_stability/repetition_rate_mean": 0.811718761920929, "eval_stability/response_length_mean": 512.0, "eval_stability/response_length_std": 0.0, "eval_stability/response_length_var": 0.0, "eval_stability/token_entropy_mean": 1.127832055091858, "eval_steps_per_second": 1.835, "step": 900 }, { "epoch": 0.9823235730670625, "grad_norm": 11.817096710205078, "learning_rate": 5.744553459100243e-10, "logits/chosen": -1.3120925426483154, "logits/rejected": -1.3330440521240234, "logps/chosen": -324.1401062011719, "logps/rejected": -261.5168762207031, "loss": 0.4964, "metrics/advantage_var": 119.01836395263672, "rewards/accuracies": 0.7562500238418579, "rewards/chosen": 0.9547117352485657, "rewards/margins": 0.9206423163414001, "rewards/rejected": 0.03406943380832672, "stability/repetition_rate_mean": 0.875195324420929, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 1.468652367591858, "step": 910 }, { "epoch": 0.993118337606261, "grad_norm": 19.269649505615234, "learning_rate": 1.1350755386951849e-10, "logits/chosen": -1.3584011793136597, "logits/rejected": -1.3463481664657593, "logps/chosen": -329.8288879394531, "logps/rejected": -261.056640625, "loss": 0.4515, "metrics/advantage_var": 127.2206802368164, "rewards/accuracies": 0.784375011920929, "rewards/chosen": 1.044679045677185, "rewards/margins": 1.0371487140655518, "rewards/rejected": 0.007530394475907087, "stability/repetition_rate_mean": 0.861328125, "stability/response_length_mean": 512.0, "stability/response_length_std": 0.0, "stability/response_length_var": 0.0, "stability/token_entropy_mean": 2.8033204078674316, "step": 920 }, { "epoch": 1.0, "step": 927, "total_flos": 1.761935585720664e+18, "train_loss": 0.5203882457111775, "train_runtime": 14354.8151, "train_samples_per_second": 4.13, "train_steps_per_second": 0.065 } ], "logging_steps": 10, "max_steps": 927, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 1.761935585720664e+18, "train_batch_size": 8, "trial_name": null, "trial_params": null }