{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 3.0, "eval_steps": 500, "global_step": 2316, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.025906735751295335, "grad_norm": 13.400735855102539, "kl": 20.154539108276367, "learning_rate": 1.8999999999999998e-07, "logits/chosen": -38125000.29806259, "logits/rejected": -37537566.6863711, "logps/chosen": -463.5167660208644, "logps/rejected": -394.78040332512313, "loss": 0.5246, "loss/base_kto": 3.8404922485351562, "loss/total_with_kl": 4.196873664855957, "metrics/advantage_var": 214.0430145263672, "regularization/advantage_var": 214.0430145263672, "regularization/kl": 0.3563816547393799, "rewards/chosen": 0.32326160777698865, "rewards/margins": 0.1113325119872719, "rewards/rejected": 0.21192909578971675, "step": 20 }, { "epoch": 0.05181347150259067, "grad_norm": 21.468503952026367, "kl": 5.3923821449279785, "learning_rate": 3.8999999999999997e-07, "logits/chosen": -37059050.17704918, "logits/rejected": -37883155.104477614, "logps/chosen": -484.46352459016396, "logps/rejected": -372.7042677238806, "loss": 0.524, "loss/base_kto": 3.8899757862091064, "loss/total_with_kl": 4.192325115203857, "metrics/advantage_var": 181.59121704101562, "regularization/advantage_var": 181.59121704101562, "regularization/kl": 0.30234935879707336, "rewards/chosen": 0.01020375548816118, "rewards/margins": 0.09601645225821714, "rewards/rejected": -0.08581269677005597, "step": 40 }, { "epoch": 0.07772020725388601, "grad_norm": 19.037933349609375, "kl": 2.5164599418640137, "learning_rate": 5.9e-07, "logits/chosen": -36605792.71111111, "logits/rejected": -37880494.86769231, "logps/chosen": -482.24598214285714, "logps/rejected": -347.82774038461537, "loss": 0.5196, "loss/base_kto": 3.826831817626953, "loss/total_with_kl": 4.156479835510254, "metrics/advantage_var": 197.98660278320312, "regularization/advantage_var": 197.98660278320312, "regularization/kl": 0.3296476900577545, "rewards/chosen": -0.061350740705217635, "rewards/margins": 0.18291101560487855, "rewards/rejected": -0.24426175631009617, "step": 60 }, { "epoch": 0.10362694300518134, "grad_norm": 18.15247344970703, "kl": 3.9609954357147217, "learning_rate": 7.9e-07, "logits/chosen": -36251251.4031746, "logits/rejected": -38429787.372307695, "logps/chosen": -491.85843253968255, "logps/rejected": -382.4696394230769, "loss": 0.5278, "loss/base_kto": 3.88032603263855, "loss/total_with_kl": 4.222623825073242, "metrics/advantage_var": 205.5841827392578, "regularization/advantage_var": 205.5841827392578, "regularization/kl": 0.3422977030277252, "rewards/chosen": -0.10211856175982763, "rewards/margins": 0.09690999330327334, "rewards/rejected": -0.19902855506310096, "step": 80 }, { "epoch": 0.12953367875647667, "grad_norm": 14.19953727722168, "kl": 2.6970603466033936, "learning_rate": 9.9e-07, "logits/chosen": -35694232.87203792, "logits/rejected": -35838502.77588872, "logps/chosen": -497.6655805687204, "logps/rejected": -364.4688465996909, "loss": 0.5163, "loss/base_kto": 3.787245273590088, "loss/total_with_kl": 4.130555152893066, "metrics/advantage_var": 206.1918182373047, "regularization/advantage_var": 206.1918182373047, "regularization/kl": 0.3433093726634979, "rewards/chosen": 0.02683106794553157, "rewards/margins": 0.20959978206025576, "rewards/rejected": -0.1827687141147242, "step": 100 }, { "epoch": 0.15544041450777202, "grad_norm": 16.51021385192871, "kl": 15.355883598327637, "learning_rate": 9.998186234298189e-07, "logits/chosen": -35762474.22288261, "logits/rejected": -37603010.84678748, "logps/chosen": -482.34193907875186, "logps/rejected": -377.04242174629326, "loss": 0.5151, "loss/base_kto": 3.736530065536499, "loss/total_with_kl": 4.121108531951904, "metrics/advantage_var": 230.9782257080078, "regularization/advantage_var": 230.9782257080078, "regularization/kl": 0.38457876443862915, "rewards/chosen": 0.3250251305227294, "rewards/margins": 0.2259323987640964, "rewards/rejected": 0.09909273175863301, "step": 120 }, { "epoch": 0.18134715025906736, "grad_norm": 12.586821556091309, "kl": 9.067645072937012, "learning_rate": 9.992359552107714e-07, "logits/chosen": -35689568.60377359, "logits/rejected": -36856714.335403726, "logps/chosen": -488.97651336477986, "logps/rejected": -407.28008540372673, "loss": 0.515, "loss/base_kto": 3.7524573802948, "loss/total_with_kl": 4.1200852394104, "metrics/advantage_var": 220.79763793945312, "regularization/advantage_var": 220.79763793945312, "regularization/kl": 0.3676280677318573, "rewards/chosen": 0.15914545119183618, "rewards/margins": 0.23662834405163274, "rewards/rejected": -0.07748289285979656, "step": 140 }, { "epoch": 0.20725388601036268, "grad_norm": 13.009500503540039, "kl": 8.946215629577637, "learning_rate": 9.982519612796161e-07, "logits/chosen": -36485641.42331288, "logits/rejected": -37813169.73248408, "logps/chosen": -460.18457630368096, "logps/rejected": -385.6954617834395, "loss": 0.521, "loss/base_kto": 3.7870709896087646, "loss/total_with_kl": 4.167824745178223, "metrics/advantage_var": 228.68077087402344, "regularization/advantage_var": 228.68077087402344, "regularization/kl": 0.38075345754623413, "rewards/chosen": 0.14907382894878737, "rewards/margins": 0.20395135918645904, "rewards/rejected": -0.054877530237671675, "step": 160 }, { "epoch": 0.23316062176165803, "grad_norm": 15.541783332824707, "kl": 13.2084379196167, "learning_rate": 9.968674326492213e-07, "logits/chosen": -36214550.225080386, "logits/rejected": -37451903.61094225, "logps/chosen": -497.6577572347267, "logps/rejected": -371.0432655775076, "loss": 0.522, "loss/base_kto": 3.7317333221435547, "loss/total_with_kl": 4.175820350646973, "metrics/advantage_var": 266.7189025878906, "regularization/advantage_var": 266.7189025878906, "regularization/kl": 0.4440869987010956, "rewards/chosen": 0.2306328433119599, "rewards/margins": 0.23384995201817044, "rewards/rejected": -0.003217108706210522, "step": 180 }, { "epoch": 0.25906735751295334, "grad_norm": 13.655057907104492, "kl": 13.225244522094727, "learning_rate": 9.950834823142198e-07, "logits/chosen": -37435065.32515337, "logits/rejected": -37178863.69426752, "logps/chosen": -475.14239838957053, "logps/rejected": -383.55662818471336, "loss": 0.5139, "loss/base_kto": 3.7183244228363037, "loss/total_with_kl": 4.111454963684082, "metrics/advantage_var": 236.11441040039062, "regularization/advantage_var": 236.11441040039062, "regularization/kl": 0.39313045144081116, "rewards/chosen": 0.2551800780501102, "rewards/margins": 0.25233012410103844, "rewards/rejected": 0.002849953949071799, "step": 200 }, { "epoch": 0.2849740932642487, "grad_norm": 16.7305908203125, "kl": 19.802534103393555, "learning_rate": 9.929015443562942e-07, "logits/chosen": -37822903.0912, "logits/rejected": -37016402.46717557, "logps/chosen": -486.791, "logps/rejected": -354.5992604961832, "loss": 0.5101, "loss/base_kto": 3.684840440750122, "loss/total_with_kl": 4.080587863922119, "metrics/advantage_var": 237.68594360351562, "regularization/advantage_var": 237.68594360351562, "regularization/kl": 0.39574703574180603, "rewards/chosen": 0.3682064208984375, "rewards/margins": 0.27839331660379896, "rewards/rejected": 0.0898131042946386, "step": 220 }, { "epoch": 0.31088082901554404, "grad_norm": 17.98505401611328, "kl": 27.593021392822266, "learning_rate": 9.90323372791347e-07, "logits/chosen": -36503874.86862442, "logits/rejected": -37995806.33175355, "logps/chosen": -463.3840803709428, "logps/rejected": -361.9566301342812, "loss": 0.5185, "loss/base_kto": 3.679857015609741, "loss/total_with_kl": 4.147706508636475, "metrics/advantage_var": 280.9908752441406, "regularization/advantage_var": 280.9908752441406, "regularization/kl": 0.46784982085227966, "rewards/chosen": 0.4872561046476526, "rewards/margins": 0.2739743436107993, "rewards/rejected": 0.21328176103685328, "step": 240 }, { "epoch": 0.33678756476683935, "grad_norm": 13.71194839477539, "kl": 8.389355659484863, "learning_rate": 9.87351040159484e-07, "logits/chosen": -37835969.11568938, "logits/rejected": -39779386.37904468, "logps/chosen": -495.0869156101426, "logps/rejected": -386.3656346302003, "loss": 0.5249, "loss/base_kto": 3.7221741676330566, "loss/total_with_kl": 4.1990790367126465, "metrics/advantage_var": 286.4291687011719, "regularization/advantage_var": 286.4291687011719, "regularization/kl": 0.4769045412540436, "rewards/chosen": 0.12366109566907686, "rewards/margins": 0.25988230099397575, "rewards/rejected": -0.13622120532489887, "step": 260 }, { "epoch": 0.3626943005181347, "grad_norm": 14.024954795837402, "kl": 8.718774795532227, "learning_rate": 9.839869358589396e-07, "logits/chosen": -36703920.10191083, "logits/rejected": -37847592.83435583, "logps/chosen": -483.0296576433121, "logps/rejected": -370.02993194018404, "loss": 0.5172, "loss/base_kto": 3.712843656539917, "loss/total_with_kl": 4.1373419761657715, "metrics/advantage_var": 254.9535675048828, "regularization/advantage_var": 254.9535675048828, "regularization/kl": 0.42449769377708435, "rewards/chosen": 0.1800000379039983, "rewards/margins": 0.2664519486601153, "rewards/rejected": -0.086451910756117, "step": 280 }, { "epoch": 0.38860103626943004, "grad_norm": 14.930802345275879, "kl": 6.597970485687256, "learning_rate": 9.80233764225289e-07, "logits/chosen": -37782321.27699531, "logits/rejected": -38477721.75975039, "logps/chosen": -485.4553012519562, "logps/rejected": -377.11095943837756, "loss": 0.5147, "loss/base_kto": 3.685213804244995, "loss/total_with_kl": 4.11769962310791, "metrics/advantage_var": 259.7512512207031, "regularization/advantage_var": 259.7512512207031, "regularization/kl": 0.43248581886291504, "rewards/chosen": 0.13042405774709018, "rewards/margins": 0.30123039500546334, "rewards/rejected": -0.17080633725837316, "step": 300 }, { "epoch": 0.41450777202072536, "grad_norm": 15.397305488586426, "kl": 14.572275161743164, "learning_rate": 9.760945423574868e-07, "logits/chosen": -35922289.00900901, "logits/rejected": -37610392.59934854, "logps/chosen": -482.9690315315315, "logps/rejected": -346.9250305374593, "loss": 0.5085, "loss/base_kto": 3.6896419525146484, "loss/total_with_kl": 4.067927360534668, "metrics/advantage_var": 227.1981964111328, "regularization/advantage_var": 227.1981964111328, "regularization/kl": 0.3782850205898285, "rewards/chosen": 0.348306604333826, "rewards/margins": 0.27785791218600875, "rewards/rejected": 0.07044869214781721, "step": 320 }, { "epoch": 0.44041450777202074, "grad_norm": 11.70887565612793, "kl": 17.5533447265625, "learning_rate": 9.71572597692482e-07, "logits/chosen": -35765926.55421687, "logits/rejected": -37370281.55844156, "logps/chosen": -475.8682228915663, "logps/rejected": -385.58502435064935, "loss": 0.5174, "loss/base_kto": 3.7124946117401123, "loss/total_with_kl": 4.139125347137451, "metrics/advantage_var": 256.23419189453125, "regularization/advantage_var": 256.23419189453125, "regularization/kl": 0.4266298711299896, "rewards/chosen": 0.38016680062535296, "rewards/margins": 0.23725575659126205, "rewards/rejected": 0.1429110440340909, "step": 340 }, { "epoch": 0.46632124352331605, "grad_norm": 20.91187858581543, "kl": 15.539642333984375, "learning_rate": 9.666715653303588e-07, "logits/chosen": -34957664.76555024, "logits/rejected": -36433938.81776416, "logps/chosen": -498.6603867623605, "logps/rejected": -354.29424291730476, "loss": 0.5111, "loss/base_kto": 3.648345708847046, "loss/total_with_kl": 4.088916301727295, "metrics/advantage_var": 264.60699462890625, "regularization/advantage_var": 264.60699462890625, "regularization/kl": 0.4405706822872162, "rewards/chosen": 0.3147894997726027, "rewards/margins": 0.31643964897779736, "rewards/rejected": -0.0016501492051946808, "step": 360 }, { "epoch": 0.49222797927461137, "grad_norm": 18.93696403503418, "kl": 5.291294097900391, "learning_rate": 9.613953851121528e-07, "logits/chosen": -36112220.16, "logits/rejected": -37810000.9038168, "logps/chosen": -495.9295, "logps/rejected": -375.96839217557255, "loss": 0.5246, "loss/base_kto": 3.72355055809021, "loss/total_with_kl": 4.196588039398193, "metrics/advantage_var": 284.10650634765625, "regularization/advantage_var": 284.10650634765625, "regularization/kl": 0.47303733229637146, "rewards/chosen": 0.058169354248046874, "rewards/margins": 0.2757499486093303, "rewards/rejected": -0.2175805943612834, "step": 380 }, { "epoch": 0.5181347150259067, "grad_norm": 16.306333541870117, "kl": 7.787692546844482, "learning_rate": 9.557482984526924e-07, "logits/chosen": -35247359.21230769, "logits/rejected": -37101408.71111111, "logps/chosen": -490.1022115384615, "logps/rejected": -384.15761408730157, "loss": 0.5195, "loss/base_kto": 3.714895725250244, "loss/total_with_kl": 4.1563310623168945, "metrics/advantage_var": 265.1262512207031, "regularization/advantage_var": 265.1262512207031, "regularization/kl": 0.44143515825271606, "rewards/chosen": 0.1767663339468149, "rewards/margins": 0.26536319262233266, "rewards/rejected": -0.08859685867551774, "step": 400 }, { "epoch": 0.5440414507772021, "grad_norm": 13.557397842407227, "kl": 17.727249145507812, "learning_rate": 9.497348449310107e-07, "logits/chosen": -35565049.67901234, "logits/rejected": -39187663.39240506, "logps/chosen": -491.52353395061726, "logps/rejected": -355.3190268987342, "loss": 0.5138, "loss/base_kto": 3.6316330432891846, "loss/total_with_kl": 4.11004114151001, "metrics/advantage_var": 287.3321838378906, "regularization/advantage_var": 287.3321838378906, "regularization/kl": 0.4784080684185028, "rewards/chosen": 0.3309371383101852, "rewards/margins": 0.3535887140988521, "rewards/rejected": -0.02265157578866693, "step": 420 }, { "epoch": 0.5699481865284974, "grad_norm": 15.566226959228516, "kl": 9.367915153503418, "learning_rate": 9.433598586410701e-07, "logits/chosen": -34480345.31189711, "logits/rejected": -36543945.5319149, "logps/chosen": -478.0792805466238, "logps/rejected": -374.4915463525836, "loss": 0.5228, "loss/base_kto": 3.7414352893829346, "loss/total_with_kl": 4.18229866027832, "metrics/advantage_var": 264.78271484375, "regularization/advantage_var": 264.78271484375, "regularization/kl": 0.4408632218837738, "rewards/chosen": 0.10756603621209933, "rewards/margins": 0.23945736218881025, "rewards/rejected": -0.1318913259767109, "step": 440 }, { "epoch": 0.5958549222797928, "grad_norm": 15.71782112121582, "kl": 14.42451000213623, "learning_rate": 9.366284643057313e-07, "logits/chosen": -35432485.236363634, "logits/rejected": -36481050.425806455, "logps/chosen": -458.8940340909091, "logps/rejected": -380.9310735887097, "loss": 0.5155, "loss/base_kto": 3.680269241333008, "loss/total_with_kl": 4.124385356903076, "metrics/advantage_var": 266.7364807128906, "regularization/advantage_var": 266.7364807128906, "regularization/kl": 0.4441162049770355, "rewards/chosen": 0.32331404252485796, "rewards/margins": 0.3041512369060796, "rewards/rejected": 0.019162805618778353, "step": 460 }, { "epoch": 0.6217616580310881, "grad_norm": 13.159608840942383, "kl": 8.973199844360352, "learning_rate": 9.295460731570917e-07, "logits/chosen": -37341114.29497569, "logits/rejected": -37675323.07692308, "logps/chosen": -500.275425445705, "logps/rejected": -365.4965592006033, "loss": 0.5141, "loss/base_kto": 3.6769776344299316, "loss/total_with_kl": 4.112514495849609, "metrics/advantage_var": 261.5839538574219, "regularization/advantage_var": 261.5839538574219, "regularization/kl": 0.4355372488498688, "rewards/chosen": 0.18383294450212723, "rewards/margins": 0.31317346009914127, "rewards/rejected": -0.12934051559701404, "step": 480 }, { "epoch": 0.6476683937823834, "grad_norm": 13.037299156188965, "kl": 9.277281761169434, "learning_rate": 9.221183785865056e-07, "logits/chosen": -35099822.06431853, "logits/rejected": -35899133.9585327, "logps/chosen": -495.6490237366003, "logps/rejected": -382.206115430622, "loss": 0.5184, "loss/base_kto": 3.6986634731292725, "loss/total_with_kl": 4.147059917449951, "metrics/advantage_var": 269.30712890625, "regularization/advantage_var": 269.30712890625, "regularization/kl": 0.44839635491371155, "rewards/chosen": 0.12724044823171898, "rewards/margins": 0.266642944228903, "rewards/rejected": -0.13940249599718402, "step": 500 }, { "epoch": 0.6735751295336787, "grad_norm": 22.929607391357422, "kl": 12.852584838867188, "learning_rate": 9.143513515677817e-07, "logits/chosen": -36335391.219512194, "logits/rejected": -37296974.76923077, "logps/chosen": -477.5895579268293, "logps/rejected": -366.32221554487177, "loss": 0.5159, "loss/base_kto": 3.6915135383605957, "loss/total_with_kl": 4.127551078796387, "metrics/advantage_var": 261.88427734375, "regularization/advantage_var": 261.88427734375, "regularization/kl": 0.4360373020172119, "rewards/chosen": 0.23083547266518198, "rewards/margins": 0.2442612217470137, "rewards/rejected": -0.013425749081831712, "step": 520 }, { "epoch": 0.6994818652849741, "grad_norm": 16.30242919921875, "kl": 22.94746208190918, "learning_rate": 9.062512358572373e-07, "logits/chosen": -37376289.81132075, "logits/rejected": -38749177.63975155, "logps/chosen": -481.67845911949684, "logps/rejected": -377.6274990295031, "loss": 0.5149, "loss/base_kto": 3.6707916259765625, "loss/total_with_kl": 4.1191840171813965, "metrics/advantage_var": 269.30450439453125, "regularization/advantage_var": 269.30450439453125, "regularization/kl": 0.44839200377464294, "rewards/chosen": 0.37893259300375887, "rewards/margins": 0.2631320908663911, "rewards/rejected": 0.11580050213736777, "step": 540 }, { "epoch": 0.7253886010362695, "grad_norm": 14.475749969482422, "kl": 16.794092178344727, "learning_rate": 8.978245429744691e-07, "logits/chosen": -35113893.83647799, "logits/rejected": -37618837.46583851, "logps/chosen": -472.98432586477986, "logps/rejected": -358.71797360248445, "loss": 0.5175, "loss/base_kto": 3.662067413330078, "loss/total_with_kl": 4.140262603759766, "metrics/advantage_var": 287.2043151855469, "regularization/advantage_var": 287.2043151855469, "regularization/kl": 0.4781951904296875, "rewards/chosen": 0.32611971681222973, "rewards/margins": 0.2916624952775481, "rewards/rejected": 0.034457221534681615, "step": 560 }, { "epoch": 0.7512953367875648, "grad_norm": 16.73085594177246, "kl": 20.94074821472168, "learning_rate": 8.890780469678738e-07, "logits/chosen": -35394108.32628399, "logits/rejected": -36212742.62783171, "logps/chosen": -481.3646148036254, "logps/rejected": -363.4192455501618, "loss": 0.515, "loss/base_kto": 3.6869637966156006, "loss/total_with_kl": 4.119616508483887, "metrics/advantage_var": 259.85150146484375, "regularization/advantage_var": 259.85150146484375, "regularization/kl": 0.43265271186828613, "rewards/chosen": 0.384003515329966, "rewards/margins": 0.24847656811095686, "rewards/rejected": 0.13552694721900915, "step": 580 }, { "epoch": 0.7772020725388601, "grad_norm": 16.116498947143555, "kl": 15.556981086730957, "learning_rate": 8.800187789691269e-07, "logits/chosen": -36604658.526315786, "logits/rejected": -37119865.139356814, "logps/chosen": -481.2325558213716, "logps/rejected": -371.94807618683, "loss": 0.5156, "loss/base_kto": 3.6460304260253906, "loss/total_with_kl": 4.1249775886535645, "metrics/advantage_var": 287.6559753417969, "regularization/advantage_var": 287.6559753417969, "regularization/kl": 0.47894716262817383, "rewards/chosen": 0.2796679531748405, "rewards/margins": 0.31520266437293165, "rewards/rejected": -0.035534711198091144, "step": 600 }, { "epoch": 0.8031088082901554, "grad_norm": 13.209074020385742, "kl": 18.143861770629883, "learning_rate": 8.706540215409981e-07, "logits/chosen": -37562311.935587764, "logits/rejected": -37459352.667678304, "logps/chosen": -504.02480877616745, "logps/rejected": -388.95549601669194, "loss": 0.5179, "loss/base_kto": 3.6370232105255127, "loss/total_with_kl": 4.143251895904541, "metrics/advantage_var": 304.0413513183594, "regularization/advantage_var": 304.0413513183594, "regularization/kl": 0.5062288641929626, "rewards/chosen": 0.2887623859104519, "rewards/margins": 0.31435846622160335, "rewards/rejected": -0.025596080311151483, "step": 620 }, { "epoch": 0.8290155440414507, "grad_norm": 14.419571876525879, "kl": 9.830085754394531, "learning_rate": 8.609913028230462e-07, "logits/chosen": -35569604.73311897, "logits/rejected": -36431735.051671736, "logps/chosen": -479.87590434083603, "logps/rejected": -383.4615311550152, "loss": 0.5181, "loss/base_kto": 3.7124288082122803, "loss/total_with_kl": 4.144567012786865, "metrics/advantage_var": 259.5424499511719, "regularization/advantage_var": 259.5424499511719, "regularization/kl": 0.4321381747722626, "rewards/chosen": 0.10982839700876708, "rewards/margins": 0.24909327771379175, "rewards/rejected": -0.13926488070502469, "step": 640 }, { "epoch": 0.8549222797927462, "grad_norm": 20.648231506347656, "kl": 14.493423461914062, "learning_rate": 8.510383904798994e-07, "logits/chosen": -36495965.65853658, "logits/rejected": -36950193.23076923, "logps/chosen": -503.938262195122, "logps/rejected": -382.3724959935897, "loss": 0.5177, "loss/base_kto": 3.6546566486358643, "loss/total_with_kl": 4.1416168212890625, "metrics/advantage_var": 292.46881103515625, "regularization/advantage_var": 292.46881103515625, "regularization/kl": 0.48696061968803406, "rewards/chosen": 0.2888377119855183, "rewards/margins": 0.31819413661062157, "rewards/rejected": -0.02935642462510329, "step": 660 }, { "epoch": 0.8808290155440415, "grad_norm": 13.785265922546387, "kl": 16.71992301940918, "learning_rate": 8.408032854569865e-07, "logits/chosen": -35666898.34394904, "logits/rejected": -35189175.754601225, "logps/chosen": -469.50577229299364, "logps/rejected": -365.37624616564415, "loss": 0.5101, "loss/base_kto": 3.6475751399993896, "loss/total_with_kl": 4.080502510070801, "metrics/advantage_var": 260.0164489746094, "regularization/advantage_var": 260.0164489746094, "regularization/kl": 0.43292737007141113, "rewards/chosen": 0.2669396856028563, "rewards/margins": 0.29887760072616976, "rewards/rejected": -0.03193791512331348, "step": 680 }, { "epoch": 0.9067357512953368, "grad_norm": 43.316898345947266, "kl": 20.487064361572266, "learning_rate": 8.302942155487377e-07, "logits/chosen": -37338285.24471299, "logits/rejected": -37374581.64401294, "logps/chosen": -475.091625755287, "logps/rejected": -369.8734324433657, "loss": 0.5204, "loss/base_kto": 3.684366226196289, "loss/total_with_kl": 4.16303014755249, "metrics/advantage_var": 287.4859313964844, "regularization/advantage_var": 287.4859313964844, "regularization/kl": 0.4786640703678131, "rewards/chosen": 0.349972796944333, "rewards/margins": 0.27057078691244457, "rewards/rejected": 0.0794020100318884, "step": 700 }, { "epoch": 0.9326424870466321, "grad_norm": 21.38011932373047, "kl": 24.615724563598633, "learning_rate": 8.195196287844278e-07, "logits/chosen": -35794931.1799687, "logits/rejected": -35510048.34945398, "logps/chosen": -491.32707355242565, "logps/rejected": -372.66712168486737, "loss": 0.5117, "loss/base_kto": 3.600149631500244, "loss/total_with_kl": 4.0938029289245605, "metrics/advantage_var": 296.4882507324219, "regularization/advantage_var": 296.4882507324219, "regularization/kl": 0.493652880191803, "rewards/chosen": 0.43623239818686427, "rewards/margins": 0.3389319683889106, "rewards/rejected": 0.09730042979795364, "step": 720 }, { "epoch": 0.9585492227979274, "grad_norm": 145.93531799316406, "kl": 19.115631103515625, "learning_rate": 8.08488186636975e-07, "logits/chosen": -36294998.420382164, "logits/rejected": -37305601.57055215, "logps/chosen": -474.3715167197452, "logps/rejected": -359.26993865030676, "loss": 0.5191, "loss/base_kto": 3.640221357345581, "loss/total_with_kl": 4.152987480163574, "metrics/advantage_var": 307.96746826171875, "regularization/advantage_var": 307.96746826171875, "regularization/kl": 0.5127658247947693, "rewards/chosen": 0.33320646832703027, "rewards/margins": 0.32371776870069274, "rewards/rejected": 0.009488699626337532, "step": 740 }, { "epoch": 0.9844559585492227, "grad_norm": 16.139719009399414, "kl": 15.433197021484375, "learning_rate": 7.972087570601576e-07, "logits/chosen": -35393484.8798752, "logits/rejected": -36672468.73239437, "logps/chosen": -507.7396158346334, "logps/rejected": -372.2236404538341, "loss": 0.5206, "loss/base_kto": 3.639004945755005, "loss/total_with_kl": 4.164422035217285, "metrics/advantage_var": 315.56585693359375, "regularization/advantage_var": 315.56585693359375, "regularization/kl": 0.525417149066925, "rewards/chosen": 0.20587270085041684, "rewards/margins": 0.3296515679003728, "rewards/rejected": -0.12377886704995598, "step": 760 }, { "epoch": 1.0103626943005182, "grad_norm": 31.113555908203125, "kl": 15.1028470993042, "learning_rate": 7.856904073598458e-07, "logits/chosen": -36330912.94851794, "logits/rejected": -37367312.87912088, "logps/chosen": -487.8967433697348, "logps/rejected": -362.51091542386183, "loss": 0.5182, "loss/base_kto": 3.4688007831573486, "loss/total_with_kl": 4.145680904388428, "metrics/advantage_var": 406.5345153808594, "regularization/advantage_var": 406.5345153808594, "regularization/kl": 0.6768800020217896, "rewards/chosen": 0.35281691052798364, "rewards/margins": 0.5150296476548356, "rewards/rejected": -0.16221273712685194, "step": 780 }, { "epoch": 1.0362694300518134, "grad_norm": 21.974807739257812, "kl": 21.096019744873047, "learning_rate": 7.739423969049761e-07, "logits/chosen": -34902750.12923077, "logits/rejected": -35730493.765079364, "logps/chosen": -483.70221153846154, "logps/rejected": -382.71502976190476, "loss": 0.5228, "loss/base_kto": 3.095506429672241, "loss/total_with_kl": 4.1820855140686035, "metrics/advantage_var": 652.60009765625, "regularization/advantage_var": 652.60009765625, "regularization/kl": 1.0865792036056519, "rewards/chosen": 0.7133823805588942, "rewards/margins": 0.9873754458025699, "rewards/rejected": -0.2739930652436756, "step": 800 }, { "epoch": 1.0621761658031088, "grad_norm": 11.445603370666504, "kl": 24.443540573120117, "learning_rate": 7.619741696841322e-07, "logits/chosen": -34818446.222222224, "logits/rejected": -36179767.088607594, "logps/chosen": -493.6755401234568, "logps/rejected": -398.9047913370253, "loss": 0.5363, "loss/base_kto": 3.096897602081299, "loss/total_with_kl": 4.290025234222412, "metrics/advantage_var": 716.59326171875, "regularization/advantage_var": 716.59326171875, "regularization/kl": 1.1931277513504028, "rewards/chosen": 0.7552757734133874, "rewards/margins": 0.9890814177746661, "rewards/rejected": -0.23380564436127868, "step": 820 }, { "epoch": 1.0880829015544042, "grad_norm": 20.024269104003906, "kl": 26.2730712890625, "learning_rate": 7.497953467137135e-07, "logits/chosen": -34908460.1095008, "logits/rejected": -35962824.060698025, "logps/chosen": -488.4697564412238, "logps/rejected": -393.1842754172989, "loss": 0.5048, "loss/base_kto": 3.1360538005828857, "loss/total_with_kl": 4.038689613342285, "metrics/advantage_var": 542.1234741210938, "regularization/advantage_var": 542.1234741210938, "regularization/kl": 0.9026355743408203, "rewards/chosen": 0.757409530369364, "rewards/margins": 0.9136508242924481, "rewards/rejected": -0.15624129392308422, "step": 840 }, { "epoch": 1.1139896373056994, "grad_norm": 21.529926300048828, "kl": 9.573410987854004, "learning_rate": 7.37415718303793e-07, "logits/chosen": -34975737.4464, "logits/rejected": -36086504.15877862, "logps/chosen": -486.6575, "logps/rejected": -389.136641221374, "loss": 0.5244, "loss/base_kto": 3.2002956867218018, "loss/total_with_kl": 4.195446968078613, "metrics/advantage_var": 597.6884765625, "regularization/advantage_var": 597.6884765625, "regularization/kl": 0.9951512217521667, "rewards/chosen": 0.388057763671875, "rewards/margins": 0.8503276937470181, "rewards/rejected": -0.4622699300751431, "step": 860 }, { "epoch": 1.1398963730569949, "grad_norm": 14.2310791015625, "kl": 16.73166847229004, "learning_rate": 7.248452361878855e-07, "logits/chosen": -35703589.717868336, "logits/rejected": -36457500.71028037, "logps/chosen": -468.9491085423198, "logps/rejected": -361.90678543613706, "loss": 0.5179, "loss/base_kto": 3.1490395069122314, "loss/total_with_kl": 4.14325475692749, "metrics/advantage_var": 597.1264038085938, "regularization/advantage_var": 597.1264038085938, "regularization/kl": 0.9942154288291931, "rewards/chosen": 0.6469504138145327, "rewards/margins": 0.93025283978136, "rewards/rejected": -0.2833024259668273, "step": 880 }, { "epoch": 1.16580310880829, "grad_norm": 19.249755859375, "kl": 10.497282981872559, "learning_rate": 7.120940055229497e-07, "logits/chosen": -35211503.45846154, "logits/rejected": -36472604.44444445, "logps/chosen": -474.0904807692308, "logps/rejected": -384.2449404761905, "loss": 0.5222, "loss/base_kto": 3.2282378673553467, "loss/total_with_kl": 4.177388668060303, "metrics/advantage_var": 570.0606689453125, "regularization/advantage_var": 570.0606689453125, "regularization/kl": 0.9491510391235352, "rewards/chosen": 0.4077448918269231, "rewards/margins": 0.8156021025009061, "rewards/rejected": -0.4078572106739831, "step": 900 }, { "epoch": 1.1917098445595855, "grad_norm": 12.673859596252441, "kl": 22.763656616210938, "learning_rate": 6.991722767660556e-07, "logits/chosen": -35433958.04608295, "logits/rejected": -36659326.98251192, "logps/chosen": -486.46409370199694, "logps/rejected": -354.6059966216216, "loss": 0.5003, "loss/base_kto": 3.0805881023406982, "loss/total_with_kl": 4.0026164054870605, "metrics/advantage_var": 553.770751953125, "regularization/advantage_var": 553.770751953125, "regularization/kl": 0.9220282435417175, "rewards/chosen": 0.7849215318530386, "rewards/margins": 0.9836626866071436, "rewards/rejected": -0.19874115475410498, "step": 920 }, { "epoch": 1.2176165803108807, "grad_norm": 17.543062210083008, "kl": 18.01152992248535, "learning_rate": 6.860904374342499e-07, "logits/chosen": -34316418.238473766, "logits/rejected": -35322412.82949309, "logps/chosen": -472.60741255961847, "logps/rejected": -347.34483006912444, "loss": 0.5108, "loss/base_kto": 3.2038142681121826, "loss/total_with_kl": 4.086097717285156, "metrics/advantage_var": 529.9000244140625, "regularization/advantage_var": 529.9000244140625, "regularization/kl": 0.8822835087776184, "rewards/chosen": 0.5735934064953051, "rewards/margins": 0.8595392226525391, "rewards/rejected": -0.2859458161572341, "step": 940 }, { "epoch": 1.2435233160621761, "grad_norm": 9.224899291992188, "kl": 18.6295223236084, "learning_rate": 6.7285900375424e-07, "logits/chosen": -34802521.6, "logits/rejected": -36403760.0, "logps/chosen": -433.265234375, "logps/rejected": -375.4929443359375, "loss": 0.5261, "loss/base_kto": 3.145615339279175, "loss/total_with_kl": 4.209061622619629, "metrics/advantage_var": 638.7062377929688, "regularization/advantage_var": 638.7062377929688, "regularization/kl": 1.063446044921875, "rewards/chosen": 0.6340777397155761, "rewards/margins": 0.9245184659957886, "rewards/rejected": -0.2904407262802124, "step": 960 }, { "epoch": 1.2694300518134716, "grad_norm": 16.165605545043945, "kl": 17.752416610717773, "learning_rate": 6.594886122086123e-07, "logits/chosen": -33469711.203852326, "logits/rejected": -36636893.32115677, "logps/chosen": -511.0252307383628, "logps/rejected": -389.2472650304414, "loss": 0.5349, "loss/base_kto": 3.1670024394989014, "loss/total_with_kl": 4.279271125793457, "metrics/advantage_var": 668.0293579101562, "regularization/advantage_var": 668.0293579101562, "regularization/kl": 1.1122688055038452, "rewards/chosen": 0.6031466709094101, "rewards/margins": 0.9128984166820985, "rewards/rejected": -0.3097517457726884, "step": 980 }, { "epoch": 1.2953367875647668, "grad_norm": 119.1275405883789, "kl": 16.88669204711914, "learning_rate": 6.459900109853766e-07, "logits/chosen": -36045253.67088608, "logits/rejected": -34577449.086419754, "logps/chosen": -467.2184038765823, "logps/rejected": -375.0442708333333, "loss": 0.665, "loss/base_kto": 3.161041021347046, "loss/total_with_kl": 5.320053577423096, "metrics/advantage_var": 1296.7041015625, "regularization/advantage_var": 1296.7041015625, "regularization/kl": 2.15901255607605, "rewards/chosen": 0.6263412282436709, "rewards/margins": 0.9371888756398057, "rewards/rejected": -0.31084764739613474, "step": 1000 }, { "epoch": 1.3212435233160622, "grad_norm": 20.290742874145508, "kl": 12.173848152160645, "learning_rate": 6.323740513377171e-07, "logits/chosen": -34316786.832072616, "logits/rejected": -36752296.323101774, "logps/chosen": -471.86497730711045, "logps/rejected": -377.8185581583199, "loss": 0.5494, "loss/base_kto": 3.226526975631714, "loss/total_with_kl": 4.395519256591797, "metrics/advantage_var": 702.097412109375, "regularization/advantage_var": 702.097412109375, "regularization/kl": 1.1689921617507935, "rewards/chosen": 0.5521525095883368, "rewards/margins": 0.8078719178608782, "rewards/rejected": -0.2557194082725414, "step": 1020 }, { "epoch": 1.3471502590673574, "grad_norm": 17.557085037231445, "kl": 25.647724151611328, "learning_rate": 6.186516788608865e-07, "logits/chosen": -34383122.583703704, "logits/rejected": -36638151.299173556, "logps/chosen": -481.93148148148146, "logps/rejected": -360.85795454545456, "loss": 0.5108, "loss/base_kto": 3.1940364837646484, "loss/total_with_kl": 4.08669900894165, "metrics/advantage_var": 536.1336059570312, "regularization/advantage_var": 536.1336059570312, "regularization/kl": 0.8926624655723572, "rewards/chosen": 0.7570982982494213, "rewards/margins": 0.8593505459573242, "rewards/rejected": -0.1022522477079029, "step": 1040 }, { "epoch": 1.3730569948186528, "grad_norm": 12.21741771697998, "kl": 22.415632247924805, "learning_rate": 6.048339246932652e-07, "logits/chosen": -35247341.742033385, "logits/rejected": -36756554.20289855, "logps/chosen": -483.63002655538696, "logps/rejected": -381.67507045088564, "loss": 0.5783, "loss/base_kto": 3.184499502182007, "loss/total_with_kl": 4.626182556152344, "metrics/advantage_var": 865.8757934570312, "regularization/advantage_var": 865.8757934570312, "regularization/kl": 1.4416831731796265, "rewards/chosen": 0.7054853656404116, "rewards/margins": 0.8441251013552366, "rewards/rejected": -0.13863973571482488, "step": 1060 }, { "epoch": 1.3989637305699483, "grad_norm": 9.0669527053833, "kl": 17.679346084594727, "learning_rate": 5.909318966486494e-07, "logits/chosen": -35166431.3609831, "logits/rejected": -37391400.699523054, "logps/chosen": -497.14170506912444, "logps/rejected": -367.95933525437204, "loss": 0.5164, "loss/base_kto": 3.18158221244812, "loss/total_with_kl": 4.131343364715576, "metrics/advantage_var": 570.4268798828125, "regularization/advantage_var": 570.4268798828125, "regularization/kl": 0.9497607350349426, "rewards/chosen": 0.6133537233882969, "rewards/margins": 0.8766409022405177, "rewards/rejected": -0.2632871788522208, "step": 1080 }, { "epoch": 1.4248704663212435, "grad_norm": 19.357873916625977, "kl": 20.663843154907227, "learning_rate": 5.769567702869052e-07, "logits/chosen": -35012993.58024691, "logits/rejected": -36169468.75949367, "logps/chosen": -477.03211805555554, "logps/rejected": -357.1598595727848, "loss": 0.502, "loss/base_kto": 3.1430957317352295, "loss/total_with_kl": 4.015756607055664, "metrics/advantage_var": 524.1205444335938, "regularization/advantage_var": 524.1205444335938, "regularization/kl": 0.872660756111145, "rewards/chosen": 0.6914439260223766, "rewards/margins": 0.8968758038346294, "rewards/rejected": -0.20543187781225278, "step": 1100 }, { "epoch": 1.450777202072539, "grad_norm": 14.7743558883667, "kl": 22.073400497436523, "learning_rate": 5.629197799301614e-07, "logits/chosen": -35463370.54945055, "logits/rejected": -36603024.92068429, "logps/chosen": -469.2876766091052, "logps/rejected": -369.5140211897356, "loss": 0.5372, "loss/base_kto": 3.152519941329956, "loss/total_with_kl": 4.297989368438721, "metrics/advantage_var": 687.9698486328125, "regularization/advantage_var": 687.9698486328125, "regularization/kl": 1.1454696655273438, "rewards/chosen": 0.6753101199163559, "rewards/margins": 0.8877968022920606, "rewards/rejected": -0.2124866823757047, "step": 1120 }, { "epoch": 1.4766839378238341, "grad_norm": 25.720378875732422, "kl": 18.7586669921875, "learning_rate": 5.488322096317633e-07, "logits/chosen": -34098366.69796557, "logits/rejected": -36353859.49453978, "logps/chosen": -496.879010172144, "logps/rejected": -393.6916195397816, "loss": 0.5312, "loss/base_kto": 3.1742589473724365, "loss/total_with_kl": 4.249287128448486, "metrics/advantage_var": 645.6625366210938, "regularization/advantage_var": 645.6625366210938, "regularization/kl": 1.0750280618667603, "rewards/chosen": 0.6056689051954959, "rewards/margins": 0.8555184359198889, "rewards/rejected": -0.24984953072439303, "step": 1140 }, { "epoch": 1.5025906735751295, "grad_norm": 13.962899208068848, "kl": 23.922412872314453, "learning_rate": 5.347053841052518e-07, "logits/chosen": -32700181.85365854, "logits/rejected": -36528456.20512821, "logps/chosen": -473.018340320122, "logps/rejected": -367.9064503205128, "loss": 0.5293, "loss/base_kto": 3.193666934967041, "loss/total_with_kl": 4.23423957824707, "metrics/advantage_var": 624.9685668945312, "regularization/advantage_var": 624.9685668945312, "regularization/kl": 1.0405725240707397, "rewards/chosen": 0.7202913237781059, "rewards/margins": 0.872715549218498, "rewards/rejected": -0.15242422544039214, "step": 1160 }, { "epoch": 1.528497409326425, "grad_norm": 34.06401062011719, "kl": 27.436681747436523, "learning_rate": 5.205506596206531e-07, "logits/chosen": -35291384.8224299, "logits/rejected": -35755377.153605014, "logps/chosen": -462.1151674454829, "logps/rejected": -369.5713166144201, "loss": 0.5394, "loss/base_kto": 3.1750638484954834, "loss/total_with_kl": 4.315387725830078, "metrics/advantage_var": 684.8792114257812, "regularization/advantage_var": 684.8792114257812, "regularization/kl": 1.1403238773345947, "rewards/chosen": 0.713312642225224, "rewards/margins": 0.8410065120760762, "rewards/rejected": -0.12769386985085227, "step": 1180 }, { "epoch": 1.5544041450777202, "grad_norm": 18.86966896057129, "kl": 15.892166137695312, "learning_rate": 5.063794148754032e-07, "logits/chosen": -35464024.376012966, "logits/rejected": -36400058.49773756, "logps/chosen": -470.35929902755265, "logps/rejected": -350.68240950226243, "loss": 0.512, "loss/base_kto": 3.220384359359741, "loss/total_with_kl": 4.095946311950684, "metrics/advantage_var": 525.8631591796875, "regularization/advantage_var": 525.8631591796875, "regularization/kl": 0.8755621314048767, "rewards/chosen": 0.6322690200187399, "rewards/margins": 0.8324806499869831, "rewards/rejected": -0.2002116299682433, "step": 1200 }, { "epoch": 1.5803108808290154, "grad_norm": 27.72390365600586, "kl": 16.47442054748535, "learning_rate": 4.922030418472422e-07, "logits/chosen": -35167950.42539682, "logits/rejected": -36979809.673846155, "logps/chosen": -497.8609623015873, "logps/rejected": -385.4046153846154, "loss": 0.5057, "loss/base_kto": 3.1687557697296143, "loss/total_with_kl": 4.045971870422363, "metrics/advantage_var": 526.8564453125, "regularization/advantage_var": 526.8564453125, "regularization/kl": 0.8772159814834595, "rewards/chosen": 0.5816710941375248, "rewards/margins": 0.8957860843718999, "rewards/rejected": -0.314114990234375, "step": 1220 }, { "epoch": 1.6062176165803108, "grad_norm": 13.024700164794922, "kl": 14.6386079788208, "learning_rate": 4.780329366364336e-07, "logits/chosen": -35346368.41830065, "logits/rejected": -36299034.05988024, "logps/chosen": -485.52359068627453, "logps/rejected": -395.9764221556886, "loss": 0.5054, "loss/base_kto": 3.2019081115722656, "loss/total_with_kl": 4.043558597564697, "metrics/advantage_var": 505.49566650390625, "regularization/advantage_var": 505.49566650390625, "regularization/kl": 0.8416501879692078, "rewards/chosen": 0.4972510244332108, "rewards/margins": 0.8567804854087206, "rewards/rejected": -0.3595294609755099, "step": 1240 }, { "epoch": 1.6321243523316062, "grad_norm": 15.260805130004883, "kl": 17.375057220458984, "learning_rate": 4.638804903046684e-07, "logits/chosen": -34435508.56426332, "logits/rejected": -35684664.62305296, "logps/chosen": -495.2782621473354, "logps/rejected": -374.00642523364485, "loss": 0.5491, "loss/base_kto": 3.1951146125793457, "loss/total_with_kl": 4.392637729644775, "metrics/advantage_var": 719.2330322265625, "regularization/advantage_var": 719.2330322265625, "regularization/kl": 1.1975229978561401, "rewards/chosen": 0.6216823852921728, "rewards/margins": 0.816191802389318, "rewards/rejected": -0.19450941709714514, "step": 1260 }, { "epoch": 1.6580310880829017, "grad_norm": 13.97712230682373, "kl": 10.879806518554688, "learning_rate": 4.497570797180217e-07, "logits/chosen": -34823172.82260597, "logits/rejected": -36052233.953343704, "logps/chosen": -498.48302590266877, "logps/rejected": -372.73617321150857, "loss": 0.5146, "loss/base_kto": 3.2071304321289062, "loss/total_with_kl": 4.116452217102051, "metrics/advantage_var": 546.1390991210938, "regularization/advantage_var": 546.1390991210938, "regularization/kl": 0.9093216061592102, "rewards/chosen": 0.4803002086501668, "rewards/margins": 0.8660376737566622, "rewards/rejected": -0.3857374651064954, "step": 1280 }, { "epoch": 1.6839378238341969, "grad_norm": 22.177412033081055, "kl": 15.012435913085938, "learning_rate": 4.3567405840131853e-07, "logits/chosen": -37029306.469135806, "logits/rejected": -36475217.01265823, "logps/chosen": -480.08256172839504, "logps/rejected": -379.47804588607596, "loss": 0.5276, "loss/base_kto": 3.254469394683838, "loss/total_with_kl": 4.220569133758545, "metrics/advantage_var": 580.240234375, "regularization/advantage_var": 580.240234375, "regularization/kl": 0.9660999178886414, "rewards/chosen": 0.5825492482126495, "rewards/margins": 0.7777795149285563, "rewards/rejected": -0.19523026671590685, "step": 1300 }, { "epoch": 1.709844559585492, "grad_norm": 23.869699478149414, "kl": 12.002751350402832, "learning_rate": 4.2164274741126506e-07, "logits/chosen": -34679942.4, "logits/rejected": -35694416.0, "logps/chosen": -508.879736328125, "logps/rejected": -363.1947265625, "loss": 0.5073, "loss/base_kto": 3.2425103187561035, "loss/total_with_kl": 4.058449745178223, "metrics/advantage_var": 490.0536193847656, "regularization/advantage_var": 490.0536193847656, "regularization/kl": 0.8159392476081848, "rewards/chosen": 0.46154537200927737, "rewards/margins": 0.7988351821899414, "rewards/rejected": -0.33728981018066406, "step": 1320 }, { "epoch": 1.7357512953367875, "grad_norm": 22.328872680664062, "kl": 11.328975677490234, "learning_rate": 4.0767442623567856e-07, "logits/chosen": -33229568.41761827, "logits/rejected": -34615025.79910045, "logps/chosen": -463.5056076672104, "logps/rejected": -372.407561844078, "loss": 0.5118, "loss/base_kto": 3.231456756591797, "loss/total_with_kl": 4.094671249389648, "metrics/advantage_var": 518.447265625, "regularization/advantage_var": 518.447265625, "regularization/kl": 0.8632146716117859, "rewards/chosen": 0.41919343584318924, "rewards/margins": 0.8015186840467242, "rewards/rejected": -0.382325248203535, "step": 1340 }, { "epoch": 1.761658031088083, "grad_norm": 17.927732467651367, "kl": 13.05345630645752, "learning_rate": 3.937803237261357e-07, "logits/chosen": -34053565.217391305, "logits/rejected": -35702049.81132075, "logps/chosen": -461.24844720496895, "logps/rejected": -407.92162932389937, "loss": 0.5019, "loss/base_kto": 3.2033932209014893, "loss/total_with_kl": 4.015069484710693, "metrics/advantage_var": 487.4932556152344, "regularization/advantage_var": 487.4932556152344, "regularization/kl": 0.8116763234138489, "rewards/chosen": 0.5069902313421972, "rewards/margins": 0.8418340408418779, "rewards/rejected": -0.3348438094996806, "step": 1360 }, { "epoch": 1.7875647668393784, "grad_norm": 20.6157283782959, "kl": 14.827483177185059, "learning_rate": 3.7997160907132456e-07, "logits/chosen": -34930153.73913044, "logits/rejected": -36487099.629742034, "logps/chosen": -484.662037037037, "logps/rejected": -393.36428300455236, "loss": 0.5146, "loss/base_kto": 3.2425010204315186, "loss/total_with_kl": 4.116556644439697, "metrics/advantage_var": 524.95849609375, "regularization/advantage_var": 524.95849609375, "regularization/kl": 0.8740558624267578, "rewards/chosen": 0.4873495547283867, "rewards/margins": 0.8078777385221912, "rewards/rejected": -0.32052818379380454, "step": 1380 }, { "epoch": 1.8134715025906736, "grad_norm": 12.403368949890137, "kl": 10.325544357299805, "learning_rate": 3.662593828183601e-07, "logits/chosen": -33934097.22691706, "logits/rejected": -34286149.49141966, "logps/chosen": -465.74623435054775, "logps/rejected": -363.0441205148206, "loss": 0.5182, "loss/base_kto": 3.248669147491455, "loss/total_with_kl": 4.1453728675842285, "metrics/advantage_var": 538.5609741210938, "regularization/advantage_var": 538.5609741210938, "regularization/kl": 0.8967038989067078, "rewards/chosen": 0.4542280743379548, "rewards/margins": 0.8177103785227003, "rewards/rejected": -0.36348230418474553, "step": 1400 }, { "epoch": 1.8393782383419688, "grad_norm": 16.25936508178711, "kl": 12.982745170593262, "learning_rate": 3.5265466794927725e-07, "logits/chosen": -33951469.83225807, "logits/rejected": -35572503.27272727, "logps/chosen": -485.821875, "logps/rejected": -373.7669270833333, "loss": 0.5075, "loss/base_kto": 3.1363184452056885, "loss/total_with_kl": 4.059669017791748, "metrics/advantage_var": 554.5648803710938, "regularization/advantage_var": 554.5648803710938, "regularization/kl": 0.9233505129814148, "rewards/chosen": 0.539885490171371, "rewards/margins": 0.9372862710514609, "rewards/rejected": -0.39740078088008995, "step": 1420 }, { "epoch": 1.8652849740932642, "grad_norm": 16.2618408203125, "kl": 15.021658897399902, "learning_rate": 3.3916840101987887e-07, "logits/chosen": -35530640.96385542, "logits/rejected": -34702435.74025974, "logps/chosen": -486.8272307981928, "logps/rejected": -384.74543425324674, "loss": 0.508, "loss/base_kto": 3.16685152053833, "loss/total_with_kl": 4.064095497131348, "metrics/advantage_var": 538.8851318359375, "regularization/advantage_var": 538.8851318359375, "regularization/kl": 0.8972436785697937, "rewards/chosen": 0.6191224247576242, "rewards/margins": 0.8982581907250553, "rewards/rejected": -0.279135765967431, "step": 1440 }, { "epoch": 1.8911917098445596, "grad_norm": 10.686800956726074, "kl": 13.021166801452637, "learning_rate": 3.258114233680583e-07, "logits/chosen": -34040484.256651014, "logits/rejected": -35551113.78471139, "logps/chosen": -484.4517312206573, "logps/rejected": -367.4321129095164, "loss": 0.4938, "loss/base_kto": 3.207645893096924, "loss/total_with_kl": 3.9501023292541504, "metrics/advantage_var": 445.91973876953125, "regularization/advantage_var": 445.91973876953125, "regularization/kl": 0.7424563765525818, "rewards/chosen": 0.5291195005318369, "rewards/margins": 0.8354601867764742, "rewards/rejected": -0.3063406862446373, "step": 1460 }, { "epoch": 1.917098445595855, "grad_norm": 16.483449935913086, "kl": 14.305689811706543, "learning_rate": 3.1259447239866796e-07, "logits/chosen": -35764896.29179332, "logits/rejected": -35511905.1318328, "logps/chosen": -478.85011398176295, "logps/rejected": -377.6265072347267, "loss": 0.5013, "loss/base_kto": 3.1524693965911865, "loss/total_with_kl": 4.010059356689453, "metrics/advantage_var": 515.0692138671875, "regularization/advantage_var": 515.0692138671875, "regularization/kl": 0.8575901985168457, "rewards/chosen": 0.5934567900776501, "rewards/margins": 0.9168128052002384, "rewards/rejected": -0.32335601512258844, "step": 1480 }, { "epoch": 1.9430051813471503, "grad_norm": 13.733931541442871, "kl": 15.422091484069824, "learning_rate": 2.9952817295193435e-07, "logits/chosen": -35332454.74829932, "logits/rejected": -36086943.8150289, "logps/chosen": -464.26589073129253, "logps/rejected": -377.4020953757225, "loss": 0.4931, "loss/base_kto": 3.217292070388794, "loss/total_with_kl": 3.944584608078003, "metrics/advantage_var": 436.8122253417969, "regularization/advantage_var": 436.8122253417969, "regularization/kl": 0.7272922396659851, "rewards/chosen": 0.48512465288849915, "rewards/margins": 0.7993540778667664, "rewards/rejected": -0.3142294249782672, "step": 1500 }, { "epoch": 1.9689119170984455, "grad_norm": 18.97408103942871, "kl": 11.096129417419434, "learning_rate": 2.866230287623651e-07, "logits/chosen": -34857671.80487805, "logits/rejected": -36135020.307692304, "logps/chosen": -495.4438833841463, "logps/rejected": -358.8788060897436, "loss": 0.5214, "loss/base_kto": 3.1606152057647705, "loss/total_with_kl": 4.170856952667236, "metrics/advantage_var": 606.7518920898438, "regularization/advantage_var": 606.7518920898438, "regularization/kl": 1.0102418661117554, "rewards/chosen": 0.5045560976354088, "rewards/margins": 0.9385915700162181, "rewards/rejected": -0.4340354723808093, "step": 1520 }, { "epoch": 1.994818652849741, "grad_norm": 26.482131958007812, "kl": 12.319792747497559, "learning_rate": 2.738894140150075e-07, "logits/chosen": -34331154.17751479, "logits/rejected": -36180422.357615896, "logps/chosen": -484.29391642011836, "logps/rejected": -370.2840438741722, "loss": 0.5038, "loss/base_kto": 3.2293007373809814, "loss/total_with_kl": 4.0306596755981445, "metrics/advantage_var": 481.29638671875, "regularization/advantage_var": 481.29638671875, "regularization/kl": 0.8013584017753601, "rewards/chosen": 0.568287414911936, "rewards/margins": 0.8108820517406301, "rewards/rejected": -0.24259463682869412, "step": 1540 }, { "epoch": 2.0207253886010363, "grad_norm": 7.906752586364746, "kl": 14.77958869934082, "learning_rate": 2.6133756500585156e-07, "logits/chosen": -34494963.26368159, "logits/rejected": -36518837.66518518, "logps/chosen": -481.1639718076285, "logps/rejected": -360.60342592592593, "loss": 0.4732, "loss/base_kto": 3.1612300872802734, "loss/total_with_kl": 3.785780429840088, "metrics/advantage_var": 375.10516357421875, "regularization/advantage_var": 375.10516357421875, "regularization/kl": 0.6245501041412354, "rewards/chosen": 0.5704252581493574, "rewards/margins": 0.8595004625041027, "rewards/rejected": -0.2890752043547454, "step": 1560 }, { "epoch": 2.0466321243523318, "grad_norm": 7.699523448944092, "kl": 7.288259983062744, "learning_rate": 2.489775719130767e-07, "logits/chosen": -34509809.15942029, "logits/rejected": -35382292.20030349, "logps/chosen": -465.41183574879227, "logps/rejected": -375.4367886949924, "loss": 0.4662, "loss/base_kto": 3.136625051498413, "loss/total_with_kl": 3.729235887527466, "metrics/advantage_var": 355.9222106933594, "regularization/advantage_var": 355.9222106933594, "regularization/kl": 0.5926105380058289, "rewards/chosen": 0.4267546673711755, "rewards/margins": 0.8816557533709146, "rewards/rejected": -0.45490108599973916, "step": 1580 }, { "epoch": 2.0725388601036268, "grad_norm": 11.611719131469727, "kl": 9.821981430053711, "learning_rate": 2.3681937068576303e-07, "logits/chosen": -33172362.598726116, "logits/rejected": -35869447.85276074, "logps/chosen": -457.77846337579615, "logps/rejected": -379.2510784125767, "loss": 0.4636, "loss/base_kto": 3.097532272338867, "loss/total_with_kl": 3.708944320678711, "metrics/advantage_var": 367.21441650390625, "regularization/advantage_var": 367.21441650390625, "regularization/kl": 0.611411988735199, "rewards/chosen": 0.5261565287401722, "rewards/margins": 0.9095836396579013, "rewards/rejected": -0.38342711091772913, "step": 1600 }, { "epoch": 2.098445595854922, "grad_norm": 10.034419059753418, "kl": 13.82537841796875, "learning_rate": 2.2487273505658e-07, "logits/chosen": -34067506.7244582, "logits/rejected": -36231445.8044164, "logps/chosen": -482.71579914860683, "logps/rejected": -377.22473876182966, "loss": 0.4638, "loss/base_kto": 3.110532760620117, "loss/total_with_kl": 3.7102158069610596, "metrics/advantage_var": 360.1698913574219, "regularization/advantage_var": 360.1698913574219, "regularization/kl": 0.5996828675270081, "rewards/chosen": 0.5588428509124661, "rewards/margins": 0.9019656521065217, "rewards/rejected": -0.3431228011940556, "step": 1620 }, { "epoch": 2.1243523316062176, "grad_norm": 9.551857948303223, "kl": 13.70224666595459, "learning_rate": 2.131472686848817e-07, "logits/chosen": -32953689.854304634, "logits/rejected": -34677993.27810651, "logps/chosen": -450.9413286423841, "logps/rejected": -373.42111224112426, "loss": 0.4644, "loss/base_kto": 3.104682207107544, "loss/total_with_kl": 3.7153890132904053, "metrics/advantage_var": 366.7907409667969, "regularization/advantage_var": 366.7907409667969, "regularization/kl": 0.610706627368927, "rewards/chosen": 0.5498176726284406, "rewards/margins": 0.910659051286216, "rewards/rejected": -0.3608413786577755, "step": 1640 }, { "epoch": 2.150259067357513, "grad_norm": 26.569332122802734, "kl": 11.5601224899292, "learning_rate": 2.016523974365188e-07, "logits/chosen": -34094174.1958457, "logits/rejected": -35801260.35643564, "logps/chosen": -467.55313427299706, "logps/rejected": -371.6885571369637, "loss": 0.477, "loss/base_kto": 3.153583526611328, "loss/total_with_kl": 3.815898895263672, "metrics/advantage_var": 397.7869567871094, "regularization/advantage_var": 397.7869567871094, "regularization/kl": 0.662315309047699, "rewards/chosen": 0.501616163848062, "rewards/margins": 0.8756453422786004, "rewards/rejected": -0.37402917843053834, "step": 1660 }, { "epoch": 2.1761658031088085, "grad_norm": 10.05427074432373, "kl": 18.795507431030273, "learning_rate": 1.9039736180657372e-07, "logits/chosen": -33660038.8982036, "logits/rejected": -35006765.176470585, "logps/chosen": -468.94217814371257, "logps/rejected": -355.3689746732026, "loss": 0.4573, "loss/base_kto": 3.1093814373016357, "loss/total_with_kl": 3.65879225730896, "metrics/advantage_var": 329.9764709472656, "regularization/advantage_var": 329.9764709472656, "regularization/kl": 0.5494108200073242, "rewards/chosen": 0.6614648510595995, "rewards/margins": 0.8575095984059664, "rewards/rejected": -0.19604474734636693, "step": 1680 }, { "epoch": 2.2020725388601035, "grad_norm": 11.636170387268066, "kl": 21.24412727355957, "learning_rate": 1.7939120949111498e-07, "logits/chosen": -34235893.71251932, "logits/rejected": -34109161.75671406, "logps/chosen": -474.5167117465224, "logps/rejected": -366.46265304107425, "loss": 0.4558, "loss/base_kto": 3.141784906387329, "loss/total_with_kl": 3.646454334259033, "metrics/advantage_var": 303.104736328125, "regularization/advantage_var": 303.104736328125, "regularization/kl": 0.5046693682670593, "rewards/chosen": 0.6264453162734255, "rewards/margins": 0.8109377438643245, "rewards/rejected": -0.18449242759089898, "step": 1700 }, { "epoch": 2.227979274611399, "grad_norm": 11.995245933532715, "kl": 22.42620849609375, "learning_rate": 1.6864278811393523e-07, "logits/chosen": -34035581.96825397, "logits/rejected": -36610854.596923076, "logps/chosen": -469.57876984126983, "logps/rejected": -351.3943269230769, "loss": 0.4613, "loss/base_kto": 3.067842721939087, "loss/total_with_kl": 3.690115451812744, "metrics/advantage_var": 373.7374572753906, "regularization/advantage_var": 373.7374572753906, "regularization/kl": 0.6222729086875916, "rewards/chosen": 0.7026108999100943, "rewards/margins": 0.9254983792520414, "rewards/rejected": -0.2228874793419471, "step": 1720 }, { "epoch": 2.2538860103626943, "grad_norm": 9.906083106994629, "kl": 18.84529685974121, "learning_rate": 1.5816073811412584e-07, "logits/chosen": -35700720.43768997, "logits/rejected": -35828755.75562701, "logps/chosen": -509.5447378419453, "logps/rejected": -365.9818127009646, "loss": 0.4602, "loss/base_kto": 3.1173386573791504, "loss/total_with_kl": 3.681575059890747, "metrics/advantage_var": 338.88067626953125, "regularization/advantage_var": 338.88067626953125, "regularization/kl": 0.5642362833023071, "rewards/chosen": 0.6589171806729673, "rewards/margins": 0.8846643167307321, "rewards/rejected": -0.22574713605776478, "step": 1740 }, { "epoch": 2.2797927461139897, "grad_norm": 13.125312805175781, "kl": 20.307037353515625, "learning_rate": 1.4795348580020207e-07, "logits/chosen": -34938205.545864664, "logits/rejected": -35919255.93495935, "logps/chosen": -477.4031015037594, "logps/rejected": -368.5592987804878, "loss": 0.4881, "loss/base_kto": 3.1182637214660645, "loss/total_with_kl": 3.9045815467834473, "metrics/advantage_var": 472.26300048828125, "regularization/advantage_var": 472.26300048828125, "regularization/kl": 0.7863179445266724, "rewards/chosen": 0.6869242044319784, "rewards/margins": 0.8638465637434621, "rewards/rejected": -0.17692235931148373, "step": 1760 }, { "epoch": 2.305699481865285, "grad_norm": 9.469399452209473, "kl": 16.510149002075195, "learning_rate": 1.3802923657636355e-07, "logits/chosen": -33766469.341732286, "logits/rejected": -35859425.835658915, "logps/chosen": -485.0823818897638, "logps/rejected": -369.6765019379845, "loss": 0.4635, "loss/base_kto": 3.0808887481689453, "loss/total_with_kl": 3.707745313644409, "metrics/advantage_var": 376.490478515625, "regularization/advantage_var": 376.490478515625, "regularization/kl": 0.6268566846847534, "rewards/chosen": 0.6425019992618111, "rewards/margins": 0.9252334739090373, "rewards/rejected": -0.28273147464722626, "step": 1780 }, { "epoch": 2.33160621761658, "grad_norm": 15.960187911987305, "kl": 20.4044246673584, "learning_rate": 1.28395968346336e-07, "logits/chosen": -35527502.76923077, "logits/rejected": -36641623.414634146, "logps/chosen": -470.1169871794872, "logps/rejected": -387.9233517530488, "loss": 0.4617, "loss/base_kto": 3.1373703479766846, "loss/total_with_kl": 3.6938633918762207, "metrics/advantage_var": 334.2300720214844, "regularization/advantage_var": 334.2300720214844, "regularization/kl": 0.5564931035041809, "rewards/chosen": 0.6184520721435547, "rewards/margins": 0.8400480689072027, "rewards/rejected": -0.22159599676364805, "step": 1800 }, { "epoch": 2.3575129533678756, "grad_norm": 21.24134635925293, "kl": 19.327011108398438, "learning_rate": 1.1906142510009415e-07, "logits/chosen": -33151336.448, "logits/rejected": -35577421.38625954, "logps/chosen": -490.10015, "logps/rejected": -381.9539599236641, "loss": 0.4592, "loss/base_kto": 3.028256893157959, "loss/total_with_kl": 3.673821210861206, "metrics/advantage_var": 387.72637939453125, "regularization/advantage_var": 387.72637939453125, "regularization/kl": 0.6455643773078918, "rewards/chosen": 0.6580330078125, "rewards/margins": 0.9728783419310592, "rewards/rejected": -0.31484533411855914, "step": 1820 }, { "epoch": 2.383419689119171, "grad_norm": 12.932014465332031, "kl": 16.66994285583496, "learning_rate": 1.1003311068862547e-07, "logits/chosen": -34448999.65644172, "logits/rejected": -36296253.96178344, "logps/chosen": -480.6158454754601, "logps/rejected": -390.64965664808915, "loss": 0.4593, "loss/base_kto": 3.0751588344573975, "loss/total_with_kl": 3.674107313156128, "metrics/advantage_var": 359.7289733886719, "regularization/advantage_var": 359.7289733886719, "regularization/kl": 0.5989487171173096, "rewards/chosen": 0.6390529702777511, "rewards/margins": 0.9042419771074685, "rewards/rejected": -0.26518900682971736, "step": 1840 }, { "epoch": 2.4093264248704664, "grad_norm": 7.143278121948242, "kl": 19.18073844909668, "learning_rate": 1.0131828279173588e-07, "logits/chosen": -35604486.131736524, "logits/rejected": -35972671.58169935, "logps/chosen": -461.79640718562877, "logps/rejected": -395.61805555555554, "loss": 0.4571, "loss/base_kto": 3.1069934368133545, "loss/total_with_kl": 3.656771183013916, "metrics/advantage_var": 330.1967468261719, "regularization/advantage_var": 330.1967468261719, "regularization/kl": 0.5497775673866272, "rewards/chosen": 0.6508044397045752, "rewards/margins": 0.8794850690608099, "rewards/rejected": -0.22868062935623468, "step": 1860 }, { "epoch": 2.4352331606217614, "grad_norm": 26.480388641357422, "kl": 20.91214370727539, "learning_rate": 9.292394708374596e-08, "logits/chosen": -35775464.18604651, "logits/rejected": -35679533.55590551, "logps/chosen": -492.06598837209305, "logps/rejected": -361.64721948818897, "loss": 0.4646, "loss/base_kto": 3.077267646789551, "loss/total_with_kl": 3.7171669006347656, "metrics/advantage_var": 384.3238220214844, "regularization/advantage_var": 384.3238220214844, "regularization/kl": 0.6398991942405701, "rewards/chosen": 0.7391787506813227, "rewards/margins": 0.9163473970657345, "rewards/rejected": -0.1771686463844119, "step": 1880 }, { "epoch": 2.461139896373057, "grad_norm": 11.588574409484863, "kl": 17.6380558013916, "learning_rate": 8.48568516017727e-08, "logits/chosen": -34916281.166917294, "logits/rejected": -35104245.177235775, "logps/chosen": -487.8825187969925, "logps/rejected": -395.2237042682927, "loss": 0.4605, "loss/base_kto": 3.077972173690796, "loss/total_with_kl": 3.6838810443878174, "metrics/advantage_var": 363.909423828125, "regularization/advantage_var": 363.909423828125, "regularization/kl": 0.6059091687202454, "rewards/chosen": 0.6262954597186325, "rewards/margins": 0.9238782442317803, "rewards/rejected": -0.29758278451314785, "step": 1900 }, { "epoch": 2.4870466321243523, "grad_norm": 14.420379638671875, "kl": 20.456974029541016, "learning_rate": 7.71234813211169e-08, "logits/chosen": -32797281.295950156, "logits/rejected": -36612009.329153605, "logps/chosen": -502.29955218068534, "logps/rejected": -359.5392829153605, "loss": 0.462, "loss/base_kto": 3.069866418838501, "loss/total_with_kl": 3.6956498622894287, "metrics/advantage_var": 375.845703125, "regularization/advantage_var": 375.845703125, "regularization/kl": 0.6257831454277039, "rewards/chosen": 0.7031409718165887, "rewards/margins": 0.9267154756524283, "rewards/rejected": -0.22357450383583954, "step": 1920 }, { "epoch": 2.5129533678756477, "grad_norm": 14.924396514892578, "kl": 18.695880889892578, "learning_rate": 6.973005294212353e-08, "logits/chosen": -34356465.12538226, "logits/rejected": -35959634.60702875, "logps/chosen": -459.5124235474006, "logps/rejected": -376.58441493610223, "loss": 0.4654, "loss/base_kto": 3.134883165359497, "loss/total_with_kl": 3.7233529090881348, "metrics/advantage_var": 353.4350280761719, "regularization/advantage_var": 353.4350280761719, "regularization/kl": 0.5884692668914795, "rewards/chosen": 0.6450473890392059, "rewards/margins": 0.8598745396217113, "rewards/rejected": -0.21482715058250548, "step": 1940 }, { "epoch": 2.538860103626943, "grad_norm": 10.060352325439453, "kl": 20.296659469604492, "learning_rate": 6.268250989270102e-08, "logits/chosen": -34791790.518053375, "logits/rejected": -35867662.33281493, "logps/chosen": -483.68754905808476, "logps/rejected": -395.350651244168, "loss": 0.4641, "loss/base_kto": 3.0956616401672363, "loss/total_with_kl": 3.712872266769409, "metrics/advantage_var": 370.6968688964844, "regularization/advantage_var": 370.6968688964844, "regularization/kl": 0.6172103881835938, "rewards/chosen": 0.6617858024369604, "rewards/margins": 0.8896421907959708, "rewards/rejected": -0.2278563883590105, "step": 1960 }, { "epoch": 2.5647668393782386, "grad_norm": 10.029075622558594, "kl": 21.600072860717773, "learning_rate": 5.598651755051975e-08, "logits/chosen": -35024255.19496855, "logits/rejected": -36128510.40993789, "logps/chosen": -474.2742727987421, "logps/rejected": -403.06766789596276, "loss": 0.4711, "loss/base_kto": 3.1070034503936768, "loss/total_with_kl": 3.7690491676330566, "metrics/advantage_var": 397.62518310546875, "regularization/advantage_var": 397.62518310546875, "regularization/kl": 0.662045955657959, "rewards/chosen": 0.6627385361389544, "rewards/margins": 0.8590032888290504, "rewards/rejected": -0.19626475269009608, "step": 1980 }, { "epoch": 2.5906735751295336, "grad_norm": 11.753561973571777, "kl": 19.34547233581543, "learning_rate": 4.964745868872933e-08, "logits/chosen": -32902551.328050714, "logits/rejected": -35807042.66255778, "logps/chosen": -508.9552297939778, "logps/rejected": -372.83573285824343, "loss": 0.4666, "loss/base_kto": 3.078979969024658, "loss/total_with_kl": 3.7330939769744873, "metrics/advantage_var": 392.8611755371094, "regularization/advantage_var": 392.8611755371094, "regularization/kl": 0.6541138887405396, "rewards/chosen": 0.6873350793321612, "rewards/margins": 0.9273398906705183, "rewards/rejected": -0.24000481133835708, "step": 2000 }, { "epoch": 2.616580310880829, "grad_norm": 10.120428085327148, "kl": 17.297834396362305, "learning_rate": 4.3670429148855493e-08, "logits/chosen": -34779033.90029325, "logits/rejected": -35680735.46488294, "logps/chosen": -478.8338526392962, "logps/rejected": -380.67605560200667, "loss": 0.468, "loss/base_kto": 3.1093106269836426, "loss/total_with_kl": 3.743926763534546, "metrics/advantage_var": 381.1507873535156, "regularization/advantage_var": 381.1507873535156, "regularization/kl": 0.6346160769462585, "rewards/chosen": 0.6901628153065432, "rewards/margins": 0.8785802563780185, "rewards/rejected": -0.18841744107147523, "step": 2020 }, { "epoch": 2.6424870466321244, "grad_norm": 9.36275863647461, "kl": 20.08066749572754, "learning_rate": 3.806023374435663e-08, "logits/chosen": -34936683.87421384, "logits/rejected": -36909545.73913044, "logps/chosen": -496.70341981132077, "logps/rejected": -388.51669254658384, "loss": 0.4685, "loss/base_kto": 3.0967319011688232, "loss/total_with_kl": 3.747715711593628, "metrics/advantage_var": 390.98138427734375, "regularization/advantage_var": 390.98138427734375, "regularization/kl": 0.650983989238739, "rewards/chosen": 0.6852142046082694, "rewards/margins": 0.9062081943513304, "rewards/rejected": -0.22099398974306095, "step": 2040 }, { "epoch": 2.66839378238342, "grad_norm": 13.986465454101562, "kl": 20.332181930541992, "learning_rate": 3.282138239813037e-08, "logits/chosen": -34612885.81350482, "logits/rejected": -37749831.58662614, "logps/chosen": -509.0453175241158, "logps/rejected": -334.9946333586626, "loss": 0.4605, "loss/base_kto": 3.1062123775482178, "loss/total_with_kl": 3.6836297512054443, "metrics/advantage_var": 346.79693603515625, "regularization/advantage_var": 346.79693603515625, "regularization/kl": 0.5774168372154236, "rewards/chosen": 0.6701935893852994, "rewards/margins": 0.8920773457967736, "rewards/rejected": -0.22188375641147418, "step": 2060 }, { "epoch": 2.694300518134715, "grad_norm": 12.418783187866211, "kl": 17.193639755249023, "learning_rate": 2.795808651707793e-08, "logits/chosen": -34308268.8, "logits/rejected": -35254160.0, "logps/chosen": -462.082470703125, "logps/rejected": -383.4199951171875, "loss": 0.4642, "loss/base_kto": 3.1278998851776123, "loss/total_with_kl": 3.7134804725646973, "metrics/advantage_var": 351.69970703125, "regularization/advantage_var": 351.69970703125, "regularization/kl": 0.585580050945282, "rewards/chosen": 0.6585504055023194, "rewards/margins": 0.8750317335128784, "rewards/rejected": -0.21648132801055908, "step": 2080 }, { "epoch": 2.7202072538860103, "grad_norm": 13.723422050476074, "kl": 18.919214248657227, "learning_rate": 2.3474255606639293e-08, "logits/chosen": -34210896.75709779, "logits/rejected": -36117884.433436535, "logps/chosen": -487.7697160883281, "logps/rejected": -371.83978328173373, "loss": 0.4621, "loss/base_kto": 3.0969226360321045, "loss/total_with_kl": 3.696964979171753, "metrics/advantage_var": 360.3858947753906, "regularization/advantage_var": 360.3858947753906, "regularization/kl": 0.6000425219535828, "rewards/chosen": 0.6286294738576992, "rewards/margins": 0.8873858929420887, "rewards/rejected": -0.2587564190843895, "step": 2100 }, { "epoch": 2.7461139896373057, "grad_norm": 10.867472648620605, "kl": 18.517934799194336, "learning_rate": 1.9373494128020195e-08, "logits/chosen": -34305692.25974026, "logits/rejected": -35613933.4939759, "logps/chosen": -469.52450284090907, "logps/rejected": -364.32163027108436, "loss": 0.457, "loss/base_kto": 3.108471632003784, "loss/total_with_kl": 3.6561081409454346, "metrics/advantage_var": 328.9109191894531, "regularization/advantage_var": 328.9109191894531, "regularization/kl": 0.5476366281509399, "rewards/chosen": 0.6623576771129261, "rewards/margins": 0.8936989816621779, "rewards/rejected": -0.2313413045492517, "step": 2120 }, { "epoch": 2.772020725388601, "grad_norm": 10.805334091186523, "kl": 17.159452438354492, "learning_rate": 1.5659098600638798e-08, "logits/chosen": -34961395.512195125, "logits/rejected": -36792411.897435896, "logps/chosen": -476.2725323932927, "logps/rejected": -384.1209685496795, "loss": 0.4627, "loss/base_kto": 3.120931625366211, "loss/total_with_kl": 3.7019202709198, "metrics/advantage_var": 348.9420471191406, "regularization/advantage_var": 348.9420471191406, "regularization/kl": 0.5809884667396545, "rewards/chosen": 0.642271274473609, "rewards/margins": 0.9047954146604078, "rewards/rejected": -0.26252414018679887, "step": 2140 }, { "epoch": 2.7979274611398965, "grad_norm": 15.718250274658203, "kl": 18.677274703979492, "learning_rate": 1.2334054952120143e-08, "logits/chosen": -34408862.8681672, "logits/rejected": -35151037.86018237, "logps/chosen": -500.32817524115757, "logps/rejected": -386.14176481762917, "loss": 0.4685, "loss/base_kto": 3.0871524810791016, "loss/total_with_kl": 3.7476539611816406, "metrics/advantage_var": 396.69757080078125, "regularization/advantage_var": 396.69757080078125, "regularization/kl": 0.6605014204978943, "rewards/chosen": 0.6487110788032556, "rewards/margins": 0.9272517625379987, "rewards/rejected": -0.27854068373474306, "step": 2160 }, { "epoch": 2.823834196891192, "grad_norm": 9.103044509887695, "kl": 17.242324829101562, "learning_rate": 9.401036117969108e-09, "logits/chosen": -34204503.17411402, "logits/rejected": -36221067.56259905, "logps/chosen": -444.0169010015408, "logps/rejected": -366.1140303090333, "loss": 0.4619, "loss/base_kto": 3.1496777534484863, "loss/total_with_kl": 3.6954143047332764, "metrics/advantage_var": 327.7698974609375, "regularization/advantage_var": 327.7698974609375, "regularization/kl": 0.5457369089126587, "rewards/chosen": 0.6019741346362192, "rewards/margins": 0.8338497885152307, "rewards/rejected": -0.2318756538790115, "step": 2180 }, { "epoch": 2.849740932642487, "grad_norm": 13.3783540725708, "kl": 16.618896484375, "learning_rate": 6.8623998928517555e-09, "logits/chosen": -35433203.18057663, "logits/rejected": -36360043.5942029, "logps/chosen": -504.1375189681335, "logps/rejected": -373.38315217391306, "loss": 0.4627, "loss/base_kto": 3.100325345993042, "loss/total_with_kl": 3.7016799449920654, "metrics/advantage_var": 361.1739196777344, "regularization/advantage_var": 361.1739196777344, "regularization/kl": 0.6013545393943787, "rewards/chosen": 0.6479189327164738, "rewards/margins": 0.9116225047179081, "rewards/rejected": -0.2637035720014342, "step": 2200 }, { "epoch": 2.8756476683937824, "grad_norm": 12.644377708435059, "kl": 18.45366859436035, "learning_rate": 4.72018703521132e-09, "logits/chosen": -33634692.6953748, "logits/rejected": -35180525.18223584, "logps/chosen": -475.40649920255186, "logps/rejected": -367.3895482388974, "loss": 0.4617, "loss/base_kto": 3.1135945320129395, "loss/total_with_kl": 3.6932201385498047, "metrics/advantage_var": 348.12353515625, "regularization/advantage_var": 348.12353515625, "regularization/kl": 0.5796257257461548, "rewards/chosen": 0.6570217978250847, "rewards/margins": 0.8777010415350053, "rewards/rejected": -0.22067924370992056, "step": 2220 }, { "epoch": 2.901554404145078, "grad_norm": 13.391596794128418, "kl": 17.913007736206055, "learning_rate": 2.976119626744267e-09, "logits/chosen": -36026576.7133758, "logits/rejected": -36553376.19631902, "logps/chosen": -481.25597133757964, "logps/rejected": -394.75723734662574, "loss": 0.467, "loss/base_kto": 3.1105620861053467, "loss/total_with_kl": 3.7358384132385254, "metrics/advantage_var": 375.5413513183594, "regularization/advantage_var": 375.5413513183594, "regularization/kl": 0.6252763867378235, "rewards/chosen": 0.6789868105748657, "rewards/margins": 0.9045335713679782, "rewards/rejected": -0.22554676079311253, "step": 2240 }, { "epoch": 2.927461139896373, "grad_norm": 13.19859504699707, "kl": 17.68929672241211, "learning_rate": 1.631599688052765e-09, "logits/chosen": -33718461.922720246, "logits/rejected": -36536048.22748815, "logps/chosen": -479.38591576506957, "logps/rejected": -387.46608412322274, "loss": 0.4602, "loss/base_kto": 3.110572576522827, "loss/total_with_kl": 3.6816444396972656, "metrics/advantage_var": 342.98614501953125, "regularization/advantage_var": 342.98614501953125, "regularization/kl": 0.5710719227790833, "rewards/chosen": 0.6081728338274005, "rewards/margins": 0.8642510386648561, "rewards/rejected": -0.2560782048374556, "step": 2260 }, { "epoch": 2.9533678756476682, "grad_norm": 15.231182098388672, "kl": 16.830408096313477, "learning_rate": 6.877080515894085e-10, "logits/chosen": -34022834.32817338, "logits/rejected": -36190615.01577287, "logps/chosen": -478.59287925696594, "logps/rejected": -363.70176951892745, "loss": 0.4581, "loss/base_kto": 3.0981926918029785, "loss/total_with_kl": 3.664703130722046, "metrics/advantage_var": 340.24658203125, "regularization/advantage_var": 340.24658203125, "regularization/kl": 0.5665105581283569, "rewards/chosen": 0.6395979842903444, "rewards/margins": 0.9017433546293379, "rewards/rejected": -0.2621453703389935, "step": 2280 }, { "epoch": 2.9792746113989637, "grad_norm": 8.46706771850586, "kl": 18.756183624267578, "learning_rate": 1.4520349279739663e-10, "logits/chosen": -33320350.2394822, "logits/rejected": -35217565.77643505, "logps/chosen": -499.474767394822, "logps/rejected": -375.9756655966767, "loss": 0.4769, "loss/base_kto": 3.155273199081421, "loss/total_with_kl": 3.815504550933838, "metrics/advantage_var": 396.5354919433594, "regularization/advantage_var": 396.5354919433594, "regularization/kl": 0.6602315902709961, "rewards/chosen": 0.6185921480740544, "rewards/margins": 0.8359393536596378, "rewards/rejected": -0.21734720558558346, "step": 2300 }, { "epoch": 3.0, "step": 2316, "total_flos": 9.960196480656998e+17, "train_loss": 0.501307312482791, "train_runtime": 11007.4603, "train_samples_per_second": 13.465, "train_steps_per_second": 0.21 } ], "logging_steps": 20, "max_steps": 2316, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 9.960196480656998e+17, "train_batch_size": 8, "trial_name": null, "trial_params": null }