{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 340,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0029411764705882353,
      "grad_norm": 2.3687267303466797,
      "kl/avg_steps": 0.0,
      "kl/beta": 0.009999999776482582,
      "kl/n_epsilon_steps": 0.5,
      "kl/p_epsilon_steps": 0.5,
      "learning_rate": 0.0,
      "logits/chosen": -0.5232092142105103,
      "logits/rejected": -0.36964714527130127,
      "logps/chosen": -69.28079223632812,
      "logps/ref_chosen": -69.2831802368164,
      "logps/ref_rejected": -69.74366760253906,
      "logps/rejected": -69.7318344116211,
      "loss": 0.6932,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 9.683193638920784e-06,
      "rewards/margins": -0.0001216536620631814,
      "rewards/rejected": 0.00013133684115018696,
      "step": 1
    },
    {
      "epoch": 0.014705882352941176,
      "grad_norm": 2.401517868041992,
      "kl/avg_steps": 0.001953125,
      "kl/beta": 0.009997854940593243,
      "kl/n_epsilon_steps": 0.498046875,
      "kl/p_epsilon_steps": 0.5,
      "learning_rate": 5.88235294117647e-08,
      "logits/chosen": -0.5336302518844604,
      "logits/rejected": -0.41014784574508667,
      "logps/chosen": -75.71084594726562,
      "logps/ref_chosen": -75.70054626464844,
      "logps/ref_rejected": -81.47293090820312,
      "logps/rejected": -81.47822570800781,
      "loss": 0.6932,
      "rewards/accuracies": 0.505859375,
      "rewards/chosen": -0.00011636512499535456,
      "rewards/margins": -7.826486398698762e-05,
      "rewards/rejected": -3.8100268284324557e-05,
      "step": 5
    },
    {
      "epoch": 0.029411764705882353,
      "grad_norm": 2.312957525253296,
      "kl/avg_steps": -0.05312500149011612,
      "kl/beta": 0.010005339980125427,
      "kl/n_epsilon_steps": 0.5234375,
      "kl/p_epsilon_steps": 0.4703125059604645,
      "learning_rate": 1.3235294117647057e-07,
      "logits/chosen": -0.5401719808578491,
      "logits/rejected": -0.4321846067905426,
      "logps/chosen": -77.008544921875,
      "logps/ref_chosen": -77.0025405883789,
      "logps/ref_rejected": -82.64138793945312,
      "logps/rejected": -82.64922332763672,
      "loss": 0.6932,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": -7.36937508918345e-05,
      "rewards/margins": -9.958527698472608e-06,
      "rewards/rejected": -6.373519863700494e-05,
      "step": 10
    },
    {
      "epoch": 0.04411764705882353,
      "grad_norm": 2.890460968017578,
      "kl/avg_steps": 0.12968750298023224,
      "kl/beta": 0.010008977726101875,
      "kl/n_epsilon_steps": 0.43437498807907104,
      "kl/p_epsilon_steps": 0.5640624761581421,
      "learning_rate": 2.0588235294117645e-07,
      "logits/chosen": -0.5125764608383179,
      "logits/rejected": -0.4432317316532135,
      "logps/chosen": -70.82783508300781,
      "logps/ref_chosen": -70.83788299560547,
      "logps/ref_rejected": -87.43305206298828,
      "logps/rejected": -87.48735809326172,
      "loss": 0.6928,
      "rewards/accuracies": 0.5718749761581421,
      "rewards/chosen": 8.707816596142948e-05,
      "rewards/margins": 0.0006134368595667183,
      "rewards/rejected": -0.0005263587227091193,
      "step": 15
    },
    {
      "epoch": 0.058823529411764705,
      "grad_norm": 2.124864101409912,
      "kl/avg_steps": 0.265625,
      "kl/beta": 0.009920386597514153,
      "kl/n_epsilon_steps": 0.3656249940395355,
      "kl/p_epsilon_steps": 0.6312500238418579,
      "learning_rate": 2.7941176470588235e-07,
      "logits/chosen": -0.5464522242546082,
      "logits/rejected": -0.4405369162559509,
      "logps/chosen": -70.1437759399414,
      "logps/ref_chosen": -70.1697006225586,
      "logps/ref_rejected": -82.27420806884766,
      "logps/rejected": -82.44139099121094,
      "loss": 0.6922,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00024056310940068215,
      "rewards/margins": 0.0018755672499537468,
      "rewards/rejected": -0.0016350041842088103,
      "step": 20
    },
    {
      "epoch": 0.07352941176470588,
      "grad_norm": 2.521970510482788,
      "kl/avg_steps": 0.543749988079071,
      "kl/beta": 0.009726567193865776,
      "kl/n_epsilon_steps": 0.22812500596046448,
      "kl/p_epsilon_steps": 0.7718750238418579,
      "learning_rate": 3.529411764705882e-07,
      "logits/chosen": -0.568504273891449,
      "logits/rejected": -0.4329379200935364,
      "logps/chosen": -74.4179458618164,
      "logps/ref_chosen": -74.5040283203125,
      "logps/ref_rejected": -89.5297622680664,
      "logps/rejected": -90.02223205566406,
      "loss": 0.6904,
      "rewards/accuracies": 0.7796875238418579,
      "rewards/chosen": 0.0008119211415760219,
      "rewards/margins": 0.005549114663153887,
      "rewards/rejected": -0.00473719323053956,
      "step": 25
    },
    {
      "epoch": 0.08823529411764706,
      "grad_norm": 2.3963418006896973,
      "kl/avg_steps": 0.567187488079071,
      "kl/beta": 0.00945484172552824,
      "kl/n_epsilon_steps": 0.21562500298023224,
      "kl/p_epsilon_steps": 0.7828124761581421,
      "learning_rate": 4.264705882352941e-07,
      "logits/chosen": -0.6653466820716858,
      "logits/rejected": -0.49282917380332947,
      "logps/chosen": -76.55107879638672,
      "logps/ref_chosen": -76.60227966308594,
      "logps/ref_rejected": -82.36322784423828,
      "logps/rejected": -83.71476745605469,
      "loss": 0.6867,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0004493276646826416,
      "rewards/margins": 0.01309473067522049,
      "rewards/rejected": -0.012645403854548931,
      "step": 30
    },
    {
      "epoch": 0.10294117647058823,
      "grad_norm": 2.311098337173462,
      "kl/avg_steps": 0.5546875,
      "kl/beta": 0.009198471903800964,
      "kl/n_epsilon_steps": 0.22187499701976776,
      "kl/p_epsilon_steps": 0.776562511920929,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6610927581787109,
      "logits/rejected": -0.5268033146858215,
      "logps/chosen": -76.14710998535156,
      "logps/ref_chosen": -75.79379272460938,
      "logps/ref_rejected": -83.69039154052734,
      "logps/rejected": -86.21320343017578,
      "loss": 0.6835,
      "rewards/accuracies": 0.809374988079071,
      "rewards/chosen": -0.003281622426584363,
      "rewards/margins": 0.019690800458192825,
      "rewards/rejected": -0.02297242358326912,
      "step": 35
    },
    {
      "epoch": 0.11764705882352941,
      "grad_norm": 2.681466817855835,
      "kl/avg_steps": 0.4078125059604645,
      "kl/beta": 0.008961381390690804,
      "kl/n_epsilon_steps": 0.2953124940395355,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.996706849759452e-07,
      "logits/chosen": -0.8570469617843628,
      "logits/rejected": -0.7218376398086548,
      "logps/chosen": -77.57659149169922,
      "logps/ref_chosen": -75.21812438964844,
      "logps/ref_rejected": -86.6792984008789,
      "logps/rejected": -93.75047302246094,
      "loss": 0.6732,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.021187324076890945,
      "rewards/margins": 0.04168969392776489,
      "rewards/rejected": -0.06287702172994614,
      "step": 40
    },
    {
      "epoch": 0.1323529411764706,
      "grad_norm": 3.2158050537109375,
      "kl/avg_steps": 0.3187499940395355,
      "kl/beta": 0.008803511038422585,
      "kl/n_epsilon_steps": 0.34062498807907104,
      "kl/p_epsilon_steps": 0.659375011920929,
      "learning_rate": 4.986836074908615e-07,
      "logits/chosen": -0.9623914957046509,
      "logits/rejected": -0.8303581476211548,
      "logps/chosen": -82.83192443847656,
      "logps/ref_chosen": -77.2712173461914,
      "logps/ref_rejected": -91.67030334472656,
      "logps/rejected": -102.5731201171875,
      "loss": 0.6715,
      "rewards/accuracies": 0.7515624761581421,
      "rewards/chosen": -0.049075882881879807,
      "rewards/margins": 0.04632042720913887,
      "rewards/rejected": -0.09539631009101868,
      "step": 45
    },
    {
      "epoch": 0.14705882352941177,
      "grad_norm": 3.3687705993652344,
      "kl/avg_steps": 0.20624999701976776,
      "kl/beta": 0.008690183982253075,
      "kl/n_epsilon_steps": 0.3968749940395355,
      "kl/p_epsilon_steps": 0.6031249761581421,
      "learning_rate": 4.970413680203148e-07,
      "logits/chosen": -1.0613696575164795,
      "logits/rejected": -0.921216607093811,
      "logps/chosen": -82.58769226074219,
      "logps/ref_chosen": -73.91633605957031,
      "logps/ref_rejected": -79.92402648925781,
      "logps/rejected": -94.2342758178711,
      "loss": 0.6712,
      "rewards/accuracies": 0.698437511920929,
      "rewards/chosen": -0.07562652230262756,
      "rewards/margins": 0.048155996948480606,
      "rewards/rejected": -0.12378251552581787,
      "step": 50
    },
    {
      "epoch": 0.16176470588235295,
      "grad_norm": 4.448400020599365,
      "kl/avg_steps": 0.2109375,
      "kl/beta": 0.00860314816236496,
      "kl/n_epsilon_steps": 0.39375001192092896,
      "kl/p_epsilon_steps": 0.604687511920929,
      "learning_rate": 4.947482930773511e-07,
      "logits/chosen": -1.1757243871688843,
      "logits/rejected": -1.0121644735336304,
      "logps/chosen": -91.91180419921875,
      "logps/ref_chosen": -79.74378204345703,
      "logps/ref_rejected": -83.18132019042969,
      "logps/rejected": -103.12516021728516,
      "loss": 0.6639,
      "rewards/accuracies": 0.6859375238418579,
      "rewards/chosen": -0.10501708835363388,
      "rewards/margins": 0.06571148335933685,
      "rewards/rejected": -0.17072856426239014,
      "step": 55
    },
    {
      "epoch": 0.17647058823529413,
      "grad_norm": 3.918736219406128,
      "kl/avg_steps": 0.171875,
      "kl/beta": 0.008520014584064484,
      "kl/n_epsilon_steps": 0.4140625,
      "kl/p_epsilon_steps": 0.5859375,
      "learning_rate": 4.918104238142103e-07,
      "logits/chosen": -1.2249476909637451,
      "logits/rejected": -1.1036134958267212,
      "logps/chosen": -98.29524993896484,
      "logps/ref_chosen": -81.61141967773438,
      "logps/ref_rejected": -80.947998046875,
      "logps/rejected": -105.27479553222656,
      "loss": 0.6663,
      "rewards/accuracies": 0.660937488079071,
      "rewards/chosen": -0.14256855845451355,
      "rewards/margins": 0.06371048837900162,
      "rewards/rejected": -0.20627903938293457,
      "step": 60
    },
    {
      "epoch": 0.19117647058823528,
      "grad_norm": 3.5865535736083984,
      "kl/avg_steps": 0.27656251192092896,
      "kl/beta": 0.00841777864843607,
      "kl/n_epsilon_steps": 0.3609375059604645,
      "kl/p_epsilon_steps": 0.637499988079071,
      "learning_rate": 4.882355001067891e-07,
      "logits/chosen": -1.2322965860366821,
      "logits/rejected": -1.151759147644043,
      "logps/chosen": -91.71420288085938,
      "logps/ref_chosen": -75.09439849853516,
      "logps/ref_rejected": -87.96830749511719,
      "logps/rejected": -117.06733703613281,
      "loss": 0.6479,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.14002391695976257,
      "rewards/margins": 0.10341048240661621,
      "rewards/rejected": -0.24343439936637878,
      "step": 65
    },
    {
      "epoch": 0.20588235294117646,
      "grad_norm": 3.871297836303711,
      "kl/avg_steps": 0.28437501192092896,
      "kl/beta": 0.008305966854095459,
      "kl/n_epsilon_steps": 0.3578124940395355,
      "kl/p_epsilon_steps": 0.6421874761581421,
      "learning_rate": 4.840329401637809e-07,
      "logits/chosen": -1.2796670198440552,
      "logits/rejected": -1.1985622644424438,
      "logps/chosen": -89.69293975830078,
      "logps/ref_chosen": -70.07804870605469,
      "logps/ref_rejected": -88.98612976074219,
      "logps/rejected": -122.0387954711914,
      "loss": 0.6462,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.16316808760166168,
      "rewards/margins": 0.10986582934856415,
      "rewards/rejected": -0.27303391695022583,
      "step": 70
    },
    {
      "epoch": 0.22058823529411764,
      "grad_norm": 3.9685800075531006,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.008209030143916607,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.792138157142157e-07,
      "logits/chosen": -1.2629064321517944,
      "logits/rejected": -1.1684788465499878,
      "logps/chosen": -101.08387756347656,
      "logps/ref_chosen": -77.74958801269531,
      "logps/ref_rejected": -82.17206573486328,
      "logps/rejected": -117.42021179199219,
      "loss": 0.6538,
      "rewards/accuracies": 0.6796875,
      "rewards/chosen": -0.191951185464859,
      "rewards/margins": 0.09596569836139679,
      "rewards/rejected": -0.2879168391227722,
      "step": 75
    },
    {
      "epoch": 0.23529411764705882,
      "grad_norm": 4.582348823547363,
      "kl/avg_steps": 0.24375000596046448,
      "kl/beta": 0.008118118159472942,
      "kl/n_epsilon_steps": 0.37812501192092896,
      "kl/p_epsilon_steps": 0.621874988079071,
      "learning_rate": 4.737908228387656e-07,
      "logits/chosen": -1.2720203399658203,
      "logits/rejected": -1.218477725982666,
      "logps/chosen": -107.53079986572266,
      "logps/ref_chosen": -81.88478088378906,
      "logps/ref_rejected": -90.519775390625,
      "logps/rejected": -131.16079711914062,
      "loss": 0.6438,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.20838662981987,
      "rewards/margins": 0.11963216215372086,
      "rewards/rejected": -0.32801881432533264,
      "step": 80
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.6524829864501953,
      "kl/avg_steps": 0.2515625059604645,
      "kl/beta": 0.0080325398594141,
      "kl/n_epsilon_steps": 0.37187498807907104,
      "kl/p_epsilon_steps": 0.6234375238418579,
      "learning_rate": 4.6777824852166437e-07,
      "logits/chosen": -1.2834303379058838,
      "logits/rejected": -1.198880672454834,
      "logps/chosen": -95.5977554321289,
      "logps/ref_chosen": -70.41683197021484,
      "logps/ref_rejected": -78.02936553955078,
      "logps/rejected": -118.98405456542969,
      "loss": 0.6418,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -0.20263484120368958,
      "rewards/margins": 0.12456460297107697,
      "rewards/rejected": -0.32719942927360535,
      "step": 85
    },
    {
      "epoch": 0.2647058823529412,
      "grad_norm": 4.22735071182251,
      "kl/avg_steps": 0.2874999940395355,
      "kl/beta": 0.007919726893305779,
      "kl/n_epsilon_steps": 0.35468751192092896,
      "kl/p_epsilon_steps": 0.6421874761581421,
      "learning_rate": 4.611919330113591e-07,
      "logits/chosen": -1.2632228136062622,
      "logits/rejected": -1.2163931131362915,
      "logps/chosen": -105.8456039428711,
      "logps/ref_chosen": -76.6160888671875,
      "logps/ref_rejected": -89.49937438964844,
      "logps/rejected": -136.4986572265625,
      "loss": 0.6361,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.23172405362129211,
      "rewards/margins": 0.13836422562599182,
      "rewards/rejected": -0.37008827924728394,
      "step": 90
    },
    {
      "epoch": 0.27941176470588236,
      "grad_norm": 4.236695766448975,
      "kl/avg_steps": 0.2750000059604645,
      "kl/beta": 0.0078009068965911865,
      "kl/n_epsilon_steps": 0.36250001192092896,
      "kl/p_epsilon_steps": 0.637499988079071,
      "learning_rate": 4.5404922808905543e-07,
      "logits/chosen": -1.2625572681427002,
      "logits/rejected": -1.2011988162994385,
      "logps/chosen": -104.29510498046875,
      "logps/ref_chosen": -73.50260162353516,
      "logps/ref_rejected": -76.48811340332031,
      "logps/rejected": -124.16410827636719,
      "loss": 0.6411,
      "rewards/accuracies": 0.7015625238418579,
      "rewards/chosen": -0.24040882289409637,
      "rewards/margins": 0.1294616460800171,
      "rewards/rejected": -0.36987045407295227,
      "step": 95
    },
    {
      "epoch": 0.29411764705882354,
      "grad_norm": 4.193100452423096,
      "kl/avg_steps": 0.3656249940395355,
      "kl/beta": 0.0076876478269696236,
      "kl/n_epsilon_steps": 0.31718748807907104,
      "kl/p_epsilon_steps": 0.682812511920929,
      "learning_rate": 4.4636895135509966e-07,
      "logits/chosen": -1.2317556142807007,
      "logits/rejected": -1.1946831941604614,
      "logps/chosen": -103.88249206542969,
      "logps/ref_chosen": -72.6116714477539,
      "logps/ref_rejected": -81.16241455078125,
      "logps/rejected": -134.57403564453125,
      "loss": 0.6236,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.24038386344909668,
      "rewards/margins": 0.16778317093849182,
      "rewards/rejected": -0.4081670641899109,
      "step": 100
    },
    {
      "epoch": 0.29411764705882354,
      "eval_kl/n_epsilon_steps": 0.4236111044883728,
      "eval_kl/p_epsilon_steps": 0.5759548544883728,
      "eval_logits/chosen": -1.2261141538619995,
      "eval_logits/rejected": -1.1807267665863037,
      "eval_logps/chosen": -127.64717864990234,
      "eval_logps/ref_chosen": -87.82356262207031,
      "eval_logps/ref_rejected": -82.81887817382812,
      "eval_logps/rejected": -134.3017578125,
      "eval_loss": 0.6636335253715515,
      "eval_rewards/accuracies": 0.6124131679534912,
      "eval_rewards/chosen": -0.30329596996307373,
      "eval_rewards/margins": 0.08622786402702332,
      "eval_rewards/rejected": -0.38952386379241943,
      "eval_runtime": 22.4366,
      "eval_samples_per_second": 104.249,
      "eval_steps_per_second": 0.847,
      "step": 100
    },
    {
      "epoch": 0.3088235294117647,
      "grad_norm": 4.206778049468994,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.007563448045402765,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.381713366536311e-07,
      "logits/chosen": -1.2459900379180908,
      "logits/rejected": -1.1858142614364624,
      "logps/chosen": -112.22574615478516,
      "logps/ref_chosen": -76.5867919921875,
      "logps/ref_rejected": -84.33440399169922,
      "logps/rejected": -140.7528533935547,
      "loss": 0.6304,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2697572112083435,
      "rewards/margins": 0.15455064177513123,
      "rewards/rejected": -0.42430782318115234,
      "step": 105
    },
    {
      "epoch": 0.3235294117647059,
      "grad_norm": 5.154345989227295,
      "kl/avg_steps": 0.28437501192092896,
      "kl/beta": 0.007447557989507914,
      "kl/n_epsilon_steps": 0.3578124940395355,
      "kl/p_epsilon_steps": 0.6421874761581421,
      "learning_rate": 4.2947798076611047e-07,
      "logits/chosen": -1.2248286008834839,
      "logits/rejected": -1.1694958209991455,
      "logps/chosen": -118.81462097167969,
      "logps/ref_chosen": -78.16385650634766,
      "logps/ref_rejected": -83.61200714111328,
      "logps/rejected": -146.4515838623047,
      "loss": 0.6294,
      "rewards/accuracies": 0.692187488079071,
      "rewards/chosen": -0.3029385209083557,
      "rewards/margins": 0.16256316006183624,
      "rewards/rejected": -0.46550169587135315,
      "step": 110
    },
    {
      "epoch": 0.3382352941176471,
      "grad_norm": 5.148464679718018,
      "kl/avg_steps": 0.35468751192092896,
      "kl/beta": 0.007336863782256842,
      "kl/n_epsilon_steps": 0.3218750059604645,
      "kl/p_epsilon_steps": 0.676562488079071,
      "learning_rate": 4.203117865141635e-07,
      "logits/chosen": -1.2170436382293701,
      "logits/rejected": -1.1504008769989014,
      "logps/chosen": -118.66552734375,
      "logps/ref_chosen": -74.8998031616211,
      "logps/ref_rejected": -85.2784652709961,
      "logps/rejected": -156.08580017089844,
      "loss": 0.618,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.3210408687591553,
      "rewards/margins": 0.19527961313724518,
      "rewards/rejected": -0.516320526599884,
      "step": 115
    },
    {
      "epoch": 0.35294117647058826,
      "grad_norm": 5.226547718048096,
      "kl/avg_steps": 0.265625,
      "kl/beta": 0.007222268730401993,
      "kl/n_epsilon_steps": 0.3671875,
      "kl/p_epsilon_steps": 0.6328125,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": -1.1989049911499023,
      "logits/rejected": -1.1565752029418945,
      "logps/chosen": -119.46983337402344,
      "logps/ref_chosen": -73.58607482910156,
      "logps/ref_rejected": -85.84365844726562,
      "logps/rejected": -158.82113647460938,
      "loss": 0.6197,
      "rewards/accuracies": 0.698437511920929,
      "rewards/chosen": -0.3315422534942627,
      "rewards/margins": 0.19263319671154022,
      "rewards/rejected": -0.5241755247116089,
      "step": 120
    },
    {
      "epoch": 0.36764705882352944,
      "grad_norm": 5.764974594116211,
      "kl/avg_steps": 0.30937498807907104,
      "kl/beta": 0.007117821369320154,
      "kl/n_epsilon_steps": 0.3453125059604645,
      "kl/p_epsilon_steps": 0.6546875238418579,
      "learning_rate": 4.006586590948141e-07,
      "logits/chosen": -1.1903568506240845,
      "logits/rejected": -1.130084753036499,
      "logps/chosen": -130.13233947753906,
      "logps/ref_chosen": -80.25770568847656,
      "logps/ref_rejected": -81.34100341796875,
      "logps/rejected": -161.29537963867188,
      "loss": 0.6139,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.3550013303756714,
      "rewards/margins": 0.21088404953479767,
      "rewards/rejected": -0.5658854246139526,
      "step": 125
    },
    {
      "epoch": 0.38235294117647056,
      "grad_norm": 5.23793363571167,
      "kl/avg_steps": 0.2718749940395355,
      "kl/beta": 0.007017888128757477,
      "kl/n_epsilon_steps": 0.36406248807907104,
      "kl/p_epsilon_steps": 0.635937511920929,
      "learning_rate": 3.9022350248844246e-07,
      "logits/chosen": -1.1651326417922974,
      "logits/rejected": -1.1263306140899658,
      "logps/chosen": -128.90423583984375,
      "logps/ref_chosen": -74.67902374267578,
      "logps/ref_rejected": -84.1854019165039,
      "logps/rejected": -167.0582733154297,
      "loss": 0.6209,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.3804508149623871,
      "rewards/margins": 0.19761842489242554,
      "rewards/rejected": -0.5780693292617798,
      "step": 130
    },
    {
      "epoch": 0.39705882352941174,
      "grad_norm": 5.766234874725342,
      "kl/avg_steps": 0.32499998807907104,
      "kl/beta": 0.006909938994795084,
      "kl/n_epsilon_steps": 0.3375000059604645,
      "kl/p_epsilon_steps": 0.6625000238418579,
      "learning_rate": 3.794189242333106e-07,
      "logits/chosen": -1.1625608205795288,
      "logits/rejected": -1.0963513851165771,
      "logps/chosen": -138.46322631835938,
      "logps/ref_chosen": -81.2975845336914,
      "logps/ref_rejected": -87.74832916259766,
      "logps/rejected": -174.3780059814453,
      "loss": 0.6207,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.39489540457725525,
      "rewards/margins": 0.20013752579689026,
      "rewards/rejected": -0.5950329303741455,
      "step": 135
    },
    {
      "epoch": 0.4117647058823529,
      "grad_norm": 4.888461112976074,
      "kl/avg_steps": 0.3828125,
      "kl/beta": 0.006796327419579029,
      "kl/n_epsilon_steps": 0.30781251192092896,
      "kl/p_epsilon_steps": 0.690625011920929,
      "learning_rate": 3.6827338920900253e-07,
      "logits/chosen": -1.172863483428955,
      "logits/rejected": -1.106768012046814,
      "logps/chosen": -121.53498840332031,
      "logps/ref_chosen": -71.20382690429688,
      "logps/ref_rejected": -84.62137603759766,
      "logps/rejected": -170.3011474609375,
      "loss": 0.6009,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.3417351245880127,
      "rewards/margins": 0.2368461638689041,
      "rewards/rejected": -0.5785812139511108,
      "step": 140
    },
    {
      "epoch": 0.4264705882352941,
      "grad_norm": 5.264912128448486,
      "kl/avg_steps": 0.390625,
      "kl/beta": 0.0066697075963020325,
      "kl/n_epsilon_steps": 0.3046875,
      "kl/p_epsilon_steps": 0.6953125,
      "learning_rate": 3.568162605525952e-07,
      "logits/chosen": -1.1562573909759521,
      "logits/rejected": -1.0977518558502197,
      "logps/chosen": -132.38858032226562,
      "logps/ref_chosen": -78.03334045410156,
      "logps/ref_rejected": -86.95343017578125,
      "logps/rejected": -178.3859405517578,
      "loss": 0.5992,
      "rewards/accuracies": 0.7421875,
      "rewards/chosen": -0.3619559407234192,
      "rewards/margins": 0.2438311129808426,
      "rewards/rejected": -0.6057869791984558,
      "step": 145
    },
    {
      "epoch": 0.4411764705882353,
      "grad_norm": 5.746659278869629,
      "kl/avg_steps": 0.328125,
      "kl/beta": 0.00654013454914093,
      "kl/n_epsilon_steps": 0.3359375,
      "kl/p_epsilon_steps": 0.6640625,
      "learning_rate": 3.4507772230088147e-07,
      "logits/chosen": -1.0893394947052002,
      "logits/rejected": -1.0510880947113037,
      "logps/chosen": -136.75088500976562,
      "logps/ref_chosen": -73.69932556152344,
      "logps/ref_rejected": -86.18521118164062,
      "logps/rejected": -184.1068878173828,
      "loss": 0.614,
      "rewards/accuracies": 0.7015625238418579,
      "rewards/chosen": -0.4121219515800476,
      "rewards/margins": 0.22426274418830872,
      "rewards/rejected": -0.6363847255706787,
      "step": 150
    },
    {
      "epoch": 0.45588235294117646,
      "grad_norm": 5.235478401184082,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.006440295372158289,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 3.3308869986991487e-07,
      "logits/chosen": -1.0995477437973022,
      "logits/rejected": -1.031232237815857,
      "logps/chosen": -144.14666748046875,
      "logps/ref_chosen": -78.81468963623047,
      "logps/ref_rejected": -82.33976745605469,
      "logps/rejected": -183.3446502685547,
      "loss": 0.6118,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.4207354485988617,
      "rewards/margins": 0.22600603103637695,
      "rewards/rejected": -0.646741509437561,
      "step": 155
    },
    {
      "epoch": 0.47058823529411764,
      "grad_norm": 5.473912239074707,
      "kl/avg_steps": 0.3812499940395355,
      "kl/beta": 0.0063315341249108315,
      "kl/n_epsilon_steps": 0.30937498807907104,
      "kl/p_epsilon_steps": 0.690625011920929,
      "learning_rate": 3.208807785813777e-07,
      "logits/chosen": -1.080108880996704,
      "logits/rejected": -1.0139106512069702,
      "logps/chosen": -132.09349060058594,
      "logps/ref_chosen": -71.280517578125,
      "logps/ref_rejected": -86.39788818359375,
      "logps/rejected": -188.9801788330078,
      "loss": 0.5951,
      "rewards/accuracies": 0.7203124761581421,
      "rewards/chosen": -0.3846417963504791,
      "rewards/margins": 0.26059722900390625,
      "rewards/rejected": -0.645238995552063,
      "step": 160
    },
    {
      "epoch": 0.4852941176470588,
      "grad_norm": 5.492692947387695,
      "kl/avg_steps": 0.33125001192092896,
      "kl/beta": 0.006211251951754093,
      "kl/n_epsilon_steps": 0.3343749940395355,
      "kl/p_epsilon_steps": 0.6656249761581421,
      "learning_rate": 3.084861204504122e-07,
      "logits/chosen": -1.064668893814087,
      "logits/rejected": -0.9995222091674805,
      "logps/chosen": -148.7730255126953,
      "logps/ref_chosen": -79.35147094726562,
      "logps/ref_rejected": -83.44163513183594,
      "logps/rejected": -191.25628662109375,
      "loss": 0.608,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.430931031703949,
      "rewards/margins": 0.23459258675575256,
      "rewards/rejected": -0.6655236482620239,
      "step": 165
    },
    {
      "epoch": 0.5,
      "grad_norm": 5.870633602142334,
      "kl/avg_steps": 0.3499999940395355,
      "kl/beta": 0.006105704233050346,
      "kl/n_epsilon_steps": 0.32343751192092896,
      "kl/p_epsilon_steps": 0.6734374761581421,
      "learning_rate": 2.959373794541426e-07,
      "logits/chosen": -1.0475225448608398,
      "logits/rejected": -1.006306529045105,
      "logps/chosen": -147.1262664794922,
      "logps/ref_chosen": -75.01612854003906,
      "logps/ref_rejected": -86.07945251464844,
      "logps/rejected": -199.21173095703125,
      "loss": 0.6032,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4399870038032532,
      "rewards/margins": 0.2465648353099823,
      "rewards/rejected": -0.6865519285202026,
      "step": 170
    },
    {
      "epoch": 0.5147058823529411,
      "grad_norm": 5.422708988189697,
      "kl/avg_steps": 0.41093748807907104,
      "kl/beta": 0.0059935590252280235,
      "kl/n_epsilon_steps": 0.29374998807907104,
      "kl/p_epsilon_steps": 0.7046874761581421,
      "learning_rate": 2.8326761550411346e-07,
      "logits/chosen": -1.037719488143921,
      "logits/rejected": -0.9720247387886047,
      "logps/chosen": -149.66494750976562,
      "logps/ref_chosen": -75.85931396484375,
      "logps/ref_rejected": -88.4763412475586,
      "logps/rejected": -206.0808563232422,
      "loss": 0.5969,
      "rewards/accuracies": 0.729687511920929,
      "rewards/chosen": -0.4419892430305481,
      "rewards/margins": 0.2586033344268799,
      "rewards/rejected": -0.7005925178527832,
      "step": 175
    },
    {
      "epoch": 0.5294117647058824,
      "grad_norm": 5.140402793884277,
      "kl/avg_steps": 0.30937498807907104,
      "kl/beta": 0.005884683690965176,
      "kl/n_epsilon_steps": 0.3453125059604645,
      "kl/p_epsilon_steps": 0.6546875238418579,
      "learning_rate": 2.7051020734928443e-07,
      "logits/chosen": -1.0452353954315186,
      "logits/rejected": -0.968549370765686,
      "logps/chosen": -143.4625701904297,
      "logps/ref_chosen": -74.5296859741211,
      "logps/ref_rejected": -78.44059753417969,
      "logps/rejected": -188.22622680664062,
      "loss": 0.6093,
      "rewards/accuracies": 0.692187488079071,
      "rewards/chosen": -0.4056355059146881,
      "rewards/margins": 0.23657508194446564,
      "rewards/rejected": -0.6422106027603149,
      "step": 180
    },
    {
      "epoch": 0.5441176470588235,
      "grad_norm": 5.03032112121582,
      "kl/avg_steps": 0.3984375,
      "kl/beta": 0.005778872407972813,
      "kl/n_epsilon_steps": 0.30000001192092896,
      "kl/p_epsilon_steps": 0.698437511920929,
      "learning_rate": 2.5769876463904263e-07,
      "logits/chosen": -1.0298566818237305,
      "logits/rejected": -0.9755008816719055,
      "logps/chosen": -137.92031860351562,
      "logps/ref_chosen": -70.28861999511719,
      "logps/ref_rejected": -85.20851135253906,
      "logps/rejected": -197.1123809814453,
      "loss": 0.5968,
      "rewards/accuracies": 0.7328125238418579,
      "rewards/chosen": -0.3904454708099365,
      "rewards/margins": 0.25234130024909973,
      "rewards/rejected": -0.6427868008613586,
      "step": 185
    },
    {
      "epoch": 0.5588235294117647,
      "grad_norm": 6.057910919189453,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.005678877234458923,
      "kl/n_epsilon_steps": 0.3203125,
      "kl/p_epsilon_steps": 0.6796875,
      "learning_rate": 2.4486703937790243e-07,
      "logits/chosen": -1.0044220685958862,
      "logits/rejected": -0.9527886509895325,
      "logps/chosen": -151.2794952392578,
      "logps/ref_chosen": -75.0217514038086,
      "logps/ref_rejected": -90.4836654663086,
      "logps/rejected": -214.67868041992188,
      "loss": 0.5951,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.43261224031448364,
      "rewards/margins": 0.2681336998939514,
      "rewards/rejected": -0.7007459402084351,
      "step": 190
    },
    {
      "epoch": 0.5735294117647058,
      "grad_norm": 5.573934555053711,
      "kl/avg_steps": 0.3499999940395355,
      "kl/beta": 0.005573070142418146,
      "kl/n_epsilon_steps": 0.32499998807907104,
      "kl/p_epsilon_steps": 0.675000011920929,
      "learning_rate": 2.320488370051681e-07,
      "logits/chosen": -0.989575207233429,
      "logits/rejected": -0.9092248678207397,
      "logps/chosen": -154.51953125,
      "logps/ref_chosen": -73.42979431152344,
      "logps/ref_rejected": -84.43408203125,
      "logps/rejected": -211.646240234375,
      "loss": 0.6019,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.4517548084259033,
      "rewards/margins": 0.25306665897369385,
      "rewards/rejected": -0.7048214673995972,
      "step": 195
    },
    {
      "epoch": 0.5882352941176471,
      "grad_norm": 5.598110198974609,
      "kl/avg_steps": 0.35468751192092896,
      "kl/beta": 0.005477838683873415,
      "kl/n_epsilon_steps": 0.3218750059604645,
      "kl/p_epsilon_steps": 0.676562488079071,
      "learning_rate": 2.192779273338215e-07,
      "logits/chosen": -0.9810283780097961,
      "logits/rejected": -0.8921745419502258,
      "logps/chosen": -159.2919464111328,
      "logps/ref_chosen": -77.8104019165039,
      "logps/ref_rejected": -86.66553497314453,
      "logps/rejected": -219.63626098632812,
      "loss": 0.5934,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.4459984302520752,
      "rewards/margins": 0.27772727608680725,
      "rewards/rejected": -0.7237256765365601,
      "step": 200
    },
    {
      "epoch": 0.5882352941176471,
      "eval_kl/n_epsilon_steps": 0.3958333432674408,
      "eval_kl/p_epsilon_steps": 0.6037326455116272,
      "eval_logits/chosen": -0.9817464351654053,
      "eval_logits/rejected": -0.8951107859611511,
      "eval_logps/chosen": -182.90843200683594,
      "eval_logps/ref_chosen": -87.82356262207031,
      "eval_logps/ref_rejected": -82.81887817382812,
      "eval_logps/rejected": -209.30340576171875,
      "eval_loss": 0.6429124474525452,
      "eval_rewards/accuracies": 0.6323784589767456,
      "eval_rewards/chosen": -0.5156466960906982,
      "eval_rewards/margins": 0.1663396805524826,
      "eval_rewards/rejected": -0.6819863319396973,
      "eval_runtime": 22.339,
      "eval_samples_per_second": 104.705,
      "eval_steps_per_second": 0.851,
      "step": 200
    },
    {
      "epoch": 0.6029411764705882,
      "grad_norm": 5.395305156707764,
      "kl/avg_steps": 0.31562501192092896,
      "kl/beta": 0.005382629111409187,
      "kl/n_epsilon_steps": 0.3421874940395355,
      "kl/p_epsilon_steps": 0.6578124761581421,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -0.9339988827705383,
      "logits/rejected": -0.8349924087524414,
      "logps/chosen": -150.00833129882812,
      "logps/ref_chosen": -71.83072662353516,
      "logps/ref_rejected": -78.26126861572266,
      "logps/rejected": -207.3239288330078,
      "loss": 0.5976,
      "rewards/accuracies": 0.7015625238418579,
      "rewards/chosen": -0.42064207792282104,
      "rewards/margins": 0.26989927887916565,
      "rewards/rejected": -0.6905413866043091,
      "step": 205
    },
    {
      "epoch": 0.6176470588235294,
      "grad_norm": 8.636336326599121,
      "kl/avg_steps": 0.3343749940395355,
      "kl/beta": 0.005294554866850376,
      "kl/n_epsilon_steps": 0.33281248807907104,
      "kl/p_epsilon_steps": 0.667187511920929,
      "learning_rate": 1.9401235374032425e-07,
      "logits/chosen": -0.940881073474884,
      "logits/rejected": -0.835827648639679,
      "logps/chosen": -169.9760284423828,
      "logps/ref_chosen": -81.13362121582031,
      "logps/ref_rejected": -83.91246032714844,
      "logps/rejected": -226.44479370117188,
      "loss": 0.5961,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4700423777103424,
      "rewards/margins": 0.28001874685287476,
      "rewards/rejected": -0.7500611543655396,
      "step": 210
    },
    {
      "epoch": 0.6323529411764706,
      "grad_norm": 5.697958946228027,
      "kl/avg_steps": 0.37968748807907104,
      "kl/beta": 0.005207170732319355,
      "kl/n_epsilon_steps": 0.30937498807907104,
      "kl/p_epsilon_steps": 0.6890624761581421,
      "learning_rate": 1.8158425248197928e-07,
      "logits/chosen": -0.9595499038696289,
      "logits/rejected": -0.8254610300064087,
      "logps/chosen": -168.97909545898438,
      "logps/ref_chosen": -79.5214614868164,
      "logps/ref_rejected": -83.58778381347656,
      "logps/rejected": -225.5334014892578,
      "loss": 0.5994,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.4653104245662689,
      "rewards/margins": 0.2690308690071106,
      "rewards/rejected": -0.7343412637710571,
      "step": 215
    },
    {
      "epoch": 0.6470588235294118,
      "grad_norm": 5.304469108581543,
      "kl/avg_steps": 0.3343749940395355,
      "kl/beta": 0.005111886188387871,
      "kl/n_epsilon_steps": 0.33281248807907104,
      "kl/p_epsilon_steps": 0.667187511920929,
      "learning_rate": 1.6933639389195134e-07,
      "logits/chosen": -0.9539089202880859,
      "logits/rejected": -0.8665965795516968,
      "logps/chosen": -166.537353515625,
      "logps/ref_chosen": -81.25938415527344,
      "logps/ref_rejected": -83.04185485839844,
      "logps/rejected": -215.3668670654297,
      "loss": 0.6056,
      "rewards/accuracies": 0.723437488079071,
      "rewards/chosen": -0.43559327721595764,
      "rewards/margins": 0.2370072603225708,
      "rewards/rejected": -0.6726005673408508,
      "step": 220
    },
    {
      "epoch": 0.6617647058823529,
      "grad_norm": 5.622444152832031,
      "kl/avg_steps": 0.4468750059604645,
      "kl/beta": 0.005018714815378189,
      "kl/n_epsilon_steps": 0.27656251192092896,
      "kl/p_epsilon_steps": 0.723437488079071,
      "learning_rate": 1.573010452010098e-07,
      "logits/chosen": -0.9484726190567017,
      "logits/rejected": -0.8518384695053101,
      "logps/chosen": -162.01535034179688,
      "logps/ref_chosen": -77.427001953125,
      "logps/ref_rejected": -89.23592376708984,
      "logps/rejected": -233.6844024658203,
      "loss": 0.5839,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.4237908720970154,
      "rewards/margins": 0.2962331175804138,
      "rewards/rejected": -0.7200239896774292,
      "step": 225
    },
    {
      "epoch": 0.6764705882352942,
      "grad_norm": 5.60673189163208,
      "kl/avg_steps": 0.48124998807907104,
      "kl/beta": 0.004900630097836256,
      "kl/n_epsilon_steps": 0.2593750059604645,
      "kl/p_epsilon_steps": 0.7406250238418579,
      "learning_rate": 1.4550991377830423e-07,
      "logits/chosen": -0.9383388757705688,
      "logits/rejected": -0.856258749961853,
      "logps/chosen": -156.29066467285156,
      "logps/ref_chosen": -70.1819839477539,
      "logps/ref_rejected": -87.79248046875,
      "logps/rejected": -232.82858276367188,
      "loss": 0.5866,
      "rewards/accuracies": 0.7671874761581421,
      "rewards/chosen": -0.42099839448928833,
      "rewards/margins": 0.2847110629081726,
      "rewards/rejected": -0.7057094573974609,
      "step": 230
    },
    {
      "epoch": 0.6911764705882353,
      "grad_norm": 5.7163004875183105,
      "kl/avg_steps": 0.4828124940395355,
      "kl/beta": 0.004785512108355761,
      "kl/n_epsilon_steps": 0.2578125,
      "kl/p_epsilon_steps": 0.7406250238418579,
      "learning_rate": 1.339940635976592e-07,
      "logits/chosen": -0.8863986134529114,
      "logits/rejected": -0.8059118390083313,
      "logps/chosen": -174.5547637939453,
      "logps/ref_chosen": -77.51251220703125,
      "logps/ref_rejected": -89.81958770751953,
      "logps/rejected": -251.2958526611328,
      "loss": 0.583,
      "rewards/accuracies": 0.7734375,
      "rewards/chosen": -0.4634285569190979,
      "rewards/margins": 0.30392760038375854,
      "rewards/rejected": -0.7673560976982117,
      "step": 235
    },
    {
      "epoch": 0.7058823529411765,
      "grad_norm": 6.860780715942383,
      "kl/avg_steps": 0.36250001192092896,
      "kl/beta": 0.004683743230998516,
      "kl/n_epsilon_steps": 0.3187499940395355,
      "kl/p_epsilon_steps": 0.6812499761581421,
      "learning_rate": 1.227838333989088e-07,
      "logits/chosen": -0.8450605273246765,
      "logits/rejected": -0.7272099256515503,
      "logps/chosen": -177.47744750976562,
      "logps/ref_chosen": -74.5803451538086,
      "logps/ref_rejected": -81.81297302246094,
      "logps/rejected": -243.76736450195312,
      "loss": 0.5968,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.4815599322319031,
      "rewards/margins": 0.27244722843170166,
      "rewards/rejected": -0.7540072202682495,
      "step": 240
    },
    {
      "epoch": 0.7205882352941176,
      "grad_norm": 5.382650852203369,
      "kl/avg_steps": 0.4000000059604645,
      "kl/beta": 0.004598929081112146,
      "kl/n_epsilon_steps": 0.30000001192092896,
      "kl/p_epsilon_steps": 0.699999988079071,
      "learning_rate": 1.1190875675987355e-07,
      "logits/chosen": -0.8378638029098511,
      "logits/rejected": -0.7307332158088684,
      "logps/chosen": -178.53826904296875,
      "logps/ref_chosen": -76.56635284423828,
      "logps/ref_rejected": -86.859130859375,
      "logps/rejected": -255.5751495361328,
      "loss": 0.5827,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.46838441491127014,
      "rewards/margins": 0.30266332626342773,
      "rewards/rejected": -0.7710477113723755,
      "step": 245
    },
    {
      "epoch": 0.7352941176470589,
      "grad_norm": 5.63203763961792,
      "kl/avg_steps": 0.3499999940395355,
      "kl/beta": 0.00451111001893878,
      "kl/n_epsilon_steps": 0.32499998807907104,
      "kl/p_epsilon_steps": 0.675000011920929,
      "learning_rate": 1.0139748428955333e-07,
      "logits/chosen": -0.8333392143249512,
      "logits/rejected": -0.7355720400810242,
      "logps/chosen": -183.86294555664062,
      "logps/ref_chosen": -77.37183380126953,
      "logps/ref_rejected": -79.96475219726562,
      "logps/rejected": -237.01980590820312,
      "loss": 0.6155,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.4800783693790436,
      "rewards/margins": 0.22398455440998077,
      "rewards/rejected": -0.7040629982948303,
      "step": 250
    },
    {
      "epoch": 0.75,
      "grad_norm": 5.822533130645752,
      "kl/avg_steps": 0.35624998807907104,
      "kl/beta": 0.004430105909705162,
      "kl/n_epsilon_steps": 0.3218750059604645,
      "kl/p_epsilon_steps": 0.6781250238418579,
      "learning_rate": 9.127770814751932e-08,
      "logits/chosen": -0.8416454195976257,
      "logits/rejected": -0.7227948904037476,
      "logps/chosen": -184.06822204589844,
      "logps/ref_chosen": -79.62632751464844,
      "logps/ref_rejected": -83.8196792602539,
      "logps/rejected": -246.44998168945312,
      "loss": 0.6013,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.46239757537841797,
      "rewards/margins": 0.25374993681907654,
      "rewards/rejected": -0.7161475419998169,
      "step": 255
    },
    {
      "epoch": 0.7647058823529411,
      "grad_norm": 5.885540008544922,
      "kl/avg_steps": 0.3687500059604645,
      "kl/beta": 0.004350547678768635,
      "kl/n_epsilon_steps": 0.31562501192092896,
      "kl/p_epsilon_steps": 0.684374988079071,
      "learning_rate": 8.15760890883607e-08,
      "logits/chosen": -0.8616160154342651,
      "logits/rejected": -0.7643041610717773,
      "logps/chosen": -184.8510284423828,
      "logps/ref_chosen": -80.03411865234375,
      "logps/ref_rejected": -85.39453125,
      "logps/rejected": -246.5211639404297,
      "loss": 0.6056,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.4556017816066742,
      "rewards/margins": 0.2411097288131714,
      "rewards/rejected": -0.6967115998268127,
      "step": 260
    },
    {
      "epoch": 0.7794117647058824,
      "grad_norm": 5.461711883544922,
      "kl/avg_steps": 0.3531250059604645,
      "kl/beta": 0.0042734695598483086,
      "kl/n_epsilon_steps": 0.32343751192092896,
      "kl/p_epsilon_steps": 0.676562488079071,
      "learning_rate": 7.231818622338822e-08,
      "logits/chosen": -0.8387966156005859,
      "logits/rejected": -0.7239198088645935,
      "logps/chosen": -178.0113067626953,
      "logps/ref_chosen": -76.63539123535156,
      "logps/ref_rejected": -79.94613647460938,
      "logps/rejected": -238.1660614013672,
      "loss": 0.603,
      "rewards/accuracies": 0.7109375,
      "rewards/chosen": -0.432711660861969,
      "rewards/margins": 0.2392859160900116,
      "rewards/rejected": -0.6719975471496582,
      "step": 265
    },
    {
      "epoch": 0.7941176470588235,
      "grad_norm": 5.6931915283203125,
      "kl/avg_steps": 0.3968749940395355,
      "kl/beta": 0.004198429174721241,
      "kl/n_epsilon_steps": 0.30156248807907104,
      "kl/p_epsilon_steps": 0.698437511920929,
      "learning_rate": 6.352838968463919e-08,
      "logits/chosen": -0.8596851229667664,
      "logits/rejected": -0.7193423509597778,
      "logps/chosen": -173.6400604248047,
      "logps/ref_chosen": -76.02762603759766,
      "logps/ref_rejected": -80.83404541015625,
      "logps/rejected": -236.97607421875,
      "loss": 0.6021,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.4092663824558258,
      "rewards/margins": 0.24205096065998077,
      "rewards/rejected": -0.6513173580169678,
      "step": 270
    },
    {
      "epoch": 0.8088235294117647,
      "grad_norm": 5.091865062713623,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.004111775197088718,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 5.5229856368582376e-08,
      "logits/chosen": -0.8502656817436218,
      "logits/rejected": -0.7681766748428345,
      "logps/chosen": -180.9755859375,
      "logps/ref_chosen": -77.58733367919922,
      "logps/ref_rejected": -88.50263214111328,
      "logps/rejected": -254.04690551757812,
      "loss": 0.5997,
      "rewards/accuracies": 0.739062488079071,
      "rewards/chosen": -0.4245019555091858,
      "rewards/margins": 0.25200843811035156,
      "rewards/rejected": -0.6765104532241821,
      "step": 275
    },
    {
      "epoch": 0.8235294117647058,
      "grad_norm": 5.737886905670166,
      "kl/avg_steps": 0.42500001192092896,
      "kl/beta": 0.004024769179522991,
      "kl/n_epsilon_steps": 0.2874999940395355,
      "kl/p_epsilon_steps": 0.7124999761581421,
      "learning_rate": 4.7444448928806615e-08,
      "logits/chosen": -0.876409649848938,
      "logits/rejected": -0.7702105641365051,
      "logps/chosen": -186.74009704589844,
      "logps/ref_chosen": -81.46415710449219,
      "logps/ref_rejected": -94.69911193847656,
      "logps/rejected": -265.301025390625,
      "loss": 0.5958,
      "rewards/accuracies": 0.729687511920929,
      "rewards/chosen": -0.4230107367038727,
      "rewards/margins": 0.2593509256839752,
      "rewards/rejected": -0.6823617219924927,
      "step": 280
    },
    {
      "epoch": 0.8382352941176471,
      "grad_norm": 5.05964469909668,
      "kl/avg_steps": 0.4000000059604645,
      "kl/beta": 0.003945710603147745,
      "kl/n_epsilon_steps": 0.30000001192092896,
      "kl/p_epsilon_steps": 0.699999988079071,
      "learning_rate": 4.019267817841834e-08,
      "logits/chosen": -0.8164280652999878,
      "logits/rejected": -0.7374383211135864,
      "logps/chosen": -183.66696166992188,
      "logps/ref_chosen": -77.9266128540039,
      "logps/ref_rejected": -85.77226257324219,
      "logps/rejected": -251.9569854736328,
      "loss": 0.6036,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.41665568947792053,
      "rewards/margins": 0.23489956557750702,
      "rewards/rejected": -0.6515552997589111,
      "step": 285
    },
    {
      "epoch": 0.8529411764705882,
      "grad_norm": 4.995400428771973,
      "kl/avg_steps": 0.37187498807907104,
      "kl/beta": 0.003868584055453539,
      "kl/n_epsilon_steps": 0.3140625059604645,
      "kl/p_epsilon_steps": 0.6859375238418579,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": -0.788993775844574,
      "logits/rejected": -0.7104808688163757,
      "logps/chosen": -178.77645874023438,
      "logps/ref_chosen": -72.49942016601562,
      "logps/ref_rejected": -83.77849578857422,
      "logps/rejected": -253.64126586914062,
      "loss": 0.6008,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.41073670983314514,
      "rewards/margins": 0.24252267181873322,
      "rewards/rejected": -0.6532593965530396,
      "step": 290
    },
    {
      "epoch": 0.8676470588235294,
      "grad_norm": 5.24601411819458,
      "kl/avg_steps": 0.4000000059604645,
      "kl/beta": 0.0037986349780112505,
      "kl/n_epsilon_steps": 0.30000001192092896,
      "kl/p_epsilon_steps": 0.699999988079071,
      "learning_rate": 2.736501028272095e-08,
      "logits/chosen": -0.7796621918678284,
      "logits/rejected": -0.7315692901611328,
      "logps/chosen": -182.55836486816406,
      "logps/ref_chosen": -72.81735229492188,
      "logps/ref_rejected": -91.62478637695312,
      "logps/rejected": -265.3271789550781,
      "loss": 0.6044,
      "rewards/accuracies": 0.739062488079071,
      "rewards/chosen": -0.4162219166755676,
      "rewards/margins": 0.23918703198432922,
      "rewards/rejected": -0.6554089784622192,
      "step": 295
    },
    {
      "epoch": 0.8823529411764706,
      "grad_norm": 5.059004306793213,
      "kl/avg_steps": 0.39375001192092896,
      "kl/beta": 0.003725191578269005,
      "kl/n_epsilon_steps": 0.3031249940395355,
      "kl/p_epsilon_steps": 0.6968749761581421,
      "learning_rate": 2.1822907887504932e-08,
      "logits/chosen": -0.7761374711990356,
      "logits/rejected": -0.6450864672660828,
      "logps/chosen": -178.6937255859375,
      "logps/ref_chosen": -70.4697265625,
      "logps/ref_rejected": -77.26274108886719,
      "logps/rejected": -241.739013671875,
      "loss": 0.6156,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.40281882882118225,
      "rewards/margins": 0.20612934231758118,
      "rewards/rejected": -0.6089481115341187,
      "step": 300
    },
    {
      "epoch": 0.8823529411764706,
      "eval_kl/n_epsilon_steps": 0.3849826455116272,
      "eval_kl/p_epsilon_steps": 0.6150173544883728,
      "eval_logits/chosen": -0.8325175046920776,
      "eval_logits/rejected": -0.7259347438812256,
      "eval_logps/chosen": -208.17991638183594,
      "eval_logps/ref_chosen": -87.82356262207031,
      "eval_logps/ref_rejected": -82.81887817382812,
      "eval_logps/rejected": -243.57456970214844,
      "eval_loss": 0.6442785263061523,
      "eval_rewards/accuracies": 0.6401909589767456,
      "eval_rewards/chosen": -0.44304588437080383,
      "eval_rewards/margins": 0.14540132880210876,
      "eval_rewards/rejected": -0.5884472131729126,
      "eval_runtime": 22.3967,
      "eval_samples_per_second": 104.435,
      "eval_steps_per_second": 0.848,
      "step": 300
    },
    {
      "epoch": 0.8970588235294118,
      "grad_norm": 5.704087734222412,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.003651682287454605,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.6881942648911074e-08,
      "logits/chosen": -0.7806903719902039,
      "logits/rejected": -0.7004286050796509,
      "logps/chosen": -181.45826721191406,
      "logps/ref_chosen": -75.5998764038086,
      "logps/ref_rejected": -86.76122283935547,
      "logps/rejected": -256.80096435546875,
      "loss": 0.6049,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3862135410308838,
      "rewards/margins": 0.23094138503074646,
      "rewards/rejected": -0.6171549558639526,
      "step": 305
    },
    {
      "epoch": 0.9117647058823529,
      "grad_norm": 5.218584060668945,
      "kl/avg_steps": 0.3968749940395355,
      "kl/beta": 0.0035780933685600758,
      "kl/n_epsilon_steps": 0.30156248807907104,
      "kl/p_epsilon_steps": 0.698437511920929,
      "learning_rate": 1.2555131639630567e-08,
      "logits/chosen": -0.7832438349723816,
      "logits/rejected": -0.6719276309013367,
      "logps/chosen": -191.44869995117188,
      "logps/ref_chosen": -78.4868392944336,
      "logps/ref_rejected": -83.08047485351562,
      "logps/rejected": -258.40545654296875,
      "loss": 0.6111,
      "rewards/accuracies": 0.7265625,
      "rewards/chosen": -0.4038282036781311,
      "rewards/margins": 0.21982701122760773,
      "rewards/rejected": -0.6236552000045776,
      "step": 310
    },
    {
      "epoch": 0.9264705882352942,
      "grad_norm": 6.10360860824585,
      "kl/avg_steps": 0.3375000059604645,
      "kl/beta": 0.0035165518056601286,
      "kl/n_epsilon_steps": 0.33125001192092896,
      "kl/p_epsilon_steps": 0.668749988079071,
      "learning_rate": 8.85387393063622e-09,
      "logits/chosen": -0.8057095408439636,
      "logits/rejected": -0.7011617422103882,
      "logps/chosen": -194.56436157226562,
      "logps/ref_chosen": -79.54651641845703,
      "logps/ref_rejected": -87.11808776855469,
      "logps/rejected": -261.40032958984375,
      "loss": 0.6153,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4042418897151947,
      "rewards/margins": 0.20526555180549622,
      "rewards/rejected": -0.6095074415206909,
      "step": 315
    },
    {
      "epoch": 0.9411764705882353,
      "grad_norm": 5.0830488204956055,
      "kl/avg_steps": 0.28437501192092896,
      "kl/beta": 0.0034615718759596348,
      "kl/n_epsilon_steps": 0.3578124940395355,
      "kl/p_epsilon_steps": 0.6421874761581421,
      "learning_rate": 5.7879205600998296e-09,
      "logits/chosen": -0.8048986196517944,
      "logits/rejected": -0.6852750778198242,
      "logps/chosen": -193.45582580566406,
      "logps/ref_chosen": -78.56401062011719,
      "logps/ref_rejected": -83.85292053222656,
      "logps/rejected": -248.977783203125,
      "loss": 0.6302,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.39771518111228943,
      "rewards/margins": 0.17076563835144043,
      "rewards/rejected": -0.5684808492660522,
      "step": 320
    },
    {
      "epoch": 0.9558823529411765,
      "grad_norm": 5.110870361328125,
      "kl/avg_steps": 0.3499999940395355,
      "kl/beta": 0.0034066252410411835,
      "kl/n_epsilon_steps": 0.32499998807907104,
      "kl/p_epsilon_steps": 0.675000011920929,
      "learning_rate": 3.3653488440851253e-09,
      "logits/chosen": -0.7829563021659851,
      "logits/rejected": -0.7219451665878296,
      "logps/chosen": -183.75088500976562,
      "logps/ref_chosen": -74.60850524902344,
      "logps/ref_rejected": -86.81698608398438,
      "logps/rejected": -254.2379150390625,
      "loss": 0.6219,
      "rewards/accuracies": 0.692187488079071,
      "rewards/chosen": -0.3717408776283264,
      "rewards/margins": 0.195209339261055,
      "rewards/rejected": -0.5669502019882202,
      "step": 325
    },
    {
      "epoch": 0.9705882352941176,
      "grad_norm": 4.562494277954102,
      "kl/avg_steps": 0.47343748807907104,
      "kl/beta": 0.003342044074088335,
      "kl/n_epsilon_steps": 0.26249998807907104,
      "kl/p_epsilon_steps": 0.7359374761581421,
      "learning_rate": 1.592541096695571e-09,
      "logits/chosen": -0.7936745882034302,
      "logits/rejected": -0.739700436592102,
      "logps/chosen": -178.63034057617188,
      "logps/ref_chosen": -74.63096618652344,
      "logps/ref_rejected": -92.50404357910156,
      "logps/rejected": -266.2847595214844,
      "loss": 0.601,
      "rewards/accuracies": 0.7578125,
      "rewards/chosen": -0.34669384360313416,
      "rewards/margins": 0.23011669516563416,
      "rewards/rejected": -0.5768105387687683,
      "step": 330
    },
    {
      "epoch": 0.9852941176470589,
      "grad_norm": 4.651317596435547,
      "kl/avg_steps": 0.35624998807907104,
      "kl/beta": 0.003271129447966814,
      "kl/n_epsilon_steps": 0.3218750059604645,
      "kl/p_epsilon_steps": 0.6781250238418579,
      "learning_rate": 4.741678157389739e-10,
      "logits/chosen": -0.8402039408683777,
      "logits/rejected": -0.7369452118873596,
      "logps/chosen": -193.51834106445312,
      "logps/ref_chosen": -81.25680541992188,
      "logps/ref_rejected": -88.71739196777344,
      "logps/rejected": -261.07110595703125,
      "loss": 0.6167,
      "rewards/accuracies": 0.7203124761581421,
      "rewards/chosen": -0.3669508695602417,
      "rewards/margins": 0.19351065158843994,
      "rewards/rejected": -0.5604615211486816,
      "step": 335
    },
    {
      "epoch": 1.0,
      "grad_norm": 4.5893425941467285,
      "kl/avg_steps": 0.37812501192092896,
      "kl/beta": 0.003211395815014839,
      "kl/n_epsilon_steps": 0.3109374940395355,
      "kl/p_epsilon_steps": 0.6890624761581421,
      "learning_rate": 1.31753782067201e-11,
      "logits/chosen": -0.7557514905929565,
      "logits/rejected": -0.6398700475692749,
      "logps/chosen": -185.0140838623047,
      "logps/ref_chosen": -72.54796600341797,
      "logps/ref_rejected": -78.83277893066406,
      "logps/rejected": -256.5284423828125,
      "loss": 0.612,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.36068642139434814,
      "rewards/margins": 0.20638315379619598,
      "rewards/rejected": -0.5670695900917053,
      "step": 340
    },
    {
      "epoch": 1.0,
      "step": 340,
      "total_flos": 0.0,
      "train_loss": 0.6232832217917723,
      "train_runtime": 1489.7896,
      "train_samples_per_second": 29.265,
      "train_steps_per_second": 0.228
    }
  ],
  "logging_steps": 5,
  "max_steps": 340,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}