{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 2130,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004694835680751174,
      "grad_norm": 1037.7874738162204,
      "learning_rate": 2.1126760563380282e-08,
      "logits/chosen": 1.66796875,
      "logits/rejected": 1.899999976158142,
      "logps/chosen": -440.3999938964844,
      "logps/rejected": -540.0,
      "loss": 2.2749,
      "rewards/accuracies": 0.19062499701976776,
      "rewards/chosen": 0.0003662109375,
      "rewards/margins": 0.01992187462747097,
      "rewards/rejected": -0.019561767578125,
      "step": 10
    },
    {
      "epoch": 0.009389671361502348,
      "grad_norm": 1098.28564587442,
      "learning_rate": 4.460093896713615e-08,
      "logits/chosen": 1.48828125,
      "logits/rejected": 1.8312499523162842,
      "logps/chosen": -462.6000061035156,
      "logps/rejected": -550.7999877929688,
      "loss": 3.3624,
      "rewards/accuracies": 0.2984375059604645,
      "rewards/chosen": -0.06291504204273224,
      "rewards/margins": -0.17246094346046448,
      "rewards/rejected": 0.10921631008386612,
      "step": 20
    },
    {
      "epoch": 0.014084507042253521,
      "grad_norm": 934.2175076110283,
      "learning_rate": 6.807511737089202e-08,
      "logits/chosen": 1.560937523841858,
      "logits/rejected": 1.8445312976837158,
      "logps/chosen": -498.0,
      "logps/rejected": -602.2000122070312,
      "loss": 3.2835,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": 0.14082030951976776,
      "rewards/margins": 0.20786133408546448,
      "rewards/rejected": -0.06682129204273224,
      "step": 30
    },
    {
      "epoch": 0.018779342723004695,
      "grad_norm": 3830.1801465342687,
      "learning_rate": 9.154929577464789e-08,
      "logits/chosen": 1.5695312023162842,
      "logits/rejected": 1.834375023841858,
      "logps/chosen": -484.20001220703125,
      "logps/rejected": -614.7999877929688,
      "loss": 3.627,
      "rewards/accuracies": 0.29765623807907104,
      "rewards/chosen": -0.15219727158546448,
      "rewards/margins": -0.3763671815395355,
      "rewards/rejected": 0.22509765625,
      "step": 40
    },
    {
      "epoch": 0.023474178403755867,
      "grad_norm": 992.3248624087067,
      "learning_rate": 1.1502347417840374e-07,
      "logits/chosen": 1.5546875,
      "logits/rejected": 1.884374976158142,
      "logps/chosen": -468.20001220703125,
      "logps/rejected": -559.0,
      "loss": 3.5195,
      "rewards/accuracies": 0.2914062440395355,
      "rewards/chosen": -0.14736327528953552,
      "rewards/margins": -0.32861328125,
      "rewards/rejected": 0.181640625,
      "step": 50
    },
    {
      "epoch": 0.028169014084507043,
      "grad_norm": 1039.4620170460248,
      "learning_rate": 1.384976525821596e-07,
      "logits/chosen": 1.5828125476837158,
      "logits/rejected": 1.8898437023162842,
      "logps/chosen": -479.6000061035156,
      "logps/rejected": -603.4000244140625,
      "loss": 3.3131,
      "rewards/accuracies": 0.30390626192092896,
      "rewards/chosen": 0.09042968600988388,
      "rewards/margins": -0.0062500000931322575,
      "rewards/rejected": 0.09687499701976776,
      "step": 60
    },
    {
      "epoch": 0.03286384976525822,
      "grad_norm": 933.991452589035,
      "learning_rate": 1.619718309859155e-07,
      "logits/chosen": 1.5125000476837158,
      "logits/rejected": 1.8132812976837158,
      "logps/chosen": -459.20001220703125,
      "logps/rejected": -549.5999755859375,
      "loss": 3.3458,
      "rewards/accuracies": 0.3335937559604645,
      "rewards/chosen": -0.27275389432907104,
      "rewards/margins": -0.12714843451976776,
      "rewards/rejected": -0.14458008110523224,
      "step": 70
    },
    {
      "epoch": 0.03755868544600939,
      "grad_norm": 852.4728601252323,
      "learning_rate": 1.8544600938967138e-07,
      "logits/chosen": 1.6749999523162842,
      "logits/rejected": 1.9812500476837158,
      "logps/chosen": -466.6000061035156,
      "logps/rejected": -581.5999755859375,
      "loss": 3.1387,
      "rewards/accuracies": 0.2945312559604645,
      "rewards/chosen": 0.08278808742761612,
      "rewards/margins": 0.18305663764476776,
      "rewards/rejected": -0.10036621242761612,
      "step": 80
    },
    {
      "epoch": 0.04225352112676056,
      "grad_norm": 811.8843803159269,
      "learning_rate": 2.089201877934272e-07,
      "logits/chosen": 1.7179687023162842,
      "logits/rejected": 1.978906273841858,
      "logps/chosen": -453.3999938964844,
      "logps/rejected": -537.7999877929688,
      "loss": 3.2373,
      "rewards/accuracies": 0.2992187440395355,
      "rewards/chosen": -0.30908203125,
      "rewards/margins": -0.090087890625,
      "rewards/rejected": -0.22043457627296448,
      "step": 90
    },
    {
      "epoch": 0.046948356807511735,
      "grad_norm": 1380.3975870343875,
      "learning_rate": 2.323943661971831e-07,
      "logits/chosen": 1.626562476158142,
      "logits/rejected": 1.857031226158142,
      "logps/chosen": -482.6000061035156,
      "logps/rejected": -528.5999755859375,
      "loss": 2.9256,
      "rewards/accuracies": 0.34453123807907104,
      "rewards/chosen": 0.37773436307907104,
      "rewards/margins": 0.913867175579071,
      "rewards/rejected": -0.535412609577179,
      "step": 100
    },
    {
      "epoch": 0.051643192488262914,
      "grad_norm": 924.5295480553729,
      "learning_rate": 2.5586854460093895e-07,
      "logits/chosen": 1.580468773841858,
      "logits/rejected": 1.86328125,
      "logps/chosen": -466.0,
      "logps/rejected": -588.0,
      "loss": 3.1722,
      "rewards/accuracies": 0.3414062559604645,
      "rewards/chosen": -0.01801757887005806,
      "rewards/margins": 1.0642821788787842,
      "rewards/rejected": -1.0830078125,
      "step": 110
    },
    {
      "epoch": 0.056338028169014086,
      "grad_norm": 892.6601780001566,
      "learning_rate": 2.7934272300469483e-07,
      "logits/chosen": 1.6085937023162842,
      "logits/rejected": 1.771875023841858,
      "logps/chosen": -476.3999938964844,
      "logps/rejected": -569.7999877929688,
      "loss": 3.3008,
      "rewards/accuracies": 0.3421874940395355,
      "rewards/chosen": 0.20747070014476776,
      "rewards/margins": 0.4571289122104645,
      "rewards/rejected": -0.24830932915210724,
      "step": 120
    },
    {
      "epoch": 0.06103286384976526,
      "grad_norm": 945.8242025750275,
      "learning_rate": 3.0281690140845066e-07,
      "logits/chosen": 1.58203125,
      "logits/rejected": 1.859375,
      "logps/chosen": -481.6000061035156,
      "logps/rejected": -585.7999877929688,
      "loss": 2.6897,
      "rewards/accuracies": 0.3382812440395355,
      "rewards/chosen": 0.4341796934604645,
      "rewards/margins": 1.186621069908142,
      "rewards/rejected": -0.75,
      "step": 130
    },
    {
      "epoch": 0.06572769953051644,
      "grad_norm": 835.0100884363507,
      "learning_rate": 3.2629107981220654e-07,
      "logits/chosen": 1.51953125,
      "logits/rejected": 1.869531273841858,
      "logps/chosen": -473.0,
      "logps/rejected": -589.7999877929688,
      "loss": 3.1885,
      "rewards/accuracies": 0.3570312559604645,
      "rewards/chosen": 0.742968738079071,
      "rewards/margins": 1.1248047351837158,
      "rewards/rejected": -0.3835388123989105,
      "step": 140
    },
    {
      "epoch": 0.07042253521126761,
      "grad_norm": 962.9529045020581,
      "learning_rate": 3.497652582159624e-07,
      "logits/chosen": 1.595312476158142,
      "logits/rejected": 1.946874976158142,
      "logps/chosen": -471.6000061035156,
      "logps/rejected": -589.5999755859375,
      "loss": 2.8585,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.007702636532485485,
      "rewards/margins": 1.1863281726837158,
      "rewards/rejected": -1.195068359375,
      "step": 150
    },
    {
      "epoch": 0.07511737089201878,
      "grad_norm": 992.1893380738026,
      "learning_rate": 3.732394366197183e-07,
      "logits/chosen": 1.6359374523162842,
      "logits/rejected": 1.904687523841858,
      "logps/chosen": -487.3999938964844,
      "logps/rejected": -586.5999755859375,
      "loss": 3.5363,
      "rewards/accuracies": 0.34062498807907104,
      "rewards/chosen": -0.21708984673023224,
      "rewards/margins": 1.580664038658142,
      "rewards/rejected": -1.797570824623108,
      "step": 160
    },
    {
      "epoch": 0.07981220657276995,
      "grad_norm": 1450.7938081305726,
      "learning_rate": 3.967136150234742e-07,
      "logits/chosen": 1.62890625,
      "logits/rejected": 1.943750023841858,
      "logps/chosen": -446.79998779296875,
      "logps/rejected": -581.0,
      "loss": 2.7451,
      "rewards/accuracies": 0.3812499940395355,
      "rewards/chosen": 0.55712890625,
      "rewards/margins": 2.714062452316284,
      "rewards/rejected": -2.1576170921325684,
      "step": 170
    },
    {
      "epoch": 0.08450704225352113,
      "grad_norm": 894.1445010625941,
      "learning_rate": 4.2018779342723e-07,
      "logits/chosen": 1.5109374523162842,
      "logits/rejected": 1.822656273841858,
      "logps/chosen": -433.79998779296875,
      "logps/rejected": -554.4000244140625,
      "loss": 2.8097,
      "rewards/accuracies": 0.41328126192092896,
      "rewards/chosen": 0.2839599549770355,
      "rewards/margins": 2.041015625,
      "rewards/rejected": -1.757226586341858,
      "step": 180
    },
    {
      "epoch": 0.0892018779342723,
      "grad_norm": 826.5671876677011,
      "learning_rate": 4.436619718309859e-07,
      "logits/chosen": 1.58984375,
      "logits/rejected": 1.8937499523162842,
      "logps/chosen": -467.0,
      "logps/rejected": -552.2000122070312,
      "loss": 2.4396,
      "rewards/accuracies": 0.4234375059604645,
      "rewards/chosen": 0.5877441167831421,
      "rewards/margins": 3.6703124046325684,
      "rewards/rejected": -3.081249952316284,
      "step": 190
    },
    {
      "epoch": 0.09389671361502347,
      "grad_norm": 679.3945446519161,
      "learning_rate": 4.671361502347418e-07,
      "logits/chosen": 1.5281250476837158,
      "logits/rejected": 1.830468773841858,
      "logps/chosen": -455.0,
      "logps/rejected": -559.5999755859375,
      "loss": 3.1477,
      "rewards/accuracies": 0.41328126192092896,
      "rewards/chosen": 0.2964843809604645,
      "rewards/margins": 2.4261717796325684,
      "rewards/rejected": -2.1273436546325684,
      "step": 200
    },
    {
      "epoch": 0.09859154929577464,
      "grad_norm": 866.7242163288782,
      "learning_rate": 4.906103286384976e-07,
      "logits/chosen": 1.5734374523162842,
      "logits/rejected": 1.8937499523162842,
      "logps/chosen": -468.3999938964844,
      "logps/rejected": -593.0,
      "loss": 2.8536,
      "rewards/accuracies": 0.43437498807907104,
      "rewards/chosen": 0.6447998285293579,
      "rewards/margins": 3.879687547683716,
      "rewards/rejected": -3.2337889671325684,
      "step": 210
    },
    {
      "epoch": 0.10328638497652583,
      "grad_norm": 858.9024176283116,
      "learning_rate": 4.984350547730829e-07,
      "logits/chosen": 1.646093726158142,
      "logits/rejected": 1.951562523841858,
      "logps/chosen": -479.20001220703125,
      "logps/rejected": -569.2000122070312,
      "loss": 3.0662,
      "rewards/accuracies": 0.43671876192092896,
      "rewards/chosen": 0.5567382574081421,
      "rewards/margins": 4.069531440734863,
      "rewards/rejected": -3.51171875,
      "step": 220
    },
    {
      "epoch": 0.107981220657277,
      "grad_norm": 859.0525179597714,
      "learning_rate": 4.958268127282212e-07,
      "logits/chosen": 1.4562499523162842,
      "logits/rejected": 1.71484375,
      "logps/chosen": -469.3999938964844,
      "logps/rejected": -594.7999877929688,
      "loss": 3.1772,
      "rewards/accuracies": 0.45703125,
      "rewards/chosen": 0.581250011920929,
      "rewards/margins": 3.7671875953674316,
      "rewards/rejected": -3.1875,
      "step": 230
    },
    {
      "epoch": 0.11267605633802817,
      "grad_norm": 831.8015203039776,
      "learning_rate": 4.932185706833594e-07,
      "logits/chosen": 1.5085937976837158,
      "logits/rejected": 1.756250023841858,
      "logps/chosen": -425.79998779296875,
      "logps/rejected": -492.3999938964844,
      "loss": 2.9827,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 0.5416259765625,
      "rewards/margins": 3.5765624046325684,
      "rewards/rejected": -3.0374999046325684,
      "step": 240
    },
    {
      "epoch": 0.11737089201877934,
      "grad_norm": 978.4326426533137,
      "learning_rate": 4.906103286384976e-07,
      "logits/chosen": 1.552343726158142,
      "logits/rejected": 1.904687523841858,
      "logps/chosen": -467.79998779296875,
      "logps/rejected": -608.5999755859375,
      "loss": 3.3828,
      "rewards/accuracies": 0.42109376192092896,
      "rewards/chosen": -0.22832031548023224,
      "rewards/margins": 3.94140625,
      "rewards/rejected": -4.170312404632568,
      "step": 250
    },
    {
      "epoch": 0.12206572769953052,
      "grad_norm": 886.4313660872724,
      "learning_rate": 4.880020865936358e-07,
      "logits/chosen": 1.552343726158142,
      "logits/rejected": 1.889062523841858,
      "logps/chosen": -437.3999938964844,
      "logps/rejected": -549.7999877929688,
      "loss": 2.7296,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.8130859136581421,
      "rewards/margins": 4.928124904632568,
      "rewards/rejected": -4.112500190734863,
      "step": 260
    },
    {
      "epoch": 0.1267605633802817,
      "grad_norm": 1113.0262682675204,
      "learning_rate": 4.853938445487741e-07,
      "logits/chosen": 1.5125000476837158,
      "logits/rejected": 1.9054687023162842,
      "logps/chosen": -465.3999938964844,
      "logps/rejected": -584.0,
      "loss": 3.1491,
      "rewards/accuracies": 0.48515623807907104,
      "rewards/chosen": 0.36933594942092896,
      "rewards/margins": 5.017187595367432,
      "rewards/rejected": -4.650781154632568,
      "step": 270
    },
    {
      "epoch": 0.13145539906103287,
      "grad_norm": 787.4680605542583,
      "learning_rate": 4.827856025039123e-07,
      "logits/chosen": 1.5234375,
      "logits/rejected": 1.884374976158142,
      "logps/chosen": -463.6000061035156,
      "logps/rejected": -587.2000122070312,
      "loss": 3.0845,
      "rewards/accuracies": 0.4585937559604645,
      "rewards/chosen": 0.706158459186554,
      "rewards/margins": 5.0498046875,
      "rewards/rejected": -4.345312595367432,
      "step": 280
    },
    {
      "epoch": 0.13615023474178403,
      "grad_norm": 830.1753520510078,
      "learning_rate": 4.801773604590506e-07,
      "logits/chosen": 1.5476562976837158,
      "logits/rejected": 1.875,
      "logps/chosen": -500.3999938964844,
      "logps/rejected": -582.2000122070312,
      "loss": 2.9557,
      "rewards/accuracies": 0.49609375,
      "rewards/chosen": 1.045263648033142,
      "rewards/margins": 5.675000190734863,
      "rewards/rejected": -4.6328125,
      "step": 290
    },
    {
      "epoch": 0.14084507042253522,
      "grad_norm": 687.2288604070689,
      "learning_rate": 4.775691184141888e-07,
      "logits/chosen": 1.4757812023162842,
      "logits/rejected": 1.872656226158142,
      "logps/chosen": -483.79998779296875,
      "logps/rejected": -565.0,
      "loss": 2.6337,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": 0.826367199420929,
      "rewards/margins": 5.732812404632568,
      "rewards/rejected": -4.903124809265137,
      "step": 300
    },
    {
      "epoch": 0.14553990610328638,
      "grad_norm": 796.7496989074315,
      "learning_rate": 4.749608763693271e-07,
      "logits/chosen": 1.59375,
      "logits/rejected": 1.877343773841858,
      "logps/chosen": -460.0,
      "logps/rejected": -565.0,
      "loss": 3.2325,
      "rewards/accuracies": 0.500781238079071,
      "rewards/chosen": 0.521484375,
      "rewards/margins": 5.595312595367432,
      "rewards/rejected": -5.076562404632568,
      "step": 310
    },
    {
      "epoch": 0.15023474178403756,
      "grad_norm": 726.3081646453343,
      "learning_rate": 4.7235263432446533e-07,
      "logits/chosen": 1.5320312976837158,
      "logits/rejected": 1.7296874523162842,
      "logps/chosen": -459.0,
      "logps/rejected": -554.7999877929688,
      "loss": 2.9645,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 0.6563476324081421,
      "rewards/margins": 7.771874904632568,
      "rewards/rejected": -7.109375,
      "step": 320
    },
    {
      "epoch": 0.15492957746478872,
      "grad_norm": 757.514944798575,
      "learning_rate": 4.6974439227960353e-07,
      "logits/chosen": 1.622656226158142,
      "logits/rejected": 1.8585937023162842,
      "logps/chosen": -475.0,
      "logps/rejected": -582.7999877929688,
      "loss": 3.1179,
      "rewards/accuracies": 0.49687498807907104,
      "rewards/chosen": 0.07988281548023224,
      "rewards/margins": 6.40625,
      "rewards/rejected": -6.329687595367432,
      "step": 330
    },
    {
      "epoch": 0.1596244131455399,
      "grad_norm": 685.1295561906059,
      "learning_rate": 4.671361502347418e-07,
      "logits/chosen": 1.591406226158142,
      "logits/rejected": 1.90234375,
      "logps/chosen": -450.6000061035156,
      "logps/rejected": -516.7999877929688,
      "loss": 2.6022,
      "rewards/accuracies": 0.4945312440395355,
      "rewards/chosen": 0.6546875238418579,
      "rewards/margins": 5.224218845367432,
      "rewards/rejected": -4.56640625,
      "step": 340
    },
    {
      "epoch": 0.1643192488262911,
      "grad_norm": 757.8463987561815,
      "learning_rate": 4.6452790818988004e-07,
      "logits/chosen": 1.560156226158142,
      "logits/rejected": 1.9054687023162842,
      "logps/chosen": -485.6000061035156,
      "logps/rejected": -589.4000244140625,
      "loss": 3.4494,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.11606445163488388,
      "rewards/margins": 7.282812595367432,
      "rewards/rejected": -7.162499904632568,
      "step": 350
    },
    {
      "epoch": 0.16901408450704225,
      "grad_norm": 828.7182059329157,
      "learning_rate": 4.6191966614501824e-07,
      "logits/chosen": 1.53125,
      "logits/rejected": 1.7117187976837158,
      "logps/chosen": -454.6000061035156,
      "logps/rejected": -551.2000122070312,
      "loss": 3.0063,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": 1.0009765625,
      "rewards/margins": 5.215624809265137,
      "rewards/rejected": -4.213281154632568,
      "step": 360
    },
    {
      "epoch": 0.17370892018779344,
      "grad_norm": 735.3391496179686,
      "learning_rate": 4.593114241001565e-07,
      "logits/chosen": 1.5007812976837158,
      "logits/rejected": 1.857812523841858,
      "logps/chosen": -488.3999938964844,
      "logps/rejected": -577.0,
      "loss": 2.911,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 0.4896484315395355,
      "rewards/margins": 6.962500095367432,
      "rewards/rejected": -6.465624809265137,
      "step": 370
    },
    {
      "epoch": 0.1784037558685446,
      "grad_norm": 926.0206844684104,
      "learning_rate": 4.5670318205529474e-07,
      "logits/chosen": 1.537500023841858,
      "logits/rejected": 1.8015625476837158,
      "logps/chosen": -466.79998779296875,
      "logps/rejected": -577.5999755859375,
      "loss": 3.0566,
      "rewards/accuracies": 0.47968751192092896,
      "rewards/chosen": -0.06386108696460724,
      "rewards/margins": 6.120312690734863,
      "rewards/rejected": -6.193749904632568,
      "step": 380
    },
    {
      "epoch": 0.18309859154929578,
      "grad_norm": 929.3329214098178,
      "learning_rate": 4.54094940010433e-07,
      "logits/chosen": 1.55078125,
      "logits/rejected": 1.896875023841858,
      "logps/chosen": -474.6000061035156,
      "logps/rejected": -592.5999755859375,
      "loss": 3.1363,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.4950195252895355,
      "rewards/margins": 6.509375095367432,
      "rewards/rejected": -6.020312309265137,
      "step": 390
    },
    {
      "epoch": 0.18779342723004694,
      "grad_norm": 838.2602252939558,
      "learning_rate": 4.514866979655712e-07,
      "logits/chosen": 1.600000023841858,
      "logits/rejected": 1.8367187976837158,
      "logps/chosen": -465.3999938964844,
      "logps/rejected": -549.2000122070312,
      "loss": 2.5334,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.385888695716858,
      "rewards/margins": 7.09375,
      "rewards/rejected": -5.701562404632568,
      "step": 400
    },
    {
      "epoch": 0.19248826291079812,
      "grad_norm": 805.2189922596,
      "learning_rate": 4.4887845592070945e-07,
      "logits/chosen": 1.501562476158142,
      "logits/rejected": 1.8367187976837158,
      "logps/chosen": -452.3999938964844,
      "logps/rejected": -604.5999755859375,
      "loss": 3.534,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.162109375,
      "rewards/margins": 6.918749809265137,
      "rewards/rejected": -6.75,
      "step": 410
    },
    {
      "epoch": 0.19718309859154928,
      "grad_norm": 845.9866611767682,
      "learning_rate": 4.462702138758477e-07,
      "logits/chosen": 1.603906273841858,
      "logits/rejected": 1.8445312976837158,
      "logps/chosen": -485.6000061035156,
      "logps/rejected": -569.2000122070312,
      "loss": 3.0954,
      "rewards/accuracies": 0.48828125,
      "rewards/chosen": 0.19809570908546448,
      "rewards/margins": 7.251562595367432,
      "rewards/rejected": -7.059374809265137,
      "step": 420
    },
    {
      "epoch": 0.20187793427230047,
      "grad_norm": 1006.975805366128,
      "learning_rate": 4.436619718309859e-07,
      "logits/chosen": 1.466406226158142,
      "logits/rejected": 1.76171875,
      "logps/chosen": -467.3999938964844,
      "logps/rejected": -603.0,
      "loss": 2.8637,
      "rewards/accuracies": 0.5054687261581421,
      "rewards/chosen": 0.823046863079071,
      "rewards/margins": 8.509374618530273,
      "rewards/rejected": -7.696875095367432,
      "step": 430
    },
    {
      "epoch": 0.20657276995305165,
      "grad_norm": 748.539225642358,
      "learning_rate": 4.4105372978612415e-07,
      "logits/chosen": 1.69140625,
      "logits/rejected": 1.90625,
      "logps/chosen": -487.0,
      "logps/rejected": -593.0,
      "loss": 2.5921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.6045898199081421,
      "rewards/margins": 7.659375190734863,
      "rewards/rejected": -7.0546875,
      "step": 440
    },
    {
      "epoch": 0.2112676056338028,
      "grad_norm": 835.2809010150727,
      "learning_rate": 4.384454877412624e-07,
      "logits/chosen": 1.6476562023162842,
      "logits/rejected": 1.96875,
      "logps/chosen": -477.0,
      "logps/rejected": -648.4000244140625,
      "loss": 3.8219,
      "rewards/accuracies": 0.46796876192092896,
      "rewards/chosen": -0.05327148362994194,
      "rewards/margins": 6.675976753234863,
      "rewards/rejected": -6.7265625,
      "step": 450
    },
    {
      "epoch": 0.215962441314554,
      "grad_norm": 775.8021637872249,
      "learning_rate": 4.358372456964006e-07,
      "logits/chosen": 1.55078125,
      "logits/rejected": 1.83984375,
      "logps/chosen": -465.3999938964844,
      "logps/rejected": -576.7999877929688,
      "loss": 2.9997,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 1.4362304210662842,
      "rewards/margins": 7.650000095367432,
      "rewards/rejected": -6.215624809265137,
      "step": 460
    },
    {
      "epoch": 0.22065727699530516,
      "grad_norm": 737.5875366189109,
      "learning_rate": 4.3322900365153886e-07,
      "logits/chosen": 1.669531226158142,
      "logits/rejected": 1.8117187023162842,
      "logps/chosen": -508.6000061035156,
      "logps/rejected": -582.2000122070312,
      "loss": 3.566,
      "rewards/accuracies": 0.49921876192092896,
      "rewards/chosen": 0.1083984375,
      "rewards/margins": 6.004687309265137,
      "rewards/rejected": -5.889062404632568,
      "step": 470
    },
    {
      "epoch": 0.22535211267605634,
      "grad_norm": 705.0595615992629,
      "learning_rate": 4.306207616066771e-07,
      "logits/chosen": 1.5984375476837158,
      "logits/rejected": 1.837499976158142,
      "logps/chosen": -460.3999938964844,
      "logps/rejected": -553.5999755859375,
      "loss": 3.2229,
      "rewards/accuracies": 0.4898437559604645,
      "rewards/chosen": 0.7978515625,
      "rewards/margins": 5.795312404632568,
      "rewards/rejected": -5.000781059265137,
      "step": 480
    },
    {
      "epoch": 0.2300469483568075,
      "grad_norm": 732.772035919026,
      "learning_rate": 4.280125195618153e-07,
      "logits/chosen": 1.4992187023162842,
      "logits/rejected": 1.7664062976837158,
      "logps/chosen": -463.6000061035156,
      "logps/rejected": -593.0,
      "loss": 2.7395,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 1.0968749523162842,
      "rewards/margins": 9.160937309265137,
      "rewards/rejected": -8.065625190734863,
      "step": 490
    },
    {
      "epoch": 0.2347417840375587,
      "grad_norm": 934.3931737047315,
      "learning_rate": 4.2540427751695357e-07,
      "logits/chosen": 1.579687476158142,
      "logits/rejected": 1.8671875,
      "logps/chosen": -474.6000061035156,
      "logps/rejected": -559.7999877929688,
      "loss": 3.0012,
      "rewards/accuracies": 0.51171875,
      "rewards/chosen": 0.6002441644668579,
      "rewards/margins": 7.6328125,
      "rewards/rejected": -7.043749809265137,
      "step": 500
    },
    {
      "epoch": 0.23943661971830985,
      "grad_norm": 782.9683758518258,
      "learning_rate": 4.227960354720918e-07,
      "logits/chosen": 1.5703125,
      "logits/rejected": 1.83984375,
      "logps/chosen": -460.0,
      "logps/rejected": -593.5999755859375,
      "loss": 3.1796,
      "rewards/accuracies": 0.4898437559604645,
      "rewards/chosen": 0.67578125,
      "rewards/margins": 7.207812309265137,
      "rewards/rejected": -6.5390625,
      "step": 510
    },
    {
      "epoch": 0.24413145539906103,
      "grad_norm": 773.7120039416785,
      "learning_rate": 4.2018779342723e-07,
      "logits/chosen": 1.517187476158142,
      "logits/rejected": 1.8093750476837158,
      "logps/chosen": -474.79998779296875,
      "logps/rejected": -557.5999755859375,
      "loss": 3.3599,
      "rewards/accuracies": 0.48906248807907104,
      "rewards/chosen": 0.20078125596046448,
      "rewards/margins": 6.559374809265137,
      "rewards/rejected": -6.362500190734863,
      "step": 520
    },
    {
      "epoch": 0.24882629107981222,
      "grad_norm": 910.9291107614686,
      "learning_rate": 4.1757955138236827e-07,
      "logits/chosen": 1.666406273841858,
      "logits/rejected": 1.93359375,
      "logps/chosen": -462.3999938964844,
      "logps/rejected": -581.5999755859375,
      "loss": 2.714,
      "rewards/accuracies": 0.5289062261581421,
      "rewards/chosen": 0.803466796875,
      "rewards/margins": 7.981249809265137,
      "rewards/rejected": -7.171875,
      "step": 530
    },
    {
      "epoch": 0.2535211267605634,
      "grad_norm": 866.1488468262313,
      "learning_rate": 4.149713093375065e-07,
      "logits/chosen": 1.5703125,
      "logits/rejected": 1.7820312976837158,
      "logps/chosen": -460.79998779296875,
      "logps/rejected": -533.7999877929688,
      "loss": 2.9371,
      "rewards/accuracies": 0.5289062261581421,
      "rewards/chosen": 0.9701873660087585,
      "rewards/margins": 7.670312404632568,
      "rewards/rejected": -6.704687595367432,
      "step": 540
    },
    {
      "epoch": 0.25821596244131456,
      "grad_norm": 682.8575410817,
      "learning_rate": 4.123630672926447e-07,
      "logits/chosen": 1.5593750476837158,
      "logits/rejected": 1.8171875476837158,
      "logps/chosen": -446.3999938964844,
      "logps/rejected": -565.7999877929688,
      "loss": 2.9894,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 0.3880859315395355,
      "rewards/margins": 7.662499904632568,
      "rewards/rejected": -7.262499809265137,
      "step": 550
    },
    {
      "epoch": 0.26291079812206575,
      "grad_norm": 770.4370087728406,
      "learning_rate": 4.09754825247783e-07,
      "logits/chosen": 1.6515624523162842,
      "logits/rejected": 1.8976562023162842,
      "logps/chosen": -471.0,
      "logps/rejected": -558.7999877929688,
      "loss": 3.0401,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.6246093511581421,
      "rewards/margins": 8.315625190734863,
      "rewards/rejected": -7.692187309265137,
      "step": 560
    },
    {
      "epoch": 0.2676056338028169,
      "grad_norm": 743.1907437640372,
      "learning_rate": 4.0714658320292123e-07,
      "logits/chosen": 1.64453125,
      "logits/rejected": 1.90234375,
      "logps/chosen": -467.0,
      "logps/rejected": -572.7999877929688,
      "loss": 3.1321,
      "rewards/accuracies": 0.5171874761581421,
      "rewards/chosen": 0.19768066704273224,
      "rewards/margins": 8.959375381469727,
      "rewards/rejected": -8.759374618530273,
      "step": 570
    },
    {
      "epoch": 0.27230046948356806,
      "grad_norm": 766.6050428052121,
      "learning_rate": 4.045383411580595e-07,
      "logits/chosen": 1.584375023841858,
      "logits/rejected": 1.9171874523162842,
      "logps/chosen": -448.6000061035156,
      "logps/rejected": -577.2000122070312,
      "loss": 3.0408,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 0.663281261920929,
      "rewards/margins": 8.190625190734863,
      "rewards/rejected": -7.534375190734863,
      "step": 580
    },
    {
      "epoch": 0.27699530516431925,
      "grad_norm": 828.5459672503498,
      "learning_rate": 4.019300991131977e-07,
      "logits/chosen": 1.5617187023162842,
      "logits/rejected": 1.838281273841858,
      "logps/chosen": -454.3999938964844,
      "logps/rejected": -567.5999755859375,
      "loss": 3.2959,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.24282225966453552,
      "rewards/margins": 7.732812404632568,
      "rewards/rejected": -7.493750095367432,
      "step": 590
    },
    {
      "epoch": 0.28169014084507044,
      "grad_norm": 768.435446486162,
      "learning_rate": 3.9932185706833594e-07,
      "logits/chosen": 1.64453125,
      "logits/rejected": 1.86328125,
      "logps/chosen": -476.3999938964844,
      "logps/rejected": -561.5999755859375,
      "loss": 2.987,
      "rewards/accuracies": 0.5093749761581421,
      "rewards/chosen": 0.6147705316543579,
      "rewards/margins": 7.693749904632568,
      "rewards/rejected": -7.078125,
      "step": 600
    },
    {
      "epoch": 0.2863849765258216,
      "grad_norm": 789.0382199214489,
      "learning_rate": 3.967136150234742e-07,
      "logits/chosen": 1.57421875,
      "logits/rejected": 1.81640625,
      "logps/chosen": -471.79998779296875,
      "logps/rejected": -576.5999755859375,
      "loss": 3.0684,
      "rewards/accuracies": 0.516406238079071,
      "rewards/chosen": 0.769726574420929,
      "rewards/margins": 8.274999618530273,
      "rewards/rejected": -7.493750095367432,
      "step": 610
    },
    {
      "epoch": 0.29107981220657275,
      "grad_norm": 772.1515414717294,
      "learning_rate": 3.941053729786124e-07,
      "logits/chosen": 1.533593773841858,
      "logits/rejected": 1.876562476158142,
      "logps/chosen": -476.3999938964844,
      "logps/rejected": -575.0,
      "loss": 3.0105,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 1.1046874523162842,
      "rewards/margins": 9.434374809265137,
      "rewards/rejected": -8.3203125,
      "step": 620
    },
    {
      "epoch": 0.29577464788732394,
      "grad_norm": 911.8430431121592,
      "learning_rate": 3.9149713093375064e-07,
      "logits/chosen": 1.6218750476837158,
      "logits/rejected": 1.892968773841858,
      "logps/chosen": -464.20001220703125,
      "logps/rejected": -593.0,
      "loss": 2.8613,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 1.171142578125,
      "rewards/margins": 9.434374809265137,
      "rewards/rejected": -8.260937690734863,
      "step": 630
    },
    {
      "epoch": 0.3004694835680751,
      "grad_norm": 644.0941254703929,
      "learning_rate": 3.888888888888889e-07,
      "logits/chosen": 1.51953125,
      "logits/rejected": 1.8992187976837158,
      "logps/chosen": -458.20001220703125,
      "logps/rejected": -607.5999755859375,
      "loss": 2.7135,
      "rewards/accuracies": 0.5492187738418579,
      "rewards/chosen": 1.617773413658142,
      "rewards/margins": 10.412500381469727,
      "rewards/rejected": -8.790624618530273,
      "step": 640
    },
    {
      "epoch": 0.3051643192488263,
      "grad_norm": 810.7073536219183,
      "learning_rate": 3.862806468440271e-07,
      "logits/chosen": 1.6749999523162842,
      "logits/rejected": 1.9382812976837158,
      "logps/chosen": -434.3999938964844,
      "logps/rejected": -536.2000122070312,
      "loss": 2.9026,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 0.59765625,
      "rewards/margins": 7.793749809265137,
      "rewards/rejected": -7.196875095367432,
      "step": 650
    },
    {
      "epoch": 0.30985915492957744,
      "grad_norm": 804.9226061849723,
      "learning_rate": 3.8367240479916535e-07,
      "logits/chosen": 1.515625,
      "logits/rejected": 1.908593773841858,
      "logps/chosen": -455.79998779296875,
      "logps/rejected": -536.2000122070312,
      "loss": 2.9334,
      "rewards/accuracies": 0.5218750238418579,
      "rewards/chosen": 0.9964843988418579,
      "rewards/margins": 7.267187595367432,
      "rewards/rejected": -6.265625,
      "step": 660
    },
    {
      "epoch": 0.3145539906103286,
      "grad_norm": 792.7033329350427,
      "learning_rate": 3.810641627543036e-07,
      "logits/chosen": 1.5226562023162842,
      "logits/rejected": 1.751562476158142,
      "logps/chosen": -466.79998779296875,
      "logps/rejected": -560.2000122070312,
      "loss": 3.1902,
      "rewards/accuracies": 0.5367187261581421,
      "rewards/chosen": 0.5681182742118835,
      "rewards/margins": 8.096875190734863,
      "rewards/rejected": -7.532812595367432,
      "step": 670
    },
    {
      "epoch": 0.3192488262910798,
      "grad_norm": 839.5333622524923,
      "learning_rate": 3.784559207094418e-07,
      "logits/chosen": 1.62890625,
      "logits/rejected": 1.8328125476837158,
      "logps/chosen": -484.6000061035156,
      "logps/rejected": -560.5999755859375,
      "loss": 3.2635,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.36542969942092896,
      "rewards/margins": 7.456250190734863,
      "rewards/rejected": -7.09375,
      "step": 680
    },
    {
      "epoch": 0.323943661971831,
      "grad_norm": 674.182509785427,
      "learning_rate": 3.7584767866458005e-07,
      "logits/chosen": 1.5859375,
      "logits/rejected": 1.830468773841858,
      "logps/chosen": -474.6000061035156,
      "logps/rejected": -539.4000244140625,
      "loss": 2.8462,
      "rewards/accuracies": 0.50390625,
      "rewards/chosen": 0.33251953125,
      "rewards/margins": 7.65625,
      "rewards/rejected": -7.323437690734863,
      "step": 690
    },
    {
      "epoch": 0.3286384976525822,
      "grad_norm": 1046.299980238175,
      "learning_rate": 3.732394366197183e-07,
      "logits/chosen": 1.5703125,
      "logits/rejected": 1.8289062976837158,
      "logps/chosen": -498.0,
      "logps/rejected": -546.0,
      "loss": 3.3674,
      "rewards/accuracies": 0.5179687738418579,
      "rewards/chosen": 0.1226806640625,
      "rewards/margins": 7.896874904632568,
      "rewards/rejected": -7.771874904632568,
      "step": 700
    },
    {
      "epoch": 0.3333333333333333,
      "grad_norm": 894.578341249558,
      "learning_rate": 3.706311945748565e-07,
      "logits/chosen": 1.6101562976837158,
      "logits/rejected": 1.8132812976837158,
      "logps/chosen": -511.20001220703125,
      "logps/rejected": -564.7999877929688,
      "loss": 3.4427,
      "rewards/accuracies": 0.5210937261581421,
      "rewards/chosen": 0.39531248807907104,
      "rewards/margins": 10.243749618530273,
      "rewards/rejected": -9.837499618530273,
      "step": 710
    },
    {
      "epoch": 0.3380281690140845,
      "grad_norm": 772.4311938642684,
      "learning_rate": 3.6802295252999476e-07,
      "logits/chosen": 1.583593726158142,
      "logits/rejected": 1.8210937976837158,
      "logps/chosen": -484.20001220703125,
      "logps/rejected": -599.4000244140625,
      "loss": 3.0442,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.864794909954071,
      "rewards/margins": 12.006250381469727,
      "rewards/rejected": -11.134374618530273,
      "step": 720
    },
    {
      "epoch": 0.3427230046948357,
      "grad_norm": 777.9591012962122,
      "learning_rate": 3.65414710485133e-07,
      "logits/chosen": 1.517968773841858,
      "logits/rejected": 1.767187476158142,
      "logps/chosen": -478.0,
      "logps/rejected": -549.7999877929688,
      "loss": 3.503,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": 0.442626953125,
      "rewards/margins": 8.203125,
      "rewards/rejected": -7.765625,
      "step": 730
    },
    {
      "epoch": 0.3474178403755869,
      "grad_norm": 701.0493602996402,
      "learning_rate": 3.6280646844027127e-07,
      "logits/chosen": 1.517187476158142,
      "logits/rejected": 1.908593773841858,
      "logps/chosen": -459.0,
      "logps/rejected": -608.7999877929688,
      "loss": 3.1922,
      "rewards/accuracies": 0.514843761920929,
      "rewards/chosen": 0.8005615472793579,
      "rewards/margins": 7.974999904632568,
      "rewards/rejected": -7.175000190734863,
      "step": 740
    },
    {
      "epoch": 0.352112676056338,
      "grad_norm": 878.3599640439072,
      "learning_rate": 3.6019822639540947e-07,
      "logits/chosen": 1.5789062976837158,
      "logits/rejected": 1.833593726158142,
      "logps/chosen": -485.20001220703125,
      "logps/rejected": -537.7999877929688,
      "loss": 2.9269,
      "rewards/accuracies": 0.51171875,
      "rewards/chosen": 1.0988280773162842,
      "rewards/margins": 8.274999618530273,
      "rewards/rejected": -7.175000190734863,
      "step": 750
    },
    {
      "epoch": 0.3568075117370892,
      "grad_norm": 700.0736655973939,
      "learning_rate": 3.575899843505477e-07,
      "logits/chosen": 1.6015625,
      "logits/rejected": 1.830468773841858,
      "logps/chosen": -468.3999938964844,
      "logps/rejected": -534.0,
      "loss": 2.9618,
      "rewards/accuracies": 0.5492187738418579,
      "rewards/chosen": 1.185449242591858,
      "rewards/margins": 8.954687118530273,
      "rewards/rejected": -7.764062404632568,
      "step": 760
    },
    {
      "epoch": 0.3615023474178404,
      "grad_norm": 822.9723785330303,
      "learning_rate": 3.5498174230568597e-07,
      "logits/chosen": 1.529687523841858,
      "logits/rejected": 1.810937523841858,
      "logps/chosen": -463.79998779296875,
      "logps/rejected": -567.2000122070312,
      "loss": 2.7354,
      "rewards/accuracies": 0.514843761920929,
      "rewards/chosen": 1.260839819908142,
      "rewards/margins": 8.181249618530273,
      "rewards/rejected": -6.918749809265137,
      "step": 770
    },
    {
      "epoch": 0.36619718309859156,
      "grad_norm": 1588.1324612607696,
      "learning_rate": 3.5237350026082417e-07,
      "logits/chosen": 1.5007812976837158,
      "logits/rejected": 1.708593726158142,
      "logps/chosen": -477.3999938964844,
      "logps/rejected": -528.4000244140625,
      "loss": 3.4354,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": 1.0634765625,
      "rewards/margins": 8.509374618530273,
      "rewards/rejected": -7.446875095367432,
      "step": 780
    },
    {
      "epoch": 0.37089201877934275,
      "grad_norm": 899.7831685923908,
      "learning_rate": 3.497652582159624e-07,
      "logits/chosen": 1.5359375476837158,
      "logits/rejected": 1.8312499523162842,
      "logps/chosen": -468.6000061035156,
      "logps/rejected": -588.4000244140625,
      "loss": 3.0563,
      "rewards/accuracies": 0.5101562738418579,
      "rewards/chosen": 1.1912109851837158,
      "rewards/margins": 8.903124809265137,
      "rewards/rejected": -7.706250190734863,
      "step": 790
    },
    {
      "epoch": 0.3755868544600939,
      "grad_norm": 764.922701258729,
      "learning_rate": 3.471570161711007e-07,
      "logits/chosen": 1.646093726158142,
      "logits/rejected": 1.868749976158142,
      "logps/chosen": -503.0,
      "logps/rejected": -593.7999877929688,
      "loss": 3.4418,
      "rewards/accuracies": 0.532031238079071,
      "rewards/chosen": 1.0953369140625,
      "rewards/margins": 9.521875381469727,
      "rewards/rejected": -8.421875,
      "step": 800
    },
    {
      "epoch": 0.38028169014084506,
      "grad_norm": 761.6914206490542,
      "learning_rate": 3.445487741262389e-07,
      "logits/chosen": 1.665624976158142,
      "logits/rejected": 1.9640624523162842,
      "logps/chosen": -458.3999938964844,
      "logps/rejected": -556.0,
      "loss": 3.7514,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 1.3761718273162842,
      "rewards/margins": 6.557812690734863,
      "rewards/rejected": -5.181250095367432,
      "step": 810
    },
    {
      "epoch": 0.38497652582159625,
      "grad_norm": 732.9553252082175,
      "learning_rate": 3.4194053208137713e-07,
      "logits/chosen": 1.578125,
      "logits/rejected": 1.853124976158142,
      "logps/chosen": -465.3999938964844,
      "logps/rejected": -543.0,
      "loss": 2.8684,
      "rewards/accuracies": 0.530468761920929,
      "rewards/chosen": 1.183374047279358,
      "rewards/margins": 8.498437881469727,
      "rewards/rejected": -7.301562309265137,
      "step": 820
    },
    {
      "epoch": 0.38967136150234744,
      "grad_norm": 801.0652817432574,
      "learning_rate": 3.393322900365154e-07,
      "logits/chosen": 1.549218773841858,
      "logits/rejected": 1.779687523841858,
      "logps/chosen": -444.3999938964844,
      "logps/rejected": -540.5999755859375,
      "loss": 3.2489,
      "rewards/accuracies": 0.547656238079071,
      "rewards/chosen": 0.30322265625,
      "rewards/margins": 8.243749618530273,
      "rewards/rejected": -7.9375,
      "step": 830
    },
    {
      "epoch": 0.39436619718309857,
      "grad_norm": 909.1627886198968,
      "learning_rate": 3.367240479916536e-07,
      "logits/chosen": 1.5359375476837158,
      "logits/rejected": 1.8390624523162842,
      "logps/chosen": -497.20001220703125,
      "logps/rejected": -576.0,
      "loss": 3.0953,
      "rewards/accuracies": 0.5367187261581421,
      "rewards/chosen": 0.684814453125,
      "rewards/margins": 8.050000190734863,
      "rewards/rejected": -7.368750095367432,
      "step": 840
    },
    {
      "epoch": 0.39906103286384975,
      "grad_norm": 649.2304694050939,
      "learning_rate": 3.3411580594679184e-07,
      "logits/chosen": 1.5320312976837158,
      "logits/rejected": 1.89453125,
      "logps/chosen": -455.20001220703125,
      "logps/rejected": -568.2000122070312,
      "loss": 2.7713,
      "rewards/accuracies": 0.547656238079071,
      "rewards/chosen": 1.004785180091858,
      "rewards/margins": 9.787500381469727,
      "rewards/rejected": -8.784375190734863,
      "step": 850
    },
    {
      "epoch": 0.40375586854460094,
      "grad_norm": 703.3592429539607,
      "learning_rate": 3.315075639019301e-07,
      "logits/chosen": 1.5812499523162842,
      "logits/rejected": 1.9992187023162842,
      "logps/chosen": -452.3999938964844,
      "logps/rejected": -566.0,
      "loss": 3.0776,
      "rewards/accuracies": 0.53515625,
      "rewards/chosen": 0.8291991949081421,
      "rewards/margins": 9.340624809265137,
      "rewards/rejected": -8.524999618530273,
      "step": 860
    },
    {
      "epoch": 0.4084507042253521,
      "grad_norm": 628.9762804773742,
      "learning_rate": 3.288993218570683e-07,
      "logits/chosen": 1.6007812023162842,
      "logits/rejected": 1.8117187023162842,
      "logps/chosen": -460.0,
      "logps/rejected": -537.5999755859375,
      "loss": 3.0393,
      "rewards/accuracies": 0.5453125238418579,
      "rewards/chosen": 1.051367163658142,
      "rewards/margins": 8.074999809265137,
      "rewards/rejected": -7.012499809265137,
      "step": 870
    },
    {
      "epoch": 0.4131455399061033,
      "grad_norm": 626.4536980455136,
      "learning_rate": 3.2629107981220654e-07,
      "logits/chosen": 1.546875,
      "logits/rejected": 1.8250000476837158,
      "logps/chosen": -474.0,
      "logps/rejected": -552.2000122070312,
      "loss": 2.6331,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 0.8148437738418579,
      "rewards/margins": 9.006250381469727,
      "rewards/rejected": -8.199999809265137,
      "step": 880
    },
    {
      "epoch": 0.41784037558685444,
      "grad_norm": 959.1083763187145,
      "learning_rate": 3.236828377673448e-07,
      "logits/chosen": 1.4874999523162842,
      "logits/rejected": 1.842187523841858,
      "logps/chosen": -497.0,
      "logps/rejected": -554.7999877929688,
      "loss": 4.1,
      "rewards/accuracies": 0.5210937261581421,
      "rewards/chosen": -0.5257812738418579,
      "rewards/margins": 7.662499904632568,
      "rewards/rejected": -8.1875,
      "step": 890
    },
    {
      "epoch": 0.4225352112676056,
      "grad_norm": 704.3169529778276,
      "learning_rate": 3.2107459572248305e-07,
      "logits/chosen": 1.545312523841858,
      "logits/rejected": 1.8367187976837158,
      "logps/chosen": -477.6000061035156,
      "logps/rejected": -564.7999877929688,
      "loss": 3.2374,
      "rewards/accuracies": 0.5523437261581421,
      "rewards/chosen": 0.975781261920929,
      "rewards/margins": 8.831250190734863,
      "rewards/rejected": -7.856249809265137,
      "step": 900
    },
    {
      "epoch": 0.4272300469483568,
      "grad_norm": 792.6037482929422,
      "learning_rate": 3.1846635367762125e-07,
      "logits/chosen": 1.537500023841858,
      "logits/rejected": 1.83984375,
      "logps/chosen": -474.20001220703125,
      "logps/rejected": -596.5999755859375,
      "loss": 3.3151,
      "rewards/accuracies": 0.5210937261581421,
      "rewards/chosen": 0.501513659954071,
      "rewards/margins": 9.609375,
      "rewards/rejected": -9.112500190734863,
      "step": 910
    },
    {
      "epoch": 0.431924882629108,
      "grad_norm": 898.4607033517732,
      "learning_rate": 3.158581116327595e-07,
      "logits/chosen": 1.53515625,
      "logits/rejected": 1.8976562023162842,
      "logps/chosen": -476.20001220703125,
      "logps/rejected": -622.0,
      "loss": 2.9233,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.7281249761581421,
      "rewards/margins": 9.449999809265137,
      "rewards/rejected": -8.728124618530273,
      "step": 920
    },
    {
      "epoch": 0.43661971830985913,
      "grad_norm": 688.2587916171982,
      "learning_rate": 3.1324986958789775e-07,
      "logits/chosen": 1.5007812976837158,
      "logits/rejected": 1.767968773841858,
      "logps/chosen": -446.20001220703125,
      "logps/rejected": -541.4000244140625,
      "loss": 3.1576,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.980664074420929,
      "rewards/margins": 8.543749809265137,
      "rewards/rejected": -7.568749904632568,
      "step": 930
    },
    {
      "epoch": 0.4413145539906103,
      "grad_norm": 643.7002888730533,
      "learning_rate": 3.1064162754303595e-07,
      "logits/chosen": 1.5851562023162842,
      "logits/rejected": 1.86328125,
      "logps/chosen": -481.20001220703125,
      "logps/rejected": -592.2000122070312,
      "loss": 3.3057,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": 0.8134765625,
      "rewards/margins": 9.981249809265137,
      "rewards/rejected": -9.168749809265137,
      "step": 940
    },
    {
      "epoch": 0.4460093896713615,
      "grad_norm": 679.3608026839896,
      "learning_rate": 3.080333854981742e-07,
      "logits/chosen": 1.4539062976837158,
      "logits/rejected": 1.7507812976837158,
      "logps/chosen": -464.0,
      "logps/rejected": -540.2000122070312,
      "loss": 2.8895,
      "rewards/accuracies": 0.539843738079071,
      "rewards/chosen": 1.134790062904358,
      "rewards/margins": 8.387499809265137,
      "rewards/rejected": -7.256249904632568,
      "step": 950
    },
    {
      "epoch": 0.4507042253521127,
      "grad_norm": 700.2351014435227,
      "learning_rate": 3.0542514345331246e-07,
      "logits/chosen": 1.5164062976837158,
      "logits/rejected": 1.8484375476837158,
      "logps/chosen": -472.6000061035156,
      "logps/rejected": -614.2000122070312,
      "loss": 3.079,
      "rewards/accuracies": 0.5328124761581421,
      "rewards/chosen": 1.40966796875,
      "rewards/margins": 9.8125,
      "rewards/rejected": -8.404687881469727,
      "step": 960
    },
    {
      "epoch": 0.45539906103286387,
      "grad_norm": 1024.5974182377843,
      "learning_rate": 3.0281690140845066e-07,
      "logits/chosen": 1.505468726158142,
      "logits/rejected": 1.75,
      "logps/chosen": -500.79998779296875,
      "logps/rejected": -581.2000122070312,
      "loss": 3.5213,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.3951171934604645,
      "rewards/margins": 8.315625190734863,
      "rewards/rejected": -7.918749809265137,
      "step": 970
    },
    {
      "epoch": 0.460093896713615,
      "grad_norm": 696.7161336287311,
      "learning_rate": 3.002086593635889e-07,
      "logits/chosen": 1.6257812976837158,
      "logits/rejected": 1.932031273841858,
      "logps/chosen": -469.3999938964844,
      "logps/rejected": -593.4000244140625,
      "loss": 2.8077,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": 1.5646483898162842,
      "rewards/margins": 9.643750190734863,
      "rewards/rejected": -8.0703125,
      "step": 980
    },
    {
      "epoch": 0.4647887323943662,
      "grad_norm": 713.0403129793203,
      "learning_rate": 2.9760041731872716e-07,
      "logits/chosen": 1.58203125,
      "logits/rejected": 1.794531226158142,
      "logps/chosen": -433.79998779296875,
      "logps/rejected": -551.0,
      "loss": 2.5298,
      "rewards/accuracies": 0.5492187738418579,
      "rewards/chosen": 1.330078125,
      "rewards/margins": 8.725000381469727,
      "rewards/rejected": -7.3984375,
      "step": 990
    },
    {
      "epoch": 0.4694835680751174,
      "grad_norm": 656.7078693914357,
      "learning_rate": 2.9499217527386536e-07,
      "logits/chosen": 1.5187499523162842,
      "logits/rejected": 1.881250023841858,
      "logps/chosen": -459.6000061035156,
      "logps/rejected": -567.5999755859375,
      "loss": 2.8577,
      "rewards/accuracies": 0.524218738079071,
      "rewards/chosen": 1.052832007408142,
      "rewards/margins": 9.234375,
      "rewards/rejected": -8.184374809265137,
      "step": 1000
    },
    {
      "epoch": 0.47417840375586856,
      "grad_norm": 774.5186023517282,
      "learning_rate": 2.923839332290036e-07,
      "logits/chosen": 1.494531273841858,
      "logits/rejected": 1.774999976158142,
      "logps/chosen": -449.6000061035156,
      "logps/rejected": -575.0,
      "loss": 3.4228,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.59814453125,
      "rewards/margins": 8.803125381469727,
      "rewards/rejected": -8.209375381469727,
      "step": 1010
    },
    {
      "epoch": 0.4788732394366197,
      "grad_norm": 787.7405837643146,
      "learning_rate": 2.8977569118414187e-07,
      "logits/chosen": 1.615625023841858,
      "logits/rejected": 1.904687523841858,
      "logps/chosen": -450.79998779296875,
      "logps/rejected": -554.0,
      "loss": 3.0063,
      "rewards/accuracies": 0.5414062738418579,
      "rewards/chosen": 1.1956055164337158,
      "rewards/margins": 9.850000381469727,
      "rewards/rejected": -8.662500381469727,
      "step": 1020
    },
    {
      "epoch": 0.4835680751173709,
      "grad_norm": 749.8440608245284,
      "learning_rate": 2.8716744913928007e-07,
      "logits/chosen": 1.6171875,
      "logits/rejected": 1.834375023841858,
      "logps/chosen": -469.0,
      "logps/rejected": -548.4000244140625,
      "loss": 2.8022,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": 0.7684081792831421,
      "rewards/margins": 9.065625190734863,
      "rewards/rejected": -8.287500381469727,
      "step": 1030
    },
    {
      "epoch": 0.48826291079812206,
      "grad_norm": 625.3221304791,
      "learning_rate": 2.845592070944183e-07,
      "logits/chosen": 1.6242187023162842,
      "logits/rejected": 1.942968726158142,
      "logps/chosen": -465.6000061035156,
      "logps/rejected": -594.5999755859375,
      "loss": 2.9849,
      "rewards/accuracies": 0.539843738079071,
      "rewards/chosen": 1.2445800304412842,
      "rewards/margins": 9.71875,
      "rewards/rejected": -8.475000381469727,
      "step": 1040
    },
    {
      "epoch": 0.49295774647887325,
      "grad_norm": 727.4033720631547,
      "learning_rate": 2.819509650495566e-07,
      "logits/chosen": 1.62890625,
      "logits/rejected": 1.8664062023162842,
      "logps/chosen": -459.6000061035156,
      "logps/rejected": -547.2000122070312,
      "loss": 3.3002,
      "rewards/accuracies": 0.555468738079071,
      "rewards/chosen": 0.604296863079071,
      "rewards/margins": 8.175000190734863,
      "rewards/rejected": -7.568749904632568,
      "step": 1050
    },
    {
      "epoch": 0.49765258215962443,
      "grad_norm": 1739.3275973497427,
      "learning_rate": 2.7934272300469483e-07,
      "logits/chosen": 1.549218773841858,
      "logits/rejected": 1.7140624523162842,
      "logps/chosen": -488.3999938964844,
      "logps/rejected": -551.0,
      "loss": 2.7104,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 1.1282241344451904,
      "rewards/margins": 9.198437690734863,
      "rewards/rejected": -8.067187309265137,
      "step": 1060
    },
    {
      "epoch": 0.5023474178403756,
      "grad_norm": 672.7097275968157,
      "learning_rate": 2.7673448095983303e-07,
      "logits/chosen": 1.59765625,
      "logits/rejected": 1.8195312023162842,
      "logps/chosen": -471.0,
      "logps/rejected": -552.4000244140625,
      "loss": 3.7168,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.34074705839157104,
      "rewards/margins": 7.849999904632568,
      "rewards/rejected": -7.506249904632568,
      "step": 1070
    },
    {
      "epoch": 0.5070422535211268,
      "grad_norm": 891.7364841137971,
      "learning_rate": 2.741262389149713e-07,
      "logits/chosen": 1.57421875,
      "logits/rejected": 1.8507812023162842,
      "logps/chosen": -470.3999938964844,
      "logps/rejected": -570.5999755859375,
      "loss": 2.7817,
      "rewards/accuracies": 0.526562511920929,
      "rewards/chosen": 0.08632812649011612,
      "rewards/margins": 9.03125,
      "rewards/rejected": -8.940625190734863,
      "step": 1080
    },
    {
      "epoch": 0.5117370892018779,
      "grad_norm": 888.6305784421587,
      "learning_rate": 2.7151799687010953e-07,
      "logits/chosen": 1.58203125,
      "logits/rejected": 1.8390624523162842,
      "logps/chosen": -474.0,
      "logps/rejected": -568.4000244140625,
      "loss": 2.871,
      "rewards/accuracies": 0.5679687261581421,
      "rewards/chosen": 0.5508788824081421,
      "rewards/margins": 9.300000190734863,
      "rewards/rejected": -8.759374618530273,
      "step": 1090
    },
    {
      "epoch": 0.5164319248826291,
      "grad_norm": 700.9468740973527,
      "learning_rate": 2.6890975482524773e-07,
      "logits/chosen": 1.564843773841858,
      "logits/rejected": 1.78515625,
      "logps/chosen": -492.3999938964844,
      "logps/rejected": -568.7999877929688,
      "loss": 2.7666,
      "rewards/accuracies": 0.535937488079071,
      "rewards/chosen": 0.734082043170929,
      "rewards/margins": 10.043749809265137,
      "rewards/rejected": -9.3125,
      "step": 1100
    },
    {
      "epoch": 0.5211267605633803,
      "grad_norm": 762.917089663846,
      "learning_rate": 2.66301512780386e-07,
      "logits/chosen": 1.595312476158142,
      "logits/rejected": 1.80078125,
      "logps/chosen": -447.0,
      "logps/rejected": -559.4000244140625,
      "loss": 3.7658,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 0.6490234136581421,
      "rewards/margins": 8.047656059265137,
      "rewards/rejected": -7.4140625,
      "step": 1110
    },
    {
      "epoch": 0.5258215962441315,
      "grad_norm": 789.342977574461,
      "learning_rate": 2.6369327073552424e-07,
      "logits/chosen": 1.59375,
      "logits/rejected": 1.828125,
      "logps/chosen": -453.79998779296875,
      "logps/rejected": -539.2000122070312,
      "loss": 2.7902,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 1.2628905773162842,
      "rewards/margins": 8.412500381469727,
      "rewards/rejected": -7.145312309265137,
      "step": 1120
    },
    {
      "epoch": 0.5305164319248826,
      "grad_norm": 705.6496485845587,
      "learning_rate": 2.6108502869066244e-07,
      "logits/chosen": 1.53515625,
      "logits/rejected": 1.7804687023162842,
      "logps/chosen": -484.20001220703125,
      "logps/rejected": -598.4000244140625,
      "loss": 2.9249,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.8404296636581421,
      "rewards/margins": 11.837499618530273,
      "rewards/rejected": -10.993749618530273,
      "step": 1130
    },
    {
      "epoch": 0.5352112676056338,
      "grad_norm": 804.1488499168772,
      "learning_rate": 2.584767866458007e-07,
      "logits/chosen": 1.6203124523162842,
      "logits/rejected": 1.802343726158142,
      "logps/chosen": -463.20001220703125,
      "logps/rejected": -560.5999755859375,
      "loss": 3.0061,
      "rewards/accuracies": 0.526562511920929,
      "rewards/chosen": 0.6224609613418579,
      "rewards/margins": 9.229687690734863,
      "rewards/rejected": -8.609375,
      "step": 1140
    },
    {
      "epoch": 0.539906103286385,
      "grad_norm": 655.984383766115,
      "learning_rate": 2.5586854460093895e-07,
      "logits/chosen": 1.568750023841858,
      "logits/rejected": 1.9226562976837158,
      "logps/chosen": -472.20001220703125,
      "logps/rejected": -580.7999877929688,
      "loss": 3.054,
      "rewards/accuracies": 0.5570312738418579,
      "rewards/chosen": 0.594482421875,
      "rewards/margins": 9.149999618530273,
      "rewards/rejected": -8.550000190734863,
      "step": 1150
    },
    {
      "epoch": 0.5446009389671361,
      "grad_norm": 884.1114157291227,
      "learning_rate": 2.5326030255607715e-07,
      "logits/chosen": 1.6203124523162842,
      "logits/rejected": 1.8718750476837158,
      "logps/chosen": -473.79998779296875,
      "logps/rejected": -570.4000244140625,
      "loss": 3.2859,
      "rewards/accuracies": 0.532031238079071,
      "rewards/chosen": 0.3929443359375,
      "rewards/margins": 8.7578125,
      "rewards/rejected": -8.368749618530273,
      "step": 1160
    },
    {
      "epoch": 0.5492957746478874,
      "grad_norm": 732.8930682291275,
      "learning_rate": 2.506520605112154e-07,
      "logits/chosen": 1.599218726158142,
      "logits/rejected": 1.888281226158142,
      "logps/chosen": -468.0,
      "logps/rejected": -565.0,
      "loss": 3.555,
      "rewards/accuracies": 0.5218750238418579,
      "rewards/chosen": 0.02622070349752903,
      "rewards/margins": 7.895312309265137,
      "rewards/rejected": -7.865624904632568,
      "step": 1170
    },
    {
      "epoch": 0.5539906103286385,
      "grad_norm": 665.7569098636599,
      "learning_rate": 2.4804381846635365e-07,
      "logits/chosen": 1.6593749523162842,
      "logits/rejected": 1.900781273841858,
      "logps/chosen": -485.20001220703125,
      "logps/rejected": -561.2000122070312,
      "loss": 3.3737,
      "rewards/accuracies": 0.530468761920929,
      "rewards/chosen": 0.03867187350988388,
      "rewards/margins": 8.526562690734863,
      "rewards/rejected": -8.478124618530273,
      "step": 1180
    },
    {
      "epoch": 0.5586854460093896,
      "grad_norm": 724.9044332663664,
      "learning_rate": 2.454355764214919e-07,
      "logits/chosen": 1.599218726158142,
      "logits/rejected": 1.83203125,
      "logps/chosen": -449.79998779296875,
      "logps/rejected": -591.5999755859375,
      "loss": 2.9518,
      "rewards/accuracies": 0.551562488079071,
      "rewards/chosen": 0.7137695550918579,
      "rewards/margins": 9.612500190734863,
      "rewards/rejected": -8.90625,
      "step": 1190
    },
    {
      "epoch": 0.5633802816901409,
      "grad_norm": 790.280889790197,
      "learning_rate": 2.4282733437663016e-07,
      "logits/chosen": 1.51953125,
      "logits/rejected": 1.817968726158142,
      "logps/chosen": -458.0,
      "logps/rejected": -550.2000122070312,
      "loss": 3.1554,
      "rewards/accuracies": 0.51171875,
      "rewards/chosen": 0.7120116949081421,
      "rewards/margins": 7.696875095367432,
      "rewards/rejected": -6.984375,
      "step": 1200
    },
    {
      "epoch": 0.568075117370892,
      "grad_norm": 710.3808271960063,
      "learning_rate": 2.4021909233176836e-07,
      "logits/chosen": 1.611718773841858,
      "logits/rejected": 1.93359375,
      "logps/chosen": -446.0,
      "logps/rejected": -569.2000122070312,
      "loss": 2.6615,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 1.167578101158142,
      "rewards/margins": 9.146875381469727,
      "rewards/rejected": -7.978125095367432,
      "step": 1210
    },
    {
      "epoch": 0.5727699530516432,
      "grad_norm": 682.4365969675591,
      "learning_rate": 2.376108502869066e-07,
      "logits/chosen": 1.599218726158142,
      "logits/rejected": 1.939062476158142,
      "logps/chosen": -477.79998779296875,
      "logps/rejected": -592.0,
      "loss": 3.048,
      "rewards/accuracies": 0.524218738079071,
      "rewards/chosen": 0.817822277545929,
      "rewards/margins": 9.426953315734863,
      "rewards/rejected": -8.604687690734863,
      "step": 1220
    },
    {
      "epoch": 0.5774647887323944,
      "grad_norm": 711.4954275649587,
      "learning_rate": 2.3500260824204484e-07,
      "logits/chosen": 1.5851562023162842,
      "logits/rejected": 1.81640625,
      "logps/chosen": -502.6000061035156,
      "logps/rejected": -591.4000244140625,
      "loss": 3.3233,
      "rewards/accuracies": 0.561718761920929,
      "rewards/chosen": 0.35295408964157104,
      "rewards/margins": 9.90625,
      "rewards/rejected": -9.543749809265137,
      "step": 1230
    },
    {
      "epoch": 0.5821596244131455,
      "grad_norm": 754.1511889684363,
      "learning_rate": 2.323943661971831e-07,
      "logits/chosen": 1.56640625,
      "logits/rejected": 1.8976562023162842,
      "logps/chosen": -466.6000061035156,
      "logps/rejected": -565.5999755859375,
      "loss": 3.1274,
      "rewards/accuracies": 0.535937488079071,
      "rewards/chosen": 0.8163086175918579,
      "rewards/margins": 8.746874809265137,
      "rewards/rejected": -7.925000190734863,
      "step": 1240
    },
    {
      "epoch": 0.5868544600938967,
      "grad_norm": 830.4278215073776,
      "learning_rate": 2.2978612415232132e-07,
      "logits/chosen": 1.575781226158142,
      "logits/rejected": 1.8679687976837158,
      "logps/chosen": -494.79998779296875,
      "logps/rejected": -599.2000122070312,
      "loss": 3.0011,
      "rewards/accuracies": 0.5328124761581421,
      "rewards/chosen": 1.1947143077850342,
      "rewards/margins": 9.309374809265137,
      "rewards/rejected": -8.106249809265137,
      "step": 1250
    },
    {
      "epoch": 0.5915492957746479,
      "grad_norm": 645.3066801609073,
      "learning_rate": 2.2717788210745957e-07,
      "logits/chosen": 1.588281273841858,
      "logits/rejected": 1.8828125,
      "logps/chosen": -466.79998779296875,
      "logps/rejected": -569.2000122070312,
      "loss": 3.3713,
      "rewards/accuracies": 0.524218738079071,
      "rewards/chosen": 0.5673828125,
      "rewards/margins": 9.260937690734863,
      "rewards/rejected": -8.696874618530273,
      "step": 1260
    },
    {
      "epoch": 0.596244131455399,
      "grad_norm": 755.0309932727065,
      "learning_rate": 2.245696400625978e-07,
      "logits/chosen": 1.5890624523162842,
      "logits/rejected": 1.8781249523162842,
      "logps/chosen": -488.6000061035156,
      "logps/rejected": -575.2000122070312,
      "loss": 3.1939,
      "rewards/accuracies": 0.5296875238418579,
      "rewards/chosen": 0.36767578125,
      "rewards/margins": 8.665624618530273,
      "rewards/rejected": -8.300000190734863,
      "step": 1270
    },
    {
      "epoch": 0.6009389671361502,
      "grad_norm": 755.2838601585016,
      "learning_rate": 2.2196139801773602e-07,
      "logits/chosen": 1.567968726158142,
      "logits/rejected": 1.736718773841858,
      "logps/chosen": -458.6000061035156,
      "logps/rejected": -538.4000244140625,
      "loss": 2.7505,
      "rewards/accuracies": 0.5210937261581421,
      "rewards/chosen": 0.5143066644668579,
      "rewards/margins": 8.268750190734863,
      "rewards/rejected": -7.762499809265137,
      "step": 1280
    },
    {
      "epoch": 0.6056338028169014,
      "grad_norm": 1059.938450366818,
      "learning_rate": 2.1935315597287428e-07,
      "logits/chosen": 1.502343773841858,
      "logits/rejected": 1.794531226158142,
      "logps/chosen": -476.79998779296875,
      "logps/rejected": -542.7999877929688,
      "loss": 3.0586,
      "rewards/accuracies": 0.546093761920929,
      "rewards/chosen": 0.6070801019668579,
      "rewards/margins": 8.703125,
      "rewards/rejected": -8.100000381469727,
      "step": 1290
    },
    {
      "epoch": 0.6103286384976526,
      "grad_norm": 649.1773353355767,
      "learning_rate": 2.167449139280125e-07,
      "logits/chosen": 1.56640625,
      "logits/rejected": 1.82421875,
      "logps/chosen": -475.79998779296875,
      "logps/rejected": -575.2000122070312,
      "loss": 2.9404,
      "rewards/accuracies": 0.546093761920929,
      "rewards/chosen": 0.762499988079071,
      "rewards/margins": 9.090624809265137,
      "rewards/rejected": -8.337499618530273,
      "step": 1300
    },
    {
      "epoch": 0.6150234741784038,
      "grad_norm": 618.2035598886883,
      "learning_rate": 2.1413667188315073e-07,
      "logits/chosen": 1.56640625,
      "logits/rejected": 1.8984375,
      "logps/chosen": -450.79998779296875,
      "logps/rejected": -538.0,
      "loss": 2.8939,
      "rewards/accuracies": 0.555468738079071,
      "rewards/chosen": 0.7720702886581421,
      "rewards/margins": 9.271875381469727,
      "rewards/rejected": -8.5,
      "step": 1310
    },
    {
      "epoch": 0.6197183098591549,
      "grad_norm": 673.7392898848505,
      "learning_rate": 2.1152842983828898e-07,
      "logits/chosen": 1.5320312976837158,
      "logits/rejected": 1.7570312023162842,
      "logps/chosen": -477.0,
      "logps/rejected": -552.7999877929688,
      "loss": 3.9292,
      "rewards/accuracies": 0.54296875,
      "rewards/chosen": -0.10249023139476776,
      "rewards/margins": 7.642187595367432,
      "rewards/rejected": -7.75,
      "step": 1320
    },
    {
      "epoch": 0.6244131455399061,
      "grad_norm": 764.2583904101606,
      "learning_rate": 2.089201877934272e-07,
      "logits/chosen": 1.6398437023162842,
      "logits/rejected": 1.8234374523162842,
      "logps/chosen": -491.20001220703125,
      "logps/rejected": -596.5999755859375,
      "loss": 2.7944,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.568066418170929,
      "rewards/margins": 10.609375,
      "rewards/rejected": -10.037500381469727,
      "step": 1330
    },
    {
      "epoch": 0.6291079812206573,
      "grad_norm": 719.026441696121,
      "learning_rate": 2.0631194574856543e-07,
      "logits/chosen": 1.5539062023162842,
      "logits/rejected": 1.849218726158142,
      "logps/chosen": -454.3999938964844,
      "logps/rejected": -513.5999755859375,
      "loss": 2.758,
      "rewards/accuracies": 0.5367187261581421,
      "rewards/chosen": 1.431249976158142,
      "rewards/margins": 7.637499809265137,
      "rewards/rejected": -6.199999809265137,
      "step": 1340
    },
    {
      "epoch": 0.6338028169014085,
      "grad_norm": 919.8027536019214,
      "learning_rate": 2.0370370370370369e-07,
      "logits/chosen": 1.5460937023162842,
      "logits/rejected": 1.7890625,
      "logps/chosen": -470.6000061035156,
      "logps/rejected": -601.4000244140625,
      "loss": 3.2054,
      "rewards/accuracies": 0.5523437261581421,
      "rewards/chosen": 0.924243152141571,
      "rewards/margins": 11.303125381469727,
      "rewards/rejected": -10.390625,
      "step": 1350
    },
    {
      "epoch": 0.6384976525821596,
      "grad_norm": 661.2693488123647,
      "learning_rate": 2.010954616588419e-07,
      "logits/chosen": 1.5476562976837158,
      "logits/rejected": 1.7546875476837158,
      "logps/chosen": -461.79998779296875,
      "logps/rejected": -543.2000122070312,
      "loss": 2.8084,
      "rewards/accuracies": 0.5210937261581421,
      "rewards/chosen": 0.8590332269668579,
      "rewards/margins": 8.684374809265137,
      "rewards/rejected": -7.824999809265137,
      "step": 1360
    },
    {
      "epoch": 0.6431924882629108,
      "grad_norm": 768.5829004979365,
      "learning_rate": 1.9848721961398017e-07,
      "logits/chosen": 1.600000023841858,
      "logits/rejected": 1.868749976158142,
      "logps/chosen": -472.6000061035156,
      "logps/rejected": -542.5999755859375,
      "loss": 2.8249,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 1.555517554283142,
      "rewards/margins": 8.481249809265137,
      "rewards/rejected": -6.934374809265137,
      "step": 1370
    },
    {
      "epoch": 0.647887323943662,
      "grad_norm": 802.4475068235987,
      "learning_rate": 1.958789775691184e-07,
      "logits/chosen": 1.5617187023162842,
      "logits/rejected": 1.7890625,
      "logps/chosen": -452.79998779296875,
      "logps/rejected": -552.2000122070312,
      "loss": 2.4807,
      "rewards/accuracies": 0.5609375238418579,
      "rewards/chosen": 1.7082030773162842,
      "rewards/margins": 9.934374809265137,
      "rewards/rejected": -8.221875190734863,
      "step": 1380
    },
    {
      "epoch": 0.6525821596244131,
      "grad_norm": 658.5852045344016,
      "learning_rate": 1.9327073552425662e-07,
      "logits/chosen": 1.5984375476837158,
      "logits/rejected": 1.864843726158142,
      "logps/chosen": -449.6000061035156,
      "logps/rejected": -542.5999755859375,
      "loss": 2.5862,
      "rewards/accuracies": 0.555468738079071,
      "rewards/chosen": 1.613867163658142,
      "rewards/margins": 9.809374809265137,
      "rewards/rejected": -8.193750381469727,
      "step": 1390
    },
    {
      "epoch": 0.6572769953051644,
      "grad_norm": 670.0717844792842,
      "learning_rate": 1.906624934793949e-07,
      "logits/chosen": 1.5593750476837158,
      "logits/rejected": 1.9406249523162842,
      "logps/chosen": -468.0,
      "logps/rejected": -590.7999877929688,
      "loss": 2.8904,
      "rewards/accuracies": 0.51953125,
      "rewards/chosen": 0.6343749761581421,
      "rewards/margins": 9.459375381469727,
      "rewards/rejected": -8.8125,
      "step": 1400
    },
    {
      "epoch": 0.6619718309859155,
      "grad_norm": 737.9126807890543,
      "learning_rate": 1.8805425143453312e-07,
      "logits/chosen": 1.548437476158142,
      "logits/rejected": 1.818750023841858,
      "logps/chosen": -449.0,
      "logps/rejected": -612.2000122070312,
      "loss": 3.066,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.760791003704071,
      "rewards/margins": 9.40625,
      "rewards/rejected": -8.640625,
      "step": 1410
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 748.077714087459,
      "learning_rate": 1.8544600938967138e-07,
      "logits/chosen": 1.501562476158142,
      "logits/rejected": 1.830468773841858,
      "logps/chosen": -460.6000061035156,
      "logps/rejected": -544.2000122070312,
      "loss": 3.0853,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 1.5924804210662842,
      "rewards/margins": 8.928125381469727,
      "rewards/rejected": -7.334374904632568,
      "step": 1420
    },
    {
      "epoch": 0.6713615023474179,
      "grad_norm": 613.0480291752376,
      "learning_rate": 1.828377673448096e-07,
      "logits/chosen": 1.5867187976837158,
      "logits/rejected": 1.8328125476837158,
      "logps/chosen": -495.20001220703125,
      "logps/rejected": -571.4000244140625,
      "loss": 3.3106,
      "rewards/accuracies": 0.5609375238418579,
      "rewards/chosen": 0.38066405057907104,
      "rewards/margins": 10.415624618530273,
      "rewards/rejected": -10.018750190734863,
      "step": 1430
    },
    {
      "epoch": 0.676056338028169,
      "grad_norm": 745.7280391813616,
      "learning_rate": 1.8022952529994783e-07,
      "logits/chosen": 1.588281273841858,
      "logits/rejected": 1.8093750476837158,
      "logps/chosen": -475.79998779296875,
      "logps/rejected": -548.7999877929688,
      "loss": 2.5514,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 1.418066382408142,
      "rewards/margins": 9.809374809265137,
      "rewards/rejected": -8.396875381469727,
      "step": 1440
    },
    {
      "epoch": 0.6807511737089202,
      "grad_norm": 811.8901833928198,
      "learning_rate": 1.7762128325508608e-07,
      "logits/chosen": 1.681249976158142,
      "logits/rejected": 1.9523437023162842,
      "logps/chosen": -486.3999938964844,
      "logps/rejected": -601.5999755859375,
      "loss": 3.3527,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 0.7126327753067017,
      "rewards/margins": 11.399999618530273,
      "rewards/rejected": -10.671875,
      "step": 1450
    },
    {
      "epoch": 0.6854460093896714,
      "grad_norm": 790.1684142661642,
      "learning_rate": 1.750130412102243e-07,
      "logits/chosen": 1.521875023841858,
      "logits/rejected": 1.838281273841858,
      "logps/chosen": -508.3999938964844,
      "logps/rejected": -572.2000122070312,
      "loss": 3.4724,
      "rewards/accuracies": 0.551562488079071,
      "rewards/chosen": 0.520368218421936,
      "rewards/margins": 8.560937881469727,
      "rewards/rejected": -8.045312881469727,
      "step": 1460
    },
    {
      "epoch": 0.6901408450704225,
      "grad_norm": 723.4478625982207,
      "learning_rate": 1.7240479916536254e-07,
      "logits/chosen": 1.5359375476837158,
      "logits/rejected": 1.850000023841858,
      "logps/chosen": -458.6000061035156,
      "logps/rejected": -586.4000244140625,
      "loss": 2.7939,
      "rewards/accuracies": 0.5367187261581421,
      "rewards/chosen": 1.3474609851837158,
      "rewards/margins": 9.318750381469727,
      "rewards/rejected": -7.978125095367432,
      "step": 1470
    },
    {
      "epoch": 0.6948356807511737,
      "grad_norm": 617.5894783901739,
      "learning_rate": 1.697965571205008e-07,
      "logits/chosen": 1.6335937976837158,
      "logits/rejected": 1.853124976158142,
      "logps/chosen": -461.6000061035156,
      "logps/rejected": -536.2000122070312,
      "loss": 2.979,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": 0.99609375,
      "rewards/margins": 8.459375381469727,
      "rewards/rejected": -7.456250190734863,
      "step": 1480
    },
    {
      "epoch": 0.6995305164319249,
      "grad_norm": 820.539046423614,
      "learning_rate": 1.6718831507563902e-07,
      "logits/chosen": 1.567968726158142,
      "logits/rejected": 1.8523437976837158,
      "logps/chosen": -469.20001220703125,
      "logps/rejected": -533.2000122070312,
      "loss": 2.424,
      "rewards/accuracies": 0.538281261920929,
      "rewards/chosen": 1.629150390625,
      "rewards/margins": 8.896875381469727,
      "rewards/rejected": -7.28125,
      "step": 1490
    },
    {
      "epoch": 0.704225352112676,
      "grad_norm": 650.0944531204904,
      "learning_rate": 1.6458007303077727e-07,
      "logits/chosen": 1.5851562023162842,
      "logits/rejected": 1.8156249523162842,
      "logps/chosen": -455.20001220703125,
      "logps/rejected": -539.2000122070312,
      "loss": 2.7878,
      "rewards/accuracies": 0.524218738079071,
      "rewards/chosen": 1.5869140625,
      "rewards/margins": 7.817187309265137,
      "rewards/rejected": -6.235937595367432,
      "step": 1500
    },
    {
      "epoch": 0.7089201877934272,
      "grad_norm": 797.6220258126673,
      "learning_rate": 1.619718309859155e-07,
      "logits/chosen": 1.576562523841858,
      "logits/rejected": 1.756250023841858,
      "logps/chosen": -472.3999938964844,
      "logps/rejected": -585.7999877929688,
      "loss": 3.1826,
      "rewards/accuracies": 0.5328124761581421,
      "rewards/chosen": 0.876171886920929,
      "rewards/margins": 8.895312309265137,
      "rewards/rejected": -8.0234375,
      "step": 1510
    },
    {
      "epoch": 0.7136150234741784,
      "grad_norm": 781.2077669116666,
      "learning_rate": 1.5936358894105372e-07,
      "logits/chosen": 1.435937523841858,
      "logits/rejected": 1.739843726158142,
      "logps/chosen": -471.3999938964844,
      "logps/rejected": -565.5999755859375,
      "loss": 2.942,
      "rewards/accuracies": 0.5257812738418579,
      "rewards/chosen": 0.7884765863418579,
      "rewards/margins": 8.628125190734863,
      "rewards/rejected": -7.845312595367432,
      "step": 1520
    },
    {
      "epoch": 0.7183098591549296,
      "grad_norm": 677.7667479888489,
      "learning_rate": 1.5675534689619197e-07,
      "logits/chosen": 1.5710937976837158,
      "logits/rejected": 1.8914062976837158,
      "logps/chosen": -475.79998779296875,
      "logps/rejected": -534.2000122070312,
      "loss": 2.9829,
      "rewards/accuracies": 0.5570312738418579,
      "rewards/chosen": 0.15742187201976776,
      "rewards/margins": 8.475000381469727,
      "rewards/rejected": -8.328125,
      "step": 1530
    },
    {
      "epoch": 0.7230046948356808,
      "grad_norm": 628.0065350069356,
      "learning_rate": 1.541471048513302e-07,
      "logits/chosen": 1.619531273841858,
      "logits/rejected": 1.896875023841858,
      "logps/chosen": -476.3999938964844,
      "logps/rejected": -563.4000244140625,
      "loss": 3.0475,
      "rewards/accuracies": 0.5367187261581421,
      "rewards/chosen": 0.950390636920929,
      "rewards/margins": 9.428125381469727,
      "rewards/rejected": -8.4765625,
      "step": 1540
    },
    {
      "epoch": 0.7276995305164319,
      "grad_norm": 624.117882163589,
      "learning_rate": 1.5153886280646843e-07,
      "logits/chosen": 1.568750023841858,
      "logits/rejected": 1.8718750476837158,
      "logps/chosen": -444.6000061035156,
      "logps/rejected": -521.7999877929688,
      "loss": 2.8055,
      "rewards/accuracies": 0.510937511920929,
      "rewards/chosen": 1.182519555091858,
      "rewards/margins": 8.168749809265137,
      "rewards/rejected": -6.985937595367432,
      "step": 1550
    },
    {
      "epoch": 0.7323943661971831,
      "grad_norm": 827.9896496843885,
      "learning_rate": 1.4893062076160668e-07,
      "logits/chosen": 1.564843773841858,
      "logits/rejected": 1.8976562023162842,
      "logps/chosen": -467.3999938964844,
      "logps/rejected": -577.5999755859375,
      "loss": 3.1996,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.8426758050918579,
      "rewards/margins": 9.931249618530273,
      "rewards/rejected": -9.079687118530273,
      "step": 1560
    },
    {
      "epoch": 0.7370892018779343,
      "grad_norm": 630.4827632909435,
      "learning_rate": 1.463223787167449e-07,
      "logits/chosen": 1.62890625,
      "logits/rejected": 1.8367187976837158,
      "logps/chosen": -474.6000061035156,
      "logps/rejected": -553.4000244140625,
      "loss": 3.0279,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 1.112060546875,
      "rewards/margins": 9.012499809265137,
      "rewards/rejected": -7.900000095367432,
      "step": 1570
    },
    {
      "epoch": 0.7417840375586855,
      "grad_norm": 678.067038170456,
      "learning_rate": 1.4371413667188313e-07,
      "logits/chosen": 1.5773437023162842,
      "logits/rejected": 1.880468726158142,
      "logps/chosen": -462.6000061035156,
      "logps/rejected": -600.0,
      "loss": 2.7917,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": 1.6378905773162842,
      "rewards/margins": 11.21875,
      "rewards/rejected": -9.581250190734863,
      "step": 1580
    },
    {
      "epoch": 0.7464788732394366,
      "grad_norm": 765.1041367943592,
      "learning_rate": 1.4110589462702139e-07,
      "logits/chosen": 1.611718773841858,
      "logits/rejected": 1.8757812976837158,
      "logps/chosen": -461.0,
      "logps/rejected": -538.2000122070312,
      "loss": 3.0238,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 1.2042968273162842,
      "rewards/margins": 7.556250095367432,
      "rewards/rejected": -6.356249809265137,
      "step": 1590
    },
    {
      "epoch": 0.7511737089201878,
      "grad_norm": 811.35173901598,
      "learning_rate": 1.384976525821596e-07,
      "logits/chosen": 1.6632812023162842,
      "logits/rejected": 1.9296875,
      "logps/chosen": -477.3999938964844,
      "logps/rejected": -582.2000122070312,
      "loss": 4.4862,
      "rewards/accuracies": 0.5289062261581421,
      "rewards/chosen": 1.419531226158142,
      "rewards/margins": 8.798437118530273,
      "rewards/rejected": -7.371874809265137,
      "step": 1600
    },
    {
      "epoch": 0.755868544600939,
      "grad_norm": 869.8983525278993,
      "learning_rate": 1.3588941053729787e-07,
      "logits/chosen": 1.639062523841858,
      "logits/rejected": 1.970312476158142,
      "logps/chosen": -481.20001220703125,
      "logps/rejected": -584.7999877929688,
      "loss": 3.1794,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": 0.962597668170929,
      "rewards/margins": 9.509374618530273,
      "rewards/rejected": -8.548437118530273,
      "step": 1610
    },
    {
      "epoch": 0.7605633802816901,
      "grad_norm": 788.7762015368364,
      "learning_rate": 1.332811684924361e-07,
      "logits/chosen": 1.596093773841858,
      "logits/rejected": 1.806249976158142,
      "logps/chosen": -486.20001220703125,
      "logps/rejected": -564.4000244140625,
      "loss": 3.0238,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 1.120703101158142,
      "rewards/margins": 8.828125,
      "rewards/rejected": -7.696875095367432,
      "step": 1620
    },
    {
      "epoch": 0.7652582159624414,
      "grad_norm": 626.3254794265277,
      "learning_rate": 1.3067292644757432e-07,
      "logits/chosen": 1.525781273841858,
      "logits/rejected": 1.892968773841858,
      "logps/chosen": -482.79998779296875,
      "logps/rejected": -595.4000244140625,
      "loss": 3.1792,
      "rewards/accuracies": 0.547656238079071,
      "rewards/chosen": 0.8549560308456421,
      "rewards/margins": 9.703125,
      "rewards/rejected": -8.859375,
      "step": 1630
    },
    {
      "epoch": 0.7699530516431925,
      "grad_norm": 818.2442525580944,
      "learning_rate": 1.2806468440271257e-07,
      "logits/chosen": 1.584375023841858,
      "logits/rejected": 1.861718773841858,
      "logps/chosen": -456.20001220703125,
      "logps/rejected": -556.4000244140625,
      "loss": 2.8125,
      "rewards/accuracies": 0.5648437738418579,
      "rewards/chosen": 1.4094727039337158,
      "rewards/margins": 9.471875190734863,
      "rewards/rejected": -8.065625190734863,
      "step": 1640
    },
    {
      "epoch": 0.7746478873239436,
      "grad_norm": 674.3784203460635,
      "learning_rate": 1.254564423578508e-07,
      "logits/chosen": 1.6171875,
      "logits/rejected": 1.8289062976837158,
      "logps/chosen": -462.20001220703125,
      "logps/rejected": -560.2000122070312,
      "loss": 2.4976,
      "rewards/accuracies": 0.57421875,
      "rewards/chosen": 1.001855492591858,
      "rewards/margins": 10.146875381469727,
      "rewards/rejected": -9.1484375,
      "step": 1650
    },
    {
      "epoch": 0.7793427230046949,
      "grad_norm": 769.9669670160146,
      "learning_rate": 1.2284820031298902e-07,
      "logits/chosen": 1.6015625,
      "logits/rejected": 1.8195312023162842,
      "logps/chosen": -497.0,
      "logps/rejected": -588.2000122070312,
      "loss": 3.3367,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.6490234136581421,
      "rewards/margins": 10.134374618530273,
      "rewards/rejected": -9.493749618530273,
      "step": 1660
    },
    {
      "epoch": 0.784037558685446,
      "grad_norm": 854.6061332401104,
      "learning_rate": 1.2023995826812728e-07,
      "logits/chosen": 1.5625,
      "logits/rejected": 1.8359375,
      "logps/chosen": -467.20001220703125,
      "logps/rejected": -572.7999877929688,
      "loss": 2.956,
      "rewards/accuracies": 0.557812511920929,
      "rewards/chosen": 0.908203125,
      "rewards/margins": 9.896875381469727,
      "rewards/rejected": -8.993749618530273,
      "step": 1670
    },
    {
      "epoch": 0.7887323943661971,
      "grad_norm": 710.7017540987989,
      "learning_rate": 1.176317162232655e-07,
      "logits/chosen": 1.5031249523162842,
      "logits/rejected": 1.83203125,
      "logps/chosen": -458.3999938964844,
      "logps/rejected": -579.4000244140625,
      "loss": 2.646,
      "rewards/accuracies": 0.557812511920929,
      "rewards/chosen": 0.8578125238418579,
      "rewards/margins": 9.703125,
      "rewards/rejected": -8.846875190734863,
      "step": 1680
    },
    {
      "epoch": 0.7934272300469484,
      "grad_norm": 865.3221932317232,
      "learning_rate": 1.1502347417840374e-07,
      "logits/chosen": 1.57421875,
      "logits/rejected": 1.896875023841858,
      "logps/chosen": -466.6000061035156,
      "logps/rejected": -560.2000122070312,
      "loss": 2.9606,
      "rewards/accuracies": 0.5296875238418579,
      "rewards/chosen": 0.635449230670929,
      "rewards/margins": 8.984375,
      "rewards/rejected": -8.34375,
      "step": 1690
    },
    {
      "epoch": 0.7981220657276995,
      "grad_norm": 947.5822987493972,
      "learning_rate": 1.1241523213354198e-07,
      "logits/chosen": 1.564843773841858,
      "logits/rejected": 1.775781273841858,
      "logps/chosen": -464.6000061035156,
      "logps/rejected": -581.7999877929688,
      "loss": 2.6536,
      "rewards/accuracies": 0.55859375,
      "rewards/chosen": 1.546289086341858,
      "rewards/margins": 9.159375190734863,
      "rewards/rejected": -7.631249904632568,
      "step": 1700
    },
    {
      "epoch": 0.8028169014084507,
      "grad_norm": 1080.8252122454785,
      "learning_rate": 1.0980699008868022e-07,
      "logits/chosen": 1.529687523841858,
      "logits/rejected": 1.7734375,
      "logps/chosen": -475.79998779296875,
      "logps/rejected": -546.5999755859375,
      "loss": 3.4411,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 0.5889648199081421,
      "rewards/margins": 8.393750190734863,
      "rewards/rejected": -7.800000190734863,
      "step": 1710
    },
    {
      "epoch": 0.8075117370892019,
      "grad_norm": 630.8786171130072,
      "learning_rate": 1.0719874804381846e-07,
      "logits/chosen": 1.4289062023162842,
      "logits/rejected": 1.7179687023162842,
      "logps/chosen": -464.20001220703125,
      "logps/rejected": -523.2000122070312,
      "loss": 2.8904,
      "rewards/accuracies": 0.532031238079071,
      "rewards/chosen": 0.9647461175918579,
      "rewards/margins": 7.910937309265137,
      "rewards/rejected": -6.939062595367432,
      "step": 1720
    },
    {
      "epoch": 0.812206572769953,
      "grad_norm": 650.5825682863951,
      "learning_rate": 1.045905059989567e-07,
      "logits/chosen": 1.5546875,
      "logits/rejected": 1.873437523841858,
      "logps/chosen": -440.0,
      "logps/rejected": -516.5999755859375,
      "loss": 2.4886,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 1.442968726158142,
      "rewards/margins": 8.578125,
      "rewards/rejected": -7.131249904632568,
      "step": 1730
    },
    {
      "epoch": 0.8169014084507042,
      "grad_norm": 655.5874686052315,
      "learning_rate": 1.0198226395409494e-07,
      "logits/chosen": 1.596093773841858,
      "logits/rejected": 1.8117187023162842,
      "logps/chosen": -445.20001220703125,
      "logps/rejected": -537.2000122070312,
      "loss": 2.4495,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": 1.2373778820037842,
      "rewards/margins": 9.649999618530273,
      "rewards/rejected": -8.428125381469727,
      "step": 1740
    },
    {
      "epoch": 0.8215962441314554,
      "grad_norm": 614.7178254340793,
      "learning_rate": 9.937402190923318e-08,
      "logits/chosen": 1.549218773841858,
      "logits/rejected": 1.896093726158142,
      "logps/chosen": -435.79998779296875,
      "logps/rejected": -575.7999877929688,
      "loss": 2.8784,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 1.5398437976837158,
      "rewards/margins": 10.762499809265137,
      "rewards/rejected": -9.231249809265137,
      "step": 1750
    },
    {
      "epoch": 0.8262910798122066,
      "grad_norm": 794.364527499517,
      "learning_rate": 9.676577986437141e-08,
      "logits/chosen": 1.525781273841858,
      "logits/rejected": 1.7976562976837158,
      "logps/chosen": -485.0,
      "logps/rejected": -551.5999755859375,
      "loss": 3.4006,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 1.199609398841858,
      "rewards/margins": 9.143750190734863,
      "rewards/rejected": -7.956250190734863,
      "step": 1760
    },
    {
      "epoch": 0.8309859154929577,
      "grad_norm": 688.1005137129989,
      "learning_rate": 9.415753781950965e-08,
      "logits/chosen": 1.564843773841858,
      "logits/rejected": 1.860937476158142,
      "logps/chosen": -455.6000061035156,
      "logps/rejected": -552.7999877929688,
      "loss": 3.0485,
      "rewards/accuracies": 0.546093761920929,
      "rewards/chosen": 1.359472632408142,
      "rewards/margins": 9.871874809265137,
      "rewards/rejected": -8.515625,
      "step": 1770
    },
    {
      "epoch": 0.8356807511737089,
      "grad_norm": 689.0317499141692,
      "learning_rate": 9.154929577464789e-08,
      "logits/chosen": 1.571874976158142,
      "logits/rejected": 1.8679687976837158,
      "logps/chosen": -465.0,
      "logps/rejected": -569.0,
      "loss": 2.6103,
      "rewards/accuracies": 0.569531261920929,
      "rewards/chosen": 1.3486816883087158,
      "rewards/margins": 11.387499809265137,
      "rewards/rejected": -10.050000190734863,
      "step": 1780
    },
    {
      "epoch": 0.8403755868544601,
      "grad_norm": 686.8662113537191,
      "learning_rate": 8.894105372978613e-08,
      "logits/chosen": 1.5539062023162842,
      "logits/rejected": 1.8984375,
      "logps/chosen": -477.79998779296875,
      "logps/rejected": -563.7999877929688,
      "loss": 2.8418,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.906665027141571,
      "rewards/margins": 8.778124809265137,
      "rewards/rejected": -7.871874809265137,
      "step": 1790
    },
    {
      "epoch": 0.8450704225352113,
      "grad_norm": 824.1563079996546,
      "learning_rate": 8.633281168492435e-08,
      "logits/chosen": 1.5515625476837158,
      "logits/rejected": 1.900781273841858,
      "logps/chosen": -462.0,
      "logps/rejected": -550.4000244140625,
      "loss": 3.2794,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.7906249761581421,
      "rewards/margins": 9.515625,
      "rewards/rejected": -8.725000381469727,
      "step": 1800
    },
    {
      "epoch": 0.8497652582159625,
      "grad_norm": 643.1282272598696,
      "learning_rate": 8.372456964006259e-08,
      "logits/chosen": 1.556249976158142,
      "logits/rejected": 1.8078124523162842,
      "logps/chosen": -463.6000061035156,
      "logps/rejected": -555.2000122070312,
      "loss": 3.2837,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 0.4295898377895355,
      "rewards/margins": 9.356249809265137,
      "rewards/rejected": -8.928125381469727,
      "step": 1810
    },
    {
      "epoch": 0.8544600938967136,
      "grad_norm": 845.3694524591884,
      "learning_rate": 8.111632759520083e-08,
      "logits/chosen": 1.6171875,
      "logits/rejected": 1.919531226158142,
      "logps/chosen": -499.6000061035156,
      "logps/rejected": -565.5999755859375,
      "loss": 3.0866,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": 0.699511706829071,
      "rewards/margins": 9.087499618530273,
      "rewards/rejected": -8.384374618530273,
      "step": 1820
    },
    {
      "epoch": 0.8591549295774648,
      "grad_norm": 665.8872746408979,
      "learning_rate": 7.850808555033907e-08,
      "logits/chosen": 1.6296875476837158,
      "logits/rejected": 1.830468773841858,
      "logps/chosen": -468.79998779296875,
      "logps/rejected": -579.5999755859375,
      "loss": 2.44,
      "rewards/accuracies": 0.55859375,
      "rewards/chosen": 1.9228515625,
      "rewards/margins": 10.4375,
      "rewards/rejected": -8.512499809265137,
      "step": 1830
    },
    {
      "epoch": 0.863849765258216,
      "grad_norm": 876.5478167256126,
      "learning_rate": 7.58998435054773e-08,
      "logits/chosen": 1.5070312023162842,
      "logits/rejected": 1.8429687023162842,
      "logps/chosen": -478.79998779296875,
      "logps/rejected": -566.0,
      "loss": 3.1495,
      "rewards/accuracies": 0.547656238079071,
      "rewards/chosen": 1.232324242591858,
      "rewards/margins": 9.034375190734863,
      "rewards/rejected": -7.8046875,
      "step": 1840
    },
    {
      "epoch": 0.8685446009389671,
      "grad_norm": 877.9509406982572,
      "learning_rate": 7.329160146061554e-08,
      "logits/chosen": 1.576562523841858,
      "logits/rejected": 1.908593773841858,
      "logps/chosen": -455.6000061035156,
      "logps/rejected": -553.0,
      "loss": 3.0998,
      "rewards/accuracies": 0.53515625,
      "rewards/chosen": 1.340234398841858,
      "rewards/margins": 8.012499809265137,
      "rewards/rejected": -6.672656059265137,
      "step": 1850
    },
    {
      "epoch": 0.8732394366197183,
      "grad_norm": 828.0040861843527,
      "learning_rate": 7.068335941575378e-08,
      "logits/chosen": 1.5906250476837158,
      "logits/rejected": 1.8195312023162842,
      "logps/chosen": -458.79998779296875,
      "logps/rejected": -545.7999877929688,
      "loss": 3.2934,
      "rewards/accuracies": 0.538281261920929,
      "rewards/chosen": 0.660937488079071,
      "rewards/margins": 8.584375381469727,
      "rewards/rejected": -7.918749809265137,
      "step": 1860
    },
    {
      "epoch": 0.8779342723004695,
      "grad_norm": 717.2949499326328,
      "learning_rate": 6.807511737089202e-08,
      "logits/chosen": 1.5398437976837158,
      "logits/rejected": 1.939062476158142,
      "logps/chosen": -457.3999938964844,
      "logps/rejected": -571.2000122070312,
      "loss": 2.7155,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 1.1306641101837158,
      "rewards/margins": 10.203125,
      "rewards/rejected": -9.065625190734863,
      "step": 1870
    },
    {
      "epoch": 0.8826291079812206,
      "grad_norm": 791.8508905436098,
      "learning_rate": 6.546687532603024e-08,
      "logits/chosen": 1.6570312976837158,
      "logits/rejected": 1.935937523841858,
      "logps/chosen": -463.20001220703125,
      "logps/rejected": -564.7999877929688,
      "loss": 2.5961,
      "rewards/accuracies": 0.5414062738418579,
      "rewards/chosen": 1.568750023841858,
      "rewards/margins": 9.295312881469727,
      "rewards/rejected": -7.7265625,
      "step": 1880
    },
    {
      "epoch": 0.8873239436619719,
      "grad_norm": 672.2240889421737,
      "learning_rate": 6.285863328116848e-08,
      "logits/chosen": 1.603124976158142,
      "logits/rejected": 1.911718726158142,
      "logps/chosen": -421.6000061035156,
      "logps/rejected": -580.0,
      "loss": 2.1885,
      "rewards/accuracies": 0.5757812261581421,
      "rewards/chosen": 1.6212890148162842,
      "rewards/margins": 10.762499809265137,
      "rewards/rejected": -9.125,
      "step": 1890
    },
    {
      "epoch": 0.892018779342723,
      "grad_norm": 817.2454327591987,
      "learning_rate": 6.025039123630672e-08,
      "logits/chosen": 1.528906226158142,
      "logits/rejected": 1.7570312023162842,
      "logps/chosen": -475.6000061035156,
      "logps/rejected": -529.4000244140625,
      "loss": 3.1626,
      "rewards/accuracies": 0.532031238079071,
      "rewards/chosen": 1.579003930091858,
      "rewards/margins": 8.490625381469727,
      "rewards/rejected": -6.907812595367432,
      "step": 1900
    },
    {
      "epoch": 0.8967136150234741,
      "grad_norm": 707.7196999294846,
      "learning_rate": 5.764214919144496e-08,
      "logits/chosen": 1.5617187023162842,
      "logits/rejected": 1.830468773841858,
      "logps/chosen": -477.6000061035156,
      "logps/rejected": -528.5999755859375,
      "loss": 2.9166,
      "rewards/accuracies": 0.5445312261581421,
      "rewards/chosen": 1.2537109851837158,
      "rewards/margins": 8.284375190734863,
      "rewards/rejected": -7.034375190734863,
      "step": 1910
    },
    {
      "epoch": 0.9014084507042254,
      "grad_norm": 708.221570255977,
      "learning_rate": 5.50339071465832e-08,
      "logits/chosen": 1.58203125,
      "logits/rejected": 1.84765625,
      "logps/chosen": -471.3999938964844,
      "logps/rejected": -530.0,
      "loss": 2.7829,
      "rewards/accuracies": 0.5609375238418579,
      "rewards/chosen": 1.7107055187225342,
      "rewards/margins": 9.350000381469727,
      "rewards/rejected": -7.631249904632568,
      "step": 1920
    },
    {
      "epoch": 0.9061032863849765,
      "grad_norm": 767.6418563087653,
      "learning_rate": 5.2425665101721436e-08,
      "logits/chosen": 1.5070312023162842,
      "logits/rejected": 1.8015625476837158,
      "logps/chosen": -475.20001220703125,
      "logps/rejected": -599.2000122070312,
      "loss": 2.1905,
      "rewards/accuracies": 0.55859375,
      "rewards/chosen": 1.959375023841858,
      "rewards/margins": 10.574999809265137,
      "rewards/rejected": -8.618749618530273,
      "step": 1930
    },
    {
      "epoch": 0.9107981220657277,
      "grad_norm": 790.87726044267,
      "learning_rate": 4.9817423056859675e-08,
      "logits/chosen": 1.5593750476837158,
      "logits/rejected": 1.8132812976837158,
      "logps/chosen": -476.3999938964844,
      "logps/rejected": -572.7999877929688,
      "loss": 2.9692,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 1.1171875,
      "rewards/margins": 9.946874618530273,
      "rewards/rejected": -8.828125,
      "step": 1940
    },
    {
      "epoch": 0.9154929577464789,
      "grad_norm": 692.161611989156,
      "learning_rate": 4.720918101199791e-08,
      "logits/chosen": 1.50390625,
      "logits/rejected": 1.841406226158142,
      "logps/chosen": -478.3999938964844,
      "logps/rejected": -623.0,
      "loss": 2.7112,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": 1.0671875476837158,
      "rewards/margins": 11.487500190734863,
      "rewards/rejected": -10.431249618530273,
      "step": 1950
    },
    {
      "epoch": 0.92018779342723,
      "grad_norm": 753.6479486460894,
      "learning_rate": 4.460093896713615e-08,
      "logits/chosen": 1.540624976158142,
      "logits/rejected": 1.786718726158142,
      "logps/chosen": -468.20001220703125,
      "logps/rejected": -575.7999877929688,
      "loss": 2.9615,
      "rewards/accuracies": 0.5523437261581421,
      "rewards/chosen": 1.4455077648162842,
      "rewards/margins": 9.521875381469727,
      "rewards/rejected": -8.0703125,
      "step": 1960
    },
    {
      "epoch": 0.9248826291079812,
      "grad_norm": 670.5588969559416,
      "learning_rate": 4.199269692227438e-08,
      "logits/chosen": 1.5539062023162842,
      "logits/rejected": 1.8718750476837158,
      "logps/chosen": -459.0,
      "logps/rejected": -524.7999877929688,
      "loss": 2.8938,
      "rewards/accuracies": 0.5289062261581421,
      "rewards/chosen": 1.475976586341858,
      "rewards/margins": 8.440625190734863,
      "rewards/rejected": -6.96875,
      "step": 1970
    },
    {
      "epoch": 0.9295774647887324,
      "grad_norm": 564.9795408907369,
      "learning_rate": 3.938445487741262e-08,
      "logits/chosen": 1.482031226158142,
      "logits/rejected": 1.6749999523162842,
      "logps/chosen": -483.6000061035156,
      "logps/rejected": -551.5999755859375,
      "loss": 2.8125,
      "rewards/accuracies": 0.551562488079071,
      "rewards/chosen": 0.8265625238418579,
      "rewards/margins": 10.959375381469727,
      "rewards/rejected": -10.121874809265137,
      "step": 1980
    },
    {
      "epoch": 0.9342723004694836,
      "grad_norm": 647.1533721452755,
      "learning_rate": 3.677621283255086e-08,
      "logits/chosen": 1.564062476158142,
      "logits/rejected": 2.0015625953674316,
      "logps/chosen": -481.79998779296875,
      "logps/rejected": -559.5999755859375,
      "loss": 2.9934,
      "rewards/accuracies": 0.55859375,
      "rewards/chosen": 1.2698242664337158,
      "rewards/margins": 9.621874809265137,
      "rewards/rejected": -8.353124618530273,
      "step": 1990
    },
    {
      "epoch": 0.9389671361502347,
      "grad_norm": 649.7646328509203,
      "learning_rate": 3.41679707876891e-08,
      "logits/chosen": 1.560937523841858,
      "logits/rejected": 1.869531273841858,
      "logps/chosen": -468.79998779296875,
      "logps/rejected": -616.0,
      "loss": 2.6607,
      "rewards/accuracies": 0.5484374761581421,
      "rewards/chosen": 1.2326171398162842,
      "rewards/margins": 10.640625,
      "rewards/rejected": -9.418749809265137,
      "step": 2000
    },
    {
      "epoch": 0.9436619718309859,
      "grad_norm": 855.6164889834444,
      "learning_rate": 3.155972874282733e-08,
      "logits/chosen": 1.567968726158142,
      "logits/rejected": 1.904687523841858,
      "logps/chosen": -450.20001220703125,
      "logps/rejected": -553.4000244140625,
      "loss": 2.9886,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 1.1906249523162842,
      "rewards/margins": 7.998437404632568,
      "rewards/rejected": -6.807812690734863,
      "step": 2010
    },
    {
      "epoch": 0.9483568075117371,
      "grad_norm": 744.3718557642948,
      "learning_rate": 2.8951486697965573e-08,
      "logits/chosen": 1.525781273841858,
      "logits/rejected": 1.7109375,
      "logps/chosen": -476.6000061035156,
      "logps/rejected": -562.7999877929688,
      "loss": 2.8297,
      "rewards/accuracies": 0.561718761920929,
      "rewards/chosen": 1.1457030773162842,
      "rewards/margins": 9.021875381469727,
      "rewards/rejected": -7.875,
      "step": 2020
    },
    {
      "epoch": 0.9530516431924883,
      "grad_norm": 735.7806750211452,
      "learning_rate": 2.634324465310381e-08,
      "logits/chosen": 1.588281273841858,
      "logits/rejected": 1.8523437976837158,
      "logps/chosen": -455.6000061035156,
      "logps/rejected": -560.5999755859375,
      "loss": 2.6506,
      "rewards/accuracies": 0.547656238079071,
      "rewards/chosen": 1.0134766101837158,
      "rewards/margins": 9.631250381469727,
      "rewards/rejected": -8.615625381469727,
      "step": 2030
    },
    {
      "epoch": 0.9577464788732394,
      "grad_norm": 703.8506265893652,
      "learning_rate": 2.3735002608242045e-08,
      "logits/chosen": 1.58203125,
      "logits/rejected": 1.888281226158142,
      "logps/chosen": -479.79998779296875,
      "logps/rejected": -539.2000122070312,
      "loss": 3.2018,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": 0.9636627435684204,
      "rewards/margins": 9.578125,
      "rewards/rejected": -8.625,
      "step": 2040
    },
    {
      "epoch": 0.9624413145539906,
      "grad_norm": 711.3587999204046,
      "learning_rate": 2.1126760563380282e-08,
      "logits/chosen": 1.540624976158142,
      "logits/rejected": 1.853124976158142,
      "logps/chosen": -463.6000061035156,
      "logps/rejected": -515.5999755859375,
      "loss": 2.801,
      "rewards/accuracies": 0.5492187738418579,
      "rewards/chosen": 1.2963378429412842,
      "rewards/margins": 8.046875,
      "rewards/rejected": -6.756249904632568,
      "step": 2050
    },
    {
      "epoch": 0.9671361502347418,
      "grad_norm": 608.753508187121,
      "learning_rate": 1.8518518518518518e-08,
      "logits/chosen": 1.599218726158142,
      "logits/rejected": 1.8507812023162842,
      "logps/chosen": -480.6000061035156,
      "logps/rejected": -556.4000244140625,
      "loss": 3.1135,
      "rewards/accuracies": 0.55859375,
      "rewards/chosen": 0.732421875,
      "rewards/margins": 8.715624809265137,
      "rewards/rejected": -7.993750095367432,
      "step": 2060
    },
    {
      "epoch": 0.971830985915493,
      "grad_norm": 628.2893922247904,
      "learning_rate": 1.5910276473656755e-08,
      "logits/chosen": 1.5812499523162842,
      "logits/rejected": 1.7625000476837158,
      "logps/chosen": -460.6000061035156,
      "logps/rejected": -555.7999877929688,
      "loss": 2.6318,
      "rewards/accuracies": 0.571093738079071,
      "rewards/chosen": 1.3933594226837158,
      "rewards/margins": 10.253125190734863,
      "rewards/rejected": -8.853124618530273,
      "step": 2070
    },
    {
      "epoch": 0.9765258215962441,
      "grad_norm": 721.8264614979432,
      "learning_rate": 1.3302034428794991e-08,
      "logits/chosen": 1.6476562023162842,
      "logits/rejected": 1.94921875,
      "logps/chosen": -444.79998779296875,
      "logps/rejected": -522.4000244140625,
      "loss": 2.8609,
      "rewards/accuracies": 0.5546875,
      "rewards/chosen": 1.316552758216858,
      "rewards/margins": 8.798437118530273,
      "rewards/rejected": -7.4765625,
      "step": 2080
    },
    {
      "epoch": 0.9812206572769953,
      "grad_norm": 702.9698466166137,
      "learning_rate": 1.0693792383933229e-08,
      "logits/chosen": 1.5539062023162842,
      "logits/rejected": 1.9421875476837158,
      "logps/chosen": -472.79998779296875,
      "logps/rejected": -576.4000244140625,
      "loss": 3.1795,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 1.106835961341858,
      "rewards/margins": 9.018750190734863,
      "rewards/rejected": -7.918749809265137,
      "step": 2090
    },
    {
      "epoch": 0.9859154929577465,
      "grad_norm": 860.8486982092512,
      "learning_rate": 8.085550339071465e-09,
      "logits/chosen": 1.4835937023162842,
      "logits/rejected": 1.857812523841858,
      "logps/chosen": -482.3999938964844,
      "logps/rejected": -598.5999755859375,
      "loss": 2.8833,
      "rewards/accuracies": 0.55078125,
      "rewards/chosen": 1.1150939464569092,
      "rewards/margins": 10.556249618530273,
      "rewards/rejected": -9.453125,
      "step": 2100
    },
    {
      "epoch": 0.9906103286384976,
      "grad_norm": 850.6963719967856,
      "learning_rate": 5.4773082942097025e-09,
      "logits/chosen": 1.5890624523162842,
      "logits/rejected": 1.834375023841858,
      "logps/chosen": -457.20001220703125,
      "logps/rejected": -536.7999877929688,
      "loss": 2.8074,
      "rewards/accuracies": 0.5609375238418579,
      "rewards/chosen": 1.2479248046875,
      "rewards/margins": 9.140625,
      "rewards/rejected": -7.868750095367432,
      "step": 2110
    },
    {
      "epoch": 0.9953051643192489,
      "grad_norm": 727.4778191630578,
      "learning_rate": 2.8690662493479393e-09,
      "logits/chosen": 1.583593726158142,
      "logits/rejected": 1.7820312976837158,
      "logps/chosen": -464.3999938964844,
      "logps/rejected": -543.7999877929688,
      "loss": 3.2388,
      "rewards/accuracies": 0.5406249761581421,
      "rewards/chosen": 0.870312511920929,
      "rewards/margins": 9.178125381469727,
      "rewards/rejected": -8.300000190734863,
      "step": 2120
    },
    {
      "epoch": 1.0,
      "grad_norm": 727.528201128329,
      "learning_rate": 2.608242044861763e-10,
      "logits/chosen": 1.57421875,
      "logits/rejected": 1.919531226158142,
      "logps/chosen": -457.3999938964844,
      "logps/rejected": -571.5999755859375,
      "loss": 3.1124,
      "rewards/accuracies": 0.5481592416763306,
      "rewards/chosen": 1.4762694835662842,
      "rewards/margins": 9.956250190734863,
      "rewards/rejected": -8.484375,
      "step": 2130
    },
    {
      "epoch": 1.0,
      "step": 2130,
      "total_flos": 0.0,
      "train_loss": 3.0287644077354754,
      "train_runtime": 5560.8629,
      "train_samples_per_second": 49.018,
      "train_steps_per_second": 0.383
    }
  ],
  "logging_steps": 10,
  "max_steps": 2130,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}