{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 400,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002093692750588851,
      "grad_norm": 23.60069859438079,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.8280014991760254,
      "logits/rejected": -2.8466408252716064,
      "logps/chosen": -1.1081000566482544,
      "logps/rejected": -1.146370530128479,
      "loss": 1.825,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -2.216200113296509,
      "rewards/margins": 0.0765407383441925,
      "rewards/rejected": -2.292741060256958,
      "step": 1
    },
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 20.150796004914934,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.8811752796173096,
      "logits/rejected": -2.847626209259033,
      "logps/chosen": -1.0211418867111206,
      "logps/rejected": -1.080704689025879,
      "loss": 1.7105,
      "rewards/accuracies": 0.5390625,
      "rewards/chosen": -2.042283773422241,
      "rewards/margins": 0.11912553012371063,
      "rewards/rejected": -2.161409378051758,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 19.30396074137424,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.992589235305786,
      "logits/rejected": -2.9415202140808105,
      "logps/chosen": -1.0235936641693115,
      "logps/rejected": -1.1658079624176025,
      "loss": 1.7413,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -2.047187328338623,
      "rewards/margins": 0.2844284772872925,
      "rewards/rejected": -2.331615924835205,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 22.37404641220351,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.9186646938323975,
      "logits/rejected": -2.905055522918701,
      "logps/chosen": -1.0526732206344604,
      "logps/rejected": -1.1032135486602783,
      "loss": 1.7229,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -2.105346441268921,
      "rewards/margins": 0.10108034312725067,
      "rewards/rejected": -2.2064270973205566,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 18.24180275182366,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.9416229724884033,
      "logits/rejected": -2.890882968902588,
      "logps/chosen": -1.0062425136566162,
      "logps/rejected": -1.1030701398849487,
      "loss": 1.6956,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.0124850273132324,
      "rewards/margins": 0.19365549087524414,
      "rewards/rejected": -2.2061402797698975,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 17.74156497951028,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.9460272789001465,
      "logits/rejected": -2.882147789001465,
      "logps/chosen": -0.9283784031867981,
      "logps/rejected": -1.1168667078018188,
      "loss": 1.664,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.8567568063735962,
      "rewards/margins": 0.37697672843933105,
      "rewards/rejected": -2.2337334156036377,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 17.414699170293087,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.919449806213379,
      "logits/rejected": -2.9000954627990723,
      "logps/chosen": -0.9085705876350403,
      "logps/rejected": -0.9413141012191772,
      "loss": 1.6764,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.8171411752700806,
      "rewards/margins": 0.06548723578453064,
      "rewards/rejected": -1.8826282024383545,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 18.306181256584207,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.929173469543457,
      "logits/rejected": -2.8954787254333496,
      "logps/chosen": -0.9194827079772949,
      "logps/rejected": -1.1145708560943604,
      "loss": 1.6503,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.8389654159545898,
      "rewards/margins": 0.39017611742019653,
      "rewards/rejected": -2.2291417121887207,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 17.72310110786498,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.920375347137451,
      "logits/rejected": -2.878530979156494,
      "logps/chosen": -0.9253458976745605,
      "logps/rejected": -1.0473893880844116,
      "loss": 1.6223,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.850691795349121,
      "rewards/margins": 0.2440868616104126,
      "rewards/rejected": -2.0947787761688232,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 16.885070690793345,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.875908851623535,
      "logits/rejected": -2.8810861110687256,
      "logps/chosen": -0.903559684753418,
      "logps/rejected": -1.0813857316970825,
      "loss": 1.5727,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.807119369506836,
      "rewards/margins": 0.3556520342826843,
      "rewards/rejected": -2.162771463394165,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 23.820902499939343,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -2.820807933807373,
      "logits/rejected": -2.832883358001709,
      "logps/chosen": -0.9573640823364258,
      "logps/rejected": -1.2130459547042847,
      "loss": 1.6274,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.9147281646728516,
      "rewards/margins": 0.5113638043403625,
      "rewards/rejected": -2.4260919094085693,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 24.790862979912625,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -2.9123668670654297,
      "logits/rejected": -2.9268505573272705,
      "logps/chosen": -1.0739004611968994,
      "logps/rejected": -1.1845686435699463,
      "loss": 1.6259,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.147800922393799,
      "rewards/margins": 0.22133660316467285,
      "rewards/rejected": -2.3691372871398926,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 30.216317311158747,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.847761869430542,
      "logits/rejected": -2.8407883644104004,
      "logps/chosen": -1.0353600978851318,
      "logps/rejected": -1.207129716873169,
      "loss": 1.5827,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.0707201957702637,
      "rewards/margins": 0.3435389995574951,
      "rewards/rejected": -2.414259433746338,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 25.74208520684934,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -2.8678090572357178,
      "logits/rejected": -2.89147686958313,
      "logps/chosen": -1.0218393802642822,
      "logps/rejected": -1.196505069732666,
      "loss": 1.5964,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -2.0436787605285645,
      "rewards/margins": 0.3493315577507019,
      "rewards/rejected": -2.393010139465332,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 26.44073609906296,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -2.86854887008667,
      "logits/rejected": -2.8683791160583496,
      "logps/chosen": -1.1050409078598022,
      "logps/rejected": -1.300787091255188,
      "loss": 1.5446,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.2100818157196045,
      "rewards/margins": 0.391492635011673,
      "rewards/rejected": -2.601574182510376,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 32.621154621425745,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -2.816270351409912,
      "logits/rejected": -2.7658441066741943,
      "logps/chosen": -1.1289862394332886,
      "logps/rejected": -1.42450749874115,
      "loss": 1.5692,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.257972478866577,
      "rewards/margins": 0.5910428762435913,
      "rewards/rejected": -2.8490149974823,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 23.011756326022333,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -2.7463507652282715,
      "logits/rejected": -2.741854667663574,
      "logps/chosen": -1.118318796157837,
      "logps/rejected": -1.486810564994812,
      "loss": 1.5494,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.236637592315674,
      "rewards/margins": 0.7369835376739502,
      "rewards/rejected": -2.973621129989624,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 28.536903966310312,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -2.8005614280700684,
      "logits/rejected": -2.7525715827941895,
      "logps/chosen": -1.1057350635528564,
      "logps/rejected": -1.4478759765625,
      "loss": 1.5328,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.211470127105713,
      "rewards/margins": 0.6842816472053528,
      "rewards/rejected": -2.895751953125,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 21.41681785684056,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.7789416313171387,
      "logits/rejected": -2.7269153594970703,
      "logps/chosen": -1.1642733812332153,
      "logps/rejected": -1.5922167301177979,
      "loss": 1.5065,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.3285467624664307,
      "rewards/margins": 0.8558871150016785,
      "rewards/rejected": -3.1844334602355957,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 31.29059195928571,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -2.705059051513672,
      "logits/rejected": -2.689885139465332,
      "logps/chosen": -1.1885052919387817,
      "logps/rejected": -1.503861665725708,
      "loss": 1.5348,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -2.3770105838775635,
      "rewards/margins": 0.6307128667831421,
      "rewards/rejected": -3.007723331451416,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 29.067491467613685,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -2.776214122772217,
      "logits/rejected": -2.7497711181640625,
      "logps/chosen": -1.157089352607727,
      "logps/rejected": -1.3819860219955444,
      "loss": 1.5864,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -2.314178705215454,
      "rewards/margins": 0.44979315996170044,
      "rewards/rejected": -2.763972043991089,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 23.067455098331433,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -2.7103278636932373,
      "logits/rejected": -2.68127703666687,
      "logps/chosen": -1.180944800376892,
      "logps/rejected": -1.535935640335083,
      "loss": 1.4346,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.361889600753784,
      "rewards/margins": 0.7099816203117371,
      "rewards/rejected": -3.071871280670166,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 38.83165169666446,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -2.6472716331481934,
      "logits/rejected": -2.6239981651306152,
      "logps/chosen": -1.1390531063079834,
      "logps/rejected": -1.4835283756256104,
      "loss": 1.4351,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -2.278106212615967,
      "rewards/margins": 0.6889505386352539,
      "rewards/rejected": -2.9670567512512207,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 28.07429755635963,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -2.6805942058563232,
      "logits/rejected": -2.6706345081329346,
      "logps/chosen": -1.1800041198730469,
      "logps/rejected": -1.4017146825790405,
      "loss": 1.527,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.3600082397460938,
      "rewards/margins": 0.4434211254119873,
      "rewards/rejected": -2.803429365158081,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 25.451878448448234,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.695066213607788,
      "logits/rejected": -2.6383955478668213,
      "logps/chosen": -1.11952805519104,
      "logps/rejected": -1.5505688190460205,
      "loss": 1.5091,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.23905611038208,
      "rewards/margins": 0.8620816469192505,
      "rewards/rejected": -3.101137638092041,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 28.525020675058567,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -2.5875375270843506,
      "logits/rejected": -2.5485615730285645,
      "logps/chosen": -1.1817591190338135,
      "logps/rejected": -1.588836669921875,
      "loss": 1.5135,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.363518238067627,
      "rewards/margins": 0.8141552209854126,
      "rewards/rejected": -3.17767333984375,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 33.5709446027395,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -2.6082234382629395,
      "logits/rejected": -2.614490032196045,
      "logps/chosen": -1.1771992444992065,
      "logps/rejected": -1.5893863439559937,
      "loss": 1.4422,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.354398488998413,
      "rewards/margins": 0.8243740200996399,
      "rewards/rejected": -3.1787726879119873,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 26.6001483024502,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -2.54782772064209,
      "logits/rejected": -2.51760196685791,
      "logps/chosen": -1.0749025344848633,
      "logps/rejected": -1.5542266368865967,
      "loss": 1.4927,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.1498050689697266,
      "rewards/margins": 0.9586483240127563,
      "rewards/rejected": -3.1084532737731934,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 37.50133455368946,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -2.544795274734497,
      "logits/rejected": -2.545607089996338,
      "logps/chosen": -1.3422894477844238,
      "logps/rejected": -1.5928326845169067,
      "loss": 1.5489,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.6845788955688477,
      "rewards/margins": 0.5010861158370972,
      "rewards/rejected": -3.1856653690338135,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 30.749771656186557,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -2.5184309482574463,
      "logits/rejected": -2.4989206790924072,
      "logps/chosen": -1.229201078414917,
      "logps/rejected": -1.7190883159637451,
      "loss": 1.4308,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.458402156829834,
      "rewards/margins": 0.9797741770744324,
      "rewards/rejected": -3.4381766319274902,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 34.705616703882306,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.5644986629486084,
      "logits/rejected": -2.5165212154388428,
      "logps/chosen": -1.2188150882720947,
      "logps/rejected": -1.596199631690979,
      "loss": 1.4579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.4376301765441895,
      "rewards/margins": 0.7547694444656372,
      "rewards/rejected": -3.192399263381958,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 52.49121820937414,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -2.475571393966675,
      "logits/rejected": -2.447605609893799,
      "logps/chosen": -1.2845265865325928,
      "logps/rejected": -1.8401352167129517,
      "loss": 1.4002,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.5690531730651855,
      "rewards/margins": 1.1112172603607178,
      "rewards/rejected": -3.6802704334259033,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 38.68489554173724,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -2.445486545562744,
      "logits/rejected": -2.4290173053741455,
      "logps/chosen": -1.2640022039413452,
      "logps/rejected": -1.90715754032135,
      "loss": 1.4271,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.5280044078826904,
      "rewards/margins": 1.2863105535507202,
      "rewards/rejected": -3.8143150806427,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 47.847635567656944,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -2.5186421871185303,
      "logits/rejected": -2.476241111755371,
      "logps/chosen": -1.2429786920547485,
      "logps/rejected": -1.6412330865859985,
      "loss": 1.3755,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -2.485957384109497,
      "rewards/margins": 0.7965084314346313,
      "rewards/rejected": -3.282466173171997,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 39.55414229808935,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -2.5068397521972656,
      "logits/rejected": -2.458843231201172,
      "logps/chosen": -1.2319917678833008,
      "logps/rejected": -1.6991866827011108,
      "loss": 1.3875,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.4639835357666016,
      "rewards/margins": 0.9343900680541992,
      "rewards/rejected": -3.3983733654022217,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 45.87476259725973,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -2.4231743812561035,
      "logits/rejected": -2.4120521545410156,
      "logps/chosen": -1.2813334465026855,
      "logps/rejected": -1.8030840158462524,
      "loss": 1.3979,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.562666893005371,
      "rewards/margins": 1.0435012578964233,
      "rewards/rejected": -3.606168031692505,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 52.555406414516895,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.438419818878174,
      "logits/rejected": -2.4260263442993164,
      "logps/chosen": -1.4097450971603394,
      "logps/rejected": -1.9904677867889404,
      "loss": 1.4005,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.8194901943206787,
      "rewards/margins": 1.1614452600479126,
      "rewards/rejected": -3.980935573577881,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 45.270627593903136,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -2.390608787536621,
      "logits/rejected": -2.4067492485046387,
      "logps/chosen": -1.4339230060577393,
      "logps/rejected": -2.0207736492156982,
      "loss": 1.3374,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.8678460121154785,
      "rewards/margins": 1.173701524734497,
      "rewards/rejected": -4.0415472984313965,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 35.97699766429664,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -2.389159679412842,
      "logits/rejected": -2.3969435691833496,
      "logps/chosen": -1.4970250129699707,
      "logps/rejected": -2.0378897190093994,
      "loss": 1.3428,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.9940500259399414,
      "rewards/margins": 1.081729531288147,
      "rewards/rejected": -4.075779438018799,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 53.06912771848456,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -2.3946540355682373,
      "logits/rejected": -2.3885226249694824,
      "logps/chosen": -1.5624958276748657,
      "logps/rejected": -2.2428553104400635,
      "loss": 1.4342,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1249916553497314,
      "rewards/margins": 1.3607194423675537,
      "rewards/rejected": -4.485710620880127,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 46.84209895719228,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -2.2939867973327637,
      "logits/rejected": -2.276329755783081,
      "logps/chosen": -1.6615016460418701,
      "logps/rejected": -2.154167652130127,
      "loss": 1.3573,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -3.3230032920837402,
      "rewards/margins": 0.9853318333625793,
      "rewards/rejected": -4.308335304260254,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 63.174735320239826,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -2.364863634109497,
      "logits/rejected": -2.3571414947509766,
      "logps/chosen": -1.7413444519042969,
      "logps/rejected": -2.463257312774658,
      "loss": 1.3484,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -3.4826889038085938,
      "rewards/margins": 1.4438252449035645,
      "rewards/rejected": -4.926514625549316,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 53.13105715426384,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.216089963912964,
      "logits/rejected": -2.2170519828796387,
      "logps/chosen": -1.8861106634140015,
      "logps/rejected": -2.334871768951416,
      "loss": 1.4309,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -3.772221326828003,
      "rewards/margins": 0.8975218534469604,
      "rewards/rejected": -4.669743537902832,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 59.15829566376919,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -2.225267171859741,
      "logits/rejected": -2.250946521759033,
      "logps/chosen": -1.865648627281189,
      "logps/rejected": -2.464839220046997,
      "loss": 1.3613,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -3.731297254562378,
      "rewards/margins": 1.198380708694458,
      "rewards/rejected": -4.929678440093994,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 60.027916344260326,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -2.2151072025299072,
      "logits/rejected": -2.2236618995666504,
      "logps/chosen": -1.9221923351287842,
      "logps/rejected": -2.5661563873291016,
      "loss": 1.2543,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -3.8443846702575684,
      "rewards/margins": 1.2879277467727661,
      "rewards/rejected": -5.132312774658203,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 58.043157481996744,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -2.1862709522247314,
      "logits/rejected": -2.1562201976776123,
      "logps/chosen": -2.2186365127563477,
      "logps/rejected": -2.7678346633911133,
      "loss": 1.3579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -4.437273025512695,
      "rewards/margins": 1.0983966588974,
      "rewards/rejected": -5.535669326782227,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 61.982027045427564,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -2.1588408946990967,
      "logits/rejected": -2.1733763217926025,
      "logps/chosen": -2.2563939094543457,
      "logps/rejected": -2.928832530975342,
      "loss": 1.2783,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -4.512787818908691,
      "rewards/margins": 1.3448779582977295,
      "rewards/rejected": -5.857665061950684,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 53.47910914358791,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -2.066746234893799,
      "logits/rejected": -2.0809385776519775,
      "logps/chosen": -2.1830739974975586,
      "logps/rejected": -2.7093000411987305,
      "loss": 1.3083,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -4.366147994995117,
      "rewards/margins": 1.0524519681930542,
      "rewards/rejected": -5.418600082397461,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 51.26667753004939,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -2.0583231449127197,
      "logits/rejected": -2.0678017139434814,
      "logps/chosen": -2.203321933746338,
      "logps/rejected": -2.9744162559509277,
      "loss": 1.3103,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -4.406643867492676,
      "rewards/margins": 1.5421884059906006,
      "rewards/rejected": -5.9488325119018555,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 54.582002602834834,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -2.076049327850342,
      "logits/rejected": -2.105649948120117,
      "logps/chosen": -2.2140307426452637,
      "logps/rejected": -3.014204740524292,
      "loss": 1.242,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -4.428061485290527,
      "rewards/margins": 1.6003482341766357,
      "rewards/rejected": -6.028409481048584,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 65.50612310325059,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -1.9772818088531494,
      "logits/rejected": -2.0134644508361816,
      "logps/chosen": -2.4714977741241455,
      "logps/rejected": -3.3275609016418457,
      "loss": 1.3888,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -4.942995548248291,
      "rewards/margins": 1.7121270895004272,
      "rewards/rejected": -6.655121803283691,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 67.4992864768084,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -2.059023141860962,
      "logits/rejected": -2.0484743118286133,
      "logps/chosen": -2.5257086753845215,
      "logps/rejected": -3.2463812828063965,
      "loss": 1.3315,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -5.051417350769043,
      "rewards/margins": 1.4413448572158813,
      "rewards/rejected": -6.492762565612793,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 69.13744141717783,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -2.006592273712158,
      "logits/rejected": -2.014500141143799,
      "logps/chosen": -2.560147762298584,
      "logps/rejected": -3.3215479850769043,
      "loss": 1.3082,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -5.120295524597168,
      "rewards/margins": 1.5228006839752197,
      "rewards/rejected": -6.643095970153809,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 58.072442212510744,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -1.9831031560897827,
      "logits/rejected": -1.9949731826782227,
      "logps/chosen": -2.81583833694458,
      "logps/rejected": -3.404637575149536,
      "loss": 1.3035,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -5.63167667388916,
      "rewards/margins": 1.1775987148284912,
      "rewards/rejected": -6.809275150299072,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 87.65400359085467,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -2.0101704597473145,
      "logits/rejected": -2.017418384552002,
      "logps/chosen": -2.8499555587768555,
      "logps/rejected": -3.4598793983459473,
      "loss": 1.2368,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -5.699911117553711,
      "rewards/margins": 1.219847559928894,
      "rewards/rejected": -6.9197587966918945,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 45.097233149719436,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -1.9900894165039062,
      "logits/rejected": -1.990896224975586,
      "logps/chosen": -2.661165714263916,
      "logps/rejected": -3.678067445755005,
      "loss": 1.1497,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -5.322331428527832,
      "rewards/margins": 2.033804416656494,
      "rewards/rejected": -7.35613489151001,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 65.10418020833369,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -2.0073654651641846,
      "logits/rejected": -2.005659580230713,
      "logps/chosen": -2.6216259002685547,
      "logps/rejected": -3.4922943115234375,
      "loss": 1.3131,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -5.243251800537109,
      "rewards/margins": 1.7413362264633179,
      "rewards/rejected": -6.984588623046875,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 61.639250570571264,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -1.9917762279510498,
      "logits/rejected": -1.982318639755249,
      "logps/chosen": -2.5954482555389404,
      "logps/rejected": -3.323718309402466,
      "loss": 1.2697,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -5.190896511077881,
      "rewards/margins": 1.4565393924713135,
      "rewards/rejected": -6.647436618804932,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 69.64758924542811,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -1.9363685846328735,
      "logits/rejected": -1.9144127368927002,
      "logps/chosen": -2.655473232269287,
      "logps/rejected": -3.300741195678711,
      "loss": 1.2942,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -5.310946464538574,
      "rewards/margins": 1.2905361652374268,
      "rewards/rejected": -6.601482391357422,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 72.7122045465263,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -1.9503307342529297,
      "logits/rejected": -1.938823938369751,
      "logps/chosen": -2.770573139190674,
      "logps/rejected": -3.3855674266815186,
      "loss": 1.2444,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -5.541146278381348,
      "rewards/margins": 1.2299891710281372,
      "rewards/rejected": -6.771134853363037,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 79.46704807847372,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -1.8155113458633423,
      "logits/rejected": -1.800158143043518,
      "logps/chosen": -2.862525463104248,
      "logps/rejected": -3.680985927581787,
      "loss": 1.2954,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -5.725050926208496,
      "rewards/margins": 1.6369216442108154,
      "rewards/rejected": -7.361971855163574,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 77.33985147922212,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -1.9133965969085693,
      "logits/rejected": -1.9274393320083618,
      "logps/chosen": -2.733630657196045,
      "logps/rejected": -3.405717134475708,
      "loss": 1.2303,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -5.46726131439209,
      "rewards/margins": 1.3441731929779053,
      "rewards/rejected": -6.811434268951416,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 87.53372002667572,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -1.8388316631317139,
      "logits/rejected": -1.8692519664764404,
      "logps/chosen": -2.9406466484069824,
      "logps/rejected": -3.684401035308838,
      "loss": 1.2409,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -5.881293296813965,
      "rewards/margins": 1.4875091314315796,
      "rewards/rejected": -7.368802070617676,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 63.982977249408776,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -1.9070667028427124,
      "logits/rejected": -1.91787850856781,
      "logps/chosen": -2.804577350616455,
      "logps/rejected": -3.586026668548584,
      "loss": 1.2734,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -5.60915470123291,
      "rewards/margins": 1.5628979206085205,
      "rewards/rejected": -7.172053337097168,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 68.2071236663429,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -1.9229049682617188,
      "logits/rejected": -1.9212894439697266,
      "logps/chosen": -2.7598495483398438,
      "logps/rejected": -3.6366069316864014,
      "loss": 1.173,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -5.5196990966796875,
      "rewards/margins": 1.7535146474838257,
      "rewards/rejected": -7.273213863372803,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 82.89442348714914,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -1.8585288524627686,
      "logits/rejected": -1.8806421756744385,
      "logps/chosen": -3.1847102642059326,
      "logps/rejected": -4.028578758239746,
      "loss": 1.319,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -6.369420528411865,
      "rewards/margins": 1.6877357959747314,
      "rewards/rejected": -8.057157516479492,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 63.652198082327615,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -1.8519096374511719,
      "logits/rejected": -1.8569042682647705,
      "logps/chosen": -3.092832565307617,
      "logps/rejected": -4.055886268615723,
      "loss": 1.2226,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -6.185665130615234,
      "rewards/margins": 1.9261070489883423,
      "rewards/rejected": -8.111772537231445,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 88.02707766997058,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -1.8144184350967407,
      "logits/rejected": -1.8429317474365234,
      "logps/chosen": -3.2898623943328857,
      "logps/rejected": -4.14624547958374,
      "loss": 1.2802,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -6.5797247886657715,
      "rewards/margins": 1.7127659320831299,
      "rewards/rejected": -8.29249095916748,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 81.51547237456022,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -1.8092453479766846,
      "logits/rejected": -1.884722113609314,
      "logps/chosen": -3.1887125968933105,
      "logps/rejected": -4.263826370239258,
      "loss": 1.2472,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -6.377425193786621,
      "rewards/margins": 2.1502270698547363,
      "rewards/rejected": -8.527652740478516,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 61.57880406452509,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -1.8666706085205078,
      "logits/rejected": -1.8893043994903564,
      "logps/chosen": -3.0370490550994873,
      "logps/rejected": -3.9561619758605957,
      "loss": 1.255,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -6.074098110198975,
      "rewards/margins": 1.838226079940796,
      "rewards/rejected": -7.912323951721191,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 58.397329958567745,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -1.895630121231079,
      "logits/rejected": -1.9232200384140015,
      "logps/chosen": -2.7903852462768555,
      "logps/rejected": -3.5561904907226562,
      "loss": 1.1821,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -5.580770492553711,
      "rewards/margins": 1.5316107273101807,
      "rewards/rejected": -7.1123809814453125,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 52.74691249746145,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -1.8785375356674194,
      "logits/rejected": -1.9092729091644287,
      "logps/chosen": -2.7013652324676514,
      "logps/rejected": -3.447122573852539,
      "loss": 1.3265,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -5.402730464935303,
      "rewards/margins": 1.4915144443511963,
      "rewards/rejected": -6.894245147705078,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 65.82463759506157,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -1.8855186700820923,
      "logits/rejected": -1.9008718729019165,
      "logps/chosen": -2.6092782020568848,
      "logps/rejected": -3.4059150218963623,
      "loss": 1.2498,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -5.2185564041137695,
      "rewards/margins": 1.5932741165161133,
      "rewards/rejected": -6.811830043792725,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 62.941225991768896,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -1.9058167934417725,
      "logits/rejected": -1.9186041355133057,
      "logps/chosen": -2.606133222579956,
      "logps/rejected": -3.5266499519348145,
      "loss": 1.2337,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -5.212266445159912,
      "rewards/margins": 1.8410332202911377,
      "rewards/rejected": -7.053299903869629,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 80.15726866517387,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -1.8724944591522217,
      "logits/rejected": -1.888501524925232,
      "logps/chosen": -2.636202573776245,
      "logps/rejected": -3.564650058746338,
      "loss": 1.1151,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -5.27240514755249,
      "rewards/margins": 1.8568944931030273,
      "rewards/rejected": -7.129300117492676,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 85.74581196541867,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -1.8323078155517578,
      "logits/rejected": -1.8531125783920288,
      "logps/chosen": -2.757204532623291,
      "logps/rejected": -3.6793594360351562,
      "loss": 1.2422,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -5.514409065246582,
      "rewards/margins": 1.8443094491958618,
      "rewards/rejected": -7.3587188720703125,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 65.11322516172854,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -1.881985068321228,
      "logits/rejected": -1.9223979711532593,
      "logps/chosen": -2.7589480876922607,
      "logps/rejected": -3.593773603439331,
      "loss": 1.2412,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -5.5178961753845215,
      "rewards/margins": 1.6696510314941406,
      "rewards/rejected": -7.187547206878662,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 70.17153962504261,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -1.8261827230453491,
      "logits/rejected": -1.8423763513565063,
      "logps/chosen": -2.850209951400757,
      "logps/rejected": -3.670152187347412,
      "loss": 1.2011,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -5.700419902801514,
      "rewards/margins": 1.639884352684021,
      "rewards/rejected": -7.340304374694824,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 68.38146040720316,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -1.8114173412322998,
      "logits/rejected": -1.8267343044281006,
      "logps/chosen": -3.0265450477600098,
      "logps/rejected": -3.899582624435425,
      "loss": 1.187,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -6.0530900955200195,
      "rewards/margins": 1.7460750341415405,
      "rewards/rejected": -7.79916524887085,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 77.09446527435051,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -1.7677643299102783,
      "logits/rejected": -1.80731999874115,
      "logps/chosen": -2.9306697845458984,
      "logps/rejected": -3.8741061687469482,
      "loss": 1.2481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.861339569091797,
      "rewards/margins": 1.8868728876113892,
      "rewards/rejected": -7.7482123374938965,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 71.1284274145702,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -1.7906978130340576,
      "logits/rejected": -1.8041505813598633,
      "logps/chosen": -2.976975202560425,
      "logps/rejected": -3.7604854106903076,
      "loss": 1.2816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -5.95395040512085,
      "rewards/margins": 1.5670208930969238,
      "rewards/rejected": -7.520970821380615,
      "step": 400
    },
    {
      "epoch": 0.8374771002355405,
      "eval_logits/chosen": -1.8679399490356445,
      "eval_logits/rejected": -1.879980444908142,
      "eval_logps/chosen": -2.9611897468566895,
      "eval_logps/rejected": -3.90179443359375,
      "eval_loss": 1.2016023397445679,
      "eval_rewards/accuracies": 0.7379999756813049,
      "eval_rewards/chosen": -5.922379493713379,
      "eval_rewards/margins": 1.8812097311019897,
      "eval_rewards/rejected": -7.8035888671875,
      "eval_runtime": 171.6266,
      "eval_samples_per_second": 11.653,
      "eval_steps_per_second": 0.728,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 63.83271294085232,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -1.8161646127700806,
      "logits/rejected": -1.8377959728240967,
      "logps/chosen": -3.039977788925171,
      "logps/rejected": -3.8821442127227783,
      "loss": 1.1734,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -6.079955577850342,
      "rewards/margins": 1.6843324899673462,
      "rewards/rejected": -7.764288425445557,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 58.30518411516352,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -1.7875831127166748,
      "logits/rejected": -1.8193409442901611,
      "logps/chosen": -3.0099644660949707,
      "logps/rejected": -3.809281587600708,
      "loss": 1.2211,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -6.019928932189941,
      "rewards/margins": 1.5986337661743164,
      "rewards/rejected": -7.618563175201416,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 75.30650643769822,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -1.8199684619903564,
      "logits/rejected": -1.8271119594573975,
      "logps/chosen": -3.101931571960449,
      "logps/rejected": -3.806241273880005,
      "loss": 1.2569,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -6.203863143920898,
      "rewards/margins": 1.4086189270019531,
      "rewards/rejected": -7.61248254776001,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 78.83242409965422,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -1.7683451175689697,
      "logits/rejected": -1.7631126642227173,
      "logps/chosen": -2.9747061729431152,
      "logps/rejected": -4.003817558288574,
      "loss": 1.1673,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -5.9494123458862305,
      "rewards/margins": 2.0582237243652344,
      "rewards/rejected": -8.007635116577148,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 78.91890630683983,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -1.794054627418518,
      "logits/rejected": -1.823462724685669,
      "logps/chosen": -2.8728091716766357,
      "logps/rejected": -3.8705430030822754,
      "loss": 1.1418,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -5.7456183433532715,
      "rewards/margins": 1.9954681396484375,
      "rewards/rejected": -7.741086006164551,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 77.02994695164918,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -1.777092695236206,
      "logits/rejected": -1.8050721883773804,
      "logps/chosen": -3.0531764030456543,
      "logps/rejected": -3.9285216331481934,
      "loss": 1.1231,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -6.106352806091309,
      "rewards/margins": 1.7506898641586304,
      "rewards/rejected": -7.857043266296387,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 78.61899751414452,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -1.783837914466858,
      "logits/rejected": -1.7817462682724,
      "logps/chosen": -2.908364772796631,
      "logps/rejected": -3.625439405441284,
      "loss": 1.1821,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -5.816729545593262,
      "rewards/margins": 1.4341497421264648,
      "rewards/rejected": -7.250878810882568,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 94.12554697658324,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -1.835268259048462,
      "logits/rejected": -1.8342323303222656,
      "logps/chosen": -3.0066416263580322,
      "logps/rejected": -3.7469406127929688,
      "loss": 1.209,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -6.0132832527160645,
      "rewards/margins": 1.4805978536605835,
      "rewards/rejected": -7.4938812255859375,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 78.08601822772991,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -1.7435308694839478,
      "logits/rejected": -1.7644704580307007,
      "logps/chosen": -3.0101089477539062,
      "logps/rejected": -3.985473155975342,
      "loss": 1.2211,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -6.0202178955078125,
      "rewards/margins": 1.9507286548614502,
      "rewards/rejected": -7.970946311950684,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 93.57542246833127,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -1.7990074157714844,
      "logits/rejected": -1.8333011865615845,
      "logps/chosen": -2.7810821533203125,
      "logps/rejected": -3.82600474357605,
      "loss": 1.1351,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -5.562164306640625,
      "rewards/margins": 2.08984375,
      "rewards/rejected": -7.6520094871521,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 70.87010486644522,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -1.7927166223526,
      "logits/rejected": -1.7781349420547485,
      "logps/chosen": -2.9593749046325684,
      "logps/rejected": -3.826608180999756,
      "loss": 1.2607,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -5.918749809265137,
      "rewards/margins": 1.7344669103622437,
      "rewards/rejected": -7.653216361999512,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 93.5003333433491,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -1.7764625549316406,
      "logits/rejected": -1.8235756158828735,
      "logps/chosen": -2.923929214477539,
      "logps/rejected": -3.9943060874938965,
      "loss": 1.2149,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -5.847858428955078,
      "rewards/margins": 2.1407532691955566,
      "rewards/rejected": -7.988612174987793,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 58.15978819844721,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -1.8141473531723022,
      "logits/rejected": -1.805326223373413,
      "logps/chosen": -2.911797046661377,
      "logps/rejected": -4.0116353034973145,
      "loss": 1.1457,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -5.823594093322754,
      "rewards/margins": 2.1996755599975586,
      "rewards/rejected": -8.023270606994629,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 66.91626514842031,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -1.7597990036010742,
      "logits/rejected": -1.7440340518951416,
      "logps/chosen": -2.7474265098571777,
      "logps/rejected": -3.8517417907714844,
      "loss": 1.1939,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -5.4948530197143555,
      "rewards/margins": 2.2086310386657715,
      "rewards/rejected": -7.703483581542969,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 95.9824717757533,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -1.8236198425292969,
      "logits/rejected": -1.8562953472137451,
      "logps/chosen": -2.951925754547119,
      "logps/rejected": -3.691092014312744,
      "loss": 1.1781,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -5.903851509094238,
      "rewards/margins": 1.4783318042755127,
      "rewards/rejected": -7.382184028625488,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 1.3635074789918944,
      "train_runtime": 22192.2901,
      "train_samples_per_second": 2.755,
      "train_steps_per_second": 0.021
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}