{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 2130,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004694835680751174,
      "grad_norm": 349.74116855206194,
      "learning_rate": 2.1126760563380282e-08,
      "logits/chosen": -4.25,
      "logits/rejected": -4.112500190734863,
      "logps/chosen": -535.5999755859375,
      "logps/rejected": -629.2000122070312,
      "loss": 2.2736,
      "rewards/accuracies": 0.17812499403953552,
      "rewards/chosen": 0.1143798828125,
      "rewards/margins": -0.07117919623851776,
      "rewards/rejected": 0.18559570610523224,
      "step": 10
    },
    {
      "epoch": 0.009389671361502348,
      "grad_norm": 376.55227149483585,
      "learning_rate": 4.460093896713615e-08,
      "logits/chosen": -4.135937690734863,
      "logits/rejected": -4.090624809265137,
      "logps/chosen": -563.5999755859375,
      "logps/rejected": -632.0,
      "loss": 3.1034,
      "rewards/accuracies": 0.22734375298023224,
      "rewards/chosen": -0.597216784954071,
      "rewards/margins": -0.2764648497104645,
      "rewards/rejected": -0.3194335997104645,
      "step": 20
    },
    {
      "epoch": 0.014084507042253521,
      "grad_norm": 300.8383614545982,
      "learning_rate": 6.807511737089202e-08,
      "logits/chosen": -4.126562595367432,
      "logits/rejected": -3.989062547683716,
      "logps/chosen": -597.2000122070312,
      "logps/rejected": -695.2000122070312,
      "loss": 2.7347,
      "rewards/accuracies": 0.23828125,
      "rewards/chosen": -0.2744140625,
      "rewards/margins": -0.08232422173023224,
      "rewards/rejected": -0.19179686903953552,
      "step": 30
    },
    {
      "epoch": 0.018779342723004695,
      "grad_norm": 350.24179262269337,
      "learning_rate": 9.154929577464789e-08,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.0546875,
      "logps/chosen": -587.4000244140625,
      "logps/rejected": -709.5999755859375,
      "loss": 3.0624,
      "rewards/accuracies": 0.21796874701976776,
      "rewards/chosen": -0.02348632737994194,
      "rewards/margins": -0.41816407442092896,
      "rewards/rejected": 0.3939453065395355,
      "step": 40
    },
    {
      "epoch": 0.023474178403755867,
      "grad_norm": 342.8146294422575,
      "learning_rate": 1.1502347417840374e-07,
      "logits/chosen": -4.123437404632568,
      "logits/rejected": -4.074999809265137,
      "logps/chosen": -564.5999755859375,
      "logps/rejected": -635.0,
      "loss": 2.6875,
      "rewards/accuracies": 0.24609375,
      "rewards/chosen": 0.10722656548023224,
      "rewards/margins": 0.19741210341453552,
      "rewards/rejected": -0.09114990383386612,
      "step": 50
    },
    {
      "epoch": 0.028169014084507043,
      "grad_norm": 339.2044897129424,
      "learning_rate": 1.384976525821596e-07,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.053124904632568,
      "logps/chosen": -579.2000122070312,
      "logps/rejected": -690.4000244140625,
      "loss": 3.0802,
      "rewards/accuracies": 0.24140624701976776,
      "rewards/chosen": -0.22633056342601776,
      "rewards/margins": -0.522045910358429,
      "rewards/rejected": 0.29487305879592896,
      "step": 60
    },
    {
      "epoch": 0.03286384976525822,
      "grad_norm": 344.8739437734243,
      "learning_rate": 1.619718309859155e-07,
      "logits/chosen": -4.1171875,
      "logits/rejected": -4.040625095367432,
      "logps/chosen": -554.4000244140625,
      "logps/rejected": -624.4000244140625,
      "loss": 2.3888,
      "rewards/accuracies": 0.24296875298023224,
      "rewards/chosen": 0.2730468809604645,
      "rewards/margins": 0.3682495057582855,
      "rewards/rejected": -0.09394530951976776,
      "step": 70
    },
    {
      "epoch": 0.03755868544600939,
      "grad_norm": 293.00995731278624,
      "learning_rate": 1.8544600938967138e-07,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.126562595367432,
      "logps/chosen": -566.2000122070312,
      "logps/rejected": -664.0,
      "loss": 2.6898,
      "rewards/accuracies": 0.23749999701976776,
      "rewards/chosen": 0.02299804612994194,
      "rewards/margins": -0.01806640625,
      "rewards/rejected": 0.04155273362994194,
      "step": 80
    },
    {
      "epoch": 0.04225352112676056,
      "grad_norm": 359.38530776576044,
      "learning_rate": 2.089201877934272e-07,
      "logits/chosen": -4.215624809265137,
      "logits/rejected": -4.115624904632568,
      "logps/chosen": -541.0,
      "logps/rejected": -614.7999877929688,
      "loss": 2.678,
      "rewards/accuracies": 0.2593750059604645,
      "rewards/chosen": -0.11171875149011612,
      "rewards/margins": 0.16367188096046448,
      "rewards/rejected": -0.2755371034145355,
      "step": 90
    },
    {
      "epoch": 0.046948356807511735,
      "grad_norm": 347.23080943128696,
      "learning_rate": 2.323943661971831e-07,
      "logits/chosen": -4.146874904632568,
      "logits/rejected": -4.09375,
      "logps/chosen": -586.0,
      "logps/rejected": -617.5999755859375,
      "loss": 2.7036,
      "rewards/accuracies": 0.24531249701976776,
      "rewards/chosen": -0.08134765923023224,
      "rewards/margins": -0.08183594048023224,
      "rewards/rejected": -0.0002929687616415322,
      "step": 100
    },
    {
      "epoch": 0.051643192488262914,
      "grad_norm": 278.29672699724216,
      "learning_rate": 2.5586854460093895e-07,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.050000190734863,
      "logps/chosen": -563.2000122070312,
      "logps/rejected": -668.4000244140625,
      "loss": 2.9664,
      "rewards/accuracies": 0.25078123807907104,
      "rewards/chosen": 0.19809570908546448,
      "rewards/margins": 0.358642578125,
      "rewards/rejected": -0.1611328125,
      "step": 110
    },
    {
      "epoch": 0.056338028169014086,
      "grad_norm": 375.61150910299864,
      "learning_rate": 2.7934272300469483e-07,
      "logits/chosen": -4.198437690734863,
      "logits/rejected": -4.079687595367432,
      "logps/chosen": -574.4000244140625,
      "logps/rejected": -653.5999755859375,
      "loss": 2.7307,
      "rewards/accuracies": 0.25703126192092896,
      "rewards/chosen": -0.01904296875,
      "rewards/margins": 0.09003905951976776,
      "rewards/rejected": -0.108123779296875,
      "step": 120
    },
    {
      "epoch": 0.06103286384976526,
      "grad_norm": 360.0074496903726,
      "learning_rate": 3.0281690140845066e-07,
      "logits/chosen": -4.167187690734863,
      "logits/rejected": -4.074999809265137,
      "logps/chosen": -582.2000122070312,
      "logps/rejected": -674.7999877929688,
      "loss": 2.4376,
      "rewards/accuracies": 0.2640624940395355,
      "rewards/chosen": 0.48076170682907104,
      "rewards/margins": 0.7203613519668579,
      "rewards/rejected": -0.24033203721046448,
      "step": 130
    },
    {
      "epoch": 0.06572769953051644,
      "grad_norm": 419.71980023046956,
      "learning_rate": 3.2629107981220654e-07,
      "logits/chosen": -4.145312309265137,
      "logits/rejected": -4.081250190734863,
      "logps/chosen": -570.4000244140625,
      "logps/rejected": -685.5999755859375,
      "loss": 2.4471,
      "rewards/accuracies": 0.26093751192092896,
      "rewards/chosen": 0.2666015625,
      "rewards/margins": 0.9779297113418579,
      "rewards/rejected": -0.7116943597793579,
      "step": 140
    },
    {
      "epoch": 0.07042253521126761,
      "grad_norm": 423.5847107716145,
      "learning_rate": 3.497652582159624e-07,
      "logits/chosen": -4.206250190734863,
      "logits/rejected": -4.15625,
      "logps/chosen": -569.5999755859375,
      "logps/rejected": -670.7999877929688,
      "loss": 2.4638,
      "rewards/accuracies": 0.2554687559604645,
      "rewards/chosen": 0.37348634004592896,
      "rewards/margins": 0.873828113079071,
      "rewards/rejected": -0.502246081829071,
      "step": 150
    },
    {
      "epoch": 0.07511737089201878,
      "grad_norm": 362.89859282348164,
      "learning_rate": 3.732394366197183e-07,
      "logits/chosen": -4.171875,
      "logits/rejected": -4.057812690734863,
      "logps/chosen": -589.2000122070312,
      "logps/rejected": -674.7999877929688,
      "loss": 2.6716,
      "rewards/accuracies": 0.2601562440395355,
      "rewards/chosen": 0.02895507775247097,
      "rewards/margins": 0.669628918170929,
      "rewards/rejected": -0.6392577886581421,
      "step": 160
    },
    {
      "epoch": 0.07981220657276995,
      "grad_norm": 537.2242571500942,
      "learning_rate": 3.967136150234742e-07,
      "logits/chosen": -4.203125,
      "logits/rejected": -4.0546875,
      "logps/chosen": -540.0,
      "logps/rejected": -656.0,
      "loss": 2.7086,
      "rewards/accuracies": 0.25390625,
      "rewards/chosen": 0.41337889432907104,
      "rewards/margins": 0.10888671875,
      "rewards/rejected": 0.30546873807907104,
      "step": 170
    },
    {
      "epoch": 0.08450704225352113,
      "grad_norm": 331.16907032576614,
      "learning_rate": 4.2018779342723e-07,
      "logits/chosen": -4.160937309265137,
      "logits/rejected": -4.0390625,
      "logps/chosen": -526.5999755859375,
      "logps/rejected": -644.4000244140625,
      "loss": 2.4346,
      "rewards/accuracies": 0.2789062559604645,
      "rewards/chosen": 0.42363280057907104,
      "rewards/margins": 0.559277355670929,
      "rewards/rejected": -0.13422851264476776,
      "step": 180
    },
    {
      "epoch": 0.0892018779342723,
      "grad_norm": 332.45267620461516,
      "learning_rate": 4.436619718309859e-07,
      "logits/chosen": -4.142187595367432,
      "logits/rejected": -4.095312595367432,
      "logps/chosen": -566.2000122070312,
      "logps/rejected": -630.4000244140625,
      "loss": 2.3268,
      "rewards/accuracies": 0.2906250059604645,
      "rewards/chosen": 0.83203125,
      "rewards/margins": 1.407812476158142,
      "rewards/rejected": -0.576489269733429,
      "step": 190
    },
    {
      "epoch": 0.09389671361502347,
      "grad_norm": 257.60750428836997,
      "learning_rate": 4.671361502347418e-07,
      "logits/chosen": -4.153124809265137,
      "logits/rejected": -4.060937404632568,
      "logps/chosen": -551.0,
      "logps/rejected": -641.7999877929688,
      "loss": 2.5818,
      "rewards/accuracies": 0.2984375059604645,
      "rewards/chosen": 0.871289074420929,
      "rewards/margins": 1.178125023841858,
      "rewards/rejected": -0.30723875761032104,
      "step": 200
    },
    {
      "epoch": 0.09859154929577464,
      "grad_norm": 281.29526176415175,
      "learning_rate": 4.906103286384976e-07,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.0546875,
      "logps/chosen": -560.7999877929688,
      "logps/rejected": -674.7999877929688,
      "loss": 2.62,
      "rewards/accuracies": 0.28593748807907104,
      "rewards/chosen": 0.8622070550918579,
      "rewards/margins": 0.7486327886581421,
      "rewards/rejected": 0.11201171576976776,
      "step": 210
    },
    {
      "epoch": 0.10328638497652583,
      "grad_norm": 308.153426987116,
      "learning_rate": 4.984350547730829e-07,
      "logits/chosen": -4.253125190734863,
      "logits/rejected": -4.131249904632568,
      "logps/chosen": -574.4000244140625,
      "logps/rejected": -650.4000244140625,
      "loss": 2.6838,
      "rewards/accuracies": 0.2945312559604645,
      "rewards/chosen": 0.77734375,
      "rewards/margins": 0.964306652545929,
      "rewards/rejected": -0.18699951469898224,
      "step": 220
    },
    {
      "epoch": 0.107981220657277,
      "grad_norm": 548.8366508731452,
      "learning_rate": 4.958268127282212e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.098437309265137,
      "logps/chosen": -563.0,
      "logps/rejected": -678.7999877929688,
      "loss": 2.3709,
      "rewards/accuracies": 0.31953126192092896,
      "rewards/chosen": 1.235449194908142,
      "rewards/margins": 1.4113280773162842,
      "rewards/rejected": -0.17470702528953552,
      "step": 230
    },
    {
      "epoch": 0.11267605633802817,
      "grad_norm": 323.1601733945601,
      "learning_rate": 4.932185706833594e-07,
      "logits/chosen": -4.165625095367432,
      "logits/rejected": -4.106249809265137,
      "logps/chosen": -518.4000244140625,
      "logps/rejected": -564.5999755859375,
      "loss": 2.1705,
      "rewards/accuracies": 0.3296875059604645,
      "rewards/chosen": 1.5910155773162842,
      "rewards/margins": 2.05267333984375,
      "rewards/rejected": -0.45966798067092896,
      "step": 240
    },
    {
      "epoch": 0.11737089201877934,
      "grad_norm": 324.84319761474575,
      "learning_rate": 4.906103286384976e-07,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.137499809265137,
      "logps/chosen": -562.7999877929688,
      "logps/rejected": -685.0,
      "loss": 2.6149,
      "rewards/accuracies": 0.3296875059604645,
      "rewards/chosen": 0.911999523639679,
      "rewards/margins": 1.7622559070587158,
      "rewards/rejected": -0.8514648675918579,
      "step": 250
    },
    {
      "epoch": 0.12206572769953052,
      "grad_norm": 363.5336462630036,
      "learning_rate": 4.880020865936358e-07,
      "logits/chosen": -4.146874904632568,
      "logits/rejected": -4.084374904632568,
      "logps/chosen": -530.5999755859375,
      "logps/rejected": -631.7999877929688,
      "loss": 2.274,
      "rewards/accuracies": 0.3671875,
      "rewards/chosen": 2.0082030296325684,
      "rewards/margins": 3.0038084983825684,
      "rewards/rejected": -0.9931640625,
      "step": 260
    },
    {
      "epoch": 0.1267605633802817,
      "grad_norm": 518.6884921778877,
      "learning_rate": 4.853938445487741e-07,
      "logits/chosen": -4.087500095367432,
      "logits/rejected": -4.048437595367432,
      "logps/chosen": -562.7999877929688,
      "logps/rejected": -662.7999877929688,
      "loss": 2.6675,
      "rewards/accuracies": 0.35078126192092896,
      "rewards/chosen": 1.9542968273162842,
      "rewards/margins": 2.491406202316284,
      "rewards/rejected": -0.5386718511581421,
      "step": 270
    },
    {
      "epoch": 0.13145539906103287,
      "grad_norm": 293.98547761001106,
      "learning_rate": 4.827856025039123e-07,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.120312690734863,
      "logps/chosen": -559.5999755859375,
      "logps/rejected": -668.0,
      "loss": 2.4366,
      "rewards/accuracies": 0.37890625,
      "rewards/chosen": 1.7419922351837158,
      "rewards/margins": 2.5335936546325684,
      "rewards/rejected": -0.7909911870956421,
      "step": 280
    },
    {
      "epoch": 0.13615023474178403,
      "grad_norm": 297.7182843714146,
      "learning_rate": 4.801773604590506e-07,
      "logits/chosen": -4.159375190734863,
      "logits/rejected": -4.084374904632568,
      "logps/chosen": -601.7999877929688,
      "logps/rejected": -663.5999755859375,
      "loss": 2.6713,
      "rewards/accuracies": 0.3890624940395355,
      "rewards/chosen": 1.753515601158142,
      "rewards/margins": 3.429882764816284,
      "rewards/rejected": -1.6768066883087158,
      "step": 290
    },
    {
      "epoch": 0.14084507042253522,
      "grad_norm": 282.45414453616195,
      "learning_rate": 4.775691184141888e-07,
      "logits/chosen": -4.157812595367432,
      "logits/rejected": -4.096875190734863,
      "logps/chosen": -579.5999755859375,
      "logps/rejected": -658.7999877929688,
      "loss": 2.2817,
      "rewards/accuracies": 0.3804687559604645,
      "rewards/chosen": 2.3656249046325684,
      "rewards/margins": 3.7265625,
      "rewards/rejected": -1.356054663658142,
      "step": 300
    },
    {
      "epoch": 0.14553990610328638,
      "grad_norm": 332.6110171070324,
      "learning_rate": 4.749608763693271e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.103125095367432,
      "logps/chosen": -554.0,
      "logps/rejected": -646.7999877929688,
      "loss": 2.4298,
      "rewards/accuracies": 0.3804687559604645,
      "rewards/chosen": 1.9066650867462158,
      "rewards/margins": 3.2503905296325684,
      "rewards/rejected": -1.341577172279358,
      "step": 310
    },
    {
      "epoch": 0.15023474178403756,
      "grad_norm": 302.2874396052593,
      "learning_rate": 4.7235263432446533e-07,
      "logits/chosen": -4.178124904632568,
      "logits/rejected": -4.078125,
      "logps/chosen": -555.5999755859375,
      "logps/rejected": -637.2000122070312,
      "loss": 2.5541,
      "rewards/accuracies": 0.3851562440395355,
      "rewards/chosen": 1.837890625,
      "rewards/margins": 3.7671875953674316,
      "rewards/rejected": -1.9250977039337158,
      "step": 320
    },
    {
      "epoch": 0.15492957746478872,
      "grad_norm": 404.9891924611719,
      "learning_rate": 4.6974439227960353e-07,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.048437595367432,
      "logps/chosen": -574.4000244140625,
      "logps/rejected": -669.5999755859375,
      "loss": 2.6687,
      "rewards/accuracies": 0.390625,
      "rewards/chosen": 2.23828125,
      "rewards/margins": 3.703125,
      "rewards/rejected": -1.466406226158142,
      "step": 330
    },
    {
      "epoch": 0.1596244131455399,
      "grad_norm": 288.37085576554466,
      "learning_rate": 4.671361502347418e-07,
      "logits/chosen": -4.229687690734863,
      "logits/rejected": -4.139062404632568,
      "logps/chosen": -545.7999877929688,
      "logps/rejected": -595.4000244140625,
      "loss": 2.1681,
      "rewards/accuracies": 0.41328126192092896,
      "rewards/chosen": 2.313281297683716,
      "rewards/margins": 3.8968749046325684,
      "rewards/rejected": -1.5778319835662842,
      "step": 340
    },
    {
      "epoch": 0.1643192488262911,
      "grad_norm": 318.64337747915073,
      "learning_rate": 4.6452790818988004e-07,
      "logits/chosen": -4.21875,
      "logits/rejected": -4.162499904632568,
      "logps/chosen": -581.5999755859375,
      "logps/rejected": -670.0,
      "loss": 2.3304,
      "rewards/accuracies": 0.40156251192092896,
      "rewards/chosen": 2.6039061546325684,
      "rewards/margins": 5.3359375,
      "rewards/rejected": -2.7320313453674316,
      "step": 350
    },
    {
      "epoch": 0.16901408450704225,
      "grad_norm": 267.92746079560277,
      "learning_rate": 4.6191966614501824e-07,
      "logits/chosen": -4.1796875,
      "logits/rejected": -4.082812309265137,
      "logps/chosen": -550.2000122070312,
      "logps/rejected": -637.2000122070312,
      "loss": 2.6561,
      "rewards/accuracies": 0.3921875059604645,
      "rewards/chosen": 2.091796875,
      "rewards/margins": 4.114062309265137,
      "rewards/rejected": -2.0250244140625,
      "step": 360
    },
    {
      "epoch": 0.17370892018779344,
      "grad_norm": 296.5721114007629,
      "learning_rate": 4.593114241001565e-07,
      "logits/chosen": -4.1328125,
      "logits/rejected": -4.067187309265137,
      "logps/chosen": -592.7999877929688,
      "logps/rejected": -667.2000122070312,
      "loss": 3.0078,
      "rewards/accuracies": 0.44453126192092896,
      "rewards/chosen": 2.4898438453674316,
      "rewards/margins": 4.842968940734863,
      "rewards/rejected": -2.357421875,
      "step": 370
    },
    {
      "epoch": 0.1784037558685446,
      "grad_norm": 284.9647809945335,
      "learning_rate": 4.5670318205529474e-07,
      "logits/chosen": -4.15625,
      "logits/rejected": -4.042187690734863,
      "logps/chosen": -561.0,
      "logps/rejected": -660.7999877929688,
      "loss": 2.4469,
      "rewards/accuracies": 0.40312498807907104,
      "rewards/chosen": 2.018749952316284,
      "rewards/margins": 3.625,
      "rewards/rejected": -1.604272484779358,
      "step": 380
    },
    {
      "epoch": 0.18309859154929578,
      "grad_norm": 244.9583744550648,
      "learning_rate": 4.54094940010433e-07,
      "logits/chosen": -4.165625095367432,
      "logits/rejected": -4.065625190734863,
      "logps/chosen": -570.7999877929688,
      "logps/rejected": -675.7999877929688,
      "loss": 2.1703,
      "rewards/accuracies": 0.4164062440395355,
      "rewards/chosen": 2.6109375953674316,
      "rewards/margins": 5.390625,
      "rewards/rejected": -2.77734375,
      "step": 390
    },
    {
      "epoch": 0.18779342723004694,
      "grad_norm": 253.4802914868594,
      "learning_rate": 4.514866979655712e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.053124904632568,
      "logps/chosen": -561.5999755859375,
      "logps/rejected": -642.4000244140625,
      "loss": 2.6224,
      "rewards/accuracies": 0.43437498807907104,
      "rewards/chosen": 2.860156297683716,
      "rewards/margins": 5.653124809265137,
      "rewards/rejected": -2.792187452316284,
      "step": 400
    },
    {
      "epoch": 0.19248826291079812,
      "grad_norm": 263.3797100162804,
      "learning_rate": 4.4887845592070945e-07,
      "logits/chosen": -4.126562595367432,
      "logits/rejected": -4.021874904632568,
      "logps/chosen": -550.2000122070312,
      "logps/rejected": -686.7999877929688,
      "loss": 2.5782,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": 1.927734375,
      "rewards/margins": 5.271874904632568,
      "rewards/rejected": -3.3382811546325684,
      "step": 410
    },
    {
      "epoch": 0.19718309859154928,
      "grad_norm": 276.9425213989649,
      "learning_rate": 4.462702138758477e-07,
      "logits/chosen": -4.131249904632568,
      "logits/rejected": -4.050000190734863,
      "logps/chosen": -587.7999877929688,
      "logps/rejected": -651.5999755859375,
      "loss": 2.6343,
      "rewards/accuracies": 0.4296875,
      "rewards/chosen": 1.989843726158142,
      "rewards/margins": 5.115624904632568,
      "rewards/rejected": -3.1265625953674316,
      "step": 420
    },
    {
      "epoch": 0.20187793427230047,
      "grad_norm": 318.8701653678301,
      "learning_rate": 4.436619718309859e-07,
      "logits/chosen": -4.115624904632568,
      "logits/rejected": -3.9749999046325684,
      "logps/chosen": -571.5999755859375,
      "logps/rejected": -694.4000244140625,
      "loss": 2.8542,
      "rewards/accuracies": 0.3929687440395355,
      "rewards/chosen": 1.5437500476837158,
      "rewards/margins": 5.095507621765137,
      "rewards/rejected": -3.548046827316284,
      "step": 430
    },
    {
      "epoch": 0.20657276995305165,
      "grad_norm": 266.8593158918352,
      "learning_rate": 4.4105372978612415e-07,
      "logits/chosen": -4.221875190734863,
      "logits/rejected": -4.103125095367432,
      "logps/chosen": -587.2000122070312,
      "logps/rejected": -678.4000244140625,
      "loss": 2.751,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": 2.030078172683716,
      "rewards/margins": 5.284375190734863,
      "rewards/rejected": -3.2515625953674316,
      "step": 440
    },
    {
      "epoch": 0.2112676056338028,
      "grad_norm": 292.8532836816308,
      "learning_rate": 4.384454877412624e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.060937404632568,
      "logps/chosen": -572.2000122070312,
      "logps/rejected": -743.2000122070312,
      "loss": 3.1477,
      "rewards/accuracies": 0.42656248807907104,
      "rewards/chosen": 2.553906202316284,
      "rewards/margins": 5.631249904632568,
      "rewards/rejected": -3.0757813453674316,
      "step": 450
    },
    {
      "epoch": 0.215962441314554,
      "grad_norm": 256.3005652537314,
      "learning_rate": 4.358372456964006e-07,
      "logits/chosen": -4.140625,
      "logits/rejected": -4.095312595367432,
      "logps/chosen": -564.0,
      "logps/rejected": -662.7999877929688,
      "loss": 2.4115,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": 2.8539061546325684,
      "rewards/margins": 5.910937309265137,
      "rewards/rejected": -3.0570311546325684,
      "step": 460
    },
    {
      "epoch": 0.22065727699530516,
      "grad_norm": 288.136171582801,
      "learning_rate": 4.3322900365153886e-07,
      "logits/chosen": -4.237500190734863,
      "logits/rejected": -4.092187404632568,
      "logps/chosen": -602.7999877929688,
      "logps/rejected": -665.5999755859375,
      "loss": 3.2443,
      "rewards/accuracies": 0.43828123807907104,
      "rewards/chosen": 1.791528344154358,
      "rewards/margins": 4.225390434265137,
      "rewards/rejected": -2.439453125,
      "step": 470
    },
    {
      "epoch": 0.22535211267605634,
      "grad_norm": 250.77020639486426,
      "learning_rate": 4.306207616066771e-07,
      "logits/chosen": -4.215624809265137,
      "logits/rejected": -4.1015625,
      "logps/chosen": -551.5999755859375,
      "logps/rejected": -638.0,
      "loss": 2.6011,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": 2.882031202316284,
      "rewards/margins": 4.392968654632568,
      "rewards/rejected": -1.5114257335662842,
      "step": 480
    },
    {
      "epoch": 0.2300469483568075,
      "grad_norm": 244.34973254893228,
      "learning_rate": 4.280125195618153e-07,
      "logits/chosen": -4.157812595367432,
      "logits/rejected": -4.017187595367432,
      "logps/chosen": -562.7999877929688,
      "logps/rejected": -672.2000122070312,
      "loss": 2.6576,
      "rewards/accuracies": 0.4117187559604645,
      "rewards/chosen": 2.296875,
      "rewards/margins": 6.037499904632568,
      "rewards/rejected": -3.7393431663513184,
      "step": 490
    },
    {
      "epoch": 0.2347417840375587,
      "grad_norm": 245.7589655838516,
      "learning_rate": 4.2540427751695357e-07,
      "logits/chosen": -4.165625095367432,
      "logits/rejected": -4.073437690734863,
      "logps/chosen": -569.2000122070312,
      "logps/rejected": -641.7999877929688,
      "loss": 3.3399,
      "rewards/accuracies": 0.4164062440395355,
      "rewards/chosen": 2.551562547683716,
      "rewards/margins": 4.9765625,
      "rewards/rejected": -2.4300780296325684,
      "step": 500
    },
    {
      "epoch": 0.23943661971830985,
      "grad_norm": 273.17619762972186,
      "learning_rate": 4.227960354720918e-07,
      "logits/chosen": -4.232812404632568,
      "logits/rejected": -4.134375095367432,
      "logps/chosen": -556.4000244140625,
      "logps/rejected": -680.4000244140625,
      "loss": 2.3361,
      "rewards/accuracies": 0.4320312440395355,
      "rewards/chosen": 3.112499952316284,
      "rewards/margins": 5.8671875,
      "rewards/rejected": -2.7515625953674316,
      "step": 510
    },
    {
      "epoch": 0.24413145539906103,
      "grad_norm": 263.0614856896538,
      "learning_rate": 4.2018779342723e-07,
      "logits/chosen": -4.182812690734863,
      "logits/rejected": -4.110937595367432,
      "logps/chosen": -573.2000122070312,
      "logps/rejected": -632.4000244140625,
      "loss": 2.7766,
      "rewards/accuracies": 0.4351562559604645,
      "rewards/chosen": 2.3656249046325684,
      "rewards/margins": 5.109375,
      "rewards/rejected": -2.745898485183716,
      "step": 520
    },
    {
      "epoch": 0.24882629107981222,
      "grad_norm": 302.5738298315578,
      "learning_rate": 4.1757955138236827e-07,
      "logits/chosen": -4.234375,
      "logits/rejected": -4.090624809265137,
      "logps/chosen": -555.5999755859375,
      "logps/rejected": -667.2000122070312,
      "loss": 2.8871,
      "rewards/accuracies": 0.47187501192092896,
      "rewards/chosen": 2.374218702316284,
      "rewards/margins": 5.859375,
      "rewards/rejected": -3.4857420921325684,
      "step": 530
    },
    {
      "epoch": 0.2535211267605634,
      "grad_norm": 352.80345813139655,
      "learning_rate": 4.149713093375065e-07,
      "logits/chosen": -4.203125,
      "logits/rejected": -4.1015625,
      "logps/chosen": -556.5999755859375,
      "logps/rejected": -612.2000122070312,
      "loss": 2.5234,
      "rewards/accuracies": 0.46015626192092896,
      "rewards/chosen": 2.77734375,
      "rewards/margins": 5.982812404632568,
      "rewards/rejected": -3.2035155296325684,
      "step": 540
    },
    {
      "epoch": 0.25821596244131456,
      "grad_norm": 260.6895436253073,
      "learning_rate": 4.123630672926447e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.110937595367432,
      "logps/chosen": -545.4000244140625,
      "logps/rejected": -660.5999755859375,
      "loss": 2.3198,
      "rewards/accuracies": 0.42656248807907104,
      "rewards/chosen": 2.5062499046325684,
      "rewards/margins": 6.728125095367432,
      "rewards/rejected": -4.215624809265137,
      "step": 550
    },
    {
      "epoch": 0.26291079812206575,
      "grad_norm": 274.729220638694,
      "learning_rate": 4.09754825247783e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.096875190734863,
      "logps/chosen": -565.5999755859375,
      "logps/rejected": -642.7999877929688,
      "loss": 2.6591,
      "rewards/accuracies": 0.43828123807907104,
      "rewards/chosen": 2.645312547683716,
      "rewards/margins": 6.806250095367432,
      "rewards/rejected": -4.172656059265137,
      "step": 560
    },
    {
      "epoch": 0.2676056338028169,
      "grad_norm": 247.46078526722945,
      "learning_rate": 4.0714658320292123e-07,
      "logits/chosen": -4.214062690734863,
      "logits/rejected": -4.089062690734863,
      "logps/chosen": -564.4000244140625,
      "logps/rejected": -660.0,
      "loss": 2.694,
      "rewards/accuracies": 0.44140625,
      "rewards/chosen": 2.1488280296325684,
      "rewards/margins": 6.8359375,
      "rewards/rejected": -4.680468559265137,
      "step": 570
    },
    {
      "epoch": 0.27230046948356806,
      "grad_norm": 306.74341657476623,
      "learning_rate": 4.045383411580595e-07,
      "logits/chosen": -4.224999904632568,
      "logits/rejected": -4.137499809265137,
      "logps/chosen": -544.5999755859375,
      "logps/rejected": -656.4000244140625,
      "loss": 2.9487,
      "rewards/accuracies": 0.4234375059604645,
      "rewards/chosen": 1.7294921875,
      "rewards/margins": 5.262499809265137,
      "rewards/rejected": -3.5355467796325684,
      "step": 580
    },
    {
      "epoch": 0.27699530516431925,
      "grad_norm": 318.14800096014113,
      "learning_rate": 4.019300991131977e-07,
      "logits/chosen": -4.159375190734863,
      "logits/rejected": -4.051562309265137,
      "logps/chosen": -547.4000244140625,
      "logps/rejected": -647.2000122070312,
      "loss": 2.5842,
      "rewards/accuracies": 0.4554687440395355,
      "rewards/chosen": 2.6656250953674316,
      "rewards/margins": 6.821875095367432,
      "rewards/rejected": -4.153515815734863,
      "step": 590
    },
    {
      "epoch": 0.28169014084507044,
      "grad_norm": 229.49976631713287,
      "learning_rate": 3.9932185706833594e-07,
      "logits/chosen": -4.21875,
      "logits/rejected": -4.112500190734863,
      "logps/chosen": -572.7999877929688,
      "logps/rejected": -647.5999755859375,
      "loss": 2.9184,
      "rewards/accuracies": 0.4546875059604645,
      "rewards/chosen": 2.377734422683716,
      "rewards/margins": 7.287499904632568,
      "rewards/rejected": -4.907812595367432,
      "step": 600
    },
    {
      "epoch": 0.2863849765258216,
      "grad_norm": 249.01193980392725,
      "learning_rate": 3.967136150234742e-07,
      "logits/chosen": -4.168749809265137,
      "logits/rejected": -4.095312595367432,
      "logps/chosen": -574.7999877929688,
      "logps/rejected": -663.0,
      "loss": 2.5218,
      "rewards/accuracies": 0.45390623807907104,
      "rewards/chosen": 2.596874952316284,
      "rewards/margins": 6.484375,
      "rewards/rejected": -3.889843702316284,
      "step": 610
    },
    {
      "epoch": 0.29107981220657275,
      "grad_norm": 225.71400979655567,
      "learning_rate": 3.941053729786124e-07,
      "logits/chosen": -4.175000190734863,
      "logits/rejected": -4.082812309265137,
      "logps/chosen": -578.0,
      "logps/rejected": -657.5999755859375,
      "loss": 2.5098,
      "rewards/accuracies": 0.4593749940395355,
      "rewards/chosen": 2.515625,
      "rewards/margins": 7.762499809265137,
      "rewards/rejected": -5.243750095367432,
      "step": 620
    },
    {
      "epoch": 0.29577464788732394,
      "grad_norm": 279.9672078725096,
      "learning_rate": 3.9149713093375064e-07,
      "logits/chosen": -4.114062309265137,
      "logits/rejected": -3.9984374046325684,
      "logps/chosen": -568.7999877929688,
      "logps/rejected": -678.0,
      "loss": 2.6434,
      "rewards/accuracies": 0.46562498807907104,
      "rewards/chosen": 2.7125000953674316,
      "rewards/margins": 7.846875190734863,
      "rewards/rejected": -5.134765625,
      "step": 630
    },
    {
      "epoch": 0.3004694835680751,
      "grad_norm": 266.7534618210619,
      "learning_rate": 3.888888888888889e-07,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.035937309265137,
      "logps/chosen": -558.5999755859375,
      "logps/rejected": -691.5999755859375,
      "loss": 2.1453,
      "rewards/accuracies": 0.47187501192092896,
      "rewards/chosen": 3.3109374046325684,
      "rewards/margins": 8.459375381469727,
      "rewards/rejected": -5.1484375,
      "step": 640
    },
    {
      "epoch": 0.3051643192488263,
      "grad_norm": 236.94805089034128,
      "learning_rate": 3.862806468440271e-07,
      "logits/chosen": -4.28125,
      "logits/rejected": -4.196875095367432,
      "logps/chosen": -516.4000244140625,
      "logps/rejected": -614.0,
      "loss": 2.1915,
      "rewards/accuracies": 0.4828124940395355,
      "rewards/chosen": 2.585156202316284,
      "rewards/margins": 6.989062309265137,
      "rewards/rejected": -4.405468940734863,
      "step": 650
    },
    {
      "epoch": 0.30985915492957744,
      "grad_norm": 305.31826133508156,
      "learning_rate": 3.8367240479916535e-07,
      "logits/chosen": -4.2109375,
      "logits/rejected": -4.159375190734863,
      "logps/chosen": -550.5999755859375,
      "logps/rejected": -619.7999877929688,
      "loss": 2.6323,
      "rewards/accuracies": 0.46171873807907104,
      "rewards/chosen": 2.649218797683716,
      "rewards/margins": 6.571875095367432,
      "rewards/rejected": -3.919921875,
      "step": 660
    },
    {
      "epoch": 0.3145539906103286,
      "grad_norm": 263.7170782636469,
      "learning_rate": 3.810641627543036e-07,
      "logits/chosen": -4.203125,
      "logits/rejected": -4.110937595367432,
      "logps/chosen": -567.2000122070312,
      "logps/rejected": -653.0,
      "loss": 2.9518,
      "rewards/accuracies": 0.46171873807907104,
      "rewards/chosen": 2.742968797683716,
      "rewards/margins": 6.548437595367432,
      "rewards/rejected": -3.8017578125,
      "step": 670
    },
    {
      "epoch": 0.3192488262910798,
      "grad_norm": 288.4732382762268,
      "learning_rate": 3.784559207094418e-07,
      "logits/chosen": -4.175000190734863,
      "logits/rejected": -4.0859375,
      "logps/chosen": -585.2000122070312,
      "logps/rejected": -644.0,
      "loss": 2.6914,
      "rewards/accuracies": 0.4468750059604645,
      "rewards/chosen": 2.3148436546325684,
      "rewards/margins": 6.192187309265137,
      "rewards/rejected": -3.8734374046325684,
      "step": 680
    },
    {
      "epoch": 0.323943661971831,
      "grad_norm": 217.38072291200496,
      "learning_rate": 3.7584767866458005e-07,
      "logits/chosen": -4.159375190734863,
      "logits/rejected": -4.060937404632568,
      "logps/chosen": -569.4000244140625,
      "logps/rejected": -616.4000244140625,
      "loss": 2.131,
      "rewards/accuracies": 0.46953123807907104,
      "rewards/chosen": 2.615234375,
      "rewards/margins": 7.329687595367432,
      "rewards/rejected": -4.721875190734863,
      "step": 690
    },
    {
      "epoch": 0.3286384976525822,
      "grad_norm": 653.7281081824505,
      "learning_rate": 3.732394366197183e-07,
      "logits/chosen": -4.224999904632568,
      "logits/rejected": -4.128125190734863,
      "logps/chosen": -598.2000122070312,
      "logps/rejected": -633.2000122070312,
      "loss": 3.6844,
      "rewards/accuracies": 0.44843751192092896,
      "rewards/chosen": 1.9679687023162842,
      "rewards/margins": 5.3046875,
      "rewards/rejected": -3.3343749046325684,
      "step": 700
    },
    {
      "epoch": 0.3333333333333333,
      "grad_norm": 297.13616489844844,
      "learning_rate": 3.706311945748565e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.0859375,
      "logps/chosen": -614.0,
      "logps/rejected": -640.0,
      "loss": 2.9492,
      "rewards/accuracies": 0.4632812440395355,
      "rewards/chosen": 2.1050782203674316,
      "rewards/margins": 6.079687595367432,
      "rewards/rejected": -3.96875,
      "step": 710
    },
    {
      "epoch": 0.3380281690140845,
      "grad_norm": 306.0595029923629,
      "learning_rate": 3.6802295252999476e-07,
      "logits/chosen": -4.159375190734863,
      "logits/rejected": -4.045312404632568,
      "logps/chosen": -581.4000244140625,
      "logps/rejected": -682.5999755859375,
      "loss": 2.7086,
      "rewards/accuracies": 0.47968751192092896,
      "rewards/chosen": 2.446093797683716,
      "rewards/margins": 7.996874809265137,
      "rewards/rejected": -5.534375190734863,
      "step": 720
    },
    {
      "epoch": 0.3427230046948357,
      "grad_norm": 358.5106606361576,
      "learning_rate": 3.65414710485133e-07,
      "logits/chosen": -4.175000190734863,
      "logits/rejected": -4.090624809265137,
      "logps/chosen": -576.2000122070312,
      "logps/rejected": -626.4000244140625,
      "loss": 3.0882,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 1.953582763671875,
      "rewards/margins": 6.353125095367432,
      "rewards/rejected": -4.404687404632568,
      "step": 730
    },
    {
      "epoch": 0.3474178403755869,
      "grad_norm": 236.0280317249739,
      "learning_rate": 3.6280646844027127e-07,
      "logits/chosen": -4.140625,
      "logits/rejected": -4.065625190734863,
      "logps/chosen": -554.7999877929688,
      "logps/rejected": -696.7999877929688,
      "loss": 2.7726,
      "rewards/accuracies": 0.47265625,
      "rewards/chosen": 2.4244141578674316,
      "rewards/margins": 8.4375,
      "rewards/rejected": -6.0234375,
      "step": 740
    },
    {
      "epoch": 0.352112676056338,
      "grad_norm": 246.58963908566164,
      "learning_rate": 3.6019822639540947e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.084374904632568,
      "logps/chosen": -589.4000244140625,
      "logps/rejected": -622.7999877929688,
      "loss": 2.3574,
      "rewards/accuracies": 0.4507812559604645,
      "rewards/chosen": 2.913281202316284,
      "rewards/margins": 6.365624904632568,
      "rewards/rejected": -3.453906297683716,
      "step": 750
    },
    {
      "epoch": 0.3568075117370892,
      "grad_norm": 289.90626390753835,
      "learning_rate": 3.575899843505477e-07,
      "logits/chosen": -4.168749809265137,
      "logits/rejected": -4.099999904632568,
      "logps/chosen": -562.2000122070312,
      "logps/rejected": -609.0,
      "loss": 3.0627,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 2.616406202316284,
      "rewards/margins": 6.787499904632568,
      "rewards/rejected": -4.173437595367432,
      "step": 760
    },
    {
      "epoch": 0.3615023474178404,
      "grad_norm": 266.4393683231025,
      "learning_rate": 3.5498174230568597e-07,
      "logits/chosen": -4.110937595367432,
      "logits/rejected": -4.034375190734863,
      "logps/chosen": -561.0,
      "logps/rejected": -649.5999755859375,
      "loss": 2.4062,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 2.5062499046325684,
      "rewards/margins": 7.526562690734863,
      "rewards/rejected": -5.020312309265137,
      "step": 770
    },
    {
      "epoch": 0.36619718309859156,
      "grad_norm": 358.3088872337145,
      "learning_rate": 3.5237350026082417e-07,
      "logits/chosen": -4.140625,
      "logits/rejected": -4.050000190734863,
      "logps/chosen": -568.4000244140625,
      "logps/rejected": -603.2000122070312,
      "loss": 2.9134,
      "rewards/accuracies": 0.4742187559604645,
      "rewards/chosen": 2.537890672683716,
      "rewards/margins": 6.315625190734863,
      "rewards/rejected": -3.782031297683716,
      "step": 780
    },
    {
      "epoch": 0.37089201877934275,
      "grad_norm": 269.79098156201337,
      "learning_rate": 3.497652582159624e-07,
      "logits/chosen": -4.146874904632568,
      "logits/rejected": -4.051562309265137,
      "logps/chosen": -572.7999877929688,
      "logps/rejected": -672.7999877929688,
      "loss": 3.0026,
      "rewards/accuracies": 0.47578126192092896,
      "rewards/chosen": 2.8570313453674316,
      "rewards/margins": 7.462500095367432,
      "rewards/rejected": -4.6015625,
      "step": 790
    },
    {
      "epoch": 0.3755868544600939,
      "grad_norm": 291.7111715601876,
      "learning_rate": 3.471570161711007e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.0546875,
      "logps/chosen": -600.0,
      "logps/rejected": -681.7999877929688,
      "loss": 3.0534,
      "rewards/accuracies": 0.4859375059604645,
      "rewards/chosen": 2.608203172683716,
      "rewards/margins": 7.348437309265137,
      "rewards/rejected": -4.737500190734863,
      "step": 800
    },
    {
      "epoch": 0.38028169014084506,
      "grad_norm": 363.3048347931511,
      "learning_rate": 3.445487741262389e-07,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.115624904632568,
      "logps/chosen": -550.4000244140625,
      "logps/rejected": -640.4000244140625,
      "loss": 3.0931,
      "rewards/accuracies": 0.4945312440395355,
      "rewards/chosen": 2.6695313453674316,
      "rewards/margins": 7.228125095367432,
      "rewards/rejected": -4.556250095367432,
      "step": 810
    },
    {
      "epoch": 0.38497652582159625,
      "grad_norm": 275.9406603434156,
      "learning_rate": 3.4194053208137713e-07,
      "logits/chosen": -4.167187690734863,
      "logits/rejected": -4.089062690734863,
      "logps/chosen": -561.2000122070312,
      "logps/rejected": -618.0,
      "loss": 2.7053,
      "rewards/accuracies": 0.48515623807907104,
      "rewards/chosen": 2.9124999046325684,
      "rewards/margins": 6.731249809265137,
      "rewards/rejected": -3.8148436546325684,
      "step": 820
    },
    {
      "epoch": 0.38967136150234744,
      "grad_norm": 268.47634850337175,
      "learning_rate": 3.393322900365154e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.025000095367432,
      "logps/chosen": -531.0,
      "logps/rejected": -618.5999755859375,
      "loss": 2.6724,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 1.915624976158142,
      "rewards/margins": 6.973437309265137,
      "rewards/rejected": -5.057812690734863,
      "step": 830
    },
    {
      "epoch": 0.39436619718309857,
      "grad_norm": 245.17621324219382,
      "learning_rate": 3.367240479916536e-07,
      "logits/chosen": -4.184374809265137,
      "logits/rejected": -4.059374809265137,
      "logps/chosen": -596.4000244140625,
      "logps/rejected": -670.4000244140625,
      "loss": 3.5967,
      "rewards/accuracies": 0.49531251192092896,
      "rewards/chosen": 2.061877489089966,
      "rewards/margins": 7.34375,
      "rewards/rejected": -5.278124809265137,
      "step": 840
    },
    {
      "epoch": 0.39906103286384975,
      "grad_norm": 237.29929143033954,
      "learning_rate": 3.3411580594679184e-07,
      "logits/chosen": -4.221875190734863,
      "logits/rejected": -4.098437309265137,
      "logps/chosen": -552.5999755859375,
      "logps/rejected": -644.0,
      "loss": 2.5816,
      "rewards/accuracies": 0.46484375,
      "rewards/chosen": 2.669726610183716,
      "rewards/margins": 7.145312309265137,
      "rewards/rejected": -4.478906154632568,
      "step": 850
    },
    {
      "epoch": 0.40375586854460094,
      "grad_norm": 279.3549218574099,
      "learning_rate": 3.315075639019301e-07,
      "logits/chosen": -4.170312404632568,
      "logits/rejected": -4.106249809265137,
      "logps/chosen": -545.2000122070312,
      "logps/rejected": -643.5999755859375,
      "loss": 2.5559,
      "rewards/accuracies": 0.48515623807907104,
      "rewards/chosen": 2.764453172683716,
      "rewards/margins": 8.717187881469727,
      "rewards/rejected": -5.959765434265137,
      "step": 860
    },
    {
      "epoch": 0.4084507042253521,
      "grad_norm": 308.47860266588174,
      "learning_rate": 3.288993218570683e-07,
      "logits/chosen": -4.1953125,
      "logits/rejected": -4.067187309265137,
      "logps/chosen": -552.4000244140625,
      "logps/rejected": -623.2000122070312,
      "loss": 2.7292,
      "rewards/accuracies": 0.4789062440395355,
      "rewards/chosen": 2.921875,
      "rewards/margins": 6.7890625,
      "rewards/rejected": -3.872265577316284,
      "step": 870
    },
    {
      "epoch": 0.4131455399061033,
      "grad_norm": 206.97891373240537,
      "learning_rate": 3.2629107981220654e-07,
      "logits/chosen": -4.182812690734863,
      "logits/rejected": -4.040625095367432,
      "logps/chosen": -576.7999877929688,
      "logps/rejected": -646.5999755859375,
      "loss": 2.8882,
      "rewards/accuracies": 0.47968751192092896,
      "rewards/chosen": 2.184375047683716,
      "rewards/margins": 7.6875,
      "rewards/rejected": -5.503125190734863,
      "step": 880
    },
    {
      "epoch": 0.41784037558685444,
      "grad_norm": 298.5862678350748,
      "learning_rate": 3.236828377673448e-07,
      "logits/chosen": -4.121874809265137,
      "logits/rejected": -4.0859375,
      "logps/chosen": -600.4000244140625,
      "logps/rejected": -642.4000244140625,
      "loss": 3.9504,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 1.659765601158142,
      "rewards/margins": 5.358593940734863,
      "rewards/rejected": -3.6957030296325684,
      "step": 890
    },
    {
      "epoch": 0.4225352112676056,
      "grad_norm": 263.44877450986564,
      "learning_rate": 3.2107459572248305e-07,
      "logits/chosen": -4.126562595367432,
      "logits/rejected": -4.053124904632568,
      "logps/chosen": -577.7999877929688,
      "logps/rejected": -647.4000244140625,
      "loss": 3.2681,
      "rewards/accuracies": 0.4984374940395355,
      "rewards/chosen": 1.71875,
      "rewards/margins": 7.209374904632568,
      "rewards/rejected": -5.490624904632568,
      "step": 900
    },
    {
      "epoch": 0.4272300469483568,
      "grad_norm": 321.2673994592104,
      "learning_rate": 3.1846635367762125e-07,
      "logits/chosen": -4.151562690734863,
      "logits/rejected": -4.056250095367432,
      "logps/chosen": -578.4000244140625,
      "logps/rejected": -677.0,
      "loss": 3.0926,
      "rewards/accuracies": 0.47343748807907104,
      "rewards/chosen": 2.348437547683716,
      "rewards/margins": 8.5,
      "rewards/rejected": -6.154687404632568,
      "step": 910
    },
    {
      "epoch": 0.431924882629108,
      "grad_norm": 521.8306086900999,
      "learning_rate": 3.158581116327595e-07,
      "logits/chosen": -4.171875,
      "logits/rejected": -4.0703125,
      "logps/chosen": -576.4000244140625,
      "logps/rejected": -704.0,
      "loss": 2.9125,
      "rewards/accuracies": 0.46796876192092896,
      "rewards/chosen": 2.3511719703674316,
      "rewards/margins": 8.321874618530273,
      "rewards/rejected": -5.971875190734863,
      "step": 920
    },
    {
      "epoch": 0.43661971830985913,
      "grad_norm": 233.362548438678,
      "learning_rate": 3.1324986958789775e-07,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.1015625,
      "logps/chosen": -537.7999877929688,
      "logps/rejected": -620.4000244140625,
      "loss": 2.7801,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 2.37890625,
      "rewards/margins": 6.845312595367432,
      "rewards/rejected": -4.46875,
      "step": 930
    },
    {
      "epoch": 0.4413145539906103,
      "grad_norm": 329.5226337227366,
      "learning_rate": 3.1064162754303595e-07,
      "logits/chosen": -4.15625,
      "logits/rejected": -4.017187595367432,
      "logps/chosen": -578.4000244140625,
      "logps/rejected": -681.2000122070312,
      "loss": 2.9188,
      "rewards/accuracies": 0.4554687440395355,
      "rewards/chosen": 1.9997069835662842,
      "rewards/margins": 7.139062404632568,
      "rewards/rejected": -5.1474609375,
      "step": 940
    },
    {
      "epoch": 0.4460093896713615,
      "grad_norm": 239.68311170701213,
      "learning_rate": 3.080333854981742e-07,
      "logits/chosen": -4.1484375,
      "logits/rejected": -4.051562309265137,
      "logps/chosen": -563.0,
      "logps/rejected": -619.5999755859375,
      "loss": 2.8894,
      "rewards/accuracies": 0.4671874940395355,
      "rewards/chosen": 2.0376954078674316,
      "rewards/margins": 6.135937690734863,
      "rewards/rejected": -4.109375,
      "step": 950
    },
    {
      "epoch": 0.4507042253521127,
      "grad_norm": 301.66609073609294,
      "learning_rate": 3.0542514345331246e-07,
      "logits/chosen": -4.175000190734863,
      "logits/rejected": -4.079687595367432,
      "logps/chosen": -573.2000122070312,
      "logps/rejected": -706.5999755859375,
      "loss": 2.8653,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 2.757031202316284,
      "rewards/margins": 9.146875381469727,
      "rewards/rejected": -6.389843940734863,
      "step": 960
    },
    {
      "epoch": 0.45539906103286387,
      "grad_norm": 323.5455994600225,
      "learning_rate": 3.0281690140845066e-07,
      "logits/chosen": -4.126562595367432,
      "logits/rejected": -4.0703125,
      "logps/chosen": -601.2000122070312,
      "logps/rejected": -667.4000244140625,
      "loss": 3.0966,
      "rewards/accuracies": 0.4898437559604645,
      "rewards/chosen": 2.321484327316284,
      "rewards/margins": 7.056250095367432,
      "rewards/rejected": -4.736523628234863,
      "step": 970
    },
    {
      "epoch": 0.460093896713615,
      "grad_norm": 421.72099522991505,
      "learning_rate": 3.002086593635889e-07,
      "logits/chosen": -4.217187404632568,
      "logits/rejected": -4.071875095367432,
      "logps/chosen": -565.0,
      "logps/rejected": -675.2000122070312,
      "loss": 2.4046,
      "rewards/accuracies": 0.4765625,
      "rewards/chosen": 2.875,
      "rewards/margins": 8.495312690734863,
      "rewards/rejected": -5.607031345367432,
      "step": 980
    },
    {
      "epoch": 0.4647887323943662,
      "grad_norm": 255.42606181348677,
      "learning_rate": 2.9760041731872716e-07,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.095312595367432,
      "logps/chosen": -528.2000122070312,
      "logps/rejected": -634.5999755859375,
      "loss": 2.2063,
      "rewards/accuracies": 0.500781238079071,
      "rewards/chosen": 3.0234375,
      "rewards/margins": 8.315625190734863,
      "rewards/rejected": -5.29296875,
      "step": 990
    },
    {
      "epoch": 0.4694835680751174,
      "grad_norm": 247.2621494249662,
      "learning_rate": 2.9499217527386536e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.09375,
      "logps/chosen": -553.4000244140625,
      "logps/rejected": -650.7999877929688,
      "loss": 2.5919,
      "rewards/accuracies": 0.46171873807907104,
      "rewards/chosen": 2.6890625953674316,
      "rewards/margins": 7.465624809265137,
      "rewards/rejected": -4.764062404632568,
      "step": 1000
    },
    {
      "epoch": 0.47417840375586856,
      "grad_norm": 241.71432645437105,
      "learning_rate": 2.923839332290036e-07,
      "logits/chosen": -4.135937690734863,
      "logits/rejected": -4.045312404632568,
      "logps/chosen": -550.7999877929688,
      "logps/rejected": -664.4000244140625,
      "loss": 2.9087,
      "rewards/accuracies": 0.48046875,
      "rewards/chosen": 1.9425780773162842,
      "rewards/margins": 7.287499904632568,
      "rewards/rejected": -5.348437309265137,
      "step": 1010
    },
    {
      "epoch": 0.4788732394366197,
      "grad_norm": 341.6647376563316,
      "learning_rate": 2.8977569118414187e-07,
      "logits/chosen": -4.256249904632568,
      "logits/rejected": -4.1171875,
      "logps/chosen": -539.2000122070312,
      "logps/rejected": -631.2000122070312,
      "loss": 2.4963,
      "rewards/accuracies": 0.49921876192092896,
      "rewards/chosen": 2.3375000953674316,
      "rewards/margins": 8.607812881469727,
      "rewards/rejected": -6.264062404632568,
      "step": 1020
    },
    {
      "epoch": 0.4835680751173709,
      "grad_norm": 278.12529305873784,
      "learning_rate": 2.8716744913928007e-07,
      "logits/chosen": -4.231249809265137,
      "logits/rejected": -4.129687309265137,
      "logps/chosen": -562.4000244140625,
      "logps/rejected": -627.5999755859375,
      "loss": 2.8734,
      "rewards/accuracies": 0.4867187440395355,
      "rewards/chosen": 2.433789014816284,
      "rewards/margins": 7.042187690734863,
      "rewards/rejected": -4.610156059265137,
      "step": 1030
    },
    {
      "epoch": 0.48826291079812206,
      "grad_norm": 222.4151347018538,
      "learning_rate": 2.845592070944183e-07,
      "logits/chosen": -4.215624809265137,
      "logits/rejected": -4.107812404632568,
      "logps/chosen": -559.7999877929688,
      "logps/rejected": -679.2000122070312,
      "loss": 2.6678,
      "rewards/accuracies": 0.48046875,
      "rewards/chosen": 2.8414063453674316,
      "rewards/margins": 7.800000190734863,
      "rewards/rejected": -4.963671684265137,
      "step": 1040
    },
    {
      "epoch": 0.49295774647887325,
      "grad_norm": 1896.746417849367,
      "learning_rate": 2.819509650495566e-07,
      "logits/chosen": -4.1953125,
      "logits/rejected": -4.046875,
      "logps/chosen": -551.5999755859375,
      "logps/rejected": -624.0,
      "loss": 2.7115,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 2.544921875,
      "rewards/margins": 7.909375190734863,
      "rewards/rejected": -5.3671875,
      "step": 1050
    },
    {
      "epoch": 0.49765258215962443,
      "grad_norm": 252.3036001566129,
      "learning_rate": 2.7934272300469483e-07,
      "logits/chosen": -4.131249904632568,
      "logits/rejected": -4.053124904632568,
      "logps/chosen": -588.0,
      "logps/rejected": -637.7999877929688,
      "loss": 2.9657,
      "rewards/accuracies": 0.4898437559604645,
      "rewards/chosen": 2.487499952316284,
      "rewards/margins": 7.415625095367432,
      "rewards/rejected": -4.921875,
      "step": 1060
    },
    {
      "epoch": 0.5023474178403756,
      "grad_norm": 254.32104609167183,
      "learning_rate": 2.7673448095983303e-07,
      "logits/chosen": -4.151562690734863,
      "logits/rejected": -4.0859375,
      "logps/chosen": -568.4000244140625,
      "logps/rejected": -637.2000122070312,
      "loss": 2.7111,
      "rewards/accuracies": 0.49609375,
      "rewards/chosen": 2.651562452316284,
      "rewards/margins": 8.143750190734863,
      "rewards/rejected": -5.495312690734863,
      "step": 1070
    },
    {
      "epoch": 0.5070422535211268,
      "grad_norm": 295.27435523917325,
      "learning_rate": 2.741262389149713e-07,
      "logits/chosen": -4.170312404632568,
      "logits/rejected": -4.001562595367432,
      "logps/chosen": -568.4000244140625,
      "logps/rejected": -658.7999877929688,
      "loss": 3.0801,
      "rewards/accuracies": 0.48906248807907104,
      "rewards/chosen": 1.9158203601837158,
      "rewards/margins": 8.071874618530273,
      "rewards/rejected": -6.151562690734863,
      "step": 1080
    },
    {
      "epoch": 0.5117370892018779,
      "grad_norm": 333.6479280985517,
      "learning_rate": 2.7151799687010953e-07,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.0625,
      "logps/chosen": -575.5999755859375,
      "logps/rejected": -661.5999755859375,
      "loss": 2.9697,
      "rewards/accuracies": 0.48906248807907104,
      "rewards/chosen": 2.176953077316284,
      "rewards/margins": 7.807812690734863,
      "rewards/rejected": -5.626562595367432,
      "step": 1090
    },
    {
      "epoch": 0.5164319248826291,
      "grad_norm": 247.17684499846777,
      "learning_rate": 2.6890975482524773e-07,
      "logits/chosen": -4.103125095367432,
      "logits/rejected": -3.9749999046325684,
      "logps/chosen": -593.4000244140625,
      "logps/rejected": -651.7999877929688,
      "loss": 2.7876,
      "rewards/accuracies": 0.49921876192092896,
      "rewards/chosen": 2.3265624046325684,
      "rewards/margins": 8.818750381469727,
      "rewards/rejected": -6.496874809265137,
      "step": 1100
    },
    {
      "epoch": 0.5211267605633803,
      "grad_norm": 400.03065742916147,
      "learning_rate": 2.66301512780386e-07,
      "logits/chosen": -4.153124809265137,
      "logits/rejected": -4.028124809265137,
      "logps/chosen": -542.2000122070312,
      "logps/rejected": -639.7999877929688,
      "loss": 2.8744,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 2.279296875,
      "rewards/margins": 9.290624618530273,
      "rewards/rejected": -7.0078125,
      "step": 1110
    },
    {
      "epoch": 0.5258215962441315,
      "grad_norm": 236.3522559902175,
      "learning_rate": 2.6369327073552424e-07,
      "logits/chosen": -4.209374904632568,
      "logits/rejected": -4.106249809265137,
      "logps/chosen": -551.5999755859375,
      "logps/rejected": -615.7999877929688,
      "loss": 2.6851,
      "rewards/accuracies": 0.510937511920929,
      "rewards/chosen": 2.870312452316284,
      "rewards/margins": 7.959374904632568,
      "rewards/rejected": -5.087500095367432,
      "step": 1120
    },
    {
      "epoch": 0.5305164319248826,
      "grad_norm": 449.0784759609624,
      "learning_rate": 2.6108502869066244e-07,
      "logits/chosen": -4.168749809265137,
      "logits/rejected": -4.073437690734863,
      "logps/chosen": -583.2000122070312,
      "logps/rejected": -682.0,
      "loss": 3.18,
      "rewards/accuracies": 0.503125011920929,
      "rewards/chosen": 2.4398436546325684,
      "rewards/margins": 7.62646484375,
      "rewards/rejected": -5.1796875,
      "step": 1130
    },
    {
      "epoch": 0.5352112676056338,
      "grad_norm": 356.684946953978,
      "learning_rate": 2.584767866458007e-07,
      "logits/chosen": -4.224999904632568,
      "logits/rejected": -4.0625,
      "logps/chosen": -558.5999755859375,
      "logps/rejected": -641.5999755859375,
      "loss": 2.7943,
      "rewards/accuracies": 0.46953123807907104,
      "rewards/chosen": 2.016796827316284,
      "rewards/margins": 7.946875095367432,
      "rewards/rejected": -5.923437595367432,
      "step": 1140
    },
    {
      "epoch": 0.539906103286385,
      "grad_norm": 255.00420136519634,
      "learning_rate": 2.5586854460093895e-07,
      "logits/chosen": -4.168749809265137,
      "logits/rejected": -4.0546875,
      "logps/chosen": -568.4000244140625,
      "logps/rejected": -667.5999755859375,
      "loss": 2.9813,
      "rewards/accuracies": 0.5179687738418579,
      "rewards/chosen": 2.18115234375,
      "rewards/margins": 8.71875,
      "rewards/rejected": -6.525000095367432,
      "step": 1150
    },
    {
      "epoch": 0.5446009389671361,
      "grad_norm": 246.04810826117472,
      "learning_rate": 2.5326030255607715e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.078125,
      "logps/chosen": -574.2000122070312,
      "logps/rejected": -651.5999755859375,
      "loss": 3.0039,
      "rewards/accuracies": 0.47968751192092896,
      "rewards/chosen": 2.3734374046325684,
      "rewards/margins": 7.978125095367432,
      "rewards/rejected": -5.609765529632568,
      "step": 1160
    },
    {
      "epoch": 0.5492957746478874,
      "grad_norm": 311.02323786782017,
      "learning_rate": 2.506520605112154e-07,
      "logits/chosen": -4.160937309265137,
      "logits/rejected": -4.112500190734863,
      "logps/chosen": -567.0,
      "logps/rejected": -654.0,
      "loss": 3.4351,
      "rewards/accuracies": 0.4593749940395355,
      "rewards/chosen": 1.73828125,
      "rewards/margins": 6.778124809265137,
      "rewards/rejected": -5.04296875,
      "step": 1170
    },
    {
      "epoch": 0.5539906103286385,
      "grad_norm": 402.5987614392135,
      "learning_rate": 2.4804381846635365e-07,
      "logits/chosen": -4.221875190734863,
      "logits/rejected": -4.131249904632568,
      "logps/chosen": -583.2000122070312,
      "logps/rejected": -644.0,
      "loss": 3.0764,
      "rewards/accuracies": 0.4632812440395355,
      "rewards/chosen": 1.592187523841858,
      "rewards/margins": 8.196874618530273,
      "rewards/rejected": -6.609375,
      "step": 1180
    },
    {
      "epoch": 0.5586854460093896,
      "grad_norm": 235.98510893801293,
      "learning_rate": 2.454355764214919e-07,
      "logits/chosen": -4.193749904632568,
      "logits/rejected": -4.110937595367432,
      "logps/chosen": -544.0,
      "logps/rejected": -671.7999877929688,
      "loss": 3.0112,
      "rewards/accuracies": 0.47578126192092896,
      "rewards/chosen": 1.999609351158142,
      "rewards/margins": 8.028124809265137,
      "rewards/rejected": -6.03125,
      "step": 1190
    },
    {
      "epoch": 0.5633802816901409,
      "grad_norm": 2471.375555054977,
      "learning_rate": 2.4282733437663016e-07,
      "logits/chosen": -4.206250190734863,
      "logits/rejected": -4.157812595367432,
      "logps/chosen": -574.4000244140625,
      "logps/rejected": -628.4000244140625,
      "loss": 2.7524,
      "rewards/accuracies": 0.4789062440395355,
      "rewards/chosen": 3.109375,
      "rewards/margins": 7.653124809265137,
      "rewards/rejected": -4.545312404632568,
      "step": 1200
    },
    {
      "epoch": 0.568075117370892,
      "grad_norm": 254.88303513035962,
      "learning_rate": 2.4021909233176836e-07,
      "logits/chosen": -4.168749809265137,
      "logits/rejected": -4.099999904632568,
      "logps/chosen": -538.7999877929688,
      "logps/rejected": -649.5999755859375,
      "loss": 2.5376,
      "rewards/accuracies": 0.45234376192092896,
      "rewards/chosen": 2.389843702316284,
      "rewards/margins": 8.425000190734863,
      "rewards/rejected": -6.039453029632568,
      "step": 1210
    },
    {
      "epoch": 0.5727699530516432,
      "grad_norm": 291.0218011211237,
      "learning_rate": 2.376108502869066e-07,
      "logits/chosen": -4.193749904632568,
      "logits/rejected": -4.146874904632568,
      "logps/chosen": -575.5999755859375,
      "logps/rejected": -673.0,
      "loss": 2.5853,
      "rewards/accuracies": 0.4945312440395355,
      "rewards/chosen": 2.547656297683716,
      "rewards/margins": 7.704687595367432,
      "rewards/rejected": -5.163281440734863,
      "step": 1220
    },
    {
      "epoch": 0.5774647887323944,
      "grad_norm": 294.9902995268332,
      "learning_rate": 2.3500260824204484e-07,
      "logits/chosen": -4.134375095367432,
      "logits/rejected": -4.028124809265137,
      "logps/chosen": -603.2000122070312,
      "logps/rejected": -683.2000122070312,
      "loss": 3.4254,
      "rewards/accuracies": 0.4976562559604645,
      "rewards/chosen": 1.564843773841858,
      "rewards/margins": 8.621874809265137,
      "rewards/rejected": -7.060937404632568,
      "step": 1230
    },
    {
      "epoch": 0.5821596244131455,
      "grad_norm": 243.17738458688876,
      "learning_rate": 2.323943661971831e-07,
      "logits/chosen": -4.237500190734863,
      "logits/rejected": -4.121874809265137,
      "logps/chosen": -563.2000122070312,
      "logps/rejected": -642.0,
      "loss": 3.0896,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": 2.217968702316284,
      "rewards/margins": 8.28125,
      "rewards/rejected": -6.0625,
      "step": 1240
    },
    {
      "epoch": 0.5868544600938967,
      "grad_norm": 279.18184598229146,
      "learning_rate": 2.2978612415232132e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.099999904632568,
      "logps/chosen": -595.5999755859375,
      "logps/rejected": -672.4000244140625,
      "loss": 2.9207,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 2.26953125,
      "rewards/margins": 7.4375,
      "rewards/rejected": -5.176562309265137,
      "step": 1250
    },
    {
      "epoch": 0.5915492957746479,
      "grad_norm": 248.25515971211874,
      "learning_rate": 2.2717788210745957e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.0625,
      "logps/chosen": -557.0,
      "logps/rejected": -658.7999877929688,
      "loss": 2.7618,
      "rewards/accuracies": 0.4828124940395355,
      "rewards/chosen": 2.1996092796325684,
      "rewards/margins": 8.904687881469727,
      "rewards/rejected": -6.709374904632568,
      "step": 1260
    },
    {
      "epoch": 0.596244131455399,
      "grad_norm": 254.76169280573018,
      "learning_rate": 2.245696400625978e-07,
      "logits/chosen": -4.224999904632568,
      "logits/rejected": -4.107812404632568,
      "logps/chosen": -587.4000244140625,
      "logps/rejected": -665.7999877929688,
      "loss": 3.5291,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 1.740332007408142,
      "rewards/margins": 7.653124809265137,
      "rewards/rejected": -5.912499904632568,
      "step": 1270
    },
    {
      "epoch": 0.6009389671361502,
      "grad_norm": 238.04558401923484,
      "learning_rate": 2.2196139801773602e-07,
      "logits/chosen": -4.153124809265137,
      "logits/rejected": -4.018750190734863,
      "logps/chosen": -552.2000122070312,
      "logps/rejected": -624.2000122070312,
      "loss": 3.1411,
      "rewards/accuracies": 0.48359376192092896,
      "rewards/chosen": 1.81640625,
      "rewards/margins": 7.315625190734863,
      "rewards/rejected": -5.5,
      "step": 1280
    },
    {
      "epoch": 0.6056338028169014,
      "grad_norm": 358.6362771378,
      "learning_rate": 2.1935315597287428e-07,
      "logits/chosen": -4.15625,
      "logits/rejected": -4.076562404632568,
      "logps/chosen": -571.0,
      "logps/rejected": -623.5999755859375,
      "loss": 2.9957,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 2.0546875,
      "rewards/margins": 7.315625190734863,
      "rewards/rejected": -5.265625,
      "step": 1290
    },
    {
      "epoch": 0.6103286384976526,
      "grad_norm": 223.73969039337766,
      "learning_rate": 2.167449139280125e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.073437690734863,
      "logps/chosen": -577.2000122070312,
      "logps/rejected": -666.0,
      "loss": 2.8525,
      "rewards/accuracies": 0.49140626192092896,
      "rewards/chosen": 2.350781202316284,
      "rewards/margins": 8.157812118530273,
      "rewards/rejected": -5.814062595367432,
      "step": 1300
    },
    {
      "epoch": 0.6150234741784038,
      "grad_norm": 259.1197329772539,
      "learning_rate": 2.1413667188315073e-07,
      "logits/chosen": -4.237500190734863,
      "logits/rejected": -4.224999904632568,
      "logps/chosen": -547.2000122070312,
      "logps/rejected": -612.4000244140625,
      "loss": 2.993,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 2.399609327316284,
      "rewards/margins": 7.756249904632568,
      "rewards/rejected": -5.3515625,
      "step": 1310
    },
    {
      "epoch": 0.6197183098591549,
      "grad_norm": 267.20628690442504,
      "learning_rate": 2.1152842983828898e-07,
      "logits/chosen": -4.123437404632568,
      "logits/rejected": -4.040625095367432,
      "logps/chosen": -579.2000122070312,
      "logps/rejected": -640.4000244140625,
      "loss": 3.6735,
      "rewards/accuracies": 0.48515623807907104,
      "rewards/chosen": 1.484765648841858,
      "rewards/margins": 6.603320121765137,
      "rewards/rejected": -5.123827934265137,
      "step": 1320
    },
    {
      "epoch": 0.6244131455399061,
      "grad_norm": 256.25318165207995,
      "learning_rate": 2.089201877934272e-07,
      "logits/chosen": -4.131249904632568,
      "logits/rejected": -4.043749809265137,
      "logps/chosen": -592.0,
      "logps/rejected": -689.7999877929688,
      "loss": 3.2943,
      "rewards/accuracies": 0.5023437738418579,
      "rewards/chosen": 1.5166504383087158,
      "rewards/margins": 10.184374809265137,
      "rewards/rejected": -8.668749809265137,
      "step": 1330
    },
    {
      "epoch": 0.6291079812206573,
      "grad_norm": 334.35995211244534,
      "learning_rate": 2.0631194574856543e-07,
      "logits/chosen": -4.220312595367432,
      "logits/rejected": -4.143750190734863,
      "logps/chosen": -547.0,
      "logps/rejected": -593.7999877929688,
      "loss": 2.622,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 2.6078124046325684,
      "rewards/margins": 6.675000190734863,
      "rewards/rejected": -4.064062595367432,
      "step": 1340
    },
    {
      "epoch": 0.6338028169014085,
      "grad_norm": 291.95519056678296,
      "learning_rate": 2.0370370370370369e-07,
      "logits/chosen": -4.112500190734863,
      "logits/rejected": -4.048437595367432,
      "logps/chosen": -572.4000244140625,
      "logps/rejected": -691.2000122070312,
      "loss": 3.244,
      "rewards/accuracies": 0.4828124940395355,
      "rewards/chosen": 1.5021483898162842,
      "rewards/margins": 7.981249809265137,
      "rewards/rejected": -6.486718654632568,
      "step": 1350
    },
    {
      "epoch": 0.6384976525821596,
      "grad_norm": 267.81172219110795,
      "learning_rate": 2.010954616588419e-07,
      "logits/chosen": -4.206250190734863,
      "logits/rejected": -4.0703125,
      "logps/chosen": -559.7999877929688,
      "logps/rejected": -640.0,
      "loss": 2.825,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 2.4400391578674316,
      "rewards/margins": 8.28125,
      "rewards/rejected": -5.84375,
      "step": 1360
    },
    {
      "epoch": 0.6431924882629108,
      "grad_norm": 268.8924951378491,
      "learning_rate": 1.9848721961398017e-07,
      "logits/chosen": -4.224999904632568,
      "logits/rejected": -4.154687404632568,
      "logps/chosen": -568.5999755859375,
      "logps/rejected": -625.2000122070312,
      "loss": 2.9559,
      "rewards/accuracies": 0.4609375,
      "rewards/chosen": 2.38671875,
      "rewards/margins": 7.073437690734863,
      "rewards/rejected": -4.682812690734863,
      "step": 1370
    },
    {
      "epoch": 0.647887323943662,
      "grad_norm": 248.8386074500955,
      "learning_rate": 1.958789775691184e-07,
      "logits/chosen": -4.173437595367432,
      "logits/rejected": -4.051562309265137,
      "logps/chosen": -553.7999877929688,
      "logps/rejected": -643.2000122070312,
      "loss": 2.5208,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 2.358203172683716,
      "rewards/margins": 8.321874618530273,
      "rewards/rejected": -5.971093654632568,
      "step": 1380
    },
    {
      "epoch": 0.6525821596244131,
      "grad_norm": 234.82733297662526,
      "learning_rate": 1.9327073552425662e-07,
      "logits/chosen": -4.162499904632568,
      "logits/rejected": -4.081250190734863,
      "logps/chosen": -545.0,
      "logps/rejected": -616.7999877929688,
      "loss": 2.687,
      "rewards/accuracies": 0.5093749761581421,
      "rewards/chosen": 2.634765625,
      "rewards/margins": 8.293749809265137,
      "rewards/rejected": -5.66015625,
      "step": 1390
    },
    {
      "epoch": 0.6572769953051644,
      "grad_norm": 310.18784061012275,
      "learning_rate": 1.906624934793949e-07,
      "logits/chosen": -4.1640625,
      "logits/rejected": -4.123437404632568,
      "logps/chosen": -564.7999877929688,
      "logps/rejected": -673.2000122070312,
      "loss": 2.9459,
      "rewards/accuracies": 0.504687488079071,
      "rewards/chosen": 2.4751954078674316,
      "rewards/margins": 9.300000190734863,
      "rewards/rejected": -6.834374904632568,
      "step": 1400
    },
    {
      "epoch": 0.6619718309859155,
      "grad_norm": 235.4558018667043,
      "learning_rate": 1.8805425143453312e-07,
      "logits/chosen": -4.206250190734863,
      "logits/rejected": -4.003125190734863,
      "logps/chosen": -543.4000244140625,
      "logps/rejected": -701.2000122070312,
      "loss": 2.668,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 2.913281202316284,
      "rewards/margins": 9.1875,
      "rewards/rejected": -6.278124809265137,
      "step": 1410
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 293.41852691409196,
      "learning_rate": 1.8544600938967138e-07,
      "logits/chosen": -4.184374809265137,
      "logits/rejected": -4.092187404632568,
      "logps/chosen": -561.5999755859375,
      "logps/rejected": -628.0,
      "loss": 2.8252,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": 2.319531202316284,
      "rewards/margins": 7.837500095367432,
      "rewards/rejected": -5.506249904632568,
      "step": 1420
    },
    {
      "epoch": 0.6713615023474179,
      "grad_norm": 318.8082210935934,
      "learning_rate": 1.828377673448096e-07,
      "logits/chosen": -4.143750190734863,
      "logits/rejected": -4.087500095367432,
      "logps/chosen": -598.4000244140625,
      "logps/rejected": -649.5999755859375,
      "loss": 3.9709,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": 1.078125,
      "rewards/margins": 7.635937690734863,
      "rewards/rejected": -6.553124904632568,
      "step": 1430
    },
    {
      "epoch": 0.676056338028169,
      "grad_norm": 214.52983321408703,
      "learning_rate": 1.8022952529994783e-07,
      "logits/chosen": -4.182812690734863,
      "logits/rejected": -4.057812690734863,
      "logps/chosen": -573.4000244140625,
      "logps/rejected": -639.0,
      "loss": 2.6869,
      "rewards/accuracies": 0.5015624761581421,
      "rewards/chosen": 2.2164063453674316,
      "rewards/margins": 9.912500381469727,
      "rewards/rejected": -7.698437690734863,
      "step": 1440
    },
    {
      "epoch": 0.6807511737089202,
      "grad_norm": 535.5466031773983,
      "learning_rate": 1.7762128325508608e-07,
      "logits/chosen": -4.215624809265137,
      "logits/rejected": -4.099999904632568,
      "logps/chosen": -583.0,
      "logps/rejected": -685.5999755859375,
      "loss": 3.44,
      "rewards/accuracies": 0.4593749940395355,
      "rewards/chosen": 1.505712866783142,
      "rewards/margins": 8.571874618530273,
      "rewards/rejected": -7.068749904632568,
      "step": 1450
    },
    {
      "epoch": 0.6854460093896714,
      "grad_norm": 286.387097411706,
      "learning_rate": 1.750130412102243e-07,
      "logits/chosen": -4.112500190734863,
      "logits/rejected": -4.050000190734863,
      "logps/chosen": -612.7999877929688,
      "logps/rejected": -664.0,
      "loss": 3.4718,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 1.973486304283142,
      "rewards/margins": 8.126562118530273,
      "rewards/rejected": -6.159375190734863,
      "step": 1460
    },
    {
      "epoch": 0.6901408450704225,
      "grad_norm": 251.16006508160885,
      "learning_rate": 1.7240479916536254e-07,
      "logits/chosen": -4.110937595367432,
      "logits/rejected": -4.043749809265137,
      "logps/chosen": -551.5999755859375,
      "logps/rejected": -679.2000122070312,
      "loss": 2.9329,
      "rewards/accuracies": 0.504687488079071,
      "rewards/chosen": 2.4937500953674316,
      "rewards/margins": 9.625,
      "rewards/rejected": -7.120312690734863,
      "step": 1470
    },
    {
      "epoch": 0.6948356807511737,
      "grad_norm": 247.36057065543977,
      "learning_rate": 1.697965571205008e-07,
      "logits/chosen": -4.209374904632568,
      "logits/rejected": -4.123437404632568,
      "logps/chosen": -560.0,
      "logps/rejected": -618.4000244140625,
      "loss": 3.0374,
      "rewards/accuracies": 0.4859375059604645,
      "rewards/chosen": 2.0550780296325684,
      "rewards/margins": 7.790625095367432,
      "rewards/rejected": -5.721875190734863,
      "step": 1480
    },
    {
      "epoch": 0.6995305164319249,
      "grad_norm": 264.09934191241626,
      "learning_rate": 1.6718831507563902e-07,
      "logits/chosen": -4.178124904632568,
      "logits/rejected": -4.128125190734863,
      "logps/chosen": -565.5999755859375,
      "logps/rejected": -610.5999755859375,
      "loss": 2.8288,
      "rewards/accuracies": 0.4906249940395355,
      "rewards/chosen": 2.4937500953674316,
      "rewards/margins": 7.323437690734863,
      "rewards/rejected": -4.831250190734863,
      "step": 1490
    },
    {
      "epoch": 0.704225352112676,
      "grad_norm": 237.52384616511833,
      "learning_rate": 1.6458007303077727e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.099999904632568,
      "logps/chosen": -553.0,
      "logps/rejected": -622.0,
      "loss": 2.7071,
      "rewards/accuracies": 0.5054687261581421,
      "rewards/chosen": 2.0841307640075684,
      "rewards/margins": 6.650000095367432,
      "rewards/rejected": -4.560937404632568,
      "step": 1500
    },
    {
      "epoch": 0.7089201877934272,
      "grad_norm": 254.89751638831584,
      "learning_rate": 1.619718309859155e-07,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.073437690734863,
      "logps/chosen": -573.7999877929688,
      "logps/rejected": -673.5999755859375,
      "loss": 2.9881,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 2.4203124046325684,
      "rewards/margins": 7.485937595367432,
      "rewards/rejected": -5.064062595367432,
      "step": 1510
    },
    {
      "epoch": 0.7136150234741784,
      "grad_norm": 242.02433017820204,
      "learning_rate": 1.5936358894105372e-07,
      "logits/chosen": -4.212500095367432,
      "logits/rejected": -4.109375,
      "logps/chosen": -574.0,
      "logps/rejected": -659.5999755859375,
      "loss": 2.6569,
      "rewards/accuracies": 0.4976562559604645,
      "rewards/chosen": 2.555468797683716,
      "rewards/margins": 9.290624618530273,
      "rewards/rejected": -6.743750095367432,
      "step": 1520
    },
    {
      "epoch": 0.7183098591549296,
      "grad_norm": 204.12681215097106,
      "learning_rate": 1.5675534689619197e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.135937690734863,
      "logps/chosen": -574.0,
      "logps/rejected": -606.7999877929688,
      "loss": 2.8257,
      "rewards/accuracies": 0.48515623807907104,
      "rewards/chosen": 1.885156273841858,
      "rewards/margins": 7.301562309265137,
      "rewards/rejected": -5.4140625,
      "step": 1530
    },
    {
      "epoch": 0.7230046948356808,
      "grad_norm": 220.69702921039521,
      "learning_rate": 1.541471048513302e-07,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.104687690734863,
      "logps/chosen": -574.0,
      "logps/rejected": -648.4000244140625,
      "loss": 2.9706,
      "rewards/accuracies": 0.504687488079071,
      "rewards/chosen": 2.0511474609375,
      "rewards/margins": 7.868750095367432,
      "rewards/rejected": -5.821875095367432,
      "step": 1540
    },
    {
      "epoch": 0.7276995305164319,
      "grad_norm": 233.43235168168627,
      "learning_rate": 1.5153886280646843e-07,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.125,
      "logps/chosen": -531.7999877929688,
      "logps/rejected": -597.0,
      "loss": 2.9134,
      "rewards/accuracies": 0.4976562559604645,
      "rewards/chosen": 2.35546875,
      "rewards/margins": 7.579687595367432,
      "rewards/rejected": -5.2265625,
      "step": 1550
    },
    {
      "epoch": 0.7323943661971831,
      "grad_norm": 242.69092831621717,
      "learning_rate": 1.4893062076160668e-07,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.125,
      "logps/chosen": -562.4000244140625,
      "logps/rejected": -658.4000244140625,
      "loss": 2.895,
      "rewards/accuracies": 0.4976562559604645,
      "rewards/chosen": 2.4921875,
      "rewards/margins": 9.215624809265137,
      "rewards/rejected": -6.720312595367432,
      "step": 1560
    },
    {
      "epoch": 0.7370892018779343,
      "grad_norm": 208.19593737431086,
      "learning_rate": 1.463223787167449e-07,
      "logits/chosen": -4.212500095367432,
      "logits/rejected": -4.107812404632568,
      "logps/chosen": -574.7999877929688,
      "logps/rejected": -638.7999877929688,
      "loss": 3.2961,
      "rewards/accuracies": 0.47734373807907104,
      "rewards/chosen": 1.5167968273162842,
      "rewards/margins": 7.256249904632568,
      "rewards/rejected": -5.734375,
      "step": 1570
    },
    {
      "epoch": 0.7417840375586855,
      "grad_norm": 310.0166836408286,
      "learning_rate": 1.4371413667188313e-07,
      "logits/chosen": -4.21875,
      "logits/rejected": -4.065625190734863,
      "logps/chosen": -559.5999755859375,
      "logps/rejected": -678.4000244140625,
      "loss": 2.7786,
      "rewards/accuracies": 0.49531251192092896,
      "rewards/chosen": 2.492968797683716,
      "rewards/margins": 9.278124809265137,
      "rewards/rejected": -6.790625095367432,
      "step": 1580
    },
    {
      "epoch": 0.7464788732394366,
      "grad_norm": 264.08280499875406,
      "learning_rate": 1.4110589462702139e-07,
      "logits/chosen": -4.237500190734863,
      "logits/rejected": -4.145312309265137,
      "logps/chosen": -556.4000244140625,
      "logps/rejected": -620.2000122070312,
      "loss": 2.9828,
      "rewards/accuracies": 0.48359376192092896,
      "rewards/chosen": 2.3646483421325684,
      "rewards/margins": 7.588281154632568,
      "rewards/rejected": -5.2265625,
      "step": 1590
    },
    {
      "epoch": 0.7511737089201878,
      "grad_norm": 261.50016438802686,
      "learning_rate": 1.384976525821596e-07,
      "logits/chosen": -4.262499809265137,
      "logits/rejected": -4.110937595367432,
      "logps/chosen": -576.0,
      "logps/rejected": -670.5999755859375,
      "loss": 2.7417,
      "rewards/accuracies": 0.510937511920929,
      "rewards/chosen": 2.551953077316284,
      "rewards/margins": 9.550000190734863,
      "rewards/rejected": -6.9921875,
      "step": 1600
    },
    {
      "epoch": 0.755868544600939,
      "grad_norm": 293.0794110532751,
      "learning_rate": 1.3588941053729787e-07,
      "logits/chosen": -4.171875,
      "logits/rejected": -4.120312690734863,
      "logps/chosen": -578.0,
      "logps/rejected": -667.2000122070312,
      "loss": 3.0631,
      "rewards/accuracies": 0.5101562738418579,
      "rewards/chosen": 2.7017579078674316,
      "rewards/margins": 9.087499618530273,
      "rewards/rejected": -6.387499809265137,
      "step": 1610
    },
    {
      "epoch": 0.7605633802816901,
      "grad_norm": 263.46147273147955,
      "learning_rate": 1.332811684924361e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.084374904632568,
      "logps/chosen": -585.4000244140625,
      "logps/rejected": -646.0,
      "loss": 3.5318,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 1.736328125,
      "rewards/margins": 7.243750095367432,
      "rewards/rejected": -5.504687309265137,
      "step": 1620
    },
    {
      "epoch": 0.7652582159624414,
      "grad_norm": 264.0354478252828,
      "learning_rate": 1.3067292644757432e-07,
      "logits/chosen": -4.114062309265137,
      "logits/rejected": -4.040625095367432,
      "logps/chosen": -588.5999755859375,
      "logps/rejected": -686.7999877929688,
      "loss": 2.6745,
      "rewards/accuracies": 0.5140625238418579,
      "rewards/chosen": 2.123046875,
      "rewards/margins": 9.359375,
      "rewards/rejected": -7.239062309265137,
      "step": 1630
    },
    {
      "epoch": 0.7699530516431925,
      "grad_norm": 338.5060307146247,
      "learning_rate": 1.2806468440271257e-07,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.099999904632568,
      "logps/chosen": -549.0,
      "logps/rejected": -630.2000122070312,
      "loss": 2.7684,
      "rewards/accuracies": 0.508593738079071,
      "rewards/chosen": 2.6273436546325684,
      "rewards/margins": 7.606249809265137,
      "rewards/rejected": -4.98046875,
      "step": 1640
    },
    {
      "epoch": 0.7746478873239436,
      "grad_norm": 222.14397678530307,
      "learning_rate": 1.254564423578508e-07,
      "logits/chosen": -4.206250190734863,
      "logits/rejected": -4.089062690734863,
      "logps/chosen": -556.7999877929688,
      "logps/rejected": -640.0,
      "loss": 2.639,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 2.7789063453674316,
      "rewards/margins": 9.15625,
      "rewards/rejected": -6.387499809265137,
      "step": 1650
    },
    {
      "epoch": 0.7793427230046949,
      "grad_norm": 283.2056849330329,
      "learning_rate": 1.2284820031298902e-07,
      "logits/chosen": -4.184374809265137,
      "logits/rejected": -4.073437690734863,
      "logps/chosen": -595.5999755859375,
      "logps/rejected": -668.7999877929688,
      "loss": 2.9856,
      "rewards/accuracies": 0.5132812261581421,
      "rewards/chosen": 2.751171827316284,
      "rewards/margins": 9.5,
      "rewards/rejected": -6.7421875,
      "step": 1660
    },
    {
      "epoch": 0.784037558685446,
      "grad_norm": 292.4005754799325,
      "learning_rate": 1.2023995826812728e-07,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.0546875,
      "logps/chosen": -560.4000244140625,
      "logps/rejected": -654.0,
      "loss": 2.7566,
      "rewards/accuracies": 0.49296873807907104,
      "rewards/chosen": 2.155956983566284,
      "rewards/margins": 8.618749618530273,
      "rewards/rejected": -6.462890625,
      "step": 1670
    },
    {
      "epoch": 0.7887323943661971,
      "grad_norm": 271.92643509309005,
      "learning_rate": 1.176317162232655e-07,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.09375,
      "logps/chosen": -556.4000244140625,
      "logps/rejected": -665.0,
      "loss": 2.3201,
      "rewards/accuracies": 0.4867187440395355,
      "rewards/chosen": 2.27734375,
      "rewards/margins": 8.940625190734863,
      "rewards/rejected": -6.668749809265137,
      "step": 1680
    },
    {
      "epoch": 0.7934272300469484,
      "grad_norm": 217.20947043092823,
      "learning_rate": 1.1502347417840374e-07,
      "logits/chosen": -4.145312309265137,
      "logits/rejected": -4.0546875,
      "logps/chosen": -563.4000244140625,
      "logps/rejected": -647.2000122070312,
      "loss": 3.0267,
      "rewards/accuracies": 0.49687498807907104,
      "rewards/chosen": 1.9630858898162842,
      "rewards/margins": 8.449999809265137,
      "rewards/rejected": -6.493750095367432,
      "step": 1690
    },
    {
      "epoch": 0.7981220657276995,
      "grad_norm": 268.4501537506006,
      "learning_rate": 1.1241523213354198e-07,
      "logits/chosen": -4.221875190734863,
      "logits/rejected": -4.081250190734863,
      "logps/chosen": -563.2000122070312,
      "logps/rejected": -670.7999877929688,
      "loss": 2.5566,
      "rewards/accuracies": 0.51953125,
      "rewards/chosen": 2.8046875,
      "rewards/margins": 9.712499618530273,
      "rewards/rejected": -6.907812595367432,
      "step": 1700
    },
    {
      "epoch": 0.8028169014084507,
      "grad_norm": 228.66370177088228,
      "learning_rate": 1.0980699008868022e-07,
      "logits/chosen": -4.1796875,
      "logits/rejected": -4.092187404632568,
      "logps/chosen": -576.7999877929688,
      "logps/rejected": -632.4000244140625,
      "loss": 2.9308,
      "rewards/accuracies": 0.5023437738418579,
      "rewards/chosen": 2.1919922828674316,
      "rewards/margins": 8.440625190734863,
      "rewards/rejected": -6.240624904632568,
      "step": 1710
    },
    {
      "epoch": 0.8075117370892019,
      "grad_norm": 262.1967296178448,
      "learning_rate": 1.0719874804381846e-07,
      "logits/chosen": -4.168749809265137,
      "logits/rejected": -4.098437309265137,
      "logps/chosen": -557.4000244140625,
      "logps/rejected": -598.7999877929688,
      "loss": 2.7061,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 2.848828077316284,
      "rewards/margins": 7.423437595367432,
      "rewards/rejected": -4.579687595367432,
      "step": 1720
    },
    {
      "epoch": 0.812206572769953,
      "grad_norm": 232.10360746298278,
      "learning_rate": 1.045905059989567e-07,
      "logits/chosen": -4.206250190734863,
      "logits/rejected": -4.162499904632568,
      "logps/chosen": -533.2000122070312,
      "logps/rejected": -592.0,
      "loss": 2.5252,
      "rewards/accuracies": 0.47968751192092896,
      "rewards/chosen": 2.7535157203674316,
      "rewards/margins": 7.306250095367432,
      "rewards/rejected": -4.551562309265137,
      "step": 1730
    },
    {
      "epoch": 0.8169014084507042,
      "grad_norm": 264.2413834021659,
      "learning_rate": 1.0198226395409494e-07,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.059374809265137,
      "logps/chosen": -541.4000244140625,
      "logps/rejected": -615.5999755859375,
      "loss": 3.0827,
      "rewards/accuracies": 0.49531251192092896,
      "rewards/chosen": 1.441796898841858,
      "rewards/margins": 7.021874904632568,
      "rewards/rejected": -5.579687595367432,
      "step": 1740
    },
    {
      "epoch": 0.8215962441314554,
      "grad_norm": 243.0050739142313,
      "learning_rate": 9.937402190923318e-08,
      "logits/chosen": -4.170312404632568,
      "logits/rejected": -4.081250190734863,
      "logps/chosen": -530.4000244140625,
      "logps/rejected": -656.4000244140625,
      "loss": 2.6591,
      "rewards/accuracies": 0.49140626192092896,
      "rewards/chosen": 2.4921875,
      "rewards/margins": 10.149999618530273,
      "rewards/rejected": -7.654687404632568,
      "step": 1750
    },
    {
      "epoch": 0.8262910798122066,
      "grad_norm": 263.15989334134304,
      "learning_rate": 9.676577986437141e-08,
      "logits/chosen": -4.131249904632568,
      "logits/rejected": -4.014062404632568,
      "logps/chosen": -590.7999877929688,
      "logps/rejected": -647.5999755859375,
      "loss": 3.2525,
      "rewards/accuracies": 0.503125011920929,
      "rewards/chosen": 2.1201171875,
      "rewards/margins": 9.371874809265137,
      "rewards/rejected": -7.251562595367432,
      "step": 1760
    },
    {
      "epoch": 0.8309859154929577,
      "grad_norm": 205.3515680856732,
      "learning_rate": 9.415753781950965e-08,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.110937595367432,
      "logps/chosen": -544.7999877929688,
      "logps/rejected": -637.5999755859375,
      "loss": 2.3893,
      "rewards/accuracies": 0.503125011920929,
      "rewards/chosen": 2.53515625,
      "rewards/margins": 9.024999618530273,
      "rewards/rejected": -6.481249809265137,
      "step": 1770
    },
    {
      "epoch": 0.8356807511737089,
      "grad_norm": 255.00260836835142,
      "learning_rate": 9.154929577464789e-08,
      "logits/chosen": -4.184374809265137,
      "logits/rejected": -4.057812690734863,
      "logps/chosen": -562.4000244140625,
      "logps/rejected": -640.0,
      "loss": 2.8412,
      "rewards/accuracies": 0.5093749761581421,
      "rewards/chosen": 2.246875047683716,
      "rewards/margins": 8.6875,
      "rewards/rejected": -6.451562404632568,
      "step": 1780
    },
    {
      "epoch": 0.8403755868544601,
      "grad_norm": 306.071482107185,
      "learning_rate": 8.894105372978613e-08,
      "logits/chosen": -4.162499904632568,
      "logits/rejected": -4.081250190734863,
      "logps/chosen": -581.0,
      "logps/rejected": -648.5999755859375,
      "loss": 2.9228,
      "rewards/accuracies": 0.47968751192092896,
      "rewards/chosen": 2.34765625,
      "rewards/margins": 8.464062690734863,
      "rewards/rejected": -6.127343654632568,
      "step": 1790
    },
    {
      "epoch": 0.8450704225352113,
      "grad_norm": 254.95417180264684,
      "learning_rate": 8.633281168492435e-08,
      "logits/chosen": -4.221875190734863,
      "logits/rejected": -4.165625095367432,
      "logps/chosen": -559.5999755859375,
      "logps/rejected": -628.5999755859375,
      "loss": 2.6415,
      "rewards/accuracies": 0.5132812261581421,
      "rewards/chosen": 2.3984375,
      "rewards/margins": 8.803125381469727,
      "rewards/rejected": -6.410937309265137,
      "step": 1800
    },
    {
      "epoch": 0.8497652582159625,
      "grad_norm": 322.4975617997682,
      "learning_rate": 8.372456964006259e-08,
      "logits/chosen": -4.1484375,
      "logits/rejected": -4.051562309265137,
      "logps/chosen": -555.0,
      "logps/rejected": -640.4000244140625,
      "loss": 3.0295,
      "rewards/accuracies": 0.51171875,
      "rewards/chosen": 1.8380858898162842,
      "rewards/margins": 9.334375381469727,
      "rewards/rejected": -7.485937595367432,
      "step": 1810
    },
    {
      "epoch": 0.8544600938967136,
      "grad_norm": 333.75848335478094,
      "learning_rate": 8.111632759520083e-08,
      "logits/chosen": -4.143750190734863,
      "logits/rejected": -4.092187404632568,
      "logps/chosen": -603.2000122070312,
      "logps/rejected": -660.7999877929688,
      "loss": 2.7073,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 2.360156297683716,
      "rewards/margins": 9.676562309265137,
      "rewards/rejected": -7.326562404632568,
      "step": 1820
    },
    {
      "epoch": 0.8591549295774648,
      "grad_norm": 773.9494252353215,
      "learning_rate": 7.850808555033907e-08,
      "logits/chosen": -4.150000095367432,
      "logits/rejected": -4.032812595367432,
      "logps/chosen": -564.0,
      "logps/rejected": -665.2000122070312,
      "loss": 2.621,
      "rewards/accuracies": 0.5093749761581421,
      "rewards/chosen": 2.0914063453674316,
      "rewards/margins": 10.021875381469727,
      "rewards/rejected": -7.934374809265137,
      "step": 1830
    },
    {
      "epoch": 0.863849765258216,
      "grad_norm": 261.68841557963896,
      "learning_rate": 7.58998435054773e-08,
      "logits/chosen": -4.15625,
      "logits/rejected": -4.051562309265137,
      "logps/chosen": -582.4000244140625,
      "logps/rejected": -662.0,
      "loss": 3.2356,
      "rewards/accuracies": 0.4828124940395355,
      "rewards/chosen": 1.8888671398162842,
      "rewards/margins": 8.040624618530273,
      "rewards/rejected": -6.140625,
      "step": 1840
    },
    {
      "epoch": 0.8685446009389671,
      "grad_norm": 293.68613390890863,
      "learning_rate": 7.329160146061554e-08,
      "logits/chosen": -4.212500095367432,
      "logits/rejected": -4.196875095367432,
      "logps/chosen": -552.0,
      "logps/rejected": -644.0,
      "loss": 3.0956,
      "rewards/accuracies": 0.504687488079071,
      "rewards/chosen": 2.5859375,
      "rewards/margins": 8.465624809265137,
      "rewards/rejected": -5.879687309265137,
      "step": 1850
    },
    {
      "epoch": 0.8732394366197183,
      "grad_norm": 266.81714304493767,
      "learning_rate": 7.068335941575378e-08,
      "logits/chosen": -4.146874904632568,
      "logits/rejected": -4.1015625,
      "logps/chosen": -552.5999755859375,
      "logps/rejected": -623.2000122070312,
      "loss": 3.391,
      "rewards/accuracies": 0.503125011920929,
      "rewards/chosen": 1.478906273841858,
      "rewards/margins": 9.1171875,
      "rewards/rejected": -7.624218940734863,
      "step": 1860
    },
    {
      "epoch": 0.8779342723004695,
      "grad_norm": 270.92543752460966,
      "learning_rate": 6.807511737089202e-08,
      "logits/chosen": -4.231249809265137,
      "logits/rejected": -4.112500190734863,
      "logps/chosen": -549.5999755859375,
      "logps/rejected": -642.4000244140625,
      "loss": 2.8628,
      "rewards/accuracies": 0.504687488079071,
      "rewards/chosen": 2.350146532058716,
      "rewards/margins": 9.634374618530273,
      "rewards/rejected": -7.279687404632568,
      "step": 1870
    },
    {
      "epoch": 0.8826291079812206,
      "grad_norm": 250.02424304627982,
      "learning_rate": 6.546687532603024e-08,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.079687595367432,
      "logps/chosen": -556.5999755859375,
      "logps/rejected": -651.2000122070312,
      "loss": 2.6825,
      "rewards/accuracies": 0.4984374940395355,
      "rewards/chosen": 2.0291991233825684,
      "rewards/margins": 7.84375,
      "rewards/rejected": -5.8046875,
      "step": 1880
    },
    {
      "epoch": 0.8873239436619719,
      "grad_norm": 231.36542794007647,
      "learning_rate": 6.285863328116848e-08,
      "logits/chosen": -4.215624809265137,
      "logits/rejected": -4.103125095367432,
      "logps/chosen": -511.20001220703125,
      "logps/rejected": -661.5999755859375,
      "loss": 2.6862,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 2.270312547683716,
      "rewards/margins": 8.731249809265137,
      "rewards/rejected": -6.454687595367432,
      "step": 1890
    },
    {
      "epoch": 0.892018779342723,
      "grad_norm": 280.50166200715563,
      "learning_rate": 6.025039123630672e-08,
      "logits/chosen": -4.178124904632568,
      "logits/rejected": -4.084374904632568,
      "logps/chosen": -576.0,
      "logps/rejected": -615.5999755859375,
      "loss": 2.8327,
      "rewards/accuracies": 0.48906248807907104,
      "rewards/chosen": 2.594921827316284,
      "rewards/margins": 8.024999618530273,
      "rewards/rejected": -5.434374809265137,
      "step": 1900
    },
    {
      "epoch": 0.8967136150234741,
      "grad_norm": 214.81430839966447,
      "learning_rate": 5.764214919144496e-08,
      "logits/chosen": -4.181250095367432,
      "logits/rejected": -4.128125190734863,
      "logps/chosen": -579.2000122070312,
      "logps/rejected": -610.7999877929688,
      "loss": 2.7742,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 2.9834961891174316,
      "rewards/margins": 7.629687309265137,
      "rewards/rejected": -4.647656440734863,
      "step": 1910
    },
    {
      "epoch": 0.9014084507042254,
      "grad_norm": 348.56386639341446,
      "learning_rate": 5.50339071465832e-08,
      "logits/chosen": -4.190625190734863,
      "logits/rejected": -4.131249904632568,
      "logps/chosen": -569.5999755859375,
      "logps/rejected": -602.7999877929688,
      "loss": 2.9703,
      "rewards/accuracies": 0.4859375059604645,
      "rewards/chosen": 2.2406249046325684,
      "rewards/margins": 6.949999809265137,
      "rewards/rejected": -4.706250190734863,
      "step": 1920
    },
    {
      "epoch": 0.9061032863849765,
      "grad_norm": 289.60524521320974,
      "learning_rate": 5.2425665101721436e-08,
      "logits/chosen": -4.134375095367432,
      "logits/rejected": -4.010937690734863,
      "logps/chosen": -581.0,
      "logps/rejected": -690.7999877929688,
      "loss": 3.0283,
      "rewards/accuracies": 0.4976562559604645,
      "rewards/chosen": 2.378124952316284,
      "rewards/margins": 9.306249618530273,
      "rewards/rejected": -6.9296875,
      "step": 1930
    },
    {
      "epoch": 0.9107981220657277,
      "grad_norm": 270.1629735994182,
      "learning_rate": 4.9817423056859675e-08,
      "logits/chosen": -4.203125,
      "logits/rejected": -4.089062690734863,
      "logps/chosen": -575.0,
      "logps/rejected": -660.0,
      "loss": 2.6411,
      "rewards/accuracies": 0.500781238079071,
      "rewards/chosen": 2.956249952316284,
      "rewards/margins": 9.699999809265137,
      "rewards/rejected": -6.737500190734863,
      "step": 1940
    },
    {
      "epoch": 0.9154929577464789,
      "grad_norm": 265.7065287868444,
      "learning_rate": 4.720918101199791e-08,
      "logits/chosen": -4.09375,
      "logits/rejected": -3.981250047683716,
      "logps/chosen": -581.7999877929688,
      "logps/rejected": -714.4000244140625,
      "loss": 2.945,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 2.404296875,
      "rewards/margins": 10.553125381469727,
      "rewards/rejected": -8.143750190734863,
      "step": 1950
    },
    {
      "epoch": 0.92018779342723,
      "grad_norm": 250.18801620417574,
      "learning_rate": 4.460093896713615e-08,
      "logits/chosen": -4.209374904632568,
      "logits/rejected": -4.074999809265137,
      "logps/chosen": -561.5999755859375,
      "logps/rejected": -666.7999877929688,
      "loss": 2.8794,
      "rewards/accuracies": 0.49609375,
      "rewards/chosen": 2.867968797683716,
      "rewards/margins": 8.409375190734863,
      "rewards/rejected": -5.537499904632568,
      "step": 1960
    },
    {
      "epoch": 0.9248826291079812,
      "grad_norm": 230.95525396987273,
      "learning_rate": 4.199269692227438e-08,
      "logits/chosen": -4.1875,
      "logits/rejected": -4.123437404632568,
      "logps/chosen": -552.7999877929688,
      "logps/rejected": -604.2000122070312,
      "loss": 3.1617,
      "rewards/accuracies": 0.500781238079071,
      "rewards/chosen": 2.604687452316284,
      "rewards/margins": 7.143750190734863,
      "rewards/rejected": -4.538281440734863,
      "step": 1970
    },
    {
      "epoch": 0.9295774647887324,
      "grad_norm": 203.14051919454678,
      "learning_rate": 3.938445487741262e-08,
      "logits/chosen": -4.123437404632568,
      "logits/rejected": -3.989062547683716,
      "logps/chosen": -579.4000244140625,
      "logps/rejected": -638.0,
      "loss": 2.8828,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 2.0751953125,
      "rewards/margins": 9.285937309265137,
      "rewards/rejected": -7.2109375,
      "step": 1980
    },
    {
      "epoch": 0.9342723004694836,
      "grad_norm": 342.2288255482226,
      "learning_rate": 3.677621283255086e-08,
      "logits/chosen": -4.21875,
      "logits/rejected": -4.134375095367432,
      "logps/chosen": -584.4000244140625,
      "logps/rejected": -638.4000244140625,
      "loss": 3.0422,
      "rewards/accuracies": 0.4867187440395355,
      "rewards/chosen": 2.6312499046325684,
      "rewards/margins": 8.221875190734863,
      "rewards/rejected": -5.589062690734863,
      "step": 1990
    },
    {
      "epoch": 0.9389671361502347,
      "grad_norm": 214.8039554016909,
      "learning_rate": 3.41679707876891e-08,
      "logits/chosen": -4.153124809265137,
      "logits/rejected": -4.042187690734863,
      "logps/chosen": -568.7999877929688,
      "logps/rejected": -698.0,
      "loss": 2.5212,
      "rewards/accuracies": 0.4921875,
      "rewards/chosen": 2.428906202316284,
      "rewards/margins": 9.993749618530273,
      "rewards/rejected": -7.5546875,
      "step": 2000
    },
    {
      "epoch": 0.9436619718309859,
      "grad_norm": 438.04502258424424,
      "learning_rate": 3.155972874282733e-08,
      "logits/chosen": -4.228125095367432,
      "logits/rejected": -4.118750095367432,
      "logps/chosen": -545.7999877929688,
      "logps/rejected": -633.5999755859375,
      "loss": 2.52,
      "rewards/accuracies": 0.516406238079071,
      "rewards/chosen": 2.612499952316284,
      "rewards/margins": 8.423437118530273,
      "rewards/rejected": -5.814843654632568,
      "step": 2010
    },
    {
      "epoch": 0.9483568075117371,
      "grad_norm": 232.40256926460418,
      "learning_rate": 2.8951486697965573e-08,
      "logits/chosen": -4.153124809265137,
      "logits/rejected": -4.071875095367432,
      "logps/chosen": -579.7999877929688,
      "logps/rejected": -654.7999877929688,
      "loss": 2.5768,
      "rewards/accuracies": 0.4945312440395355,
      "rewards/chosen": 2.94921875,
      "rewards/margins": 9.087499618530273,
      "rewards/rejected": -6.140625,
      "step": 2020
    },
    {
      "epoch": 0.9530516431924883,
      "grad_norm": 247.32217363390706,
      "learning_rate": 2.634324465310381e-08,
      "logits/chosen": -4.201562404632568,
      "logits/rejected": -4.079687595367432,
      "logps/chosen": -554.4000244140625,
      "logps/rejected": -640.7999877929688,
      "loss": 2.808,
      "rewards/accuracies": 0.5078125,
      "rewards/chosen": 2.303906202316284,
      "rewards/margins": 8.643750190734863,
      "rewards/rejected": -6.345312595367432,
      "step": 2030
    },
    {
      "epoch": 0.9577464788732394,
      "grad_norm": 216.75114769995488,
      "learning_rate": 2.3735002608242045e-08,
      "logits/chosen": -4.1796875,
      "logits/rejected": -4.096875190734863,
      "logps/chosen": -573.5999755859375,
      "logps/rejected": -616.7999877929688,
      "loss": 2.5154,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 2.692578077316284,
      "rewards/margins": 8.834375381469727,
      "rewards/rejected": -6.143750190734863,
      "step": 2040
    },
    {
      "epoch": 0.9624413145539906,
      "grad_norm": 244.4652512645769,
      "learning_rate": 2.1126760563380282e-08,
      "logits/chosen": -4.184374809265137,
      "logits/rejected": -4.109375,
      "logps/chosen": -559.5999755859375,
      "logps/rejected": -592.7999877929688,
      "loss": 2.6575,
      "rewards/accuracies": 0.4828124940395355,
      "rewards/chosen": 2.5406250953674316,
      "rewards/margins": 7.303124904632568,
      "rewards/rejected": -4.754687309265137,
      "step": 2050
    },
    {
      "epoch": 0.9671361502347418,
      "grad_norm": 1006.669099795599,
      "learning_rate": 1.8518518518518518e-08,
      "logits/chosen": -4.175000190734863,
      "logits/rejected": -4.060937404632568,
      "logps/chosen": -584.7999877929688,
      "logps/rejected": -644.7999877929688,
      "loss": 2.9606,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 2.246875047683716,
      "rewards/margins": 7.487500190734863,
      "rewards/rejected": -5.237500190734863,
      "step": 2060
    },
    {
      "epoch": 0.971830985915493,
      "grad_norm": 245.9856583194556,
      "learning_rate": 1.5910276473656755e-08,
      "logits/chosen": -4.143750190734863,
      "logits/rejected": -4.0078125,
      "logps/chosen": -559.7999877929688,
      "logps/rejected": -639.7999877929688,
      "loss": 2.6479,
      "rewards/accuracies": 0.507031261920929,
      "rewards/chosen": 2.8199219703674316,
      "rewards/margins": 9.907812118530273,
      "rewards/rejected": -7.089062690734863,
      "step": 2070
    },
    {
      "epoch": 0.9765258215962441,
      "grad_norm": 219.87542070354826,
      "learning_rate": 1.3302034428794991e-08,
      "logits/chosen": -4.224999904632568,
      "logits/rejected": -4.196875095367432,
      "logps/chosen": -540.0,
      "logps/rejected": -604.5999755859375,
      "loss": 2.5598,
      "rewards/accuracies": 0.5289062261581421,
      "rewards/chosen": 2.6328125,
      "rewards/margins": 9.142187118530273,
      "rewards/rejected": -6.515625,
      "step": 2080
    },
    {
      "epoch": 0.9812206572769953,
      "grad_norm": 319.2812742395816,
      "learning_rate": 1.0693792383933229e-08,
      "logits/chosen": -4.232812404632568,
      "logits/rejected": -4.173437595367432,
      "logps/chosen": -572.4000244140625,
      "logps/rejected": -652.4000244140625,
      "loss": 2.8662,
      "rewards/accuracies": 0.4867187440395355,
      "rewards/chosen": 2.889843702316284,
      "rewards/margins": 9.271875381469727,
      "rewards/rejected": -6.3828125,
      "step": 2090
    },
    {
      "epoch": 0.9859154929577465,
      "grad_norm": 289.0276689267499,
      "learning_rate": 8.085550339071465e-09,
      "logits/chosen": -4.175000190734863,
      "logits/rejected": -4.076562404632568,
      "logps/chosen": -587.5999755859375,
      "logps/rejected": -685.0,
      "loss": 3.2285,
      "rewards/accuracies": 0.49296873807907104,
      "rewards/chosen": 2.369921922683716,
      "rewards/margins": 8.143750190734863,
      "rewards/rejected": -5.768750190734863,
      "step": 2100
    },
    {
      "epoch": 0.9906103286384976,
      "grad_norm": 318.56380856780027,
      "learning_rate": 5.4773082942097025e-09,
      "logits/chosen": -4.196875095367432,
      "logits/rejected": -4.131249904632568,
      "logps/chosen": -556.4000244140625,
      "logps/rejected": -619.5999755859375,
      "loss": 2.3993,
      "rewards/accuracies": 0.5234375,
      "rewards/chosen": 2.5599608421325684,
      "rewards/margins": 8.771875381469727,
      "rewards/rejected": -6.2109375,
      "step": 2110
    },
    {
      "epoch": 0.9953051643192489,
      "grad_norm": 236.34850676325576,
      "learning_rate": 2.8690662493479393e-09,
      "logits/chosen": -4.240624904632568,
      "logits/rejected": -4.079687595367432,
      "logps/chosen": -557.5999755859375,
      "logps/rejected": -629.5999755859375,
      "loss": 3.6049,
      "rewards/accuracies": 0.5023437738418579,
      "rewards/chosen": 2.610546827316284,
      "rewards/margins": 7.922461032867432,
      "rewards/rejected": -5.306640625,
      "step": 2120
    },
    {
      "epoch": 1.0,
      "grad_norm": 221.64813357708303,
      "learning_rate": 2.608242044861763e-10,
      "logits/chosen": -4.199999809265137,
      "logits/rejected": -4.160937309265137,
      "logps/chosen": -556.2000122070312,
      "logps/rejected": -654.4000244140625,
      "loss": 2.6604,
      "rewards/accuracies": 0.5190496444702148,
      "rewards/chosen": 3.0367188453674316,
      "rewards/margins": 10.324999809265137,
      "rewards/rejected": -7.287499904632568,
      "step": 2130
    },
    {
      "epoch": 1.0,
      "step": 2130,
      "total_flos": 0.0,
      "train_loss": 2.805410225738382,
      "train_runtime": 6798.7376,
      "train_samples_per_second": 40.093,
      "train_steps_per_second": 0.313
    }
  ],
  "logging_steps": 10,
  "max_steps": 2130,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}