{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.999244142101285,
  "eval_steps": 100,
  "global_step": 661,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0015117157974300832,
      "grad_norm": 17.933454513549805,
      "kl/avg_steps": 0.09375,
      "kl/beta": 0.10000000149011612,
      "kl/n_epsilon_steps": 0.453125,
      "kl/p_epsilon_steps": 0.546875,
      "learning_rate": 0.0,
      "logits/chosen": -0.8771844506263733,
      "logits/rejected": -0.7888585329055786,
      "logps/chosen": -80.20932006835938,
      "logps/ref_chosen": -80.27740478515625,
      "logps/ref_rejected": -83.5943374633789,
      "logps/rejected": -83.52326965332031,
      "loss": 1.3873,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.006630806718021631,
      "rewards/margins": -0.0005992341320961714,
      "rewards/rejected": 0.007230041082948446,
      "step": 1
    },
    {
      "epoch": 0.0030234315948601664,
      "grad_norm": 21.353334426879883,
      "kl/avg_steps": -0.15625,
      "kl/beta": 0.09990634024143219,
      "kl/n_epsilon_steps": 0.578125,
      "kl/p_epsilon_steps": 0.421875,
      "learning_rate": 7.462686567164179e-09,
      "logits/chosen": -0.6832054853439331,
      "logits/rejected": -0.5088719129562378,
      "logps/chosen": -74.510986328125,
      "logps/ref_chosen": -74.56095886230469,
      "logps/ref_rejected": -83.53636169433594,
      "logps/rejected": -83.51570892333984,
      "loss": 1.3843,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0048660230822861195,
      "rewards/margins": 0.0025865831412374973,
      "rewards/rejected": 0.002279440173879266,
      "step": 2
    },
    {
      "epoch": 0.0045351473922902496,
      "grad_norm": 19.950443267822266,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.10006268322467804,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 1.4925373134328357e-08,
      "logits/chosen": -0.6054874658584595,
      "logits/rejected": -0.3736334443092346,
      "logps/chosen": -82.1410140991211,
      "logps/ref_chosen": -82.15100860595703,
      "logps/ref_rejected": -109.82986450195312,
      "logps/rejected": -109.80192565917969,
      "loss": 1.3887,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 0.0008957167156040668,
      "rewards/margins": -0.0020755126606673002,
      "rewards/rejected": 0.002971230074763298,
      "step": 3
    },
    {
      "epoch": 0.006046863189720333,
      "grad_norm": 19.876798629760742,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.10003142803907394,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 2.2388059701492534e-08,
      "logits/chosen": -0.4454895853996277,
      "logits/rejected": -0.3323523998260498,
      "logps/chosen": -92.34318542480469,
      "logps/ref_chosen": -92.37549591064453,
      "logps/ref_rejected": -99.59554290771484,
      "logps/rejected": -99.51423645019531,
      "loss": 1.3919,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0031029037199914455,
      "rewards/margins": -0.005186422728002071,
      "rewards/rejected": 0.00828932598233223,
      "step": 4
    },
    {
      "epoch": 0.007558578987150416,
      "grad_norm": 18.935115814208984,
      "kl/avg_steps": 0.09375,
      "kl/beta": 0.0999065414071083,
      "kl/n_epsilon_steps": 0.453125,
      "kl/p_epsilon_steps": 0.546875,
      "learning_rate": 2.9850746268656714e-08,
      "logits/chosen": -0.6434583067893982,
      "logits/rejected": -0.43680721521377563,
      "logps/chosen": -78.93097686767578,
      "logps/ref_chosen": -78.84872436523438,
      "logps/ref_rejected": -97.88040161132812,
      "logps/rejected": -97.91473388671875,
      "loss": 1.3919,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -0.00838147010654211,
      "rewards/margins": -0.005112465005367994,
      "rewards/rejected": -0.00326900533400476,
      "step": 5
    },
    {
      "epoch": 0.009070294784580499,
      "grad_norm": 18.06861686706543,
      "kl/avg_steps": 0.0,
      "kl/beta": 0.09981296956539154,
      "kl/n_epsilon_steps": 0.5,
      "kl/p_epsilon_steps": 0.5,
      "learning_rate": 3.731343283582089e-08,
      "logits/chosen": -0.7716882228851318,
      "logits/rejected": -0.5386408567428589,
      "logps/chosen": -68.30958557128906,
      "logps/ref_chosen": -68.34607696533203,
      "logps/ref_rejected": -99.24613952636719,
      "logps/rejected": -99.24362182617188,
      "loss": 1.3836,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.00351733504794538,
      "rewards/margins": 0.0031151659786701202,
      "rewards/rejected": 0.0004021693021059036,
      "step": 6
    },
    {
      "epoch": 0.010582010582010581,
      "grad_norm": 17.43248748779297,
      "kl/avg_steps": -0.1875,
      "kl/beta": 0.09981296956539154,
      "kl/n_epsilon_steps": 0.59375,
      "kl/p_epsilon_steps": 0.40625,
      "learning_rate": 4.477611940298507e-08,
      "logits/chosen": -1.039565086364746,
      "logits/rejected": -0.6296759843826294,
      "logps/chosen": -69.15159606933594,
      "logps/ref_chosen": -69.11282348632812,
      "logps/ref_rejected": -84.01641845703125,
      "logps/rejected": -83.97854614257812,
      "loss": 1.3945,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.0039845979772508144,
      "rewards/margins": -0.00791984610259533,
      "rewards/rejected": 0.00393524719402194,
      "step": 7
    },
    {
      "epoch": 0.012093726379440665,
      "grad_norm": 18.484458923339844,
      "kl/avg_steps": -0.15625,
      "kl/beta": 0.10000047087669373,
      "kl/n_epsilon_steps": 0.578125,
      "kl/p_epsilon_steps": 0.421875,
      "learning_rate": 5.223880597014925e-08,
      "logits/chosen": -0.7085280418395996,
      "logits/rejected": -0.4177365303039551,
      "logps/chosen": -78.41571044921875,
      "logps/ref_chosen": -78.3912353515625,
      "logps/ref_rejected": -91.06254577636719,
      "logps/rejected": -91.00235748291016,
      "loss": 1.3954,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.0025808759965002537,
      "rewards/margins": -0.008768673986196518,
      "rewards/rejected": 0.006187797989696264,
      "step": 8
    },
    {
      "epoch": 0.013605442176870748,
      "grad_norm": 19.37607192993164,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.10015696287155151,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 5.970149253731343e-08,
      "logits/chosen": -0.5926854610443115,
      "logits/rejected": -0.6044590473175049,
      "logps/chosen": -69.67474365234375,
      "logps/ref_chosen": -69.67422485351562,
      "logps/ref_rejected": -105.00473022460938,
      "logps/rejected": -105.07916259765625,
      "loss": 1.3798,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.00018217615433968604,
      "rewards/margins": 0.0070345159620046616,
      "rewards/rejected": -0.007216691970825195,
      "step": 9
    },
    {
      "epoch": 0.015117157974300832,
      "grad_norm": 18.984508514404297,
      "kl/avg_steps": -0.09375,
      "kl/beta": 0.0999695211648941,
      "kl/n_epsilon_steps": 0.546875,
      "kl/p_epsilon_steps": 0.453125,
      "learning_rate": 6.71641791044776e-08,
      "logits/chosen": -0.67566978931427,
      "logits/rejected": -0.4178224802017212,
      "logps/chosen": -79.67657470703125,
      "logps/ref_chosen": -79.730712890625,
      "logps/ref_rejected": -105.50645446777344,
      "logps/rejected": -105.46436309814453,
      "loss": 1.3858,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.005256780423223972,
      "rewards/margins": 0.0008970340131781995,
      "rewards/rejected": 0.004359746817499399,
      "step": 10
    },
    {
      "epoch": 0.016628873771730914,
      "grad_norm": 17.404315948486328,
      "kl/avg_steps": -0.0625,
      "kl/beta": 0.10006333142518997,
      "kl/n_epsilon_steps": 0.53125,
      "kl/p_epsilon_steps": 0.46875,
      "learning_rate": 7.462686567164178e-08,
      "logits/chosen": -0.682112455368042,
      "logits/rejected": -0.7254103422164917,
      "logps/chosen": -85.43687438964844,
      "logps/ref_chosen": -85.41248321533203,
      "logps/ref_rejected": -86.50241088867188,
      "logps/rejected": -86.51531219482422,
      "loss": 1.3882,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -0.0025822517927736044,
      "rewards/margins": -0.0014805782120674849,
      "rewards/rejected": -0.0011016735807061195,
      "step": 11
    },
    {
      "epoch": 0.018140589569160998,
      "grad_norm": 17.35363006591797,
      "kl/avg_steps": 0.09375,
      "kl/beta": 0.10012590885162354,
      "kl/n_epsilon_steps": 0.453125,
      "kl/p_epsilon_steps": 0.546875,
      "learning_rate": 8.208955223880596e-08,
      "logits/chosen": -0.48884809017181396,
      "logits/rejected": -0.3806966543197632,
      "logps/chosen": -81.39530944824219,
      "logps/ref_chosen": -81.38086700439453,
      "logps/ref_rejected": -89.88151550292969,
      "logps/rejected": -89.9115219116211,
      "loss": 1.3855,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.0015910749789327383,
      "rewards/margins": 0.0012488359352573752,
      "rewards/rejected": -0.0028399100992828608,
      "step": 12
    },
    {
      "epoch": 0.019652305366591082,
      "grad_norm": 17.843292236328125,
      "kl/avg_steps": 0.0625,
      "kl/beta": 0.10003212839365005,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.53125,
      "learning_rate": 8.955223880597014e-08,
      "logits/chosen": -1.0486931800842285,
      "logits/rejected": -0.7209100723266602,
      "logps/chosen": -63.15821075439453,
      "logps/ref_chosen": -63.17030715942383,
      "logps/ref_rejected": -105.61166381835938,
      "logps/rejected": -105.63218688964844,
      "loss": 1.3835,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0010905354283750057,
      "rewards/margins": 0.0030125719495117664,
      "rewards/rejected": -0.0019220358226448298,
      "step": 13
    },
    {
      "epoch": 0.021164021164021163,
      "grad_norm": 20.182865142822266,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.09996964782476425,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 9.701492537313432e-08,
      "logits/chosen": -0.6667978763580322,
      "logits/rejected": -0.4419565498828888,
      "logps/chosen": -80.64845275878906,
      "logps/ref_chosen": -80.71014404296875,
      "logps/ref_rejected": -89.86041259765625,
      "logps/rejected": -89.85292053222656,
      "loss": 1.3814,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.006051511503756046,
      "rewards/margins": 0.005182279273867607,
      "rewards/rejected": 0.0008692322298884392,
      "step": 14
    },
    {
      "epoch": 0.022675736961451247,
      "grad_norm": 20.247482299804688,
      "kl/avg_steps": -0.125,
      "kl/beta": 0.09981369227170944,
      "kl/n_epsilon_steps": 0.5625,
      "kl/p_epsilon_steps": 0.4375,
      "learning_rate": 1.044776119402985e-07,
      "logits/chosen": -0.8077883720397949,
      "logits/rejected": -0.4688650667667389,
      "logps/chosen": -82.10345458984375,
      "logps/ref_chosen": -82.00294494628906,
      "logps/ref_rejected": -106.43550109863281,
      "logps/rejected": -106.45130157470703,
      "loss": 1.3954,
      "rewards/accuracies": 0.421875,
      "rewards/chosen": -0.010214600712060928,
      "rewards/margins": -0.008754991926252842,
      "rewards/rejected": -0.00145960901863873,
      "step": 15
    },
    {
      "epoch": 0.02418745275888133,
      "grad_norm": 17.199460983276367,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.09993861615657806,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 1.1194029850746268e-07,
      "logits/chosen": -0.6257915496826172,
      "logits/rejected": -0.41689813137054443,
      "logps/chosen": -62.30339813232422,
      "logps/ref_chosen": -62.308345794677734,
      "logps/ref_rejected": -89.6508560180664,
      "logps/rejected": -89.64524841308594,
      "loss": 1.387,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 0.0003566534724086523,
      "rewards/margins": -0.00036794866900891066,
      "rewards/rejected": 0.0007246022578328848,
      "step": 16
    },
    {
      "epoch": 0.025699168556311415,
      "grad_norm": 18.40418243408203,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09990739077329636,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 1.1940298507462686e-07,
      "logits/chosen": -0.6596513390541077,
      "logits/rejected": -0.38339459896087646,
      "logps/chosen": -85.23394775390625,
      "logps/ref_chosen": -85.16903686523438,
      "logps/ref_rejected": -102.57087707519531,
      "logps/rejected": -102.61199951171875,
      "loss": 1.3894,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.006622787099331617,
      "rewards/margins": -0.0026723374612629414,
      "rewards/rejected": -0.003950449638068676,
      "step": 17
    },
    {
      "epoch": 0.027210884353741496,
      "grad_norm": 17.053964614868164,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.09978266060352325,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 1.2686567164179106e-07,
      "logits/chosen": -0.8401739597320557,
      "logits/rejected": -0.48542100191116333,
      "logps/chosen": -63.1472282409668,
      "logps/ref_chosen": -63.17793273925781,
      "logps/ref_rejected": -86.06461334228516,
      "logps/rejected": -86.12118530273438,
      "loss": 1.3781,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.002997747389599681,
      "rewards/margins": 0.008445605635643005,
      "rewards/rejected": -0.005447858478873968,
      "step": 18
    },
    {
      "epoch": 0.02872260015117158,
      "grad_norm": 19.71549415588379,
      "kl/avg_steps": -0.078125,
      "kl/beta": 0.09953382611274719,
      "kl/n_epsilon_steps": 0.53125,
      "kl/p_epsilon_steps": 0.453125,
      "learning_rate": 1.343283582089552e-07,
      "logits/chosen": -0.5802021026611328,
      "logits/rejected": -0.36945077776908875,
      "logps/chosen": -85.82483673095703,
      "logps/ref_chosen": -85.82405090332031,
      "logps/ref_rejected": -100.07136535644531,
      "logps/rejected": -100.070556640625,
      "loss": 1.3872,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.00019685056759044528,
      "rewards/margins": -0.00046660611405968666,
      "rewards/rejected": 0.00026975583750754595,
      "step": 19
    },
    {
      "epoch": 0.030234315948601664,
      "grad_norm": 18.155420303344727,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.09961164742708206,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 1.4179104477611938e-07,
      "logits/chosen": -0.5410428643226624,
      "logits/rejected": -0.44256073236465454,
      "logps/chosen": -73.6261978149414,
      "logps/ref_chosen": -73.58621215820312,
      "logps/ref_rejected": -91.21690368652344,
      "logps/rejected": -91.28337860107422,
      "loss": 1.3843,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.004116400144994259,
      "rewards/margins": 0.002343452535569668,
      "rewards/rejected": -0.006459852214902639,
      "step": 20
    },
    {
      "epoch": 0.031746031746031744,
      "grad_norm": 18.056482315063477,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.09958053380250931,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 1.4925373134328355e-07,
      "logits/chosen": -0.615408182144165,
      "logits/rejected": -0.520226776599884,
      "logps/chosen": -81.95823669433594,
      "logps/ref_chosen": -81.97251892089844,
      "logps/ref_rejected": -98.05976867675781,
      "logps/rejected": -98.11122131347656,
      "loss": 1.3804,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.0012293007457628846,
      "rewards/margins": 0.006264813244342804,
      "rewards/rejected": -0.005035512149333954,
      "step": 21
    },
    {
      "epoch": 0.03325774754346183,
      "grad_norm": 18.43136978149414,
      "kl/avg_steps": -0.03125,
      "kl/beta": 0.09942518174648285,
      "kl/n_epsilon_steps": 0.515625,
      "kl/p_epsilon_steps": 0.484375,
      "learning_rate": 1.5671641791044775e-07,
      "logits/chosen": -0.7960255742073059,
      "logits/rejected": -0.3484349548816681,
      "logps/chosen": -76.95167541503906,
      "logps/ref_chosen": -76.99579620361328,
      "logps/ref_rejected": -95.76089477539062,
      "logps/rejected": -95.7391357421875,
      "loss": 1.3847,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": 0.004259100183844566,
      "rewards/margins": 0.0019459626637399197,
      "rewards/rejected": 0.0023131368216127157,
      "step": 22
    },
    {
      "epoch": 0.03476946334089191,
      "grad_norm": 18.915191650390625,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.09945625811815262,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 1.6417910447761193e-07,
      "logits/chosen": -0.5395127534866333,
      "logits/rejected": -0.37187278270721436,
      "logps/chosen": -84.71544647216797,
      "logps/ref_chosen": -84.76856994628906,
      "logps/ref_rejected": -107.28266906738281,
      "logps/rejected": -107.30066680908203,
      "loss": 1.3798,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.005197981372475624,
      "rewards/margins": 0.006793309934437275,
      "rewards/rejected": -0.0015953283291310072,
      "step": 23
    },
    {
      "epoch": 0.036281179138321996,
      "grad_norm": 17.060243606567383,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.09917732328176498,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 1.716417910447761e-07,
      "logits/chosen": -0.8160616159439087,
      "logits/rejected": -0.6523994207382202,
      "logps/chosen": -69.83349609375,
      "logps/ref_chosen": -69.87112426757812,
      "logps/ref_rejected": -84.02084350585938,
      "logps/rejected": -83.9853744506836,
      "loss": 1.3866,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0035986052826046944,
      "rewards/margins": -3.9631209801882505e-05,
      "rewards/rejected": 0.003638236550614238,
      "step": 24
    },
    {
      "epoch": 0.03779289493575208,
      "grad_norm": 19.301118850708008,
      "kl/avg_steps": 0.265625,
      "kl/beta": 0.09914634376764297,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 1.7910447761194027e-07,
      "logits/chosen": -0.5545772314071655,
      "logits/rejected": -0.5116233825683594,
      "logps/chosen": -78.25363159179688,
      "logps/ref_chosen": -78.22694396972656,
      "logps/ref_rejected": -106.65234375,
      "logps/rejected": -106.760986328125,
      "loss": 1.379,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.002777719870209694,
      "rewards/margins": 0.007790995761752129,
      "rewards/rejected": -0.010568715631961823,
      "step": 25
    },
    {
      "epoch": 0.039304610733182165,
      "grad_norm": 17.692121505737305,
      "kl/avg_steps": 0.0625,
      "kl/beta": 0.09888368099927902,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.53125,
      "learning_rate": 1.8656716417910447e-07,
      "logits/chosen": -0.431125283241272,
      "logits/rejected": -0.19942858815193176,
      "logps/chosen": -74.57691192626953,
      "logps/ref_chosen": -74.59750366210938,
      "logps/ref_rejected": -93.57858276367188,
      "logps/rejected": -93.59805297851562,
      "loss": 1.3829,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0019292905926704407,
      "rewards/margins": 0.003692114260047674,
      "rewards/rejected": -0.0017628234345465899,
      "step": 26
    },
    {
      "epoch": 0.04081632653061224,
      "grad_norm": 18.422821044921875,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09882191568613052,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 1.9402985074626865e-07,
      "logits/chosen": -0.6598864793777466,
      "logits/rejected": -0.3999100923538208,
      "logps/chosen": -78.63863372802734,
      "logps/ref_chosen": -78.64625549316406,
      "logps/ref_rejected": -92.33645629882812,
      "logps/rejected": -92.38688659667969,
      "loss": 1.3812,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.0006331197218969464,
      "rewards/margins": 0.0054315500892698765,
      "rewards/rejected": -0.004798430018126965,
      "step": 27
    },
    {
      "epoch": 0.042328042328042326,
      "grad_norm": 17.46875762939453,
      "kl/avg_steps": -0.03125,
      "kl/beta": 0.09869854152202606,
      "kl/n_epsilon_steps": 0.515625,
      "kl/p_epsilon_steps": 0.484375,
      "learning_rate": 2.0149253731343282e-07,
      "logits/chosen": -0.8402580618858337,
      "logits/rejected": -0.7967926263809204,
      "logps/chosen": -76.87187957763672,
      "logps/ref_chosen": -76.91271209716797,
      "logps/ref_rejected": -88.48194885253906,
      "logps/rejected": -88.45233154296875,
      "loss": 1.3859,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0038373656570911407,
      "rewards/margins": 0.00078454555477947,
      "rewards/rejected": 0.003052819985896349,
      "step": 28
    },
    {
      "epoch": 0.04383975812547241,
      "grad_norm": 20.94962501525879,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.09872939437627792,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 2.08955223880597e-07,
      "logits/chosen": -0.38888347148895264,
      "logits/rejected": -0.36869269609451294,
      "logps/chosen": -89.60147094726562,
      "logps/ref_chosen": -89.62060546875,
      "logps/ref_rejected": -100.57090759277344,
      "logps/rejected": -100.54659271240234,
      "loss": 1.3879,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0016972769517451525,
      "rewards/margins": -0.0009317040676251054,
      "rewards/rejected": 0.0026289813686162233,
      "step": 29
    },
    {
      "epoch": 0.045351473922902494,
      "grad_norm": 18.70415687561035,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09854462742805481,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 2.1641791044776117e-07,
      "logits/chosen": -0.8333492279052734,
      "logits/rejected": -0.5384379625320435,
      "logps/chosen": -68.81825256347656,
      "logps/ref_chosen": -68.82381439208984,
      "logps/ref_rejected": -104.7047119140625,
      "logps/rejected": -104.7557373046875,
      "loss": 1.3817,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.0004366333014331758,
      "rewards/margins": 0.005281157325953245,
      "rewards/rejected": -0.0048445239663124084,
      "step": 30
    },
    {
      "epoch": 0.04686318972033258,
      "grad_norm": 20.447372436523438,
      "kl/avg_steps": -0.046875,
      "kl/beta": 0.0984215959906578,
      "kl/n_epsilon_steps": 0.515625,
      "kl/p_epsilon_steps": 0.46875,
      "learning_rate": 2.2388059701492537e-07,
      "logits/chosen": -0.7040875554084778,
      "logits/rejected": -0.4996650815010071,
      "logps/chosen": -86.09111022949219,
      "logps/ref_chosen": -86.06916809082031,
      "logps/ref_rejected": -116.66395568847656,
      "logps/rejected": -116.6534423828125,
      "loss": 1.3903,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.002270359545946121,
      "rewards/margins": -0.0035220435820519924,
      "rewards/rejected": 0.0012516845017671585,
      "step": 31
    },
    {
      "epoch": 0.04837490551776266,
      "grad_norm": 18.30170440673828,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09846775233745575,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 2.3134328358208954e-07,
      "logits/chosen": -1.0229980945587158,
      "logits/rejected": -0.5279667377471924,
      "logps/chosen": -87.55634307861328,
      "logps/ref_chosen": -87.59809112548828,
      "logps/ref_rejected": -100.26905822753906,
      "logps/rejected": -100.24147033691406,
      "loss": 1.3857,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.003950035199522972,
      "rewards/margins": 0.0010602055117487907,
      "rewards/rejected": 0.0028898296877741814,
      "step": 32
    },
    {
      "epoch": 0.049886621315192746,
      "grad_norm": 19.37946319580078,
      "kl/avg_steps": -0.0625,
      "kl/beta": 0.0983448252081871,
      "kl/n_epsilon_steps": 0.53125,
      "kl/p_epsilon_steps": 0.46875,
      "learning_rate": 2.388059701492537e-07,
      "logits/chosen": -0.8228363394737244,
      "logits/rejected": -0.6981616616249084,
      "logps/chosen": -83.27375793457031,
      "logps/ref_chosen": -83.29850769042969,
      "logps/ref_rejected": -94.60990142822266,
      "logps/rejected": -94.55514526367188,
      "loss": 1.3899,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.0023084937129169703,
      "rewards/margins": -0.0032619782723486423,
      "rewards/rejected": 0.005570471752434969,
      "step": 33
    },
    {
      "epoch": 0.05139833711262283,
      "grad_norm": 17.70010757446289,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.09840632975101471,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 2.4626865671641786e-07,
      "logits/chosen": -0.6430321335792542,
      "logits/rejected": -0.46988445520401,
      "logps/chosen": -70.10933685302734,
      "logps/ref_chosen": -70.15070343017578,
      "logps/ref_rejected": -84.4693832397461,
      "logps/rejected": -84.48771667480469,
      "loss": 1.381,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003950835205614567,
      "rewards/margins": 0.0056062545627355576,
      "rewards/rejected": -0.001655419822782278,
      "step": 34
    },
    {
      "epoch": 0.05291005291005291,
      "grad_norm": 17.64505386352539,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.09816092252731323,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 2.537313432835821e-07,
      "logits/chosen": -0.7675759792327881,
      "logits/rejected": -0.5877223014831543,
      "logps/chosen": -78.1800537109375,
      "logps/ref_chosen": -78.25238037109375,
      "logps/ref_rejected": -91.06356811523438,
      "logps/rejected": -91.0887680053711,
      "loss": 1.3775,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.006912318058311939,
      "rewards/margins": 0.009255615994334221,
      "rewards/rejected": -0.002343298401683569,
      "step": 35
    },
    {
      "epoch": 0.05442176870748299,
      "grad_norm": 17.668521881103516,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.09800779074430466,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 2.611940298507462e-07,
      "logits/chosen": -0.9533746838569641,
      "logits/rejected": -0.4546297788619995,
      "logps/chosen": -67.0625228881836,
      "logps/ref_chosen": -67.06676483154297,
      "logps/ref_rejected": -99.34661865234375,
      "logps/rejected": -99.37528228759766,
      "loss": 1.3839,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.0003041817108169198,
      "rewards/margins": 0.0029306563083082438,
      "rewards/rejected": -0.0026264747139066458,
      "step": 36
    },
    {
      "epoch": 0.055933484504913075,
      "grad_norm": 23.152936935424805,
      "kl/avg_steps": -0.03125,
      "kl/beta": 0.0979771688580513,
      "kl/n_epsilon_steps": 0.515625,
      "kl/p_epsilon_steps": 0.484375,
      "learning_rate": 2.686567164179104e-07,
      "logits/chosen": -0.739529550075531,
      "logits/rejected": -0.6823672652244568,
      "logps/chosen": -75.89892578125,
      "logps/ref_chosen": -75.92698669433594,
      "logps/ref_rejected": -130.34371948242188,
      "logps/rejected": -130.28778076171875,
      "loss": 1.3899,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.002641711849719286,
      "rewards/margins": -0.0030628300737589598,
      "rewards/rejected": 0.005704541690647602,
      "step": 37
    },
    {
      "epoch": 0.05744520030234316,
      "grad_norm": 17.994104385375977,
      "kl/avg_steps": 0.015625,
      "kl/beta": 0.09800779819488525,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.5,
      "learning_rate": 2.761194029850746e-07,
      "logits/chosen": -0.3506224751472473,
      "logits/rejected": -0.31289514899253845,
      "logps/chosen": -83.65824127197266,
      "logps/ref_chosen": -83.65460205078125,
      "logps/ref_rejected": -89.15221405029297,
      "logps/rejected": -89.1767349243164,
      "loss": 1.385,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -0.00048196763964369893,
      "rewards/margins": 0.0017501943511888385,
      "rewards/rejected": -0.0022321625147014856,
      "step": 38
    },
    {
      "epoch": 0.05895691609977324,
      "grad_norm": 18.549318313598633,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09799248725175858,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 2.8358208955223876e-07,
      "logits/chosen": -0.5775608420372009,
      "logits/rejected": -0.3955717086791992,
      "logps/chosen": -76.12467956542969,
      "logps/ref_chosen": -76.18706512451172,
      "logps/ref_rejected": -94.39262390136719,
      "logps/rejected": -94.3853530883789,
      "loss": 1.3816,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.005926312878727913,
      "rewards/margins": 0.005100561771541834,
      "rewards/rejected": 0.0008257507579401135,
      "step": 39
    },
    {
      "epoch": 0.06046863189720333,
      "grad_norm": 17.475339889526367,
      "kl/avg_steps": 0.140625,
      "kl/beta": 0.09787014871835709,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 2.9104477611940296e-07,
      "logits/chosen": -0.5079457759857178,
      "logits/rejected": -0.4361386001110077,
      "logps/chosen": -77.43675231933594,
      "logps/ref_chosen": -77.43476867675781,
      "logps/ref_rejected": -98.58720397949219,
      "logps/rejected": -98.69015502929688,
      "loss": 1.377,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.00033624155912548304,
      "rewards/margins": 0.009605846367776394,
      "rewards/rejected": -0.009942087344825268,
      "step": 40
    },
    {
      "epoch": 0.06198034769463341,
      "grad_norm": 18.129384994506836,
      "kl/avg_steps": 0.203125,
      "kl/beta": 0.09773271530866623,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 2.985074626865671e-07,
      "logits/chosen": -0.6582231521606445,
      "logits/rejected": -0.6316337585449219,
      "logps/chosen": -86.85847473144531,
      "logps/ref_chosen": -86.87641143798828,
      "logps/ref_rejected": -101.0856704711914,
      "logps/rejected": -101.16006469726562,
      "loss": 1.3779,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0016077004838734865,
      "rewards/margins": 0.008732382208108902,
      "rewards/rejected": -0.007124680560082197,
      "step": 41
    },
    {
      "epoch": 0.06349206349206349,
      "grad_norm": 17.808015823364258,
      "kl/avg_steps": -0.078125,
      "kl/beta": 0.09753459692001343,
      "kl/n_epsilon_steps": 0.53125,
      "kl/p_epsilon_steps": 0.453125,
      "learning_rate": 3.059701492537313e-07,
      "logits/chosen": -0.6215388774871826,
      "logits/rejected": -0.5111829042434692,
      "logps/chosen": -79.35958099365234,
      "logps/ref_chosen": -79.35625457763672,
      "logps/ref_rejected": -91.5488052368164,
      "logps/rejected": -91.5380859375,
      "loss": 1.3884,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.0004895464517176151,
      "rewards/margins": -0.0016764979809522629,
      "rewards/rejected": 0.001186951994895935,
      "step": 42
    },
    {
      "epoch": 0.06500377928949358,
      "grad_norm": 19.075096130371094,
      "kl/avg_steps": -0.0625,
      "kl/beta": 0.09761085361242294,
      "kl/n_epsilon_steps": 0.53125,
      "kl/p_epsilon_steps": 0.46875,
      "learning_rate": 3.134328358208955e-07,
      "logits/chosen": -0.2507287263870239,
      "logits/rejected": -0.5635038614273071,
      "logps/chosen": -90.81982421875,
      "logps/ref_chosen": -90.81220245361328,
      "logps/ref_rejected": -94.16317749023438,
      "logps/rejected": -94.09054565429688,
      "loss": 1.3948,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.0008870699675753713,
      "rewards/margins": -0.008112498559057713,
      "rewards/rejected": 0.007225428242236376,
      "step": 43
    },
    {
      "epoch": 0.06651549508692366,
      "grad_norm": 18.593828201293945,
      "kl/avg_steps": 0.046875,
      "kl/beta": 0.09767189621925354,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 3.2089552238805965e-07,
      "logits/chosen": -0.8580632209777832,
      "logits/rejected": -0.6987817287445068,
      "logps/chosen": -88.23231506347656,
      "logps/ref_chosen": -88.27932739257812,
      "logps/ref_rejected": -101.14324951171875,
      "logps/rejected": -101.09764099121094,
      "loss": 1.3868,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.004492661450058222,
      "rewards/margins": -0.00015667756088078022,
      "rewards/rejected": 0.004649339243769646,
      "step": 44
    },
    {
      "epoch": 0.06802721088435375,
      "grad_norm": 18.8914852142334,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09762613475322723,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 3.2835820895522385e-07,
      "logits/chosen": -0.7590723037719727,
      "logits/rejected": -0.40023982524871826,
      "logps/chosen": -78.38350677490234,
      "logps/ref_chosen": -78.40264892578125,
      "logps/ref_rejected": -109.39339447021484,
      "logps/rejected": -109.42718505859375,
      "loss": 1.382,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0017465527635067701,
      "rewards/margins": 0.004819178022444248,
      "rewards/rejected": -0.003072625258937478,
      "step": 45
    },
    {
      "epoch": 0.06953892668178382,
      "grad_norm": 17.96132469177246,
      "kl/avg_steps": 0.0625,
      "kl/beta": 0.09750425815582275,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.53125,
      "learning_rate": 3.3582089552238805e-07,
      "logits/chosen": -0.6594468355178833,
      "logits/rejected": -0.7436229586601257,
      "logps/chosen": -77.98482513427734,
      "logps/ref_chosen": -78.08491516113281,
      "logps/ref_rejected": -97.42544555664062,
      "logps/rejected": -97.40345764160156,
      "loss": 1.3795,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.009642795659601688,
      "rewards/margins": 0.007294987328350544,
      "rewards/rejected": 0.002347808564081788,
      "step": 46
    },
    {
      "epoch": 0.0710506424792139,
      "grad_norm": 18.438560485839844,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.0974433571100235,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 3.432835820895522e-07,
      "logits/chosen": -0.5594514012336731,
      "logits/rejected": -0.30259305238723755,
      "logps/chosen": -70.72454833984375,
      "logps/ref_chosen": -70.78988647460938,
      "logps/ref_rejected": -91.17266845703125,
      "logps/rejected": -91.22081756591797,
      "loss": 1.3759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006220364943146706,
      "rewards/margins": 0.010731121525168419,
      "rewards/rejected": -0.0045107570476830006,
      "step": 47
    },
    {
      "epoch": 0.07256235827664399,
      "grad_norm": 16.562816619873047,
      "kl/avg_steps": 0.0625,
      "kl/beta": 0.09720035642385483,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.53125,
      "learning_rate": 3.507462686567164e-07,
      "logits/chosen": -0.771056056022644,
      "logits/rejected": -0.6000367403030396,
      "logps/chosen": -66.6251220703125,
      "logps/ref_chosen": -66.67327880859375,
      "logps/ref_rejected": -79.28543090820312,
      "logps/rejected": -79.26315307617188,
      "loss": 1.3842,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004568049218505621,
      "rewards/margins": 0.0022856390569359064,
      "rewards/rejected": 0.002282409928739071,
      "step": 48
    },
    {
      "epoch": 0.07407407407407407,
      "grad_norm": 17.03924560546875,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.0971396416425705,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 3.5820895522388055e-07,
      "logits/chosen": -0.5533872842788696,
      "logits/rejected": -0.48292940855026245,
      "logps/chosen": -75.0789794921875,
      "logps/ref_chosen": -75.17504119873047,
      "logps/ref_rejected": -80.5369873046875,
      "logps/rejected": -80.46534729003906,
      "loss": 1.3845,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.009206226095557213,
      "rewards/margins": 0.0020941859111189842,
      "rewards/rejected": 0.007112039718776941,
      "step": 49
    },
    {
      "epoch": 0.07558578987150416,
      "grad_norm": 17.23259925842285,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09701836854219437,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 3.6567164179104475e-07,
      "logits/chosen": -0.6811853647232056,
      "logits/rejected": -0.4545362591743469,
      "logps/chosen": -71.20023345947266,
      "logps/ref_chosen": -71.2314224243164,
      "logps/ref_rejected": -87.59088134765625,
      "logps/rejected": -87.6037368774414,
      "loss": 1.3825,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0029269284568727016,
      "rewards/margins": 0.004028110299259424,
      "rewards/rejected": -0.001101181609556079,
      "step": 50
    },
    {
      "epoch": 0.07709750566893424,
      "grad_norm": 18.064058303833008,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.09689724445343018,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 3.7313432835820895e-07,
      "logits/chosen": -0.7829879522323608,
      "logits/rejected": -0.6104872226715088,
      "logps/chosen": -78.68687438964844,
      "logps/ref_chosen": -78.69171142578125,
      "logps/ref_rejected": -100.78950500488281,
      "logps/rejected": -100.80244445800781,
      "loss": 1.3853,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": 0.00030520849395543337,
      "rewards/margins": 0.0013936648610979319,
      "rewards/rejected": -0.0010884563671424985,
      "step": 51
    },
    {
      "epoch": 0.07860922146636433,
      "grad_norm": 19.703731536865234,
      "kl/avg_steps": 0.0,
      "kl/beta": 0.09674607962369919,
      "kl/n_epsilon_steps": 0.5,
      "kl/p_epsilon_steps": 0.5,
      "learning_rate": 3.805970149253731e-07,
      "logits/chosen": -0.6990611553192139,
      "logits/rejected": -0.5098797678947449,
      "logps/chosen": -89.09143829345703,
      "logps/ref_chosen": -89.09419250488281,
      "logps/ref_rejected": -116.87468719482422,
      "logps/rejected": -116.89561462402344,
      "loss": 1.3848,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00012077903375029564,
      "rewards/margins": 0.0019711265340447426,
      "rewards/rejected": -0.001850348082371056,
      "step": 52
    },
    {
      "epoch": 0.0801209372637944,
      "grad_norm": 16.809465408325195,
      "kl/avg_steps": -0.09375,
      "kl/beta": 0.09674607962369919,
      "kl/n_epsilon_steps": 0.546875,
      "kl/p_epsilon_steps": 0.453125,
      "learning_rate": 3.880597014925373e-07,
      "logits/chosen": -0.8433110117912292,
      "logits/rejected": -0.894573450088501,
      "logps/chosen": -74.12965393066406,
      "logps/ref_chosen": -74.21418762207031,
      "logps/ref_rejected": -75.71167755126953,
      "logps/rejected": -75.67427062988281,
      "loss": 1.3826,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.00799822248518467,
      "rewards/margins": 0.004199557937681675,
      "rewards/rejected": 0.00379866361618042,
      "step": 53
    },
    {
      "epoch": 0.08163265306122448,
      "grad_norm": 15.830225944519043,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.0968368649482727,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 3.9552238805970144e-07,
      "logits/chosen": -0.8751938939094543,
      "logits/rejected": -0.7324712872505188,
      "logps/chosen": -65.55152893066406,
      "logps/ref_chosen": -65.63475799560547,
      "logps/ref_rejected": -76.4462890625,
      "logps/rejected": -76.46516418457031,
      "loss": 1.3771,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.007865255698561668,
      "rewards/margins": 0.009572221897542477,
      "rewards/rejected": -0.0017069653840735555,
      "step": 54
    },
    {
      "epoch": 0.08314436885865457,
      "grad_norm": 19.163211822509766,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.09662549942731857,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.0298507462686564e-07,
      "logits/chosen": -0.4196467101573944,
      "logits/rejected": -0.2608944773674011,
      "logps/chosen": -68.71656799316406,
      "logps/ref_chosen": -68.7640380859375,
      "logps/ref_rejected": -108.80075073242188,
      "logps/rejected": -108.87657928466797,
      "loss": 1.3752,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.0043769595213234425,
      "rewards/margins": 0.011570327915251255,
      "rewards/rejected": -0.007193367928266525,
      "step": 55
    },
    {
      "epoch": 0.08465608465608465,
      "grad_norm": 16.41425132751465,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.09641458839178085,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 4.1044776119402984e-07,
      "logits/chosen": -0.6438100337982178,
      "logits/rejected": -0.563813328742981,
      "logps/chosen": -74.7386703491211,
      "logps/ref_chosen": -74.7939453125,
      "logps/ref_rejected": -81.83535766601562,
      "logps/rejected": -81.83403015136719,
      "loss": 1.3819,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005185459740459919,
      "rewards/margins": 0.0048543717712163925,
      "rewards/rejected": 0.0003310886677354574,
      "step": 56
    },
    {
      "epoch": 0.08616780045351474,
      "grad_norm": 18.06746482849121,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.09638447314500809,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.17910447761194e-07,
      "logits/chosen": -0.8478030562400818,
      "logits/rejected": -0.8586157560348511,
      "logps/chosen": -74.48088073730469,
      "logps/ref_chosen": -74.5794677734375,
      "logps/ref_rejected": -105.61981964111328,
      "logps/rejected": -105.72442626953125,
      "loss": 1.3678,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.009298881515860558,
      "rewards/margins": 0.019189005717635155,
      "rewards/rejected": -0.009890124201774597,
      "step": 57
    },
    {
      "epoch": 0.08767951625094482,
      "grad_norm": 18.519271850585938,
      "kl/avg_steps": -0.125,
      "kl/beta": 0.09602437913417816,
      "kl/n_epsilon_steps": 0.5625,
      "kl/p_epsilon_steps": 0.4375,
      "learning_rate": 4.253731343283582e-07,
      "logits/chosen": -0.589920163154602,
      "logits/rejected": -0.5984715819358826,
      "logps/chosen": -92.21888732910156,
      "logps/ref_chosen": -92.24464416503906,
      "logps/ref_rejected": -103.18975830078125,
      "logps/rejected": -103.20128631591797,
      "loss": 1.3835,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0023432730231434107,
      "rewards/margins": 0.0032569742761552334,
      "rewards/rejected": -0.0009137009037658572,
      "step": 58
    },
    {
      "epoch": 0.08919123204837491,
      "grad_norm": 16.03529930114746,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.09614455699920654,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.3283582089552234e-07,
      "logits/chosen": -0.4612119793891907,
      "logits/rejected": -0.7404814958572388,
      "logps/chosen": -66.95186614990234,
      "logps/ref_chosen": -67.12688446044922,
      "logps/ref_rejected": -91.69569396972656,
      "logps/rejected": -91.7687759399414,
      "loss": 1.3636,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01665889285504818,
      "rewards/margins": 0.023446228355169296,
      "rewards/rejected": -0.006787334103137255,
      "step": 59
    },
    {
      "epoch": 0.09070294784580499,
      "grad_norm": 17.765792846679688,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.09578536450862885,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 4.4029850746268654e-07,
      "logits/chosen": -0.7013375759124756,
      "logits/rejected": -0.5072432160377502,
      "logps/chosen": -79.59794616699219,
      "logps/ref_chosen": -79.74327850341797,
      "logps/ref_rejected": -77.89244079589844,
      "logps/rejected": -77.80746459960938,
      "loss": 1.3818,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.013693554326891899,
      "rewards/margins": 0.005363212898373604,
      "rewards/rejected": 0.00833034235984087,
      "step": 60
    },
    {
      "epoch": 0.09221466364323508,
      "grad_norm": 15.700193405151367,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.09566578269004822,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.4776119402985074e-07,
      "logits/chosen": -1.1494168043136597,
      "logits/rejected": -0.5027548670768738,
      "logps/chosen": -65.95541381835938,
      "logps/ref_chosen": -66.08685302734375,
      "logps/ref_rejected": -88.1458740234375,
      "logps/rejected": -88.13238525390625,
      "loss": 1.3758,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012441026046872139,
      "rewards/margins": 0.01096752192825079,
      "rewards/rejected": 0.0014735042350366712,
      "step": 61
    },
    {
      "epoch": 0.09372637944066516,
      "grad_norm": 16.980772018432617,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.09545697271823883,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.552238805970149e-07,
      "logits/chosen": -0.628500759601593,
      "logits/rejected": -0.4428566098213196,
      "logps/chosen": -80.88948059082031,
      "logps/ref_chosen": -81.0108871459961,
      "logps/ref_rejected": -95.50444793701172,
      "logps/rejected": -95.56391143798828,
      "loss": 1.37,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.011412292718887329,
      "rewards/margins": 0.016900725662708282,
      "rewards/rejected": -0.005488432943820953,
      "step": 62
    },
    {
      "epoch": 0.09523809523809523,
      "grad_norm": 18.504274368286133,
      "kl/avg_steps": 0.234375,
      "kl/beta": 0.09515959769487381,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.626865671641791e-07,
      "logits/chosen": -0.5986104011535645,
      "logits/rejected": -0.4796867370605469,
      "logps/chosen": -78.40611267089844,
      "logps/ref_chosen": -78.57593536376953,
      "logps/ref_rejected": -99.71000671386719,
      "logps/rejected": -99.65341186523438,
      "loss": 1.3766,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.01594492793083191,
      "rewards/margins": 0.01040346547961235,
      "rewards/rejected": 0.005541461519896984,
      "step": 63
    },
    {
      "epoch": 0.09674981103552532,
      "grad_norm": 15.803566932678223,
      "kl/avg_steps": 0.09375,
      "kl/beta": 0.09493708610534668,
      "kl/n_epsilon_steps": 0.453125,
      "kl/p_epsilon_steps": 0.546875,
      "learning_rate": 4.701492537313433e-07,
      "logits/chosen": -0.7529503703117371,
      "logits/rejected": -0.6055833697319031,
      "logps/chosen": -69.16105651855469,
      "logps/ref_chosen": -69.24063110351562,
      "logps/ref_rejected": -84.14842987060547,
      "logps/rejected": -84.07394409179688,
      "loss": 1.387,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007316782139241695,
      "rewards/margins": 8.302222704514861e-05,
      "rewards/rejected": 0.007233759853988886,
      "step": 64
    },
    {
      "epoch": 0.0982615268329554,
      "grad_norm": 17.927310943603516,
      "kl/avg_steps": 0.0625,
      "kl/beta": 0.0948481634259224,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.53125,
      "learning_rate": 4.776119402985074e-07,
      "logits/chosen": -0.7595020532608032,
      "logits/rejected": -0.637890100479126,
      "logps/chosen": -83.99519348144531,
      "logps/ref_chosen": -84.0351333618164,
      "logps/ref_rejected": -96.42926788330078,
      "logps/rejected": -96.46531677246094,
      "loss": 1.3802,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": 0.0035090160090476274,
      "rewards/margins": 0.006791706196963787,
      "rewards/rejected": -0.0032826901879161596,
      "step": 65
    },
    {
      "epoch": 0.09977324263038549,
      "grad_norm": 17.392560958862305,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.09478892385959625,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.850746268656717e-07,
      "logits/chosen": -0.8872799277305603,
      "logits/rejected": -0.9815646409988403,
      "logps/chosen": -87.69140625,
      "logps/ref_chosen": -87.79239654541016,
      "logps/ref_rejected": -95.26547241210938,
      "logps/rejected": -95.44229888916016,
      "loss": 1.3614,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.009347852319478989,
      "rewards/margins": 0.02584635280072689,
      "rewards/rejected": -0.016498498618602753,
      "step": 66
    },
    {
      "epoch": 0.10128495842781557,
      "grad_norm": 17.885372161865234,
      "kl/avg_steps": 0.171875,
      "kl/beta": 0.09446420520544052,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.925373134328357e-07,
      "logits/chosen": -0.8714014887809753,
      "logits/rejected": -0.7710602283477783,
      "logps/chosen": -77.86466979980469,
      "logps/ref_chosen": -78.00114440917969,
      "logps/ref_rejected": -96.03421020507812,
      "logps/rejected": -96.05657958984375,
      "loss": 1.3726,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.012656296603381634,
      "rewards/margins": 0.014563833363354206,
      "rewards/rejected": -0.001907536992803216,
      "step": 67
    },
    {
      "epoch": 0.10279667422524566,
      "grad_norm": 18.616788864135742,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.09430211782455444,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 5e-07,
      "logits/chosen": -0.5614684820175171,
      "logits/rejected": -0.508707582950592,
      "logps/chosen": -95.97196960449219,
      "logps/ref_chosen": -96.04268646240234,
      "logps/ref_rejected": -110.91169738769531,
      "logps/rejected": -111.02496337890625,
      "loss": 1.3705,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.006450447719544172,
      "rewards/margins": 0.01685434952378273,
      "rewards/rejected": -0.010403899475932121,
      "step": 68
    },
    {
      "epoch": 0.10430839002267574,
      "grad_norm": 18.57022476196289,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.0939498096704483,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.999965034812934e-07,
      "logits/chosen": -0.6774875521659851,
      "logits/rejected": -0.5702620148658752,
      "logps/chosen": -84.87574768066406,
      "logps/ref_chosen": -85.11125183105469,
      "logps/ref_rejected": -107.57357025146484,
      "logps/rejected": -107.622802734375,
      "loss": 1.3609,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.02191595546901226,
      "rewards/margins": 0.026317887008190155,
      "rewards/rejected": -0.004401930142194033,
      "step": 69
    },
    {
      "epoch": 0.10582010582010581,
      "grad_norm": 17.01119041442871,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.09356968104839325,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.999860140229787e-07,
      "logits/chosen": -0.5370590686798096,
      "logits/rejected": -0.4853627681732178,
      "logps/chosen": -81.58915710449219,
      "logps/ref_chosen": -81.87960815429688,
      "logps/ref_rejected": -92.63243103027344,
      "logps/rejected": -92.62098693847656,
      "loss": 1.3617,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.026898501440882683,
      "rewards/margins": 0.025649238377809525,
      "rewards/rejected": 0.001249261200428009,
      "step": 70
    },
    {
      "epoch": 0.1073318216175359,
      "grad_norm": 16.591291427612305,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.0933072566986084,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.999685319184688e-07,
      "logits/chosen": -0.8710042238235474,
      "logits/rejected": -0.695549488067627,
      "logps/chosen": -79.66912841796875,
      "logps/ref_chosen": -79.74766540527344,
      "logps/ref_rejected": -83.39110565185547,
      "logps/rejected": -83.38461303710938,
      "loss": 1.3808,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.007052659057080746,
      "rewards/margins": 0.006284390110522509,
      "rewards/rejected": 0.0007682680152356625,
      "step": 71
    },
    {
      "epoch": 0.10884353741496598,
      "grad_norm": 17.93051528930664,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.0931616947054863,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.999440576567755e-07,
      "logits/chosen": -0.8033581376075745,
      "logits/rejected": -0.8477033376693726,
      "logps/chosen": -72.75706481933594,
      "logps/ref_chosen": -73.04458618164062,
      "logps/ref_rejected": -92.64720153808594,
      "logps/rejected": -92.60933685302734,
      "loss": 1.3645,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.02642909437417984,
      "rewards/margins": 0.022784588858485222,
      "rewards/rejected": 0.0036445085424929857,
      "step": 72
    },
    {
      "epoch": 0.11035525321239607,
      "grad_norm": 18.030593872070312,
      "kl/avg_steps": -0.0625,
      "kl/beta": 0.09275588393211365,
      "kl/n_epsilon_steps": 0.53125,
      "kl/p_epsilon_steps": 0.46875,
      "learning_rate": 4.999125919224965e-07,
      "logits/chosen": -0.773788571357727,
      "logits/rejected": -0.8637920022010803,
      "logps/chosen": -87.6549072265625,
      "logps/ref_chosen": -87.71681213378906,
      "logps/ref_rejected": -96.93572998046875,
      "logps/rejected": -96.90829467773438,
      "loss": 1.3848,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.005404962692409754,
      "rewards/margins": 0.0026616945397108793,
      "rewards/rejected": 0.0027432686183601618,
      "step": 73
    },
    {
      "epoch": 0.11186696900982615,
      "grad_norm": 16.754348754882812,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.0928138941526413,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.998741355957963e-07,
      "logits/chosen": -0.7274940013885498,
      "logits/rejected": -0.4740767776966095,
      "logps/chosen": -66.65380859375,
      "logps/ref_chosen": -67.07321166992188,
      "logps/ref_rejected": -96.53402709960938,
      "logps/rejected": -96.39031982421875,
      "loss": 1.3624,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03866203874349594,
      "rewards/margins": 0.025037020444869995,
      "rewards/rejected": 0.013625022023916245,
      "step": 74
    },
    {
      "epoch": 0.11337868480725624,
      "grad_norm": 15.731212615966797,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.09252475202083588,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.998286897523808e-07,
      "logits/chosen": -0.85367751121521,
      "logits/rejected": -0.7722653150558472,
      "logps/chosen": -61.570716857910156,
      "logps/ref_chosen": -61.80186462402344,
      "logps/ref_rejected": -82.37368774414062,
      "logps/rejected": -82.36898803710938,
      "loss": 1.367,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.021144213154911995,
      "rewards/margins": 0.020450696349143982,
      "rewards/rejected": 0.0006935172714293003,
      "step": 75
    },
    {
      "epoch": 0.11489040060468632,
      "grad_norm": 16.436174392700195,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.09229401499032974,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.997762556634679e-07,
      "logits/chosen": -0.8176724314689636,
      "logits/rejected": -0.6024997234344482,
      "logps/chosen": -69.61163330078125,
      "logps/ref_chosen": -69.92233276367188,
      "logps/ref_rejected": -97.08378601074219,
      "logps/rejected": -97.02426147460938,
      "loss": 1.3652,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.028404513373970985,
      "rewards/margins": 0.022574743255972862,
      "rewards/rejected": 0.005829768255352974,
      "step": 76
    },
    {
      "epoch": 0.1164021164021164,
      "grad_norm": 16.809951782226562,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.09206385910511017,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.99716834795752e-07,
      "logits/chosen": -1.176077127456665,
      "logits/rejected": -0.774753749370575,
      "logps/chosen": -70.80705261230469,
      "logps/ref_chosen": -71.206298828125,
      "logps/ref_rejected": -95.22071075439453,
      "logps/rejected": -95.2851791381836,
      "loss": 1.346,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.036420077085494995,
      "rewards/margins": 0.04206133261322975,
      "rewards/rejected": -0.005641256459057331,
      "step": 77
    },
    {
      "epoch": 0.11791383219954649,
      "grad_norm": 16.423564910888672,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.09177705645561218,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.996504288113623e-07,
      "logits/chosen": -0.6873102188110352,
      "logits/rejected": -0.5276945233345032,
      "logps/chosen": -84.00184631347656,
      "logps/ref_chosen": -84.40055847167969,
      "logps/ref_rejected": -95.41949462890625,
      "logps/rejected": -95.31796264648438,
      "loss": 1.3609,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.03620798885822296,
      "rewards/margins": 0.026718970388174057,
      "rewards/rejected": 0.009489016607403755,
      "step": 78
    },
    {
      "epoch": 0.11942554799697656,
      "grad_norm": 17.653905868530273,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.09157673269510269,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.995770395678171e-07,
      "logits/chosen": -0.7041028738021851,
      "logits/rejected": -0.6291458010673523,
      "logps/chosen": -65.53144836425781,
      "logps/ref_chosen": -65.93923950195312,
      "logps/ref_rejected": -102.92240905761719,
      "logps/rejected": -102.9657211303711,
      "loss": 1.3484,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.036987803876399994,
      "rewards/margins": 0.040554363280534744,
      "rewards/rejected": -0.0035665626637637615,
      "step": 79
    },
    {
      "epoch": 0.12093726379440665,
      "grad_norm": 15.999527931213379,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.09126301109790802,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.994966691179711e-07,
      "logits/chosen": -0.6971664428710938,
      "logits/rejected": -0.7449191212654114,
      "logps/chosen": -78.33244323730469,
      "logps/ref_chosen": -78.61624908447266,
      "logps/ref_rejected": -99.9122314453125,
      "logps/rejected": -99.92466735839844,
      "loss": 1.3617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02554541453719139,
      "rewards/margins": 0.02635783888399601,
      "rewards/rejected": -0.0008124255109578371,
      "step": 80
    },
    {
      "epoch": 0.12244897959183673,
      "grad_norm": 16.314882278442383,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.09106381237506866,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.994093197099587e-07,
      "logits/chosen": -0.8343544006347656,
      "logits/rejected": -0.7506792545318604,
      "logps/chosen": -79.16024780273438,
      "logps/ref_chosen": -79.49640655517578,
      "logps/ref_rejected": -94.52413940429688,
      "logps/rejected": -94.53294372558594,
      "loss": 1.3575,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.030200045555830002,
      "rewards/margins": 0.030745631083846092,
      "rewards/rejected": -0.0005455873906612396,
      "step": 81
    },
    {
      "epoch": 0.12396069538926682,
      "grad_norm": 16.32975959777832,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.09080841392278671,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.993149937871306e-07,
      "logits/chosen": -0.7907916903495789,
      "logits/rejected": -0.6614448428153992,
      "logps/chosen": -64.36497497558594,
      "logps/ref_chosen": -64.97168731689453,
      "logps/ref_rejected": -86.69085693359375,
      "logps/rejected": -86.62161254882812,
      "loss": 1.3408,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.054587192833423615,
      "rewards/margins": 0.04804161936044693,
      "rewards/rejected": 0.0065455688163638115,
      "step": 82
    },
    {
      "epoch": 0.1254724111866969,
      "grad_norm": 16.859764099121094,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.09035663306713104,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.992136939879856e-07,
      "logits/chosen": -0.8817363977432251,
      "logits/rejected": -0.891059398651123,
      "logps/chosen": -72.4176254272461,
      "logps/ref_chosen": -72.92498779296875,
      "logps/ref_rejected": -92.27165222167969,
      "logps/rejected": -92.21333312988281,
      "loss": 1.3487,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.04530956968665123,
      "rewards/margins": 0.03982119634747505,
      "rewards/rejected": 0.005488371476531029,
      "step": 83
    },
    {
      "epoch": 0.12698412698412698,
      "grad_norm": 17.500118255615234,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.08999104052782059,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.991054231460969e-07,
      "logits/chosen": -0.6272699236869812,
      "logits/rejected": -0.5706059336662292,
      "logps/chosen": -81.32762145996094,
      "logps/ref_chosen": -81.79109191894531,
      "logps/ref_rejected": -99.20896911621094,
      "logps/rejected": -99.24348449707031,
      "loss": 1.3445,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.041328877210617065,
      "rewards/margins": 0.04405710846185684,
      "rewards/rejected": -0.0027282284572720528,
      "step": 84
    },
    {
      "epoch": 0.12849584278155707,
      "grad_norm": 15.726845741271973,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.08957117795944214,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.989901842900325e-07,
      "logits/chosen": -1.0663063526153564,
      "logits/rejected": -1.011609435081482,
      "logps/chosen": -67.32447814941406,
      "logps/ref_chosen": -67.94148254394531,
      "logps/ref_rejected": -85.76875305175781,
      "logps/rejected": -85.65890502929688,
      "loss": 1.3438,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.054649144411087036,
      "rewards/margins": 0.0446481890976429,
      "rewards/rejected": 0.010000954382121563,
      "step": 85
    },
    {
      "epoch": 0.13000755857898716,
      "grad_norm": 15.644314765930176,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.08912555128335953,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.988679806432711e-07,
      "logits/chosen": -0.9160196781158447,
      "logits/rejected": -0.9157437086105347,
      "logps/chosen": -78.93154907226562,
      "logps/ref_chosen": -79.21485900878906,
      "logps/ref_rejected": -88.69877624511719,
      "logps/rejected": -88.68402099609375,
      "loss": 1.3645,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.024859676137566566,
      "rewards/margins": 0.023322567343711853,
      "rewards/rejected": 0.0015371122863143682,
      "step": 86
    },
    {
      "epoch": 0.13151927437641722,
      "grad_norm": 16.862993240356445,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.08898650854825974,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.987388156241114e-07,
      "logits/chosen": -0.8834874629974365,
      "logits/rejected": -1.0349664688110352,
      "logps/chosen": -83.93854522705078,
      "logps/ref_chosen": -84.45362854003906,
      "logps/ref_rejected": -103.438232421875,
      "logps/rejected": -103.58685302734375,
      "loss": 1.3317,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.04530634358525276,
      "rewards/margins": 0.05805563926696777,
      "rewards/rejected": -0.012749293819069862,
      "step": 87
    },
    {
      "epoch": 0.1330309901738473,
      "grad_norm": 16.096044540405273,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.08862645924091339,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.986026928455767e-07,
      "logits/chosen": -1.078216314315796,
      "logits/rejected": -0.7487344741821289,
      "logps/chosen": -80.88137817382812,
      "logps/ref_chosen": -81.27230834960938,
      "logps/ref_rejected": -89.51646423339844,
      "logps/rejected": -89.49003601074219,
      "loss": 1.3573,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.03411562368273735,
      "rewards/margins": 0.031519923359155655,
      "rewards/rejected": 0.0025957003235816956,
      "step": 88
    },
    {
      "epoch": 0.1345427059712774,
      "grad_norm": 16.19232749938965,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.08837790042161942,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.984596161153135e-07,
      "logits/chosen": -0.9088029861450195,
      "logits/rejected": -0.9495557546615601,
      "logps/chosen": -57.38954162597656,
      "logps/ref_chosen": -58.142333984375,
      "logps/ref_rejected": -102.53756713867188,
      "logps/rejected": -102.58346557617188,
      "loss": 1.321,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06586841493844986,
      "rewards/margins": 0.06961293518543243,
      "rewards/rejected": -0.003744515124708414,
      "step": 89
    },
    {
      "epoch": 0.1360544217687075,
      "grad_norm": 17.48473358154297,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.08799292892217636,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.983095894354857e-07,
      "logits/chosen": -0.8699663281440735,
      "logits/rejected": -0.855407178401947,
      "logps/chosen": -74.75665283203125,
      "logps/ref_chosen": -75.26505279541016,
      "logps/ref_rejected": -104.32841491699219,
      "logps/rejected": -104.36683654785156,
      "loss": 1.3422,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.04425486922264099,
      "rewards/margins": 0.04724379628896713,
      "rewards/rejected": -0.00298893079161644,
      "step": 90
    },
    {
      "epoch": 0.13756613756613756,
      "grad_norm": 15.36704158782959,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.08763690292835236,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.98152617002662e-07,
      "logits/chosen": -0.9106171131134033,
      "logits/rejected": -1.0182747840881348,
      "logps/chosen": -68.7718505859375,
      "logps/ref_chosen": -69.33902740478516,
      "logps/ref_rejected": -90.31411743164062,
      "logps/rejected": -90.28929901123047,
      "loss": 1.3438,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.04909837990999222,
      "rewards/margins": 0.04654748737812042,
      "rewards/rejected": 0.0025508906692266464,
      "step": 91
    },
    {
      "epoch": 0.13907785336356765,
      "grad_norm": 16.497882843017578,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.08747289329767227,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.979887032076988e-07,
      "logits/chosen": -0.8337477445602417,
      "logits/rejected": -0.7373151779174805,
      "logps/chosen": -71.86427307128906,
      "logps/ref_chosen": -72.4566650390625,
      "logps/ref_rejected": -91.6706771850586,
      "logps/rejected": -91.6868896484375,
      "loss": 1.3376,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.051354095339775085,
      "rewards/margins": 0.052345022559165955,
      "rewards/rejected": -0.00099092535674572,
      "step": 92
    },
    {
      "epoch": 0.14058956916099774,
      "grad_norm": 14.10353946685791,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.08722756803035736,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.978178526356172e-07,
      "logits/chosen": -1.1317577362060547,
      "logits/rejected": -0.7728261947631836,
      "logps/chosen": -63.39311218261719,
      "logps/ref_chosen": -64.08897399902344,
      "logps/ref_rejected": -75.09095764160156,
      "logps/rejected": -74.87936401367188,
      "loss": 1.3502,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.06010336056351662,
      "rewards/margins": 0.04119066148996353,
      "rewards/rejected": 0.018912700936198235,
      "step": 93
    },
    {
      "epoch": 0.1421012849584278,
      "grad_norm": 18.503324508666992,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.08701004087924957,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.976400700654751e-07,
      "logits/chosen": -1.0736751556396484,
      "logits/rejected": -1.2253625392913818,
      "logps/chosen": -78.85262298583984,
      "logps/ref_chosen": -79.67372131347656,
      "logps/ref_rejected": -94.64076232910156,
      "logps/rejected": -94.75550842285156,
      "loss": 1.3126,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07063993811607361,
      "rewards/margins": 0.08020106703042984,
      "rewards/rejected": -0.009561131708323956,
      "step": 94
    },
    {
      "epoch": 0.1436130007558579,
      "grad_norm": 16.122169494628906,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.08655024319887161,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.974553604702332e-07,
      "logits/chosen": -0.7826769351959229,
      "logits/rejected": -0.6646933555603027,
      "logps/chosen": -78.21084594726562,
      "logps/ref_chosen": -78.65760803222656,
      "logps/ref_rejected": -109.40481567382812,
      "logps/rejected": -109.658203125,
      "loss": 1.3318,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.03827132284641266,
      "rewards/margins": 0.05950234830379486,
      "rewards/rejected": -0.021231018006801605,
      "step": 95
    },
    {
      "epoch": 0.14512471655328799,
      "grad_norm": 16.467144012451172,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.08630750328302383,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.972637290166157e-07,
      "logits/chosen": -1.1068183183670044,
      "logits/rejected": -0.791649580001831,
      "logps/chosen": -77.20147705078125,
      "logps/ref_chosen": -77.70825958251953,
      "logps/ref_rejected": -104.36044311523438,
      "logps/rejected": -104.63987731933594,
      "loss": 1.3252,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.0430414117872715,
      "rewards/margins": 0.06671500205993652,
      "rewards/rejected": -0.02367359772324562,
      "step": 96
    },
    {
      "epoch": 0.14663643235071808,
      "grad_norm": 16.57407569885254,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.0860922709107399,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.970651810649666e-07,
      "logits/chosen": -0.7208718061447144,
      "logits/rejected": -0.7762876152992249,
      "logps/chosen": -84.24117279052734,
      "logps/ref_chosen": -84.58918762207031,
      "logps/ref_rejected": -99.25704956054688,
      "logps/rejected": -99.26144409179688,
      "loss": 1.3618,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.02926001325249672,
      "rewards/margins": 0.02920977585017681,
      "rewards/rejected": 5.023973062634468e-05,
      "step": 97
    },
    {
      "epoch": 0.14814814814814814,
      "grad_norm": 15.020103454589844,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.08593115210533142,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.968597221690985e-07,
      "logits/chosen": -0.9258188009262085,
      "logits/rejected": -0.6664605140686035,
      "logps/chosen": -74.06013488769531,
      "logps/ref_chosen": -74.42477416992188,
      "logps/ref_rejected": -88.93840026855469,
      "logps/rejected": -88.95329284667969,
      "loss": 1.3584,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.030702810734510422,
      "rewards/margins": 0.03164215385913849,
      "rewards/rejected": -0.0009393435902893543,
      "step": 98
    },
    {
      "epoch": 0.14965986394557823,
      "grad_norm": 15.487700462341309,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.08566345274448395,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.966473580761389e-07,
      "logits/chosen": -0.9926242232322693,
      "logits/rejected": -0.7351720333099365,
      "logps/chosen": -75.0016098022461,
      "logps/ref_chosen": -75.5974349975586,
      "logps/ref_rejected": -98.2310791015625,
      "logps/rejected": -98.35629272460938,
      "loss": 1.3342,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.05039945989847183,
      "rewards/margins": 0.06047532707452774,
      "rewards/rejected": -0.010075867176055908,
      "step": 99
    },
    {
      "epoch": 0.15117157974300832,
      "grad_norm": 16.720571517944336,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.08542320132255554,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.964280947263676e-07,
      "logits/chosen": -0.7586959004402161,
      "logits/rejected": -0.699402391910553,
      "logps/chosen": -98.01969909667969,
      "logps/ref_chosen": -98.55859375,
      "logps/ref_rejected": -106.01295471191406,
      "logps/rejected": -106.32823181152344,
      "loss": 1.3249,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.04501022771000862,
      "rewards/margins": 0.07151120156049728,
      "rewards/rejected": -0.026500973850488663,
      "step": 100
    },
    {
      "epoch": 0.15117157974300832,
      "eval_kl/n_epsilon_steps": 0.32262325286865234,
      "eval_kl/p_epsilon_steps": 0.6760563254356384,
      "eval_logits/chosen": -0.7699905037879944,
      "eval_logits/rejected": -0.7965146899223328,
      "eval_logps/chosen": -86.22242736816406,
      "eval_logps/ref_chosen": -86.90177917480469,
      "eval_logps/ref_rejected": -96.69639587402344,
      "eval_logps/rejected": -96.91645812988281,
      "eval_loss": 0.6599990129470825,
      "eval_rewards/accuracies": 0.6822183132171631,
      "eval_rewards/chosen": 0.05698024854063988,
      "eval_rewards/margins": 0.07509617507457733,
      "eval_rewards/rejected": -0.018115932121872902,
      "eval_runtime": 46.9988,
      "eval_samples_per_second": 49.001,
      "eval_steps_per_second": 1.532,
      "step": 100
    },
    {
      "epoch": 0.15268329554043839,
      "grad_norm": 13.873627662658691,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.08507757633924484,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 4.96201938253052e-07,
      "logits/chosen": -1.3703796863555908,
      "logits/rejected": -0.8814256191253662,
      "logps/chosen": -68.59088134765625,
      "logps/ref_chosen": -69.45216369628906,
      "logps/ref_rejected": -88.04588317871094,
      "logps/rejected": -88.23822021484375,
      "loss": 1.3053,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.07237797975540161,
      "rewards/margins": 0.08829944580793381,
      "rewards/rejected": -0.015921467915177345,
      "step": 101
    },
    {
      "epoch": 0.15419501133786848,
      "grad_norm": 14.738540649414062,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.08460168540477753,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.959688949822748e-07,
      "logits/chosen": -0.8969976305961609,
      "logits/rejected": -0.8516714572906494,
      "logps/chosen": -79.81106567382812,
      "logps/ref_chosen": -80.35308837890625,
      "logps/ref_rejected": -90.61380004882812,
      "logps/rejected": -90.76178741455078,
      "loss": 1.337,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.045297279953956604,
      "rewards/margins": 0.057028163224458694,
      "rewards/rejected": -0.011730881407856941,
      "step": 102
    },
    {
      "epoch": 0.15570672713529857,
      "grad_norm": 14.868295669555664,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.08436441421508789,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.957289714327572e-07,
      "logits/chosen": -1.2872700691223145,
      "logits/rejected": -0.8925309777259827,
      "logps/chosen": -78.43871307373047,
      "logps/ref_chosen": -79.30392456054688,
      "logps/ref_rejected": -93.745361328125,
      "logps/rejected": -94.05059814453125,
      "loss": 1.2989,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.0718870759010315,
      "rewards/margins": 0.09726110845804214,
      "rewards/rejected": -0.025374025106430054,
      "step": 103
    },
    {
      "epoch": 0.15721844293272866,
      "grad_norm": 15.962653160095215,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.08394469320774078,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.954821743156767e-07,
      "logits/chosen": -0.6702672243118286,
      "logits/rejected": -0.5458837747573853,
      "logps/chosen": -73.63473510742188,
      "logps/ref_chosen": -74.50674438476562,
      "logps/ref_rejected": -116.09912872314453,
      "logps/rejected": -116.38943481445312,
      "loss": 1.3024,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.07236441969871521,
      "rewards/margins": 0.09585089981555939,
      "rewards/rejected": -0.023486483842134476,
      "step": 104
    },
    {
      "epoch": 0.15873015873015872,
      "grad_norm": 15.859411239624023,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.08360504359006882,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.952285105344791e-07,
      "logits/chosen": -0.8980883955955505,
      "logits/rejected": -0.9743342995643616,
      "logps/chosen": -87.27688598632812,
      "logps/ref_chosen": -87.76654815673828,
      "logps/ref_rejected": -108.07927703857422,
      "logps/rejected": -108.57989501953125,
      "loss": 1.3207,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03998672217130661,
      "rewards/margins": 0.08090025186538696,
      "rewards/rejected": -0.04091353714466095,
      "step": 105
    },
    {
      "epoch": 0.1602418745275888,
      "grad_norm": 14.278593063354492,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.08334459364414215,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.949679871846857e-07,
      "logits/chosen": -0.9452663660049438,
      "logits/rejected": -0.9867152571678162,
      "logps/chosen": -75.48688507080078,
      "logps/ref_chosen": -76.38548278808594,
      "logps/ref_rejected": -81.63407897949219,
      "logps/rejected": -81.66070556640625,
      "loss": 1.3223,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07385978102684021,
      "rewards/margins": 0.07553449273109436,
      "rewards/rejected": -0.001674711238592863,
      "step": 106
    },
    {
      "epoch": 0.1617535903250189,
      "grad_norm": 15.913289070129395,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.08303321897983551,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.947006115536947e-07,
      "logits/chosen": -1.1638550758361816,
      "logits/rejected": -0.8853709697723389,
      "logps/chosen": -95.84579467773438,
      "logps/ref_chosen": -96.14849853515625,
      "logps/ref_rejected": -107.0481185913086,
      "logps/rejected": -107.37059020996094,
      "loss": 1.346,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.024201638996601105,
      "rewards/margins": 0.05040515959262848,
      "rewards/rejected": -0.026203524321317673,
      "step": 107
    },
    {
      "epoch": 0.16326530612244897,
      "grad_norm": 14.315526008605957,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.08274877071380615,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.944263911205772e-07,
      "logits/chosen": -1.0581843852996826,
      "logits/rejected": -1.0409530401229858,
      "logps/chosen": -84.61740112304688,
      "logps/ref_chosen": -85.39241027832031,
      "logps/ref_rejected": -97.79592895507812,
      "logps/rejected": -97.88604736328125,
      "loss": 1.3261,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.06313550472259521,
      "rewards/margins": 0.07004686444997787,
      "rewards/rejected": -0.0069113681092858315,
      "step": 108
    },
    {
      "epoch": 0.16477702191987906,
      "grad_norm": 15.245014190673828,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.08249098807573318,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.941453335558681e-07,
      "logits/chosen": -1.0355124473571777,
      "logits/rejected": -1.1666637659072876,
      "logps/chosen": -78.03228759765625,
      "logps/ref_chosen": -78.99874877929688,
      "logps/ref_rejected": -100.79278564453125,
      "logps/rejected": -101.17771911621094,
      "loss": 1.2925,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0786685198545456,
      "rewards/margins": 0.10959336161613464,
      "rewards/rejected": -0.03092484548687935,
      "step": 109
    },
    {
      "epoch": 0.16628873771730915,
      "grad_norm": 16.8065242767334,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.08231092989444733,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.938574467213517e-07,
      "logits/chosen": -1.206465244293213,
      "logits/rejected": -1.1248822212219238,
      "logps/chosen": -96.55227661132812,
      "logps/ref_chosen": -96.9527816772461,
      "logps/ref_rejected": -91.44450378417969,
      "logps/rejected": -91.25479125976562,
      "loss": 1.3809,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.031805217266082764,
      "rewards/margins": 0.015741102397441864,
      "rewards/rejected": 0.0160641111433506,
      "step": 110
    },
    {
      "epoch": 0.16780045351473924,
      "grad_norm": 13.76848030090332,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.08218251913785934,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 4.935627386698418e-07,
      "logits/chosen": -1.2548036575317383,
      "logits/rejected": -1.29953134059906,
      "logps/chosen": -69.22273254394531,
      "logps/ref_chosen": -70.01641845703125,
      "logps/ref_rejected": -92.87696838378906,
      "logps/rejected": -93.05228424072266,
      "loss": 1.3207,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.06431609392166138,
      "rewards/margins": 0.07787607610225677,
      "rewards/rejected": -0.013559989631175995,
      "step": 111
    },
    {
      "epoch": 0.1693121693121693,
      "grad_norm": 16.01951026916504,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.0820799171924591,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.932612176449559e-07,
      "logits/chosen": -1.020503044128418,
      "logits/rejected": -0.8089717626571655,
      "logps/chosen": -76.8432846069336,
      "logps/ref_chosen": -77.80027770996094,
      "logps/ref_rejected": -123.10624694824219,
      "logps/rejected": -123.45081329345703,
      "loss": 1.2966,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.07723334431648254,
      "rewards/margins": 0.10491590946912766,
      "rewards/rejected": -0.02768256887793541,
      "step": 112
    },
    {
      "epoch": 0.1708238851095994,
      "grad_norm": 13.632536888122559,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.08169696480035782,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.929528920808854e-07,
      "logits/chosen": -1.1828765869140625,
      "logits/rejected": -0.7840421199798584,
      "logps/chosen": -69.23200225830078,
      "logps/ref_chosen": -70.54346466064453,
      "logps/ref_rejected": -88.79286193847656,
      "logps/rejected": -88.31471252441406,
      "loss": 1.3326,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.10599475353956223,
      "rewards/margins": 0.06640227884054184,
      "rewards/rejected": 0.039592474699020386,
      "step": 113
    },
    {
      "epoch": 0.17233560090702948,
      "grad_norm": 15.452795028686523,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.08154407143592834,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.92637770602159e-07,
      "logits/chosen": -0.8517060875892639,
      "logits/rejected": -1.1199955940246582,
      "logps/chosen": -82.729736328125,
      "logps/ref_chosen": -83.9239501953125,
      "logps/ref_rejected": -92.85765075683594,
      "logps/rejected": -93.04132843017578,
      "loss": 1.296,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.09607648849487305,
      "rewards/margins": 0.11020876467227936,
      "rewards/rejected": -0.014132272452116013,
      "step": 114
    },
    {
      "epoch": 0.17384731670445955,
      "grad_norm": 13.919018745422363,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.08129003643989563,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.923158620234019e-07,
      "logits/chosen": -1.2890362739562988,
      "logits/rejected": -1.096760869026184,
      "logps/chosen": -68.5827407836914,
      "logps/ref_chosen": -69.82767486572266,
      "logps/ref_rejected": -96.51564025878906,
      "logps/rejected": -96.44905090332031,
      "loss": 1.3068,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.10026754438877106,
      "rewards/margins": 0.09398935735225677,
      "rewards/rejected": 0.006278195418417454,
      "step": 115
    },
    {
      "epoch": 0.17535903250188964,
      "grad_norm": 15.033320426940918,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.08108732104301453,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.91987175349089e-07,
      "logits/chosen": -1.1350982189178467,
      "logits/rejected": -0.7556982636451721,
      "logps/chosen": -64.77731323242188,
      "logps/ref_chosen": -66.19773864746094,
      "logps/ref_rejected": -90.88304138183594,
      "logps/rejected": -91.1404800415039,
      "loss": 1.2686,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11404135823249817,
      "rewards/margins": 0.13410945236682892,
      "rewards/rejected": -0.020068105310201645,
      "step": 116
    },
    {
      "epoch": 0.17687074829931973,
      "grad_norm": 13.566264152526855,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.08070899546146393,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.916517197732933e-07,
      "logits/chosen": -1.1341867446899414,
      "logits/rejected": -1.0868911743164062,
      "logps/chosen": -70.42806243896484,
      "logps/ref_chosen": -72.15988159179688,
      "logps/ref_rejected": -85.30296325683594,
      "logps/rejected": -85.21932220458984,
      "loss": 1.2738,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.13824619352817535,
      "rewards/margins": 0.13081501424312592,
      "rewards/rejected": 0.007431183010339737,
      "step": 117
    },
    {
      "epoch": 0.17838246409674982,
      "grad_norm": 13.367738723754883,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.0803324356675148,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.913095046794281e-07,
      "logits/chosen": -0.7425364851951599,
      "logits/rejected": -0.7578328847885132,
      "logps/chosen": -69.99385833740234,
      "logps/ref_chosen": -71.47773742675781,
      "logps/ref_rejected": -96.95051574707031,
      "logps/rejected": -96.75798034667969,
      "loss": 1.3011,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.11790382117033005,
      "rewards/margins": 0.10169126838445663,
      "rewards/rejected": 0.016212543472647667,
      "step": 118
    },
    {
      "epoch": 0.17989417989417988,
      "grad_norm": 13.800806045532227,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.08005724102258682,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 4.909605396399855e-07,
      "logits/chosen": -1.1756548881530762,
      "logits/rejected": -1.0593156814575195,
      "logps/chosen": -76.794677734375,
      "logps/ref_chosen": -78.2727279663086,
      "logps/ref_rejected": -94.71317291259766,
      "logps/rejected": -94.62212371826172,
      "loss": 1.3024,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": 0.11696292459964752,
      "rewards/margins": 0.10860587656497955,
      "rewards/rejected": 0.008357064798474312,
      "step": 119
    },
    {
      "epoch": 0.18140589569160998,
      "grad_norm": 14.784750938415527,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.07995729893445969,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.906048344162676e-07,
      "logits/chosen": -0.9472384452819824,
      "logits/rejected": -0.9431383013725281,
      "logps/chosen": -76.48007202148438,
      "logps/ref_chosen": -78.43108367919922,
      "logps/ref_rejected": -100.2771987915039,
      "logps/rejected": -100.35806274414062,
      "loss": 1.2469,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1544308066368103,
      "rewards/margins": 0.1602558195590973,
      "rewards/rejected": -0.005825025029480457,
      "step": 120
    },
    {
      "epoch": 0.18291761148904007,
      "grad_norm": 15.232147216796875,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.07963377982378006,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.902423989581143e-07,
      "logits/chosen": -1.2334175109863281,
      "logits/rejected": -0.9664556384086609,
      "logps/chosen": -72.31959533691406,
      "logps/ref_chosen": -74.08768463134766,
      "logps/ref_rejected": -118.6731948852539,
      "logps/rejected": -118.37752532958984,
      "loss": 1.2912,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.13956449925899506,
      "rewards/margins": 0.11507533490657806,
      "rewards/rejected": 0.0244891494512558,
      "step": 121
    },
    {
      "epoch": 0.18442932728647016,
      "grad_norm": 14.288521766662598,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.07938570529222488,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.898732434036243e-07,
      "logits/chosen": -1.187713384628296,
      "logits/rejected": -1.2080814838409424,
      "logps/chosen": -77.47132873535156,
      "logps/ref_chosen": -79.36761474609375,
      "logps/ref_rejected": -92.42371368408203,
      "logps/rejected": -92.08001708984375,
      "loss": 1.2906,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1492442637681961,
      "rewards/margins": 0.1209423691034317,
      "rewards/rejected": 0.028301900252699852,
      "step": 122
    },
    {
      "epoch": 0.18594104308390022,
      "grad_norm": 13.947713851928711,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.07911375164985657,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.894973780788722e-07,
      "logits/chosen": -1.215649127960205,
      "logits/rejected": -1.056673526763916,
      "logps/chosen": -69.75860595703125,
      "logps/ref_chosen": -71.91705322265625,
      "logps/ref_rejected": -96.36418151855469,
      "logps/rejected": -95.97323608398438,
      "loss": 1.2698,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.16937178373336792,
      "rewards/margins": 0.13755583763122559,
      "rewards/rejected": 0.031815946102142334,
      "step": 123
    },
    {
      "epoch": 0.1874527588813303,
      "grad_norm": 14.389055252075195,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.07874463498592377,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.89114813497619e-07,
      "logits/chosen": -1.3429356813430786,
      "logits/rejected": -0.8390483856201172,
      "logps/chosen": -69.33705139160156,
      "logps/ref_chosen": -71.72529602050781,
      "logps/ref_rejected": -111.17984771728516,
      "logps/rejected": -110.77123260498047,
      "loss": 1.2621,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1867365539073944,
      "rewards/margins": 0.15325342118740082,
      "rewards/rejected": 0.0334831178188324,
      "step": 124
    },
    {
      "epoch": 0.1889644746787604,
      "grad_norm": 15.154690742492676,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.07840162515640259,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.887255603610184e-07,
      "logits/chosen": -1.1709169149398804,
      "logits/rejected": -1.1800987720489502,
      "logps/chosen": -78.87779235839844,
      "logps/ref_chosen": -81.55532836914062,
      "logps/ref_rejected": -110.9144287109375,
      "logps/rejected": -110.4371566772461,
      "loss": 1.2443,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.20813393592834473,
      "rewards/margins": 0.16979669034481049,
      "rewards/rejected": 0.03833724558353424,
      "step": 125
    },
    {
      "epoch": 0.19047619047619047,
      "grad_norm": 14.45041275024414,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.07798732072114944,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.883296295573176e-07,
      "logits/chosen": -1.2791764736175537,
      "logits/rejected": -0.9790908098220825,
      "logps/chosen": -83.41371154785156,
      "logps/ref_chosen": -87.07349395751953,
      "logps/ref_rejected": -85.05271911621094,
      "logps/rejected": -83.06053161621094,
      "loss": 1.2928,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.28314852714538574,
      "rewards/margins": 0.12732331454753876,
      "rewards/rejected": 0.15582521259784698,
      "step": 126
    },
    {
      "epoch": 0.19198790627362056,
      "grad_norm": 13.543227195739746,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.07781709730625153,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.87927032161552e-07,
      "logits/chosen": -1.2604224681854248,
      "logits/rejected": -1.135559320449829,
      "logps/chosen": -76.91162109375,
      "logps/ref_chosen": -80.4578857421875,
      "logps/ref_rejected": -90.50740051269531,
      "logps/rejected": -89.30085754394531,
      "loss": 1.2325,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.2738530933856964,
      "rewards/margins": 0.17955777049064636,
      "rewards/rejected": 0.09429533034563065,
      "step": 127
    },
    {
      "epoch": 0.19349962207105065,
      "grad_norm": 15.481466293334961,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.07750224322080612,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.875177794352363e-07,
      "logits/chosen": -1.0098241567611694,
      "logits/rejected": -1.1978508234024048,
      "logps/chosen": -82.52719116210938,
      "logps/ref_chosen": -85.77519226074219,
      "logps/ref_rejected": -112.63516235351562,
      "logps/rejected": -110.98802185058594,
      "loss": 1.307,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.24998116493225098,
      "rewards/margins": 0.12104400992393494,
      "rewards/rejected": 0.12893712520599365,
      "step": 128
    },
    {
      "epoch": 0.19501133786848074,
      "grad_norm": 14.332625389099121,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.07723674178123474,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.871018828260491e-07,
      "logits/chosen": -1.243614673614502,
      "logits/rejected": -1.0912466049194336,
      "logps/chosen": -81.95756530761719,
      "logps/ref_chosen": -84.94615173339844,
      "logps/ref_rejected": -85.36473846435547,
      "logps/rejected": -84.05223083496094,
      "loss": 1.2993,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.2291855812072754,
      "rewards/margins": 0.12623238563537598,
      "rewards/rejected": 0.10295319557189941,
      "step": 129
    },
    {
      "epoch": 0.1965230536659108,
      "grad_norm": 14.440750122070312,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.07709219306707382,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.866793539675126e-07,
      "logits/chosen": -1.017459750175476,
      "logits/rejected": -0.9253628253936768,
      "logps/chosen": -75.57014465332031,
      "logps/ref_chosen": -79.01844787597656,
      "logps/ref_rejected": -97.63998413085938,
      "logps/rejected": -96.27738952636719,
      "loss": 1.2618,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.2643080949783325,
      "rewards/margins": 0.15799805521965027,
      "rewards/rejected": 0.10631005465984344,
      "step": 130
    },
    {
      "epoch": 0.1980347694633409,
      "grad_norm": 13.493081092834473,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.07687597721815109,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.86250204678667e-07,
      "logits/chosen": -1.1057569980621338,
      "logits/rejected": -1.128687858581543,
      "logps/chosen": -64.50733184814453,
      "logps/ref_chosen": -68.24565887451172,
      "logps/ref_rejected": -97.99555969238281,
      "logps/rejected": -96.99533081054688,
      "loss": 1.2242,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.28541046380996704,
      "rewards/margins": 0.2072276771068573,
      "rewards/rejected": 0.07818278670310974,
      "step": 131
    },
    {
      "epoch": 0.19954648526077098,
      "grad_norm": 13.580437660217285,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.07658877223730087,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.858144469637408e-07,
      "logits/chosen": -1.3431222438812256,
      "logits/rejected": -1.4877030849456787,
      "logps/chosen": -78.46906280517578,
      "logps/ref_chosen": -82.0653305053711,
      "logps/ref_rejected": -89.47691345214844,
      "logps/rejected": -87.79734802246094,
      "loss": 1.2726,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.27282848954200745,
      "rewards/margins": 0.14421497285366058,
      "rewards/rejected": 0.12861351668834686,
      "step": 132
    },
    {
      "epoch": 0.20105820105820105,
      "grad_norm": 13.570717811584473,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.07625515758991241,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.853720930118138e-07,
      "logits/chosen": -1.475085735321045,
      "logits/rejected": -1.0344434976577759,
      "logps/chosen": -79.60618591308594,
      "logps/ref_chosen": -83.70661163330078,
      "logps/ref_rejected": -89.3868179321289,
      "logps/rejected": -87.3953628540039,
      "loss": 1.2664,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3101305663585663,
      "rewards/margins": 0.15772530436515808,
      "rewards/rejected": 0.1524052619934082,
      "step": 133
    },
    {
      "epoch": 0.20256991685563114,
      "grad_norm": 12.60647964477539,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.0760175958275795,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.849231551964771e-07,
      "logits/chosen": -1.4141769409179688,
      "logits/rejected": -1.1397137641906738,
      "logps/chosen": -66.85868835449219,
      "logps/ref_chosen": -71.57601165771484,
      "logps/ref_rejected": -92.34259033203125,
      "logps/rejected": -90.73544311523438,
      "loss": 1.2025,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.3563517928123474,
      "rewards/margins": 0.23303887248039246,
      "rewards/rejected": 0.12331293523311615,
      "step": 134
    },
    {
      "epoch": 0.20408163265306123,
      "grad_norm": 12.527703285217285,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.07563940435647964,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.844676460754862e-07,
      "logits/chosen": -1.4790236949920654,
      "logits/rejected": -1.222767949104309,
      "logps/chosen": -61.59791946411133,
      "logps/ref_chosen": -66.39884948730469,
      "logps/ref_rejected": -81.38636016845703,
      "logps/rejected": -78.73994445800781,
      "loss": 1.2616,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.36091580986976624,
      "rewards/margins": 0.16017356514930725,
      "rewards/rejected": 0.2007422298192978,
      "step": 135
    },
    {
      "epoch": 0.20559334845049132,
      "grad_norm": 13.989602088928223,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.07535681128501892,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.840055783904106e-07,
      "logits/chosen": -1.0112760066986084,
      "logits/rejected": -1.174780011177063,
      "logps/chosen": -83.18777465820312,
      "logps/ref_chosen": -86.75381469726562,
      "logps/ref_rejected": -113.35548400878906,
      "logps/rejected": -112.59575653076172,
      "loss": 1.2478,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.267022967338562,
      "rewards/margins": 0.20755240321159363,
      "rewards/rejected": 0.05947057530283928,
      "step": 136
    },
    {
      "epoch": 0.20710506424792138,
      "grad_norm": 12.38171100616455,
      "kl/avg_steps": 0.328125,
      "kl/beta": 0.07521578669548035,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.835369650662767e-07,
      "logits/chosen": -1.440555453300476,
      "logits/rejected": -1.2636154890060425,
      "logps/chosen": -67.68683624267578,
      "logps/ref_chosen": -72.21119689941406,
      "logps/ref_rejected": -88.30802917480469,
      "logps/rejected": -86.6800765991211,
      "loss": 1.2184,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.3381859362125397,
      "rewards/margins": 0.2144036889076233,
      "rewards/rejected": 0.12378223240375519,
      "step": 137
    },
    {
      "epoch": 0.20861678004535147,
      "grad_norm": 12.980399131774902,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.07496979087591171,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.830618192112065e-07,
      "logits/chosen": -1.4844608306884766,
      "logits/rejected": -1.660388469696045,
      "logps/chosen": -71.28809356689453,
      "logps/ref_chosen": -74.54273223876953,
      "logps/ref_rejected": -84.63614654541016,
      "logps/rejected": -83.41443634033203,
      "loss": 1.2868,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.24151670932769775,
      "rewards/margins": 0.1492181122303009,
      "rewards/rejected": 0.09229859709739685,
      "step": 138
    },
    {
      "epoch": 0.21012849584278157,
      "grad_norm": 15.18370246887207,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.07471296191215515,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.825801541160509e-07,
      "logits/chosen": -1.3889563083648682,
      "logits/rejected": -1.331032156944275,
      "logps/chosen": -85.1347427368164,
      "logps/ref_chosen": -87.63740539550781,
      "logps/ref_rejected": -101.3896484375,
      "logps/rejected": -101.09443664550781,
      "loss": 1.2746,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.18462873995304108,
      "rewards/margins": 0.16127237677574158,
      "rewards/rejected": 0.023356378078460693,
      "step": 139
    },
    {
      "epoch": 0.21164021164021163,
      "grad_norm": 15.858207702636719,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.0744338408112526,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -1.2838711738586426,
      "logits/rejected": -1.091439127922058,
      "logps/chosen": -77.99700927734375,
      "logps/ref_chosen": -81.32339477539062,
      "logps/ref_rejected": -99.72755432128906,
      "logps/rejected": -99.94612121582031,
      "loss": 1.1982,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.24433737993240356,
      "rewards/margins": 0.25949618220329285,
      "rewards/rejected": -0.015158784575760365,
      "step": 140
    },
    {
      "epoch": 0.21315192743764172,
      "grad_norm": 12.245986938476562,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.07422508299350739,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.815973202802966e-07,
      "logits/chosen": -1.2865171432495117,
      "logits/rejected": -1.0338687896728516,
      "logps/chosen": -75.33793640136719,
      "logps/ref_chosen": -78.08534240722656,
      "logps/ref_rejected": -101.70516967773438,
      "logps/rejected": -102.56867218017578,
      "loss": 1.195,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.20178569853305817,
      "rewards/margins": 0.2635989189147949,
      "rewards/rejected": -0.06181325763463974,
      "step": 141
    },
    {
      "epoch": 0.2146636432350718,
      "grad_norm": 12.931549072265625,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.07394777238368988,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.810961790316729e-07,
      "logits/chosen": -1.2304003238677979,
      "logits/rejected": -1.1563172340393066,
      "logps/chosen": -80.3508529663086,
      "logps/ref_chosen": -82.84616088867188,
      "logps/ref_rejected": -95.14713287353516,
      "logps/rejected": -95.14163970947266,
      "loss": 1.2561,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1831914186477661,
      "rewards/margins": 0.1807531714439392,
      "rewards/rejected": 0.0024382397532463074,
      "step": 142
    },
    {
      "epoch": 0.2161753590325019,
      "grad_norm": 15.332486152648926,
      "kl/avg_steps": 0.171875,
      "kl/beta": 0.07369445264339447,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.805885735261454e-07,
      "logits/chosen": -1.1605253219604492,
      "logits/rejected": -1.1731536388397217,
      "logps/chosen": -78.10973358154297,
      "logps/ref_chosen": -80.29791259765625,
      "logps/ref_rejected": -87.44291687011719,
      "logps/rejected": -87.5304946899414,
      "loss": 1.2913,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.15912334620952606,
      "rewards/margins": 0.16363541781902313,
      "rewards/rejected": -0.004512062296271324,
      "step": 143
    },
    {
      "epoch": 0.21768707482993196,
      "grad_norm": 18.32091522216797,
      "kl/avg_steps": 0.03125,
      "kl/beta": 0.07356800884008408,
      "kl/n_epsilon_steps": 0.484375,
      "kl/p_epsilon_steps": 0.515625,
      "learning_rate": 4.800745179625307e-07,
      "logits/chosen": -1.2224823236465454,
      "logits/rejected": -0.8715156316757202,
      "logps/chosen": -77.77732849121094,
      "logps/ref_chosen": -79.09428405761719,
      "logps/ref_rejected": -92.42912292480469,
      "logps/rejected": -93.04374694824219,
      "loss": 1.3382,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09343221783638,
      "rewards/margins": 0.1373026967048645,
      "rewards/rejected": -0.04387049004435539,
      "step": 144
    },
    {
      "epoch": 0.21919879062736206,
      "grad_norm": 15.246169090270996,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.07354502379894257,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.795540267200686e-07,
      "logits/chosen": -1.3926103115081787,
      "logits/rejected": -1.1138077974319458,
      "logps/chosen": -95.6426773071289,
      "logps/ref_chosen": -97.7087173461914,
      "logps/ref_rejected": -97.63011169433594,
      "logps/rejected": -98.70494079589844,
      "loss": 1.2545,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.14838823676109314,
      "rewards/margins": 0.22605197131633759,
      "rewards/rejected": -0.07766373455524445,
      "step": 145
    },
    {
      "epoch": 0.22071050642479215,
      "grad_norm": 13.784503936767578,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.07320188730955124,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.790271143580173e-07,
      "logits/chosen": -1.4019311666488647,
      "logits/rejected": -1.174739122390747,
      "logps/chosen": -74.28959655761719,
      "logps/ref_chosen": -76.56293487548828,
      "logps/ref_rejected": -83.78160095214844,
      "logps/rejected": -83.92266845703125,
      "loss": 1.2839,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.16467270255088806,
      "rewards/margins": 0.1724810153245926,
      "rewards/rejected": -0.007808296009898186,
      "step": 146
    },
    {
      "epoch": 0.2222222222222222,
      "grad_norm": 15.15932559967041,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.072996586561203,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 4.784937956152489e-07,
      "logits/chosen": -1.3146681785583496,
      "logits/rejected": -1.2722773551940918,
      "logps/chosen": -82.0851821899414,
      "logps/ref_chosen": -83.24113464355469,
      "logps/ref_rejected": -97.50960540771484,
      "logps/rejected": -98.47145080566406,
      "loss": 1.3017,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.08227235823869705,
      "rewards/margins": 0.15057937800884247,
      "rewards/rejected": -0.06830701977014542,
      "step": 147
    },
    {
      "epoch": 0.2237339380196523,
      "grad_norm": 12.173563957214355,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.07290545105934143,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.779540854098347e-07,
      "logits/chosen": -1.4075052738189697,
      "logits/rejected": -1.5484192371368408,
      "logps/chosen": -64.48944091796875,
      "logps/ref_chosen": -66.36277770996094,
      "logps/ref_rejected": -87.66487121582031,
      "logps/rejected": -89.90396118164062,
      "loss": 1.1962,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.13385669887065887,
      "rewards/margins": 0.2946438789367676,
      "rewards/rejected": -0.1607871800661087,
      "step": 148
    },
    {
      "epoch": 0.2252456538170824,
      "grad_norm": 13.464912414550781,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.07258788496255875,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.774079988386296e-07,
      "logits/chosen": -1.3259451389312744,
      "logits/rejected": -1.4282548427581787,
      "logps/chosen": -72.2156982421875,
      "logps/ref_chosen": -72.0576171875,
      "logps/ref_rejected": -83.94097900390625,
      "logps/rejected": -86.57736206054688,
      "loss": 1.273,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.013719309121370316,
      "rewards/margins": 0.17610222101211548,
      "rewards/rejected": -0.1898215413093567,
      "step": 149
    },
    {
      "epoch": 0.22675736961451248,
      "grad_norm": 12.965806007385254,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.0723842978477478,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.768555511768486e-07,
      "logits/chosen": -1.2750177383422852,
      "logits/rejected": -0.8741225004196167,
      "logps/chosen": -85.07838439941406,
      "logps/ref_chosen": -85.52684783935547,
      "logps/ref_rejected": -108.37450408935547,
      "logps/rejected": -112.33922576904297,
      "loss": 1.1922,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.02943560853600502,
      "rewards/margins": 0.3137895464897156,
      "rewards/rejected": -0.28435391187667847,
      "step": 150
    },
    {
      "epoch": 0.22826908541194255,
      "grad_norm": 12.630627632141113,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.07213633507490158,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.762967578776406e-07,
      "logits/chosen": -1.2928177118301392,
      "logits/rejected": -1.1232836246490479,
      "logps/chosen": -66.61726379394531,
      "logps/ref_chosen": -69.160888671875,
      "logps/ref_rejected": -91.42207336425781,
      "logps/rejected": -93.68106079101562,
      "loss": 1.1445,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.17991137504577637,
      "rewards/margins": 0.34157001972198486,
      "rewards/rejected": -0.1616586446762085,
      "step": 151
    },
    {
      "epoch": 0.22978080120937264,
      "grad_norm": 13.797011375427246,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.07188921421766281,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.757316345716553e-07,
      "logits/chosen": -1.2502892017364502,
      "logits/rejected": -1.2125251293182373,
      "logps/chosen": -71.50172424316406,
      "logps/ref_chosen": -72.48135375976562,
      "logps/ref_rejected": -94.44818878173828,
      "logps/rejected": -97.02812957763672,
      "loss": 1.232,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.06772876530885696,
      "rewards/margins": 0.2509690523147583,
      "rewards/rejected": -0.18324029445648193,
      "step": 152
    },
    {
      "epoch": 0.23129251700680273,
      "grad_norm": 12.633346557617188,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.0716206356883049,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.751601970666064e-07,
      "logits/chosen": -1.2384766340255737,
      "logits/rejected": -1.0395536422729492,
      "logps/chosen": -89.7574691772461,
      "logps/ref_chosen": -89.6655044555664,
      "logps/ref_rejected": -90.67737579345703,
      "logps/rejected": -94.66575622558594,
      "loss": 1.1962,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.008383519947528839,
      "rewards/margins": 0.2745283842086792,
      "rewards/rejected": -0.28291189670562744,
      "step": 153
    },
    {
      "epoch": 0.2328042328042328,
      "grad_norm": 14.03810977935791,
      "kl/avg_steps": 0.328125,
      "kl/beta": 0.07141976803541183,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.745824613468292e-07,
      "logits/chosen": -1.3895330429077148,
      "logits/rejected": -1.4276092052459717,
      "logps/chosen": -75.47744750976562,
      "logps/ref_chosen": -76.58096313476562,
      "logps/ref_rejected": -78.18670654296875,
      "logps/rejected": -79.85231018066406,
      "loss": 1.2873,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07643654942512512,
      "rewards/margins": 0.19298705458641052,
      "rewards/rejected": -0.1165504902601242,
      "step": 154
    },
    {
      "epoch": 0.23431594860166288,
      "grad_norm": 18.652502059936523,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.07118618488311768,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.7399844357283393e-07,
      "logits/chosen": -1.4403748512268066,
      "logits/rejected": -1.173947811126709,
      "logps/chosen": -83.46440124511719,
      "logps/ref_chosen": -82.65617370605469,
      "logps/ref_rejected": -95.52484130859375,
      "logps/rejected": -100.07963562011719,
      "loss": 1.2294,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0596730001270771,
      "rewards/margins": 0.2618723511695862,
      "rewards/rejected": -0.3215453624725342,
      "step": 155
    },
    {
      "epoch": 0.23582766439909297,
      "grad_norm": 14.09681510925293,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.0709865391254425,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.7340816008085305e-07,
      "logits/chosen": -1.2728259563446045,
      "logits/rejected": -1.0183078050613403,
      "logps/chosen": -87.7572250366211,
      "logps/ref_chosen": -87.66494750976562,
      "logps/ref_rejected": -108.24376678466797,
      "logps/rejected": -113.68663787841797,
      "loss": 1.12,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.009032588452100754,
      "rewards/margins": 0.37457698583602905,
      "rewards/rejected": -0.3836095631122589,
      "step": 156
    },
    {
      "epoch": 0.23733938019652306,
      "grad_norm": 12.62127685546875,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.070721335709095,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.728116273823847e-07,
      "logits/chosen": -1.6038012504577637,
      "logits/rejected": -1.3034052848815918,
      "logps/chosen": -69.88165283203125,
      "logps/ref_chosen": -70.77095794677734,
      "logps/ref_rejected": -78.78271484375,
      "logps/rejected": -81.07191467285156,
      "loss": 1.2581,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.060784123837947845,
      "rewards/margins": 0.2202034294605255,
      "rewards/rejected": -0.15941932797431946,
      "step": 157
    },
    {
      "epoch": 0.23885109599395313,
      "grad_norm": 14.012279510498047,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.07061100751161575,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.7220886216373085e-07,
      "logits/chosen": -1.6464459896087646,
      "logits/rejected": -1.4208977222442627,
      "logps/chosen": -80.11083984375,
      "logps/ref_chosen": -81.21516418457031,
      "logps/ref_rejected": -97.8381118774414,
      "logps/rejected": -100.73680877685547,
      "loss": 1.229,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.07514195144176483,
      "rewards/margins": 0.277182936668396,
      "rewards/rejected": -0.20204098522663116,
      "step": 158
    },
    {
      "epoch": 0.24036281179138322,
      "grad_norm": 12.303658485412598,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.07041297107934952,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.715998812855304e-07,
      "logits/chosen": -1.1418673992156982,
      "logits/rejected": -1.0512657165527344,
      "logps/chosen": -71.02163696289062,
      "logps/ref_chosen": -72.33412170410156,
      "logps/ref_rejected": -89.49591064453125,
      "logps/rejected": -92.77830505371094,
      "loss": 1.179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08971140533685684,
      "rewards/margins": 0.31856340169906616,
      "rewards/rejected": -0.22885200381278992,
      "step": 159
    },
    {
      "epoch": 0.2418745275888133,
      "grad_norm": 11.714944839477539,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.07006265223026276,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.7098470178228755e-07,
      "logits/chosen": -1.4561386108398438,
      "logits/rejected": -1.5497028827667236,
      "logps/chosen": -62.24732971191406,
      "logps/ref_chosen": -63.26386260986328,
      "logps/ref_rejected": -82.2786636352539,
      "logps/rejected": -85.34880828857422,
      "loss": 1.1985,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.0691169872879982,
      "rewards/margins": 0.28153932094573975,
      "rewards/rejected": -0.21242231130599976,
      "step": 160
    },
    {
      "epoch": 0.24338624338624337,
      "grad_norm": 11.859792709350586,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.06980089843273163,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.703633408618955e-07,
      "logits/chosen": -1.2435733079910278,
      "logits/rejected": -1.202815294265747,
      "logps/chosen": -69.17000579833984,
      "logps/ref_chosen": -70.69304656982422,
      "logps/ref_rejected": -82.73606872558594,
      "logps/rejected": -85.28659057617188,
      "loss": 1.2059,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.10437165200710297,
      "rewards/margins": 0.2798129916191101,
      "rewards/rejected": -0.17544135451316833,
      "step": 161
    },
    {
      "epoch": 0.24489795918367346,
      "grad_norm": 13.439538955688477,
      "kl/avg_steps": 0.546875,
      "kl/beta": 0.06951848417520523,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.697358159051549e-07,
      "logits/chosen": -1.508550763130188,
      "logits/rejected": -1.2809114456176758,
      "logps/chosen": -88.38290405273438,
      "logps/ref_chosen": -89.3046646118164,
      "logps/ref_rejected": -114.05778503417969,
      "logps/rejected": -119.21257781982422,
      "loss": 1.087,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.0630064308643341,
      "rewards/margins": 0.4171755909919739,
      "rewards/rejected": -0.35416918992996216,
      "step": 162
    },
    {
      "epoch": 0.24640967498110355,
      "grad_norm": 12.317628860473633,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.06914036720991135,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.691021444652876e-07,
      "logits/chosen": -1.2010798454284668,
      "logits/rejected": -0.9982933402061462,
      "logps/chosen": -65.79276275634766,
      "logps/ref_chosen": -68.61221313476562,
      "logps/ref_rejected": -89.03155517578125,
      "logps/rejected": -91.13777160644531,
      "loss": 1.1662,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.19236725568771362,
      "rewards/margins": 0.33524468541145325,
      "rewards/rejected": -0.14287742972373962,
      "step": 163
    },
    {
      "epoch": 0.24792139077853365,
      "grad_norm": 13.418086051940918,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.06890351325273514,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.6846234426744624e-07,
      "logits/chosen": -1.324411392211914,
      "logits/rejected": -1.2855700254440308,
      "logps/chosen": -70.44233703613281,
      "logps/ref_chosen": -73.55903625488281,
      "logps/ref_rejected": -94.16201782226562,
      "logps/rejected": -97.29086303710938,
      "loss": 1.1162,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.21184256672859192,
      "rewards/margins": 0.42402780055999756,
      "rewards/rejected": -0.21218520402908325,
      "step": 164
    },
    {
      "epoch": 0.2494331065759637,
      "grad_norm": 12.428820610046387,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.06862472742795944,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.678164332082175e-07,
      "logits/chosen": -1.278984785079956,
      "logits/rejected": -1.2863500118255615,
      "logps/chosen": -66.50820922851562,
      "logps/ref_chosen": -68.67132568359375,
      "logps/ref_rejected": -85.95690155029297,
      "logps/rejected": -89.47266387939453,
      "loss": 1.1273,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.14679834246635437,
      "rewards/margins": 0.38445547223091125,
      "rewards/rejected": -0.23765714466571808,
      "step": 165
    },
    {
      "epoch": 0.2509448223733938,
      "grad_norm": 13.696086883544922,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.06834706664085388,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.6716442935512214e-07,
      "logits/chosen": -1.145601511001587,
      "logits/rejected": -1.1215341091156006,
      "logps/chosen": -79.69151306152344,
      "logps/ref_chosen": -80.89754486083984,
      "logps/ref_rejected": -111.91075134277344,
      "logps/rejected": -115.30693817138672,
      "loss": 1.1943,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.08006976544857025,
      "rewards/margins": 0.3093419671058655,
      "rewards/rejected": -0.22927218675613403,
      "step": 166
    },
    {
      "epoch": 0.25245653817082386,
      "grad_norm": 11.959296226501465,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.06811293214559555,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.6650635094610966e-07,
      "logits/chosen": -1.280385136604309,
      "logits/rejected": -1.2254626750946045,
      "logps/chosen": -73.93049621582031,
      "logps/ref_chosen": -76.73136138916016,
      "logps/ref_rejected": -92.57389068603516,
      "logps/rejected": -95.07466888427734,
      "loss": 1.1425,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.18968580663204193,
      "rewards/margins": 0.356110155582428,
      "rewards/rejected": -0.16642434895038605,
      "step": 167
    },
    {
      "epoch": 0.25396825396825395,
      "grad_norm": 12.643539428710938,
      "kl/avg_steps": 0.296875,
      "kl/beta": 0.06783734261989594,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.6584221638904767e-07,
      "logits/chosen": -1.1431384086608887,
      "logits/rejected": -1.2114157676696777,
      "logps/chosen": -81.43280029296875,
      "logps/ref_chosen": -82.63671112060547,
      "logps/ref_rejected": -96.72691345214844,
      "logps/rejected": -99.67520904541016,
      "loss": 1.1902,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.08026599138975143,
      "rewards/margins": 0.2774825096130371,
      "rewards/rejected": -0.19721652567386627,
      "step": 168
    },
    {
      "epoch": 0.25547996976568405,
      "grad_norm": 13.48161792755127,
      "kl/avg_steps": 0.390625,
      "kl/beta": 0.06763654202222824,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.651720442612075e-07,
      "logits/chosen": -1.4378364086151123,
      "logits/rejected": -1.0551780462265015,
      "logps/chosen": -76.00579833984375,
      "logps/ref_chosen": -78.87673950195312,
      "logps/ref_rejected": -94.18919372558594,
      "logps/rejected": -97.01301574707031,
      "loss": 1.1499,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.19150575995445251,
      "rewards/margins": 0.37936002016067505,
      "rewards/rejected": -0.18785430490970612,
      "step": 169
    },
    {
      "epoch": 0.25699168556311414,
      "grad_norm": 15.122897148132324,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.0673733651638031,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.6449585330874425e-07,
      "logits/chosen": -1.3555153608322144,
      "logits/rejected": -1.1191372871398926,
      "logps/chosen": -71.77020263671875,
      "logps/ref_chosen": -73.35820007324219,
      "logps/ref_rejected": -76.85078430175781,
      "logps/rejected": -78.78128814697266,
      "loss": 1.3308,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.10407811403274536,
      "rewards/margins": 0.23043477535247803,
      "rewards/rejected": -0.12635666131973267,
      "step": 170
    },
    {
      "epoch": 0.2585034013605442,
      "grad_norm": 12.071377754211426,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.06720535457134247,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.6381366244617224e-07,
      "logits/chosen": -1.0791172981262207,
      "logits/rejected": -1.18746817111969,
      "logps/chosen": -79.33969116210938,
      "logps/ref_chosen": -80.43226623535156,
      "logps/ref_rejected": -96.99999237060547,
      "logps/rejected": -102.03294372558594,
      "loss": 1.1541,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06989102065563202,
      "rewards/margins": 0.4047943949699402,
      "rewards/rejected": -0.334903359413147,
      "step": 171
    },
    {
      "epoch": 0.2600151171579743,
      "grad_norm": 11.951263427734375,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.06691261380910873,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.631254907558365e-07,
      "logits/chosen": -1.3960208892822266,
      "logits/rejected": -1.1161054372787476,
      "logps/chosen": -70.59374237060547,
      "logps/ref_chosen": -70.45406341552734,
      "logps/ref_rejected": -99.85604095458984,
      "logps/rejected": -105.46135711669922,
      "loss": 1.1674,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.012096043676137924,
      "rewards/margins": 0.3599253296852112,
      "rewards/rejected": -0.3720213770866394,
      "step": 172
    },
    {
      "epoch": 0.2615268329554044,
      "grad_norm": 12.691923141479492,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.066704161465168,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.624313574873786e-07,
      "logits/chosen": -1.4205788373947144,
      "logits/rejected": -1.4068093299865723,
      "logps/chosen": -72.95709228515625,
      "logps/ref_chosen": -72.15026092529297,
      "logps/ref_rejected": -94.10212707519531,
      "logps/rejected": -100.36970520019531,
      "loss": 1.2102,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.055580783635377884,
      "rewards/margins": 0.35744237899780273,
      "rewards/rejected": -0.4130231440067291,
      "step": 173
    },
    {
      "epoch": 0.26303854875283444,
      "grad_norm": 12.431907653808594,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.0664963573217392,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.61731282057198e-07,
      "logits/chosen": -1.0261675119400024,
      "logits/rejected": -1.1597849130630493,
      "logps/chosen": -76.77759552001953,
      "logps/ref_chosen": -75.99628448486328,
      "logps/ref_rejected": -106.2359619140625,
      "logps/rejected": -114.14846801757812,
      "loss": 1.1048,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.054633866995573044,
      "rewards/margins": 0.4673388600349426,
      "rewards/rejected": -0.5219727158546448,
      "step": 174
    },
    {
      "epoch": 0.26455026455026454,
      "grad_norm": 13.340142250061035,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.06614496558904648,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.6102528404790965e-07,
      "logits/chosen": -1.486039400100708,
      "logits/rejected": -1.1863949298858643,
      "logps/chosen": -86.00851440429688,
      "logps/ref_chosen": -84.51177978515625,
      "logps/ref_rejected": -104.46299743652344,
      "logps/rejected": -111.91407775878906,
      "loss": 1.1833,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.10132718086242676,
      "rewards/margins": 0.38702377676963806,
      "rewards/rejected": -0.4883509576320648,
      "step": 175
    },
    {
      "epoch": 0.2660619803476946,
      "grad_norm": 15.169533729553223,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.06589784473180771,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.603133832077953e-07,
      "logits/chosen": -1.232452392578125,
      "logits/rejected": -1.3203933238983154,
      "logps/chosen": -102.6416244506836,
      "logps/ref_chosen": -98.2034912109375,
      "logps/ref_rejected": -103.2023696899414,
      "logps/rejected": -111.3276596069336,
      "loss": 1.2962,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.294032484292984,
      "rewards/margins": 0.23688749969005585,
      "rewards/rejected": -0.5309199690818787,
      "step": 176
    },
    {
      "epoch": 0.2675736961451247,
      "grad_norm": 18.41356086730957,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.06573351472616196,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.5959559945025183e-07,
      "logits/chosen": -1.1291613578796387,
      "logits/rejected": -1.0879511833190918,
      "logps/chosen": -77.68388366699219,
      "logps/ref_chosen": -78.029541015625,
      "logps/ref_rejected": -112.57099914550781,
      "logps/rejected": -122.57363891601562,
      "loss": 0.9551,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.021007169038057327,
      "rewards/margins": 0.6723326444625854,
      "rewards/rejected": -0.6513254642486572,
      "step": 177
    },
    {
      "epoch": 0.2690854119425548,
      "grad_norm": 16.913576126098633,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.06540647894144058,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.588719528532341e-07,
      "logits/chosen": -1.6763261556625366,
      "logits/rejected": -1.5302388668060303,
      "logps/chosen": -81.11467742919922,
      "logps/ref_chosen": -79.48869323730469,
      "logps/ref_rejected": -96.62449645996094,
      "logps/rejected": -104.88471984863281,
      "loss": 1.1116,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.10739612579345703,
      "rewards/margins": 0.4278530478477478,
      "rewards/rejected": -0.5352491736412048,
      "step": 178
    },
    {
      "epoch": 0.2705971277399849,
      "grad_norm": 15.363626480102539,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.06510131806135178,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.581424636586928e-07,
      "logits/chosen": -1.5718255043029785,
      "logits/rejected": -1.4922645092010498,
      "logps/chosen": -84.71263885498047,
      "logps/ref_chosen": -84.5088119506836,
      "logps/ref_rejected": -93.07945251464844,
      "logps/rejected": -99.22616577148438,
      "loss": 1.19,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.01574593409895897,
      "rewards/margins": 0.37997791171073914,
      "rewards/rejected": -0.3957238495349884,
      "step": 179
    },
    {
      "epoch": 0.272108843537415,
      "grad_norm": 13.242083549499512,
      "kl/avg_steps": 0.125,
      "kl/beta": 0.0648581013083458,
      "kl/n_epsilon_steps": 0.4375,
      "kl/p_epsilon_steps": 0.5625,
      "learning_rate": 4.5740715227200897e-07,
      "logits/chosen": -1.630727767944336,
      "logits/rejected": -1.671911597251892,
      "logps/chosen": -74.23190307617188,
      "logps/ref_chosen": -74.56455993652344,
      "logps/ref_rejected": -81.02265930175781,
      "logps/rejected": -86.11099243164062,
      "loss": 1.1924,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.019411645829677582,
      "rewards/margins": 0.34585025906562805,
      "rewards/rejected": -0.3264386057853699,
      "step": 180
    },
    {
      "epoch": 0.273620559334845,
      "grad_norm": 16.082420349121094,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.06477712839841843,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.566660392614228e-07,
      "logits/chosen": -1.7474174499511719,
      "logits/rejected": -1.6977311372756958,
      "logps/chosen": -76.37571716308594,
      "logps/ref_chosen": -78.77166748046875,
      "logps/ref_rejected": -98.29750061035156,
      "logps/rejected": -103.26253509521484,
      "loss": 1.0762,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.15446849167346954,
      "rewards/margins": 0.47106197476387024,
      "rewards/rejected": -0.3165934979915619,
      "step": 181
    },
    {
      "epoch": 0.2751322751322751,
      "grad_norm": 14.591492652893066,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.06459545344114304,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 4.5591914535745817e-07,
      "logits/chosen": -1.095639944076538,
      "logits/rejected": -1.1765986680984497,
      "logps/chosen": -75.13510131835938,
      "logps/ref_chosen": -75.67765045166016,
      "logps/ref_rejected": -107.47894287109375,
      "logps/rejected": -116.218017578125,
      "loss": 1.0182,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.03239423781633377,
      "rewards/margins": 0.5916929244995117,
      "rewards/rejected": -0.5592987537384033,
      "step": 182
    },
    {
      "epoch": 0.2766439909297052,
      "grad_norm": 18.293054580688477,
      "kl/avg_steps": 0.09375,
      "kl/beta": 0.06423413753509521,
      "kl/n_epsilon_steps": 0.453125,
      "kl/p_epsilon_steps": 0.546875,
      "learning_rate": 4.551664914523433e-07,
      "logits/chosen": -1.3249409198760986,
      "logits/rejected": -1.243220567703247,
      "logps/chosen": -83.7846908569336,
      "logps/ref_chosen": -79.99969482421875,
      "logps/ref_rejected": -89.35220336914062,
      "logps/rejected": -96.42832946777344,
      "loss": 1.3087,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24508771300315857,
      "rewards/margins": 0.20615342259407043,
      "rewards/rejected": -0.451241135597229,
      "step": 183
    },
    {
      "epoch": 0.2781557067271353,
      "grad_norm": 11.336007118225098,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.06417397409677505,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.544080985994258e-07,
      "logits/chosen": -1.4438279867172241,
      "logits/rejected": -1.3008147478103638,
      "logps/chosen": -60.90853500366211,
      "logps/ref_chosen": -62.133941650390625,
      "logps/ref_rejected": -84.44404602050781,
      "logps/rejected": -90.04169464111328,
      "loss": 1.0963,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07818011939525604,
      "rewards/margins": 0.43237122893333435,
      "rewards/rejected": -0.3541911244392395,
      "step": 184
    },
    {
      "epoch": 0.2796674225245654,
      "grad_norm": 11.554147720336914,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.06391432136297226,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.5364398801258394e-07,
      "logits/chosen": -1.1598920822143555,
      "logits/rejected": -1.2415722608566284,
      "logps/chosen": -69.17321014404297,
      "logps/ref_chosen": -67.93175506591797,
      "logps/ref_rejected": -83.76744079589844,
      "logps/rejected": -92.03234100341797,
      "loss": 1.1778,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08316050469875336,
      "rewards/margins": 0.44052523374557495,
      "rewards/rejected": -0.5236857533454895,
      "step": 185
    },
    {
      "epoch": 0.2811791383219955,
      "grad_norm": 13.363798141479492,
      "kl/avg_steps": 0.234375,
      "kl/beta": 0.06367553770542145,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.5287418106563354e-07,
      "logits/chosen": -1.4864418506622314,
      "logits/rejected": -1.4083607196807861,
      "logps/chosen": -86.89867401123047,
      "logps/ref_chosen": -86.22174072265625,
      "logps/ref_rejected": -100.42019653320312,
      "logps/rejected": -108.28703308105469,
      "loss": 1.1692,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.045348234474658966,
      "rewards/margins": 0.4500497281551361,
      "rewards/rejected": -0.49539801478385925,
      "step": 186
    },
    {
      "epoch": 0.28269085411942557,
      "grad_norm": 14.167108535766602,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.06352664530277252,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.520986992917297e-07,
      "logits/chosen": -1.370615005493164,
      "logits/rejected": -1.32820463180542,
      "logps/chosen": -96.74392700195312,
      "logps/ref_chosen": -92.81202697753906,
      "logps/ref_rejected": -117.28926086425781,
      "logps/rejected": -128.07684326171875,
      "loss": 1.1352,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2524537742137909,
      "rewards/margins": 0.42868465185165405,
      "rewards/rejected": -0.6811383962631226,
      "step": 187
    },
    {
      "epoch": 0.2842025699168556,
      "grad_norm": 13.300924301147461,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.06321059167385101,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.5131756438276466e-07,
      "logits/chosen": -1.242083191871643,
      "logits/rejected": -1.1624287366867065,
      "logps/chosen": -90.72341918945312,
      "logps/ref_chosen": -87.85247802734375,
      "logps/ref_rejected": -94.58252716064453,
      "logps/rejected": -103.83467102050781,
      "loss": 1.1445,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.18243508040905,
      "rewards/margins": 0.397586464881897,
      "rewards/rejected": -0.5800215005874634,
      "step": 188
    },
    {
      "epoch": 0.2857142857142857,
      "grad_norm": 16.380733489990234,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.06299405544996262,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.5053079818876096e-07,
      "logits/chosen": -0.905377984046936,
      "logits/rejected": -1.0313442945480347,
      "logps/chosen": -96.79391479492188,
      "logps/ref_chosen": -95.00414276123047,
      "logps/ref_rejected": -90.50090789794922,
      "logps/rejected": -97.81324768066406,
      "loss": 1.2432,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11515786498785019,
      "rewards/margins": 0.34047895669937134,
      "rewards/rejected": -0.4556368589401245,
      "step": 189
    },
    {
      "epoch": 0.2872260015117158,
      "grad_norm": 15.347829818725586,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.06276848167181015,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 4.4973842271726024e-07,
      "logits/chosen": -1.1943118572235107,
      "logits/rejected": -1.4775378704071045,
      "logps/chosen": -71.35322570800781,
      "logps/ref_chosen": -70.79264831542969,
      "logps/ref_rejected": -122.56155395507812,
      "logps/rejected": -132.76144409179688,
      "loss": 1.004,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.035754382610321045,
      "rewards/margins": 0.5974862575531006,
      "rewards/rejected": -0.6332406401634216,
      "step": 190
    },
    {
      "epoch": 0.2887377173091459,
      "grad_norm": 15.84787654876709,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.06237861141562462,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.48940460132708e-07,
      "logits/chosen": -1.1038767099380493,
      "logits/rejected": -1.179697871208191,
      "logps/chosen": -97.53190612792969,
      "logps/ref_chosen": -92.15048217773438,
      "logps/ref_rejected": -106.4153060913086,
      "logps/rejected": -117.19906616210938,
      "loss": 1.2247,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.33690476417541504,
      "rewards/margins": 0.3303537666797638,
      "rewards/rejected": -0.6672585010528564,
      "step": 191
    },
    {
      "epoch": 0.29024943310657597,
      "grad_norm": 11.231852531433105,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.062126222997903824,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.481369327558329e-07,
      "logits/chosen": -1.3626068830490112,
      "logits/rejected": -1.3309613466262817,
      "logps/chosen": -74.0158920288086,
      "logps/ref_chosen": -69.51527404785156,
      "logps/ref_rejected": -80.15898132324219,
      "logps/rejected": -88.65760040283203,
      "loss": 1.2594,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.2809726595878601,
      "rewards/margins": 0.243299663066864,
      "rewards/rejected": -0.5242723226547241,
      "step": 192
    },
    {
      "epoch": 0.29176114890400606,
      "grad_norm": 11.15489673614502,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.06197129562497139,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.47327863063023e-07,
      "logits/chosen": -1.5196974277496338,
      "logits/rejected": -1.1110320091247559,
      "logps/chosen": -75.85136413574219,
      "logps/ref_chosen": -73.43277740478516,
      "logps/ref_rejected": -77.81238555908203,
      "logps/rejected": -88.97630310058594,
      "loss": 1.0604,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.15187571942806244,
      "rewards/margins": 0.5345999002456665,
      "rewards/rejected": -0.6864755749702454,
      "step": 193
    },
    {
      "epoch": 0.29327286470143615,
      "grad_norm": 14.358479499816895,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.06173977255821228,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": -1.535353660583496,
      "logits/rejected": -1.5219361782073975,
      "logps/chosen": -83.10891723632812,
      "logps/ref_chosen": -76.63236999511719,
      "logps/ref_rejected": -85.67449188232422,
      "logps/rejected": -96.16249084472656,
      "loss": 1.3277,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4020426273345947,
      "rewards/margins": 0.24061188101768494,
      "rewards/rejected": -0.642654538154602,
      "step": 194
    },
    {
      "epoch": 0.2947845804988662,
      "grad_norm": 12.973784446716309,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.06162422522902489,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.4569318740967043e-07,
      "logits/chosen": -1.4127130508422852,
      "logits/rejected": -1.2396118640899658,
      "logps/chosen": -97.07945251464844,
      "logps/ref_chosen": -89.43354797363281,
      "logps/ref_rejected": -91.25908660888672,
      "logps/rejected": -105.1020278930664,
      "loss": 1.1539,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.47174888849258423,
      "rewards/margins": 0.3756607174873352,
      "rewards/rejected": -0.8474096059799194,
      "step": 195
    },
    {
      "epoch": 0.2962962962962963,
      "grad_norm": 12.20384693145752,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.061355795711278915,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.448676271745197e-07,
      "logits/chosen": -1.1774715185165405,
      "logits/rejected": -1.1764438152313232,
      "logps/chosen": -81.13077545166016,
      "logps/ref_chosen": -75.47528076171875,
      "logps/ref_rejected": -99.37582397460938,
      "logps/rejected": -110.72274780273438,
      "loss": 1.19,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.34683603048324585,
      "rewards/margins": 0.3434637486934662,
      "rewards/rejected": -0.6902998089790344,
      "step": 196
    },
    {
      "epoch": 0.29780801209372637,
      "grad_norm": 16.509716033935547,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.06108853220939636,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.440366160729392e-07,
      "logits/chosen": -1.4987233877182007,
      "logits/rejected": -1.3290870189666748,
      "logps/chosen": -72.13752746582031,
      "logps/ref_chosen": -67.57392883300781,
      "logps/ref_rejected": -89.97993469238281,
      "logps/rejected": -102.06908416748047,
      "loss": 1.1802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2799578905105591,
      "rewards/margins": 0.4513537287712097,
      "rewards/rejected": -0.7313116788864136,
      "step": 197
    },
    {
      "epoch": 0.29931972789115646,
      "grad_norm": 12.162671089172363,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.06076571345329285,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.432001773500957e-07,
      "logits/chosen": -1.3527452945709229,
      "logits/rejected": -1.3725178241729736,
      "logps/chosen": -82.77947998046875,
      "logps/ref_chosen": -77.36013793945312,
      "logps/ref_rejected": -90.55670166015625,
      "logps/rejected": -103.86079406738281,
      "loss": 1.089,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3297055959701538,
      "rewards/margins": 0.47221100330352783,
      "rewards/rejected": -0.8019165992736816,
      "step": 198
    },
    {
      "epoch": 0.30083144368858655,
      "grad_norm": 13.684673309326172,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.060463398694992065,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.4235833440297856e-07,
      "logits/chosen": -1.1889785528182983,
      "logits/rejected": -1.427518606185913,
      "logps/chosen": -79.80760955810547,
      "logps/ref_chosen": -73.050048828125,
      "logps/ref_rejected": -95.21923065185547,
      "logps/rejected": -108.72315216064453,
      "loss": 1.2043,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.41020143032073975,
      "rewards/margins": 0.40017008781433105,
      "rewards/rejected": -0.8103715181350708,
      "step": 199
    },
    {
      "epoch": 0.30234315948601664,
      "grad_norm": 12.483220100402832,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.060256268829107285,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.415111107797445e-07,
      "logits/chosen": -1.449273705482483,
      "logits/rejected": -1.5978740453720093,
      "logps/chosen": -77.9295883178711,
      "logps/ref_chosen": -73.75833129882812,
      "logps/ref_rejected": -105.00157165527344,
      "logps/rejected": -117.43846893310547,
      "loss": 1.1555,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2541738748550415,
      "rewards/margins": 0.4897688627243042,
      "rewards/rejected": -0.7439427375793457,
      "step": 200
    },
    {
      "epoch": 0.30234315948601664,
      "eval_kl/n_epsilon_steps": 0.3050176203250885,
      "eval_kl/p_epsilon_steps": 0.6923415660858154,
      "eval_logits/chosen": -1.2620441913604736,
      "eval_logits/rejected": -1.321791410446167,
      "eval_logps/chosen": -91.24237060546875,
      "eval_logps/ref_chosen": -86.90177917480469,
      "eval_logps/ref_rejected": -96.69639587402344,
      "eval_logps/rejected": -108.5374526977539,
      "eval_loss": 0.5709093809127808,
      "eval_rewards/accuracies": 0.7090669274330139,
      "eval_rewards/chosen": -0.2619553506374359,
      "eval_rewards/margins": 0.4430922269821167,
      "eval_rewards/rejected": -0.7050475478172302,
      "eval_runtime": 46.8793,
      "eval_samples_per_second": 49.126,
      "eval_steps_per_second": 1.536,
      "step": 200
    },
    {
      "epoch": 0.30385487528344673,
      "grad_norm": 13.348389625549316,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.060012467205524445,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.4065853017905953e-07,
      "logits/chosen": -1.494527816772461,
      "logits/rejected": -1.5776469707489014,
      "logps/chosen": -84.34037017822266,
      "logps/ref_chosen": -79.4841079711914,
      "logps/ref_rejected": -100.94434356689453,
      "logps/rejected": -114.36886596679688,
      "loss": 1.0716,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.29187309741973877,
      "rewards/margins": 0.50721275806427,
      "rewards/rejected": -0.7990858554840088,
      "step": 201
    },
    {
      "epoch": 0.30536659108087677,
      "grad_norm": 15.9328031539917,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.05969533324241638,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.3980061644943575e-07,
      "logits/chosen": -1.2178311347961426,
      "logits/rejected": -1.3383135795593262,
      "logps/chosen": -69.4677963256836,
      "logps/ref_chosen": -66.83952331542969,
      "logps/ref_rejected": -93.05116271972656,
      "logps/rejected": -103.94529724121094,
      "loss": 1.0943,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.15965795516967773,
      "rewards/margins": 0.48617392778396606,
      "rewards/rejected": -0.645831823348999,
      "step": 202
    },
    {
      "epoch": 0.30687830687830686,
      "grad_norm": 13.11849594116211,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.05948157235980034,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.3893739358856455e-07,
      "logits/chosen": -1.241285800933838,
      "logits/rejected": -1.5145585536956787,
      "logps/chosen": -84.9322509765625,
      "logps/ref_chosen": -80.32998657226562,
      "logps/ref_rejected": -113.52803039550781,
      "logps/rejected": -125.73237609863281,
      "loss": 1.1076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.27520662546157837,
      "rewards/margins": 0.4461778998374939,
      "rewards/rejected": -0.7213845252990723,
      "step": 203
    },
    {
      "epoch": 0.30839002267573695,
      "grad_norm": 14.792080879211426,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.05918564647436142,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.380688857426449e-07,
      "logits/chosen": -1.321890115737915,
      "logits/rejected": -1.2647485733032227,
      "logps/chosen": -69.00665283203125,
      "logps/ref_chosen": -66.68875885009766,
      "logps/ref_rejected": -85.07586669921875,
      "logps/rejected": -95.97193145751953,
      "loss": 1.0778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13773512840270996,
      "rewards/margins": 0.5010201930999756,
      "rewards/rejected": -0.6387553215026855,
      "step": 204
    },
    {
      "epoch": 0.30990173847316704,
      "grad_norm": 13.94343376159668,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.058927834033966064,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.3719511720570814e-07,
      "logits/chosen": -1.4380276203155518,
      "logits/rejected": -1.2264991998672485,
      "logps/chosen": -90.58882141113281,
      "logps/ref_chosen": -86.5195083618164,
      "logps/ref_rejected": -112.55375671386719,
      "logps/rejected": -123.56883239746094,
      "loss": 1.1952,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.24183359742164612,
      "rewards/margins": 0.4022985100746155,
      "rewards/rejected": -0.6441320180892944,
      "step": 205
    },
    {
      "epoch": 0.31141345427059713,
      "grad_norm": 13.377684593200684,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.05874425917863846,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.363161124189387e-07,
      "logits/chosen": -1.1816599369049072,
      "logits/rejected": -1.1749916076660156,
      "logps/chosen": -92.03233337402344,
      "logps/ref_chosen": -88.68557739257812,
      "logps/ref_rejected": -97.75945281982422,
      "logps/rejected": -106.03539276123047,
      "loss": 1.2678,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.199264794588089,
      "rewards/margins": 0.2834780514240265,
      "rewards/rejected": -0.4827428460121155,
      "step": 206
    },
    {
      "epoch": 0.3129251700680272,
      "grad_norm": 13.501809120178223,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.05857950448989868,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.3543189596998986e-07,
      "logits/chosen": -1.3738679885864258,
      "logits/rejected": -1.7910804748535156,
      "logps/chosen": -90.10077667236328,
      "logps/ref_chosen": -85.12134552001953,
      "logps/ref_rejected": -103.34955596923828,
      "logps/rejected": -116.64409637451172,
      "loss": 1.0961,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.2933153212070465,
      "rewards/margins": 0.4806947708129883,
      "rewards/rejected": -0.7740100622177124,
      "step": 207
    },
    {
      "epoch": 0.3144368858654573,
      "grad_norm": 13.599878311157227,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.058306194841861725,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.3454249259229664e-07,
      "logits/chosen": -1.3307597637176514,
      "logits/rejected": -1.007921576499939,
      "logps/chosen": -79.94390869140625,
      "logps/ref_chosen": -78.84121704101562,
      "logps/ref_rejected": -89.8250503540039,
      "logps/rejected": -95.58148193359375,
      "loss": 1.2927,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.06700462847948074,
      "rewards/margins": 0.2648945748806,
      "rewards/rejected": -0.3318992257118225,
      "step": 208
    },
    {
      "epoch": 0.31594860166288735,
      "grad_norm": 14.028180122375488,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.05817892774939537,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.336479271643833e-07,
      "logits/chosen": -1.3499562740325928,
      "logits/rejected": -1.2825746536254883,
      "logps/chosen": -86.71353149414062,
      "logps/ref_chosen": -85.98588562011719,
      "logps/ref_rejected": -107.1638412475586,
      "logps/rejected": -117.89958190917969,
      "loss": 1.0535,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.045121632516384125,
      "rewards/margins": 0.5746475458145142,
      "rewards/rejected": -0.6197690963745117,
      "step": 209
    },
    {
      "epoch": 0.31746031746031744,
      "grad_norm": 14.91519546508789,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.057970594614744186,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.327482247091679e-07,
      "logits/chosen": -1.2512688636779785,
      "logits/rejected": -1.3389875888824463,
      "logps/chosen": -72.40476989746094,
      "logps/ref_chosen": -71.75653076171875,
      "logps/ref_rejected": -102.47966003417969,
      "logps/rejected": -113.26123046875,
      "loss": 1.0252,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.0397222638130188,
      "rewards/margins": 0.5798994302749634,
      "rewards/rejected": -0.6196216940879822,
      "step": 210
    },
    {
      "epoch": 0.31897203325774753,
      "grad_norm": 12.08520221710205,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.05773604288697243,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.3184341039326217e-07,
      "logits/chosen": -1.4723682403564453,
      "logits/rejected": -1.5072834491729736,
      "logps/chosen": -70.78600311279297,
      "logps/ref_chosen": -70.95170593261719,
      "logps/ref_rejected": -108.51902770996094,
      "logps/rejected": -116.66868591308594,
      "loss": 1.1059,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.008105363696813583,
      "rewards/margins": 0.4733632802963257,
      "rewards/rejected": -0.4652579426765442,
      "step": 211
    },
    {
      "epoch": 0.3204837490551776,
      "grad_norm": 15.995363235473633,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.05753825604915619,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.309335095262675e-07,
      "logits/chosen": -1.3575096130371094,
      "logits/rejected": -1.1737537384033203,
      "logps/chosen": -72.88034057617188,
      "logps/ref_chosen": -74.34010314941406,
      "logps/ref_rejected": -97.58259582519531,
      "logps/rejected": -106.42656707763672,
      "loss": 1.0578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08113045990467072,
      "rewards/margins": 0.5850059390068054,
      "rewards/rejected": -0.5038754940032959,
      "step": 212
    },
    {
      "epoch": 0.3219954648526077,
      "grad_norm": 12.164731979370117,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.057359009981155396,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.3001854756006724e-07,
      "logits/chosen": -1.9412446022033691,
      "logits/rejected": -1.4302163124084473,
      "logps/chosen": -78.09148406982422,
      "logps/ref_chosen": -80.2526626586914,
      "logps/ref_rejected": -94.76947021484375,
      "logps/rejected": -100.47601318359375,
      "loss": 1.1675,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.12260451167821884,
      "rewards/margins": 0.44440123438835144,
      "rewards/rejected": -0.321796715259552,
      "step": 213
    },
    {
      "epoch": 0.3235071806500378,
      "grad_norm": 22.89181137084961,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.05721597000956535,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.290985500881143e-07,
      "logits/chosen": -1.61836576461792,
      "logits/rejected": -1.6363917589187622,
      "logps/chosen": -76.35650634765625,
      "logps/ref_chosen": -77.9675064086914,
      "logps/ref_rejected": -84.0354232788086,
      "logps/rejected": -89.8424072265625,
      "loss": 1.1613,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.08922252058982849,
      "rewards/margins": 0.41794130206108093,
      "rewards/rejected": -0.32871878147125244,
      "step": 214
    },
    {
      "epoch": 0.3250188964474679,
      "grad_norm": 11.631714820861816,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.05705549940466881,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.281735428447157e-07,
      "logits/chosen": -1.3567637205123901,
      "logits/rejected": -1.3765416145324707,
      "logps/chosen": -81.66747283935547,
      "logps/ref_chosen": -81.2047348022461,
      "logps/ref_rejected": -116.18414306640625,
      "logps/rejected": -127.70268249511719,
      "loss": 1.0129,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.027690857648849487,
      "rewards/margins": 0.6228872537612915,
      "rewards/rejected": -0.6505780220031738,
      "step": 215
    },
    {
      "epoch": 0.32653061224489793,
      "grad_norm": 13.688364028930664,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.05680697038769722,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.2724355170431247e-07,
      "logits/chosen": -1.2972124814987183,
      "logits/rejected": -1.4084941148757935,
      "logps/chosen": -84.6046142578125,
      "logps/ref_chosen": -83.57113647460938,
      "logps/ref_rejected": -112.51902770996094,
      "logps/rejected": -122.14108276367188,
      "loss": 1.1093,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06136210635304451,
      "rewards/margins": 0.48042571544647217,
      "rewards/rejected": -0.54178786277771,
      "step": 216
    },
    {
      "epoch": 0.328042328042328,
      "grad_norm": 13.525312423706055,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.05652434751391411,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.26308602680756e-07,
      "logits/chosen": -1.4486957788467407,
      "logits/rejected": -1.5017526149749756,
      "logps/chosen": -80.17437744140625,
      "logps/ref_chosen": -77.01390075683594,
      "logps/ref_rejected": -105.28099822998047,
      "logps/rejected": -118.03376770019531,
      "loss": 1.0978,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.1803884506225586,
      "rewards/margins": 0.5344241261482239,
      "rewards/rejected": -0.7148125767707825,
      "step": 217
    },
    {
      "epoch": 0.3295540438397581,
      "grad_norm": 13.993760108947754,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.05626062676310539,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.253687219265803e-07,
      "logits/chosen": -1.5941836833953857,
      "logits/rejected": -1.464249610900879,
      "logps/chosen": -95.8490219116211,
      "logps/ref_chosen": -92.47299194335938,
      "logps/ref_rejected": -92.80751037597656,
      "logps/rejected": -102.18400573730469,
      "loss": 1.2869,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.19337476789951324,
      "rewards/margins": 0.32971978187561035,
      "rewards/rejected": -0.5230945944786072,
      "step": 218
    },
    {
      "epoch": 0.3310657596371882,
      "grad_norm": 12.02961254119873,
      "kl/avg_steps": 0.484375,
      "kl/beta": 0.056137826293706894,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.2442393573227043e-07,
      "logits/chosen": -1.676293134689331,
      "logits/rejected": -1.6631247997283936,
      "logps/chosen": -79.24311828613281,
      "logps/ref_chosen": -77.10382080078125,
      "logps/ref_rejected": -92.34390258789062,
      "logps/rejected": -103.1561279296875,
      "loss": 1.0932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12065555900335312,
      "rewards/margins": 0.48062241077423096,
      "rewards/rejected": -0.6012779474258423,
      "step": 219
    },
    {
      "epoch": 0.3325774754346183,
      "grad_norm": 12.431059837341309,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.05586721748113632,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.234742705255272e-07,
      "logits/chosen": -1.2905054092407227,
      "logits/rejected": -1.2640047073364258,
      "logps/chosen": -63.75226974487305,
      "logps/ref_chosen": -62.48020935058594,
      "logps/ref_rejected": -86.93277740478516,
      "logps/rejected": -95.7583236694336,
      "loss": 1.1614,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07252918183803558,
      "rewards/margins": 0.415139377117157,
      "rewards/rejected": -0.48766857385635376,
      "step": 220
    },
    {
      "epoch": 0.3340891912320484,
      "grad_norm": 11.70767879486084,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.055658500641584396,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.22519752870528e-07,
      "logits/chosen": -1.5557457208633423,
      "logits/rejected": -1.4986720085144043,
      "logps/chosen": -80.43194580078125,
      "logps/ref_chosen": -78.35491943359375,
      "logps/ref_rejected": -108.17631530761719,
      "logps/rejected": -118.90814971923828,
      "loss": 1.1518,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.11834853887557983,
      "rewards/margins": 0.47405463457107544,
      "rewards/rejected": -0.5924031734466553,
      "step": 221
    },
    {
      "epoch": 0.3356009070294785,
      "grad_norm": 14.381481170654297,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.05542467534542084,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.2156040946718343e-07,
      "logits/chosen": -1.5387976169586182,
      "logits/rejected": -1.5302550792694092,
      "logps/chosen": -80.1806640625,
      "logps/ref_chosen": -77.2734375,
      "logps/ref_rejected": -126.41007995605469,
      "logps/rejected": -140.3523406982422,
      "loss": 1.0157,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.16337642073631287,
      "rewards/margins": 0.6037923097610474,
      "rewards/rejected": -0.7671687602996826,
      "step": 222
    },
    {
      "epoch": 0.3371126228269085,
      "grad_norm": 10.387847900390625,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.055183250457048416,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 4.2059626715039065e-07,
      "logits/chosen": -1.778557538986206,
      "logits/rejected": -1.7442163228988647,
      "logps/chosen": -81.94721984863281,
      "logps/ref_chosen": -78.4210205078125,
      "logps/ref_rejected": -101.38420867919922,
      "logps/rejected": -116.384033203125,
      "loss": 1.0063,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.19502390921115875,
      "rewards/margins": 0.6252959966659546,
      "rewards/rejected": -0.8203198909759521,
      "step": 223
    },
    {
      "epoch": 0.3386243386243386,
      "grad_norm": 15.818403244018555,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.05487458035349846,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.1962735288928304e-07,
      "logits/chosen": -1.4391138553619385,
      "logits/rejected": -1.5035314559936523,
      "logps/chosen": -84.6730728149414,
      "logps/ref_chosen": -79.36337280273438,
      "logps/ref_rejected": -89.99789428710938,
      "logps/rejected": -102.62284088134766,
      "loss": 1.1549,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2928454875946045,
      "rewards/margins": 0.39513200521469116,
      "rewards/rejected": -0.6879774928092957,
      "step": 224
    },
    {
      "epoch": 0.3401360544217687,
      "grad_norm": 14.311567306518555,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.054584600031375885,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.186536937864752e-07,
      "logits/chosen": -1.5679316520690918,
      "logits/rejected": -1.7045098543167114,
      "logps/chosen": -94.7939453125,
      "logps/ref_chosen": -88.9960708618164,
      "logps/ref_rejected": -127.55032348632812,
      "logps/rejected": -143.58074951171875,
      "loss": 1.1249,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.3172207772731781,
      "rewards/margins": 0.5503044128417969,
      "rewards/rejected": -0.8675251603126526,
      "step": 225
    },
    {
      "epoch": 0.3416477702191988,
      "grad_norm": 10.593521118164062,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.05438067018985748,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.176753170773052e-07,
      "logits/chosen": -1.5852404832839966,
      "logits/rejected": -1.4178290367126465,
      "logps/chosen": -71.80659484863281,
      "logps/ref_chosen": -68.68444061279297,
      "logps/ref_rejected": -85.81898498535156,
      "logps/rejected": -99.06565856933594,
      "loss": 1.1023,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.17008624970912933,
      "rewards/margins": 0.5425100922584534,
      "rewards/rejected": -0.7125963568687439,
      "step": 226
    },
    {
      "epoch": 0.3431594860166289,
      "grad_norm": 12.188491821289062,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.054177507758140564,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.166922501290729e-07,
      "logits/chosen": -1.364269733428955,
      "logits/rejected": -1.418736457824707,
      "logps/chosen": -77.14056396484375,
      "logps/ref_chosen": -72.52030181884766,
      "logps/ref_rejected": -90.7720718383789,
      "logps/rejected": -105.46800231933594,
      "loss": 1.1484,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.2539520263671875,
      "rewards/margins": 0.5371447205543518,
      "rewards/rejected": -0.7910966873168945,
      "step": 227
    },
    {
      "epoch": 0.34467120181405897,
      "grad_norm": 12.944748878479004,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.05399191007018089,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.1570452044027405e-07,
      "logits/chosen": -1.4697837829589844,
      "logits/rejected": -1.390408992767334,
      "logps/chosen": -77.01600646972656,
      "logps/ref_chosen": -72.23167419433594,
      "logps/ref_rejected": -95.45873260498047,
      "logps/rejected": -109.11170196533203,
      "loss": 1.1582,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.26035553216934204,
      "rewards/margins": 0.4708428382873535,
      "rewards/rejected": -0.7311983108520508,
      "step": 228
    },
    {
      "epoch": 0.34618291761148906,
      "grad_norm": 11.364020347595215,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.05382370948791504,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.147121556398312e-07,
      "logits/chosen": -1.5286439657211304,
      "logits/rejected": -1.5656533241271973,
      "logps/chosen": -66.92510986328125,
      "logps/ref_chosen": -66.88822174072266,
      "logps/ref_rejected": -92.27890014648438,
      "logps/rejected": -102.7301025390625,
      "loss": 1.0666,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.002183683216571808,
      "rewards/margins": 0.5534392595291138,
      "rewards/rejected": -0.5556229948997498,
      "step": 229
    },
    {
      "epoch": 0.3476946334089191,
      "grad_norm": 14.49431324005127,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.05363932624459267,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -1.7045170068740845,
      "logits/rejected": -1.6729419231414795,
      "logps/chosen": -79.54010009765625,
      "logps/ref_chosen": -76.12332153320312,
      "logps/ref_rejected": -78.19171905517578,
      "logps/rejected": -89.6407241821289,
      "loss": 1.1833,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.1854729950428009,
      "rewards/margins": 0.4231482148170471,
      "rewards/rejected": -0.6086212396621704,
      "step": 230
    },
    {
      "epoch": 0.3492063492063492,
      "grad_norm": 13.21183967590332,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.053438927978277206,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.1271363186719835e-07,
      "logits/chosen": -1.4684646129608154,
      "logits/rejected": -1.1993520259857178,
      "logps/chosen": -96.73280334472656,
      "logps/ref_chosen": -92.45181274414062,
      "logps/ref_rejected": -100.89735412597656,
      "logps/rejected": -117.29405212402344,
      "loss": 1.0262,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.23050335049629211,
      "rewards/margins": 0.6388251781463623,
      "rewards/rejected": -0.8693285584449768,
      "step": 231
    },
    {
      "epoch": 0.3507180650037793,
      "grad_norm": 14.258382797241211,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.05318960174918175,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.1170752879801436e-07,
      "logits/chosen": -1.6453282833099365,
      "logits/rejected": -1.5070879459381104,
      "logps/chosen": -89.7475357055664,
      "logps/ref_chosen": -86.75383758544922,
      "logps/ref_rejected": -98.16909790039062,
      "logps/rejected": -109.11927795410156,
      "loss": 1.2166,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16051898896694183,
      "rewards/margins": 0.41557577252388,
      "rewards/rejected": -0.5760947465896606,
      "step": 232
    },
    {
      "epoch": 0.35222978080120937,
      "grad_norm": 11.350086212158203,
      "kl/avg_steps": 0.0625,
      "kl/beta": 0.05304042622447014,
      "kl/n_epsilon_steps": 0.46875,
      "kl/p_epsilon_steps": 0.53125,
      "learning_rate": 4.106969024216348e-07,
      "logits/chosen": -1.3634660243988037,
      "logits/rejected": -1.2045012712478638,
      "logps/chosen": -74.8712158203125,
      "logps/ref_chosen": -72.87556457519531,
      "logps/ref_rejected": -85.22943115234375,
      "logps/rejected": -95.50105285644531,
      "loss": 1.1816,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.10806849598884583,
      "rewards/margins": 0.43179118633270264,
      "rewards/rejected": -0.5398597121238708,
      "step": 233
    },
    {
      "epoch": 0.35374149659863946,
      "grad_norm": 11.72382926940918,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.05300729721784592,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 4.09681781007452e-07,
      "logits/chosen": -1.547611951828003,
      "logits/rejected": -1.6473437547683716,
      "logps/chosen": -71.36154174804688,
      "logps/ref_chosen": -70.05477905273438,
      "logps/ref_rejected": -68.7240982055664,
      "logps/rejected": -78.51712799072266,
      "loss": 1.1585,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.07237260043621063,
      "rewards/margins": 0.4424641728401184,
      "rewards/rejected": -0.5148367881774902,
      "step": 234
    },
    {
      "epoch": 0.35525321239606955,
      "grad_norm": 14.6141996383667,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.052908092737197876,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.08662192950594e-07,
      "logits/chosen": -1.7234680652618408,
      "logits/rejected": -1.5612242221832275,
      "logps/chosen": -85.46180725097656,
      "logps/ref_chosen": -85.86051940917969,
      "logps/ref_rejected": -96.14663696289062,
      "logps/rejected": -106.57968139648438,
      "loss": 1.0272,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.020824704319238663,
      "rewards/margins": 0.5663049221038818,
      "rewards/rejected": -0.5454801917076111,
      "step": 235
    },
    {
      "epoch": 0.35676492819349964,
      "grad_norm": 11.916207313537598,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.052685827016830444,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 4.076381667711306e-07,
      "logits/chosen": -1.7236182689666748,
      "logits/rejected": -1.3514024019241333,
      "logps/chosen": -92.91275787353516,
      "logps/ref_chosen": -89.75252532958984,
      "logps/ref_rejected": -99.28534698486328,
      "logps/rejected": -112.02670288085938,
      "loss": 1.1745,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.17047369480133057,
      "rewards/margins": 0.4959697127342224,
      "rewards/rejected": -0.6664433479309082,
      "step": 236
    },
    {
      "epoch": 0.35827664399092973,
      "grad_norm": 13.740058898925781,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.05255443975329399,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 4.066097311132753e-07,
      "logits/chosen": -1.4527928829193115,
      "logits/rejected": -1.2798683643341064,
      "logps/chosen": -95.31475067138672,
      "logps/ref_chosen": -92.59001922607422,
      "logps/ref_rejected": -101.45585632324219,
      "logps/rejected": -110.77891540527344,
      "loss": 1.2174,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.14533735811710358,
      "rewards/margins": 0.3403947353363037,
      "rewards/rejected": -0.4857320785522461,
      "step": 237
    },
    {
      "epoch": 0.35978835978835977,
      "grad_norm": 10.330193519592285,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.05247244983911514,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.0557691474458414e-07,
      "logits/chosen": -1.3112481832504272,
      "logits/rejected": -1.2185739278793335,
      "logps/chosen": -83.69696807861328,
      "logps/ref_chosen": -82.2470474243164,
      "logps/ref_rejected": -92.59944152832031,
      "logps/rejected": -104.04986572265625,
      "loss": 1.0835,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07855997234582901,
      "rewards/margins": 0.5173219442367554,
      "rewards/rejected": -0.5958819389343262,
      "step": 238
    },
    {
      "epoch": 0.36130007558578986,
      "grad_norm": 12.408441543579102,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.05227641388773918,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.045397465551513e-07,
      "logits/chosen": -1.2152026891708374,
      "logits/rejected": -1.4065308570861816,
      "logps/chosen": -79.02906799316406,
      "logps/ref_chosen": -75.30878448486328,
      "logps/ref_rejected": -131.2318115234375,
      "logps/rejected": -145.25335693359375,
      "loss": 1.1178,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19433492422103882,
      "rewards/margins": 0.5304996967315674,
      "rewards/rejected": -0.7248346209526062,
      "step": 239
    },
    {
      "epoch": 0.36281179138321995,
      "grad_norm": 15.577049255371094,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.05203251168131828,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.0349825555680045e-07,
      "logits/chosen": -1.8081055879592896,
      "logits/rejected": -1.8345035314559937,
      "logps/chosen": -72.12168884277344,
      "logps/ref_chosen": -70.81785583496094,
      "logps/ref_rejected": -98.53778839111328,
      "logps/rejected": -114.63734436035156,
      "loss": 0.95,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06976085901260376,
      "rewards/margins": 0.7603122591972351,
      "rewards/rejected": -0.8300731778144836,
      "step": 240
    },
    {
      "epoch": 0.36432350718065004,
      "grad_norm": 13.830245971679688,
      "kl/avg_steps": 0.234375,
      "kl/beta": 0.05175755172967911,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 4.0245247088227377e-07,
      "logits/chosen": -1.710863709449768,
      "logits/rejected": -1.5587537288665771,
      "logps/chosen": -91.37069702148438,
      "logps/ref_chosen": -88.60260009765625,
      "logps/ref_rejected": -101.42214965820312,
      "logps/rejected": -111.60231018066406,
      "loss": 1.1998,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.14622732996940613,
      "rewards/margins": 0.37693488597869873,
      "rewards/rejected": -0.5231622457504272,
      "step": 241
    },
    {
      "epoch": 0.36583522297808013,
      "grad_norm": 12.737060546875,
      "kl/avg_steps": 0.640625,
      "kl/beta": 0.051636528223752975,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 4.0140242178441665e-07,
      "logits/chosen": -1.5724875926971436,
      "logits/rejected": -1.6078267097473145,
      "logps/chosen": -78.00882720947266,
      "logps/ref_chosen": -77.34109497070312,
      "logps/ref_rejected": -84.76332092285156,
      "logps/rejected": -97.80973815917969,
      "loss": 1.0196,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.03599818795919418,
      "rewards/margins": 0.6313221454620361,
      "rewards/rejected": -0.6673203110694885,
      "step": 242
    },
    {
      "epoch": 0.3673469387755102,
      "grad_norm": 13.271928787231445,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.051307834684848785,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 4.003481376353596e-07,
      "logits/chosen": -1.4511834383010864,
      "logits/rejected": -1.342944622039795,
      "logps/chosen": -98.63224792480469,
      "logps/ref_chosen": -93.55897521972656,
      "logps/ref_rejected": -89.33551025390625,
      "logps/rejected": -103.51713562011719,
      "loss": 1.155,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.26296675205230713,
      "rewards/margins": 0.4598330557346344,
      "rewards/rejected": -0.7227997779846191,
      "step": 243
    },
    {
      "epoch": 0.3688586545729403,
      "grad_norm": 10.046283721923828,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.05114799737930298,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 3.9928964792569654e-07,
      "logits/chosen": -1.482820749282837,
      "logits/rejected": -1.142737865447998,
      "logps/chosen": -71.78289031982422,
      "logps/ref_chosen": -69.82603454589844,
      "logps/ref_rejected": -92.47640991210938,
      "logps/rejected": -110.2171630859375,
      "loss": 0.8783,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.10063998401165009,
      "rewards/margins": 0.7985442876815796,
      "rewards/rejected": -0.8991843461990356,
      "step": 244
    },
    {
      "epoch": 0.37037037037037035,
      "grad_norm": 11.417145729064941,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.050846099853515625,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.982269822636601e-07,
      "logits/chosen": -1.7022724151611328,
      "logits/rejected": -1.54628586769104,
      "logps/chosen": -91.77505493164062,
      "logps/ref_chosen": -85.68216705322266,
      "logps/ref_rejected": -93.8754653930664,
      "logps/rejected": -114.80189514160156,
      "loss": 0.9433,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.3088444173336029,
      "rewards/margins": 0.7455431222915649,
      "rewards/rejected": -1.0543875694274902,
      "step": 245
    },
    {
      "epoch": 0.37188208616780044,
      "grad_norm": 12.649994850158691,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.05056168884038925,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.971601703742932e-07,
      "logits/chosen": -1.5699760913848877,
      "logits/rejected": -1.451397180557251,
      "logps/chosen": -100.3763656616211,
      "logps/ref_chosen": -90.05093383789062,
      "logps/ref_rejected": -112.77645874023438,
      "logps/rejected": -136.43222045898438,
      "loss": 1.0232,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5240607857704163,
      "rewards/margins": 0.6649122834205627,
      "rewards/rejected": -1.1889731884002686,
      "step": 246
    },
    {
      "epoch": 0.37339380196523053,
      "grad_norm": 19.564983367919922,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.05032578855752945,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 3.960892420986177e-07,
      "logits/chosen": -1.52877676486969,
      "logits/rejected": -1.4984183311462402,
      "logps/chosen": -115.11454772949219,
      "logps/ref_chosen": -103.23979187011719,
      "logps/ref_rejected": -105.26278686523438,
      "logps/rejected": -123.63394165039062,
      "loss": 1.2711,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.5987535715103149,
      "rewards/margins": 0.3198391795158386,
      "rewards/rejected": -0.9185927510261536,
      "step": 247
    },
    {
      "epoch": 0.3749055177626606,
      "grad_norm": 15.282120704650879,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.050200287252664566,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.9501422739279953e-07,
      "logits/chosen": -1.6684272289276123,
      "logits/rejected": -1.437838077545166,
      "logps/chosen": -98.20668029785156,
      "logps/ref_chosen": -88.16007995605469,
      "logps/ref_rejected": -75.11514282226562,
      "logps/rejected": -96.37853240966797,
      "loss": 1.1208,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5064951777458191,
      "rewards/margins": 0.5542141199111938,
      "rewards/rejected": -1.0607093572616577,
      "step": 248
    },
    {
      "epoch": 0.3764172335600907,
      "grad_norm": 17.950796127319336,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.04996607080101967,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 3.9393515632731094e-07,
      "logits/chosen": -1.2981152534484863,
      "logits/rejected": -1.335099458694458,
      "logps/chosen": -105.10684204101562,
      "logps/ref_chosen": -91.01773071289062,
      "logps/ref_rejected": -80.51113891601562,
      "logps/rejected": -98.92596435546875,
      "loss": 1.4253,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.7059436440467834,
      "rewards/margins": 0.20767927169799805,
      "rewards/rejected": -0.9136229157447815,
      "step": 249
    },
    {
      "epoch": 0.3779289493575208,
      "grad_norm": 17.256160736083984,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.049841467291116714,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.9285205908608934e-07,
      "logits/chosen": -1.6364936828613281,
      "logits/rejected": -1.335681438446045,
      "logps/chosen": -93.50019836425781,
      "logps/ref_chosen": -80.5888671875,
      "logps/ref_rejected": -90.15093994140625,
      "logps/rejected": -117.05657958984375,
      "loss": 1.0234,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.6429185271263123,
      "rewards/margins": 0.6881313323974609,
      "rewards/rejected": -1.331049919128418,
      "step": 250
    },
    {
      "epoch": 0.3794406651549509,
      "grad_norm": 14.625908851623535,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.04957808554172516,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.9176496596569265e-07,
      "logits/chosen": -1.4772026538848877,
      "logits/rejected": -1.676422357559204,
      "logps/chosen": -95.66470336914062,
      "logps/ref_chosen": -82.70405578613281,
      "logps/ref_rejected": -98.94266510009766,
      "logps/rejected": -120.11300659179688,
      "loss": 1.2047,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6434235572814941,
      "rewards/margins": 0.39970919489860535,
      "rewards/rejected": -1.0431327819824219,
      "step": 251
    },
    {
      "epoch": 0.38095238095238093,
      "grad_norm": 11.416725158691406,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.049392860382795334,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.9067390737445254e-07,
      "logits/chosen": -1.717487096786499,
      "logits/rejected": -1.6335628032684326,
      "logps/chosen": -83.50773620605469,
      "logps/ref_chosen": -73.10369110107422,
      "logps/ref_rejected": -94.90235900878906,
      "logps/rejected": -113.88876342773438,
      "loss": 1.2021,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.514301598072052,
      "rewards/margins": 0.41625702381134033,
      "rewards/rejected": -0.9305586814880371,
      "step": 252
    },
    {
      "epoch": 0.382464096749811,
      "grad_norm": 17.22648811340332,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.049177709966897964,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.8957891383162304e-07,
      "logits/chosen": -1.2668952941894531,
      "logits/rejected": -1.309208869934082,
      "logps/chosen": -80.58470153808594,
      "logps/ref_chosen": -68.7789535522461,
      "logps/ref_rejected": -75.98162078857422,
      "logps/rejected": -95.89788818359375,
      "loss": 1.1948,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.580489993095398,
      "rewards/margins": 0.3920561969280243,
      "rewards/rejected": -0.9725462198257446,
      "step": 253
    },
    {
      "epoch": 0.3839758125472411,
      "grad_norm": 15.513288497924805,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.04899398237466812,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 3.884800159665276e-07,
      "logits/chosen": -1.7666159868240356,
      "logits/rejected": -1.7136458158493042,
      "logps/chosen": -94.73291778564453,
      "logps/ref_chosen": -81.49362182617188,
      "logps/ref_rejected": -101.43673706054688,
      "logps/rejected": -125.28010559082031,
      "loss": 1.1055,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6512777805328369,
      "rewards/margins": 0.5118330717086792,
      "rewards/rejected": -1.1631108522415161,
      "step": 254
    },
    {
      "epoch": 0.3854875283446712,
      "grad_norm": 18.647626876831055,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.04884135350584984,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.873772445177015e-07,
      "logits/chosen": -1.631853699684143,
      "logits/rejected": -1.4802442789077759,
      "logps/chosen": -102.59626770019531,
      "logps/ref_chosen": -90.46350860595703,
      "logps/ref_rejected": -105.32445526123047,
      "logps/rejected": -128.67465209960938,
      "loss": 1.1135,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.5910813808441162,
      "rewards/margins": 0.5390965938568115,
      "rewards/rejected": -1.1301779747009277,
      "step": 255
    },
    {
      "epoch": 0.3869992441421013,
      "grad_norm": 11.703644752502441,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.048568155616521835,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.862706303320329e-07,
      "logits/chosen": -1.4065661430358887,
      "logits/rejected": -1.6162680387496948,
      "logps/chosen": -95.55255126953125,
      "logps/ref_chosen": -81.56578826904297,
      "logps/ref_rejected": -108.58460998535156,
      "logps/rejected": -134.34637451171875,
      "loss": 1.1095,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.6802552938461304,
      "rewards/margins": 0.5633381605148315,
      "rewards/rejected": -1.243593454360962,
      "step": 256
    },
    {
      "epoch": 0.3885109599395314,
      "grad_norm": 16.859773635864258,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.048341553658246994,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.851602043638994e-07,
      "logits/chosen": -1.5432624816894531,
      "logits/rejected": -1.2107794284820557,
      "logps/chosen": -105.39306640625,
      "logps/ref_chosen": -89.57557678222656,
      "logps/ref_rejected": -123.74462127685547,
      "logps/rejected": -150.5018310546875,
      "loss": 1.1676,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7634068727493286,
      "rewards/margins": 0.52019864320755,
      "rewards/rejected": -1.2836055755615234,
      "step": 257
    },
    {
      "epoch": 0.3900226757369615,
      "grad_norm": 15.744192123413086,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.048116009682416916,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 3.840459976743023e-07,
      "logits/chosen": -1.4739046096801758,
      "logits/rejected": -1.587180256843567,
      "logps/chosen": -92.76885986328125,
      "logps/ref_chosen": -77.34173583984375,
      "logps/ref_rejected": -99.5709228515625,
      "logps/rejected": -127.6431655883789,
      "loss": 0.9873,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7396783828735352,
      "rewards/margins": 0.601047158241272,
      "rewards/rejected": -1.3407254219055176,
      "step": 258
    },
    {
      "epoch": 0.3915343915343915,
      "grad_norm": 12.638566970825195,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.047817155718803406,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.8292804142999796e-07,
      "logits/chosen": -1.365210771560669,
      "logits/rejected": -1.1926498413085938,
      "logps/chosen": -91.97223663330078,
      "logps/ref_chosen": -82.39556121826172,
      "logps/ref_rejected": -113.73309326171875,
      "logps/rejected": -140.43887329101562,
      "loss": 0.9534,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4582955837249756,
      "rewards/margins": 0.808417797088623,
      "rewards/rejected": -1.2667133808135986,
      "step": 259
    },
    {
      "epoch": 0.3930461073318216,
      "grad_norm": 22.389368057250977,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.04754968732595444,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.818063669026256e-07,
      "logits/chosen": -1.1842126846313477,
      "logits/rejected": -1.1899852752685547,
      "logps/chosen": -79.66688537597656,
      "logps/ref_chosen": -65.98947143554688,
      "logps/ref_rejected": -94.59706115722656,
      "logps/rejected": -119.67667388916016,
      "loss": 1.1858,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.6499999761581421,
      "rewards/margins": 0.5323763489723206,
      "rewards/rejected": -1.1823763847351074,
      "step": 260
    },
    {
      "epoch": 0.3945578231292517,
      "grad_norm": 14.604168891906738,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.04734256491065025,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 3.806810054678331e-07,
      "logits/chosen": -1.3006818294525146,
      "logits/rejected": -1.2254362106323242,
      "logps/chosen": -101.1712646484375,
      "logps/ref_chosen": -88.87684631347656,
      "logps/ref_rejected": -82.348388671875,
      "logps/rejected": -103.30619049072266,
      "loss": 1.2224,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5821806192398071,
      "rewards/margins": 0.40273311734199524,
      "rewards/rejected": -0.98491370677948,
      "step": 261
    },
    {
      "epoch": 0.3960695389266818,
      "grad_norm": 11.085594177246094,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.047180380672216415,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.7955198860439887e-07,
      "logits/chosen": -1.4064021110534668,
      "logits/rejected": -1.547346830368042,
      "logps/chosen": -94.58271789550781,
      "logps/ref_chosen": -85.81719970703125,
      "logps/ref_rejected": -105.49027252197266,
      "logps/rejected": -125.6654281616211,
      "loss": 1.0612,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4132178723812103,
      "rewards/margins": 0.5312476754188538,
      "rewards/rejected": -0.9444655179977417,
      "step": 262
    },
    {
      "epoch": 0.3975812547241119,
      "grad_norm": 11.340027809143066,
      "kl/avg_steps": 0.390625,
      "kl/beta": 0.04697486758232117,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.784193478933516e-07,
      "logits/chosen": -1.2239423990249634,
      "logits/rejected": -1.7259702682495117,
      "logps/chosen": -82.4855728149414,
      "logps/ref_chosen": -73.61693572998047,
      "logps/ref_rejected": -102.39161682128906,
      "logps/rejected": -123.17403411865234,
      "loss": 1.0883,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4169412851333618,
      "rewards/margins": 0.5516859889030457,
      "rewards/rejected": -0.9686272740364075,
      "step": 263
    },
    {
      "epoch": 0.39909297052154197,
      "grad_norm": 10.866528511047363,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.04679208621382713,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.7728311501708674e-07,
      "logits/chosen": -1.3833404779434204,
      "logits/rejected": -1.5971425771713257,
      "logps/chosen": -112.40450286865234,
      "logps/ref_chosen": -101.57856750488281,
      "logps/ref_rejected": -111.6573486328125,
      "logps/rejected": -135.07891845703125,
      "loss": 1.0381,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5059612989425659,
      "rewards/margins": 0.5817750096321106,
      "rewards/rejected": -1.0877363681793213,
      "step": 264
    },
    {
      "epoch": 0.40060468631897206,
      "grad_norm": 12.059004783630371,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.04655928909778595,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.7614332175848027e-07,
      "logits/chosen": -1.4162359237670898,
      "logits/rejected": -1.36917245388031,
      "logps/chosen": -72.67507934570312,
      "logps/ref_chosen": -65.76426696777344,
      "logps/ref_rejected": -85.19627380371094,
      "logps/rejected": -107.6723861694336,
      "loss": 1.0147,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.3222898840904236,
      "rewards/margins": 0.7146923542022705,
      "rewards/rejected": -1.0369821786880493,
      "step": 265
    },
    {
      "epoch": 0.4021164021164021,
      "grad_norm": 11.972450256347656,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.046298857778310776,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.75e-07,
      "logits/chosen": -1.4073446989059448,
      "logits/rejected": -1.5456604957580566,
      "logps/chosen": -80.41755676269531,
      "logps/ref_chosen": -75.05682373046875,
      "logps/ref_rejected": -97.52758026123047,
      "logps/rejected": -116.14582824707031,
      "loss": 1.0522,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.2491399049758911,
      "rewards/margins": 0.6057909727096558,
      "rewards/rejected": -0.8549308776855469,
      "step": 266
    },
    {
      "epoch": 0.4036281179138322,
      "grad_norm": 11.077033042907715,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.0460398830473423,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.738531817228131e-07,
      "logits/chosen": -1.2547085285186768,
      "logits/rejected": -1.262139916419983,
      "logps/chosen": -75.00776672363281,
      "logps/ref_chosen": -71.13494110107422,
      "logps/ref_rejected": -81.14566040039062,
      "logps/rejected": -96.05207824707031,
      "loss": 1.1117,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18061110377311707,
      "rewards/margins": 0.5004351735115051,
      "rewards/rejected": -0.6810463666915894,
      "step": 267
    },
    {
      "epoch": 0.4051398337112623,
      "grad_norm": 10.130515098571777,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.0458393357694149,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 3.7270289900589204e-07,
      "logits/chosen": -1.469900369644165,
      "logits/rejected": -1.4314866065979004,
      "logps/chosen": -85.93333435058594,
      "logps/ref_chosen": -80.06082153320312,
      "logps/ref_rejected": -87.43035888671875,
      "logps/rejected": -101.26762390136719,
      "loss": 1.2189,
      "rewards/accuracies": 0.578125,
      "rewards/chosen": -0.2703646123409271,
      "rewards/margins": 0.359006404876709,
      "rewards/rejected": -0.6293710470199585,
      "step": 268
    },
    {
      "epoch": 0.40665154950869237,
      "grad_norm": 10.948187828063965,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.04575354605913162,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.7154918402511714e-07,
      "logits/chosen": -1.5809710025787354,
      "logits/rejected": -1.2682452201843262,
      "logps/chosen": -90.92002868652344,
      "logps/ref_chosen": -83.36943817138672,
      "logps/ref_rejected": -100.66839599609375,
      "logps/rejected": -119.58181762695312,
      "loss": 1.0943,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.34568658471107483,
      "rewards/margins": 0.5124187469482422,
      "rewards/rejected": -0.8581053018569946,
      "step": 269
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 11.601164817810059,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.045547161251306534,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.7039206905237656e-07,
      "logits/chosen": -1.4058010578155518,
      "logits/rejected": -1.5133944749832153,
      "logps/chosen": -91.64334106445312,
      "logps/ref_chosen": -85.35945129394531,
      "logps/ref_rejected": -104.47489929199219,
      "logps/rejected": -121.33040618896484,
      "loss": 1.1188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2873057723045349,
      "rewards/margins": 0.4743884801864624,
      "rewards/rejected": -0.7616941928863525,
      "step": 270
    },
    {
      "epoch": 0.40967498110355255,
      "grad_norm": 12.935283660888672,
      "kl/avg_steps": 0.1875,
      "kl/beta": 0.04533465579152107,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.59375,
      "learning_rate": 3.692315864546635e-07,
      "logits/chosen": -1.7285494804382324,
      "logits/rejected": -1.6788990497589111,
      "logps/chosen": -93.83595275878906,
      "logps/ref_chosen": -86.01373291015625,
      "logps/ref_rejected": -109.99561309814453,
      "logps/rejected": -125.69892120361328,
      "loss": 1.2685,
      "rewards/accuracies": 0.546875,
      "rewards/chosen": -0.35634368658065796,
      "rewards/margins": 0.3500638008117676,
      "rewards/rejected": -0.7064074873924255,
      "step": 271
    },
    {
      "epoch": 0.41118669690098264,
      "grad_norm": 14.665283203125,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.0452498123049736,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.6806776869317067e-07,
      "logits/chosen": -1.4215366840362549,
      "logits/rejected": -1.2945075035095215,
      "logps/chosen": -89.55738830566406,
      "logps/ref_chosen": -86.3701400756836,
      "logps/ref_rejected": -85.74638366699219,
      "logps/rejected": -104.7607421875,
      "loss": 0.9318,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.145250603556633,
      "rewards/margins": 0.7082393169403076,
      "rewards/rejected": -0.853489875793457,
      "step": 272
    },
    {
      "epoch": 0.4126984126984127,
      "grad_norm": 17.91227912902832,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.04501069337129593,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.669006483223828e-07,
      "logits/chosen": -1.6112267971038818,
      "logits/rejected": -1.544572114944458,
      "logps/chosen": -82.86746215820312,
      "logps/ref_chosen": -75.51087951660156,
      "logps/ref_rejected": -101.60345458984375,
      "logps/rejected": -121.30127716064453,
      "loss": 1.1397,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.33276399970054626,
      "rewards/margins": 0.5460294485092163,
      "rewards/rejected": -0.8787934184074402,
      "step": 273
    },
    {
      "epoch": 0.41421012849584277,
      "grad_norm": 10.048867225646973,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.04475892335176468,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.657302579891656e-07,
      "logits/chosen": -1.2657063007354736,
      "logits/rejected": -1.2205548286437988,
      "logps/chosen": -86.40071105957031,
      "logps/ref_chosen": -79.040283203125,
      "logps/ref_rejected": -86.31329345703125,
      "logps/rejected": -106.71549987792969,
      "loss": 1.0445,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3302973508834839,
      "rewards/margins": 0.5760163068771362,
      "rewards/rejected": -0.9063136577606201,
      "step": 274
    },
    {
      "epoch": 0.41572184429327286,
      "grad_norm": 10.825162887573242,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.044563956558704376,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.645566304318526e-07,
      "logits/chosen": -1.4031257629394531,
      "logits/rejected": -1.5135775804519653,
      "logps/chosen": -75.87445831298828,
      "logps/ref_chosen": -71.82034301757812,
      "logps/ref_rejected": -94.29946899414062,
      "logps/rejected": -114.9261474609375,
      "loss": 0.9428,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.1804373562335968,
      "rewards/margins": 0.7301989793777466,
      "rewards/rejected": -0.910636305809021,
      "step": 275
    },
    {
      "epoch": 0.41723356009070295,
      "grad_norm": 14.92078685760498,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.044314686208963394,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.633797984793294e-07,
      "logits/chosen": -1.5658140182495117,
      "logits/rejected": -1.4458580017089844,
      "logps/chosen": -74.48373413085938,
      "logps/ref_chosen": -69.54020690917969,
      "logps/ref_rejected": -78.59674072265625,
      "logps/rejected": -96.87590026855469,
      "loss": 1.0668,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.22064831852912903,
      "rewards/margins": 0.582705020904541,
      "rewards/rejected": -0.8033533096313477,
      "step": 276
    },
    {
      "epoch": 0.41874527588813304,
      "grad_norm": 12.200444221496582,
      "kl/avg_steps": 0.171875,
      "kl/beta": 0.04410793259739876,
      "kl/n_epsilon_steps": 0.40625,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 3.6219979505011555e-07,
      "logits/chosen": -1.7910408973693848,
      "logits/rejected": -1.4916061162948608,
      "logps/chosen": -105.21025085449219,
      "logps/ref_chosen": -94.4896240234375,
      "logps/ref_rejected": -85.45901489257812,
      "logps/rejected": -103.62118530273438,
      "loss": 1.2871,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.47567400336265564,
      "rewards/margins": 0.3210156559944153,
      "rewards/rejected": -0.7966896891593933,
      "step": 277
    },
    {
      "epoch": 0.42025699168556313,
      "grad_norm": 13.179181098937988,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.044032249599695206,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.6101665315144353e-07,
      "logits/chosen": -1.5639655590057373,
      "logits/rejected": -1.7161953449249268,
      "logps/chosen": -97.54840850830078,
      "logps/ref_chosen": -87.42613220214844,
      "logps/ref_rejected": -105.44854736328125,
      "logps/rejected": -126.17444610595703,
      "loss": 1.183,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.44696807861328125,
      "rewards/margins": 0.45861679315567017,
      "rewards/rejected": -0.9055849313735962,
      "step": 278
    },
    {
      "epoch": 0.4217687074829932,
      "grad_norm": 12.131983757019043,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.04384727030992508,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 3.5983040587833563e-07,
      "logits/chosen": -1.8436882495880127,
      "logits/rejected": -1.4893114566802979,
      "logps/chosen": -72.60688781738281,
      "logps/ref_chosen": -70.516845703125,
      "logps/ref_rejected": -86.04248809814453,
      "logps/rejected": -106.16204071044922,
      "loss": 0.9083,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.09204297512769699,
      "rewards/margins": 0.7816708087921143,
      "rewards/rejected": -0.8737137913703918,
      "step": 279
    },
    {
      "epoch": 0.42328042328042326,
      "grad_norm": 17.89347267150879,
      "kl/avg_steps": 0.578125,
      "kl/beta": 0.043574925512075424,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.586410864126781e-07,
      "logits/chosen": -1.7019422054290771,
      "logits/rejected": -1.617280125617981,
      "logps/chosen": -81.91899108886719,
      "logps/ref_chosen": -76.5021743774414,
      "logps/ref_rejected": -94.2752685546875,
      "logps/rejected": -116.81834411621094,
      "loss": 0.9263,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.23566317558288574,
      "rewards/margins": 0.7379894256591797,
      "rewards/rejected": -0.9736526608467102,
      "step": 280
    },
    {
      "epoch": 0.42479213907785335,
      "grad_norm": 10.356485366821289,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.04332445561885834,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.574487280222929e-07,
      "logits/chosen": -1.6503106355667114,
      "logits/rejected": -1.4820497035980225,
      "logps/chosen": -84.52981567382812,
      "logps/ref_chosen": -77.50468444824219,
      "logps/ref_rejected": -79.05716705322266,
      "logps/rejected": -99.70474243164062,
      "loss": 1.0542,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3050358295440674,
      "rewards/margins": 0.5821521282196045,
      "rewards/rejected": -0.8871879577636719,
      "step": 281
    },
    {
      "epoch": 0.42630385487528344,
      "grad_norm": 14.454909324645996,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.04314916208386421,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.562533640600075e-07,
      "logits/chosen": -1.5837814807891846,
      "logits/rejected": -1.5102015733718872,
      "logps/chosen": -89.67692565917969,
      "logps/ref_chosen": -80.31298065185547,
      "logps/ref_rejected": -83.72120666503906,
      "logps/rejected": -108.5927505493164,
      "loss": 1.032,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.40527933835983276,
      "rewards/margins": 0.6601608991622925,
      "rewards/rejected": -1.06544029712677,
      "step": 282
    },
    {
      "epoch": 0.42781557067271353,
      "grad_norm": 12.454751968383789,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.04294784367084503,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.550550279627215e-07,
      "logits/chosen": -1.2369191646575928,
      "logits/rejected": -1.814032793045044,
      "logps/chosen": -93.92501831054688,
      "logps/ref_chosen": -80.72602844238281,
      "logps/ref_rejected": -115.68379211425781,
      "logps/rejected": -142.0513916015625,
      "loss": 1.1111,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5656991004943848,
      "rewards/margins": 0.5568342804908752,
      "rewards/rejected": -1.1225333213806152,
      "step": 283
    },
    {
      "epoch": 0.4293272864701436,
      "grad_norm": 10.446672439575195,
      "kl/avg_steps": 0.578125,
      "kl/beta": 0.04273417592048645,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.5385375325047163e-07,
      "logits/chosen": -1.236511468887329,
      "logits/rejected": -1.3110226392745972,
      "logps/chosen": -86.48180389404297,
      "logps/ref_chosen": -77.5223388671875,
      "logps/ref_rejected": -104.1847152709961,
      "logps/rejected": -130.71986389160156,
      "loss": 0.9483,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.38278716802597046,
      "rewards/margins": 0.742293119430542,
      "rewards/rejected": -1.1250803470611572,
      "step": 284
    },
    {
      "epoch": 0.4308390022675737,
      "grad_norm": 14.101186752319336,
      "kl/avg_steps": 0.328125,
      "kl/beta": 0.04248853772878647,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 3.5264957352549375e-07,
      "logits/chosen": -1.3226224184036255,
      "logits/rejected": -1.2694287300109863,
      "logps/chosen": -105.07521057128906,
      "logps/ref_chosen": -85.79348754882812,
      "logps/ref_rejected": -96.46463775634766,
      "logps/rejected": -126.5572509765625,
      "loss": 1.2016,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.8215754628181458,
      "rewards/margins": 0.45096495747566223,
      "rewards/rejected": -1.2725403308868408,
      "step": 285
    },
    {
      "epoch": 0.4323507180650038,
      "grad_norm": 10.810320854187012,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.04234957695007324,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.514425224712835e-07,
      "logits/chosen": -1.5429000854492188,
      "logits/rejected": -1.6007463932037354,
      "logps/chosen": -94.66502380371094,
      "logps/ref_chosen": -77.86268615722656,
      "logps/ref_rejected": -110.77134704589844,
      "logps/rejected": -144.87844848632812,
      "loss": 0.997,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.7112575769424438,
      "rewards/margins": 0.723479151725769,
      "rewards/rejected": -1.434736728668213,
      "step": 286
    },
    {
      "epoch": 0.43386243386243384,
      "grad_norm": 11.010448455810547,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.04212578386068344,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 3.502326338516534e-07,
      "logits/chosen": -1.6829123497009277,
      "logits/rejected": -1.173173189163208,
      "logps/chosen": -73.6214599609375,
      "logps/ref_chosen": -62.552825927734375,
      "logps/ref_rejected": -77.7650146484375,
      "logps/rejected": -112.00250244140625,
      "loss": 0.8527,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4675137996673584,
      "rewards/margins": 0.9647963047027588,
      "rewards/rejected": -1.4323101043701172,
      "step": 287
    },
    {
      "epoch": 0.43537414965986393,
      "grad_norm": 15.223966598510742,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.0418771393597126,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.490199415097892e-07,
      "logits/chosen": -1.740882396697998,
      "logits/rejected": -1.5782954692840576,
      "logps/chosen": -103.83071899414062,
      "logps/ref_chosen": -83.74117279052734,
      "logps/ref_rejected": -106.93913269042969,
      "logps/rejected": -139.09048461914062,
      "loss": 1.1656,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.841812252998352,
      "rewards/margins": 0.49655526876449585,
      "rewards/rejected": -1.3383675813674927,
      "step": 288
    },
    {
      "epoch": 0.436885865457294,
      "grad_norm": 10.957082748413086,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.041720688343048096,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 3.4780447936730247e-07,
      "logits/chosen": -1.3379182815551758,
      "logits/rejected": -1.224417805671692,
      "logps/chosen": -92.84049987792969,
      "logps/ref_chosen": -73.04204559326172,
      "logps/ref_rejected": -88.07904052734375,
      "logps/rejected": -124.24183654785156,
      "loss": 1.0359,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8277724981307983,
      "rewards/margins": 0.6740319728851318,
      "rewards/rejected": -1.5018043518066406,
      "step": 289
    },
    {
      "epoch": 0.4383975812547241,
      "grad_norm": 11.407926559448242,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.04159071668982506,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.465862814232821e-07,
      "logits/chosen": -1.1827163696289062,
      "logits/rejected": -1.036186695098877,
      "logps/chosen": -104.59332275390625,
      "logps/ref_chosen": -78.60614013671875,
      "logps/ref_rejected": -108.50082397460938,
      "logps/rejected": -148.4783935546875,
      "loss": 1.1027,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.0788097381591797,
      "rewards/margins": 0.5728899836540222,
      "rewards/rejected": -1.6516997814178467,
      "step": 290
    },
    {
      "epoch": 0.4399092970521542,
      "grad_norm": 13.589823722839355,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.041396670043468475,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.4536538175334343e-07,
      "logits/chosen": -1.402148723602295,
      "logits/rejected": -1.925370216369629,
      "logps/chosen": -89.59666442871094,
      "logps/ref_chosen": -66.71226501464844,
      "logps/ref_rejected": -96.14028930664062,
      "logps/rejected": -136.31503295898438,
      "loss": 1.0914,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9476636648178101,
      "rewards/margins": 0.7052263021469116,
      "rewards/rejected": -1.6528899669647217,
      "step": 291
    },
    {
      "epoch": 0.4414210128495843,
      "grad_norm": 12.945611000061035,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.04122917354106903,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.4414181450867465e-07,
      "logits/chosen": -1.4477874040603638,
      "logits/rejected": -1.6135427951812744,
      "logps/chosen": -103.1048583984375,
      "logps/ref_chosen": -80.3355484008789,
      "logps/ref_rejected": -90.44906616210938,
      "logps/rejected": -128.4977569580078,
      "loss": 1.0903,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9375072121620178,
      "rewards/margins": 0.6204345226287842,
      "rewards/rejected": -1.5579416751861572,
      "step": 292
    },
    {
      "epoch": 0.4429327286470144,
      "grad_norm": 12.051216125488281,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.04104958474636078,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.4291561391508185e-07,
      "logits/chosen": -1.367284893989563,
      "logits/rejected": -1.1402729749679565,
      "logps/chosen": -95.94146728515625,
      "logps/ref_chosen": -71.69970703125,
      "logps/ref_rejected": -102.13948059082031,
      "logps/rejected": -145.989013671875,
      "loss": 1.0684,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9954971075057983,
      "rewards/margins": 0.7931400537490845,
      "rewards/rejected": -1.7886371612548828,
      "step": 293
    },
    {
      "epoch": 0.4444444444444444,
      "grad_norm": 17.355674743652344,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.0408453568816185,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.4168681427203153e-07,
      "logits/chosen": -1.38045072555542,
      "logits/rejected": -1.2437189817428589,
      "logps/chosen": -95.20700073242188,
      "logps/ref_chosen": -70.73458862304688,
      "logps/ref_rejected": -86.68821716308594,
      "logps/rejected": -126.80274963378906,
      "loss": 1.0621,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.99778813123703,
      "rewards/margins": 0.6298106908798218,
      "rewards/rejected": -1.6275988817214966,
      "step": 294
    },
    {
      "epoch": 0.4459561602418745,
      "grad_norm": 15.311004638671875,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.040692757815122604,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.4045544995169125e-07,
      "logits/chosen": -1.6135108470916748,
      "logits/rejected": -1.862878680229187,
      "logps/chosen": -92.92829895019531,
      "logps/ref_chosen": -66.42643737792969,
      "logps/ref_rejected": -99.58766174316406,
      "logps/rejected": -138.8087158203125,
      "loss": 1.1684,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.0746097564697266,
      "rewards/margins": 0.5088898539543152,
      "rewards/rejected": -1.5834996700286865,
      "step": 295
    },
    {
      "epoch": 0.4474678760393046,
      "grad_norm": 12.159005165100098,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.04047771915793419,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.392215553979679e-07,
      "logits/chosen": -1.4763997793197632,
      "logits/rejected": -1.5790629386901855,
      "logps/chosen": -112.81022644042969,
      "logps/ref_chosen": -87.47459411621094,
      "logps/ref_rejected": -103.96894836425781,
      "logps/rejected": -148.70249938964844,
      "loss": 1.0083,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.024478793144226,
      "rewards/margins": 0.7745535373687744,
      "rewards/rejected": -1.7990323305130005,
      "step": 296
    },
    {
      "epoch": 0.4489795918367347,
      "grad_norm": 17.412349700927734,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.04028886556625366,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.3798516512554485e-07,
      "logits/chosen": -1.5393011569976807,
      "logits/rejected": -1.604128122329712,
      "logps/chosen": -100.6285171508789,
      "logps/ref_chosen": -73.46731567382812,
      "logps/ref_rejected": -88.22674560546875,
      "logps/rejected": -134.59902954101562,
      "loss": 0.9386,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.0900453329086304,
      "rewards/margins": 0.7648427486419678,
      "rewards/rejected": -1.8548879623413086,
      "step": 297
    },
    {
      "epoch": 0.4504913076341648,
      "grad_norm": 15.284070014953613,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.04006350785493851,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.367463137189156e-07,
      "logits/chosen": -1.5410802364349365,
      "logits/rejected": -1.2973120212554932,
      "logps/chosen": -97.686279296875,
      "logps/ref_chosen": -73.21676635742188,
      "logps/ref_rejected": -84.9563217163086,
      "logps/rejected": -126.0870590209961,
      "loss": 1.1325,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9782853722572327,
      "rewards/margins": 0.6568226218223572,
      "rewards/rejected": -1.6351079940795898,
      "step": 298
    },
    {
      "epoch": 0.4520030234315949,
      "grad_norm": 12.5038480758667,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.0398641899228096,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 3.355050358314172e-07,
      "logits/chosen": -1.4272797107696533,
      "logits/rejected": -1.3961483240127563,
      "logps/chosen": -104.31736755371094,
      "logps/ref_chosen": -76.9534912109375,
      "logps/ref_rejected": -87.53433227539062,
      "logps/rejected": -127.41835021972656,
      "loss": 1.2044,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.0907185077667236,
      "rewards/margins": 0.48999127745628357,
      "rewards/rejected": -1.58070969581604,
      "step": 299
    },
    {
      "epoch": 0.45351473922902497,
      "grad_norm": 13.767475128173828,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.039727624505758286,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.3426136618426043e-07,
      "logits/chosen": -1.284356951713562,
      "logits/rejected": -1.4688689708709717,
      "logps/chosen": -105.30261993408203,
      "logps/ref_chosen": -78.36398315429688,
      "logps/ref_rejected": -97.03912353515625,
      "logps/rejected": -137.79039001464844,
      "loss": 1.1837,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0687929391860962,
      "rewards/margins": 0.5387318730354309,
      "rewards/rejected": -1.6075247526168823,
      "step": 300
    },
    {
      "epoch": 0.45351473922902497,
      "eval_kl/n_epsilon_steps": 0.26892605423927307,
      "eval_kl/p_epsilon_steps": 0.7306337952613831,
      "eval_logits/chosen": -1.303465723991394,
      "eval_logits/rejected": -1.4157907962799072,
      "eval_logps/chosen": -110.7581558227539,
      "eval_logps/ref_chosen": -86.90177917480469,
      "eval_logps/ref_rejected": -96.69639587402344,
      "eval_logps/rejected": -137.05419921875,
      "eval_loss": 0.5443636178970337,
      "eval_rewards/accuracies": 0.7376760840415955,
      "eval_rewards/chosen": -0.9429383873939514,
      "eval_rewards/margins": 0.6431780457496643,
      "eval_rewards/rejected": -1.5861164331436157,
      "eval_runtime": 46.97,
      "eval_samples_per_second": 49.031,
      "eval_steps_per_second": 1.533,
      "step": 300
    },
    {
      "epoch": 0.455026455026455,
      "grad_norm": 10.284299850463867,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.03956688567996025,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 3.3301533956555885e-07,
      "logits/chosen": -1.5722053050994873,
      "logits/rejected": -1.5532550811767578,
      "logps/chosen": -96.18799591064453,
      "logps/ref_chosen": -70.6719741821289,
      "logps/ref_rejected": -87.11650848388672,
      "logps/rejected": -127.36128234863281,
      "loss": 1.1416,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0108939409255981,
      "rewards/margins": 0.5733522772789001,
      "rewards/rejected": -1.584246277809143,
      "step": 301
    },
    {
      "epoch": 0.4565381708238851,
      "grad_norm": 12.417190551757812,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.03945591300725937,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 3.317669908293554e-07,
      "logits/chosen": -1.6059229373931885,
      "logits/rejected": -1.5071606636047363,
      "logps/chosen": -115.22109985351562,
      "logps/ref_chosen": -85.29096221923828,
      "logps/ref_rejected": -106.22589874267578,
      "logps/rejected": -144.5008087158203,
      "loss": 1.2904,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.1810417175292969,
      "rewards/margins": 0.32185858488082886,
      "rewards/rejected": -1.5029003620147705,
      "step": 302
    },
    {
      "epoch": 0.4580498866213152,
      "grad_norm": 11.21825122833252,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.039333000779151917,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.3051635489464793e-07,
      "logits/chosen": -1.2936410903930664,
      "logits/rejected": -1.1640466451644897,
      "logps/chosen": -109.26667785644531,
      "logps/ref_chosen": -83.90058898925781,
      "logps/ref_rejected": -104.7340087890625,
      "logps/rejected": -149.0026397705078,
      "loss": 1.0433,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9966724514961243,
      "rewards/margins": 0.7328269481658936,
      "rewards/rejected": -1.729499340057373,
      "step": 303
    },
    {
      "epoch": 0.4595616024187453,
      "grad_norm": 10.65771770477295,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.03913731127977371,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.292634667444117e-07,
      "logits/chosen": -1.5216636657714844,
      "logits/rejected": -1.3718217611312866,
      "logps/chosen": -99.35061645507812,
      "logps/ref_chosen": -77.39997100830078,
      "logps/ref_rejected": -94.21647644042969,
      "logps/rejected": -134.24893188476562,
      "loss": 1.0042,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8575640320777893,
      "rewards/margins": 0.6984165906906128,
      "rewards/rejected": -1.5559806823730469,
      "step": 304
    },
    {
      "epoch": 0.46107331821617537,
      "grad_norm": 11.185667037963867,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.038918398320674896,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.280083614246217e-07,
      "logits/chosen": -1.4200937747955322,
      "logits/rejected": -1.178731918334961,
      "logps/chosen": -116.03550720214844,
      "logps/ref_chosen": -90.90805053710938,
      "logps/ref_rejected": -85.84992980957031,
      "logps/rejected": -128.09902954101562,
      "loss": 1.0985,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9784424304962158,
      "rewards/margins": 0.6563032269477844,
      "rewards/rejected": -1.6347455978393555,
      "step": 305
    },
    {
      "epoch": 0.46258503401360546,
      "grad_norm": 12.585004806518555,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.03876093029975891,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.267510740432719e-07,
      "logits/chosen": -1.1087026596069336,
      "logits/rejected": -1.497624397277832,
      "logps/chosen": -93.91549682617188,
      "logps/ref_chosen": -71.7261962890625,
      "logps/ref_rejected": -97.70491027832031,
      "logps/rejected": -135.10858154296875,
      "loss": 1.0697,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8565143346786499,
      "rewards/margins": 0.581344723701477,
      "rewards/rejected": -1.437859058380127,
      "step": 306
    },
    {
      "epoch": 0.46409674981103555,
      "grad_norm": 11.676605224609375,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.03855610266327858,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 3.2549163976939285e-07,
      "logits/chosen": -1.4268429279327393,
      "logits/rejected": -1.6580591201782227,
      "logps/chosen": -92.18916320800781,
      "logps/ref_chosen": -74.38668823242188,
      "logps/ref_rejected": -84.16001892089844,
      "logps/rejected": -113.77175903320312,
      "loss": 1.2034,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.6898114085197449,
      "rewards/margins": 0.4468283951282501,
      "rewards/rejected": -1.1366398334503174,
      "step": 307
    },
    {
      "epoch": 0.4656084656084656,
      "grad_norm": 10.842734336853027,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.03847194463014603,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.2423009383206874e-07,
      "logits/chosen": -2.018758773803711,
      "logits/rejected": -1.362054705619812,
      "logps/chosen": -107.61516571044922,
      "logps/ref_chosen": -87.50894165039062,
      "logps/ref_rejected": -94.80848693847656,
      "logps/rejected": -130.3218231201172,
      "loss": 1.1014,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7732656002044678,
      "rewards/margins": 0.5833746194839478,
      "rewards/rejected": -1.356640338897705,
      "step": 308
    },
    {
      "epoch": 0.4671201814058957,
      "grad_norm": 11.334769248962402,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.03832821175456047,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.229664715194511e-07,
      "logits/chosen": -1.5512263774871826,
      "logits/rejected": -1.5080833435058594,
      "logps/chosen": -105.232421875,
      "logps/ref_chosen": -82.15191650390625,
      "logps/ref_rejected": -95.03496551513672,
      "logps/rejected": -134.20103454589844,
      "loss": 1.02,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8828413486480713,
      "rewards/margins": 0.6087982654571533,
      "rewards/rejected": -1.4916396141052246,
      "step": 309
    },
    {
      "epoch": 0.46863189720332576,
      "grad_norm": 18.294967651367188,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.0381493866443634,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 3.2170080817777257e-07,
      "logits/chosen": -1.8603748083114624,
      "logits/rejected": -1.6818149089813232,
      "logps/chosen": -117.48551940917969,
      "logps/ref_chosen": -93.7555160522461,
      "logps/ref_rejected": -96.93236541748047,
      "logps/rejected": -127.7294921875,
      "loss": 1.3625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9083326458930969,
      "rewards/margins": 0.2615802586078644,
      "rewards/rejected": -1.1699128150939941,
      "step": 310
    },
    {
      "epoch": 0.47014361300075586,
      "grad_norm": 9.972360610961914,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.038089871406555176,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.204331392103574e-07,
      "logits/chosen": -1.3419418334960938,
      "logits/rejected": -1.67874276638031,
      "logps/chosen": -90.90428161621094,
      "logps/ref_chosen": -76.20762634277344,
      "logps/ref_rejected": -110.48141479492188,
      "logps/rejected": -139.69790649414062,
      "loss": 1.1054,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5603994131088257,
      "rewards/margins": 0.5446619987487793,
      "rewards/rejected": -1.105061411857605,
      "step": 311
    },
    {
      "epoch": 0.47165532879818595,
      "grad_norm": 9.703566551208496,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.03792395442724228,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.1916350007663176e-07,
      "logits/chosen": -1.3120102882385254,
      "logits/rejected": -1.1737221479415894,
      "logps/chosen": -89.21003723144531,
      "logps/ref_chosen": -69.08878326416016,
      "logps/ref_rejected": -91.84494018554688,
      "logps/rejected": -130.23162841796875,
      "loss": 1.0066,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.7615611553192139,
      "rewards/margins": 0.6839256286621094,
      "rewards/rejected": -1.4454867839813232,
      "step": 312
    },
    {
      "epoch": 0.47316704459561604,
      "grad_norm": 11.096412658691406,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.03772354871034622,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 3.178919262911314e-07,
      "logits/chosen": -1.3597538471221924,
      "logits/rejected": -1.3801279067993164,
      "logps/chosen": -92.74771118164062,
      "logps/ref_chosen": -78.20826721191406,
      "logps/ref_rejected": -86.90351867675781,
      "logps/rejected": -113.30249786376953,
      "loss": 1.2374,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.5501388311386108,
      "rewards/margins": 0.43886855244636536,
      "rewards/rejected": -0.9890073537826538,
      "step": 313
    },
    {
      "epoch": 0.47467876039304613,
      "grad_norm": 12.68870735168457,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.03760603070259094,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 3.166184534225087e-07,
      "logits/chosen": -1.4780869483947754,
      "logits/rejected": -1.4974141120910645,
      "logps/chosen": -106.15601348876953,
      "logps/ref_chosen": -90.41890716552734,
      "logps/ref_rejected": -84.33525848388672,
      "logps/rejected": -117.95146942138672,
      "loss": 1.0187,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.592218279838562,
      "rewards/margins": 0.6636958122253418,
      "rewards/rejected": -1.2559140920639038,
      "step": 314
    },
    {
      "epoch": 0.47619047619047616,
      "grad_norm": 10.189184188842773,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.03747720271348953,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.1534311709253723e-07,
      "logits/chosen": -1.4552478790283203,
      "logits/rejected": -1.5378687381744385,
      "logps/chosen": -100.55183410644531,
      "logps/ref_chosen": -87.32842254638672,
      "logps/ref_rejected": -93.71661376953125,
      "logps/rejected": -122.59095764160156,
      "loss": 1.0451,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.4959626793861389,
      "rewards/margins": 0.5785229206085205,
      "rewards/rejected": -1.0744855403900146,
      "step": 315
    },
    {
      "epoch": 0.47770219198790626,
      "grad_norm": 11.919107437133789,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.037302348762750626,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.1406595297511564e-07,
      "logits/chosen": -1.3762269020080566,
      "logits/rejected": -1.3930758237838745,
      "logps/chosen": -89.23201751708984,
      "logps/ref_chosen": -73.898681640625,
      "logps/ref_rejected": -115.42668151855469,
      "logps/rejected": -147.5204315185547,
      "loss": 1.0206,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5697667598724365,
      "rewards/margins": 0.617039680480957,
      "rewards/rejected": -1.1868064403533936,
      "step": 316
    },
    {
      "epoch": 0.47921390778533635,
      "grad_norm": 13.636117935180664,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.03711676225066185,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 3.1278699679526975e-07,
      "logits/chosen": -1.3755412101745605,
      "logits/rejected": -1.2973852157592773,
      "logps/chosen": -86.28968048095703,
      "logps/ref_chosen": -75.42947387695312,
      "logps/ref_rejected": -90.60166931152344,
      "logps/rejected": -119.53173828125,
      "loss": 1.0444,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4039730429649353,
      "rewards/margins": 0.6617342233657837,
      "rewards/rejected": -1.0657072067260742,
      "step": 317
    },
    {
      "epoch": 0.48072562358276644,
      "grad_norm": 9.761408805847168,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.03690914809703827,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.1150628432815336e-07,
      "logits/chosen": -1.304398775100708,
      "logits/rejected": -1.2814505100250244,
      "logps/chosen": -82.9610366821289,
      "logps/ref_chosen": -70.3831787109375,
      "logps/ref_rejected": -98.19901275634766,
      "logps/rejected": -124.93925476074219,
      "loss": 1.1756,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.46455252170562744,
      "rewards/margins": 0.5136125683784485,
      "rewards/rejected": -0.9781651496887207,
      "step": 318
    },
    {
      "epoch": 0.48223733938019653,
      "grad_norm": 10.158917427062988,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.03674837574362755,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.1022385139804707e-07,
      "logits/chosen": -1.8672878742218018,
      "logits/rejected": -1.6835671663284302,
      "logps/chosen": -97.37528228759766,
      "logps/ref_chosen": -83.40225982666016,
      "logps/ref_rejected": -95.40069580078125,
      "logps/rejected": -126.68914794921875,
      "loss": 1.0576,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5123413801193237,
      "rewards/margins": 0.6273802518844604,
      "rewards/rejected": -1.1397216320037842,
      "step": 319
    },
    {
      "epoch": 0.4837490551776266,
      "grad_norm": 11.402392387390137,
      "kl/avg_steps": 0.15625,
      "kl/beta": 0.03658830001950264,
      "kl/n_epsilon_steps": 0.421875,
      "kl/p_epsilon_steps": 0.578125,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -1.3285362720489502,
      "logits/rejected": -1.4290411472320557,
      "logps/chosen": -80.62210083007812,
      "logps/ref_chosen": -68.70979309082031,
      "logps/ref_rejected": -87.00541687011719,
      "logps/rejected": -108.64424133300781,
      "loss": 1.2827,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.43682974576950073,
      "rewards/margins": 0.34812477231025696,
      "rewards/rejected": -0.7849545478820801,
      "step": 320
    },
    {
      "epoch": 0.4852607709750567,
      "grad_norm": 13.148420333862305,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.036531221121549606,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.0765396768561004e-07,
      "logits/chosen": -1.6233880519866943,
      "logits/rejected": -1.6341774463653564,
      "logps/chosen": -77.85541534423828,
      "logps/ref_chosen": -66.48135375976562,
      "logps/ref_rejected": -71.84545135498047,
      "logps/rejected": -98.65225982666016,
      "loss": 1.1251,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4173020124435425,
      "rewards/margins": 0.5548455119132996,
      "rewards/rejected": -0.9721474647521973,
      "step": 321
    },
    {
      "epoch": 0.48677248677248675,
      "grad_norm": 11.733065605163574,
      "kl/avg_steps": 0.640625,
      "kl/beta": 0.03633817285299301,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 3.063665887884511e-07,
      "logits/chosen": -1.226560354232788,
      "logits/rejected": -1.4153425693511963,
      "logps/chosen": -79.17440795898438,
      "logps/ref_chosen": -65.94654846191406,
      "logps/ref_rejected": -94.26603698730469,
      "logps/rejected": -128.4293670654297,
      "loss": 0.9387,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.47956448793411255,
      "rewards/margins": 0.7515709400177002,
      "rewards/rejected": -1.231135368347168,
      "step": 322
    },
    {
      "epoch": 0.48828420256991684,
      "grad_norm": 14.1713228225708,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.03610686585307121,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.0507763319663517e-07,
      "logits/chosen": -1.614477276802063,
      "logits/rejected": -1.2867920398712158,
      "logps/chosen": -102.02337646484375,
      "logps/ref_chosen": -86.5498046875,
      "logps/ref_rejected": -110.39498901367188,
      "logps/rejected": -141.76101684570312,
      "loss": 1.1979,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5602794885635376,
      "rewards/margins": 0.5638449192047119,
      "rewards/rejected": -1.1241244077682495,
      "step": 323
    },
    {
      "epoch": 0.4897959183673469,
      "grad_norm": 13.024452209472656,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.03597196936607361,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.0378713696502097e-07,
      "logits/chosen": -1.6353603601455688,
      "logits/rejected": -1.6581999063491821,
      "logps/chosen": -85.73527526855469,
      "logps/ref_chosen": -74.44218444824219,
      "logps/ref_rejected": -85.76464080810547,
      "logps/rejected": -114.40110778808594,
      "loss": 1.0649,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.4083707630634308,
      "rewards/margins": 0.6154605746269226,
      "rewards/rejected": -1.0238313674926758,
      "step": 324
    },
    {
      "epoch": 0.491307634164777,
      "grad_norm": 11.41663932800293,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.0358097068965435,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.0249513619156206e-07,
      "logits/chosen": -1.4231340885162354,
      "logits/rejected": -1.398125410079956,
      "logps/chosen": -100.60375213623047,
      "logps/ref_chosen": -81.43812561035156,
      "logps/ref_rejected": -97.04302978515625,
      "logps/rejected": -132.8642578125,
      "loss": 1.0998,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6865410804748535,
      "rewards/margins": 0.5874417424201965,
      "rewards/rejected": -1.2739828824996948,
      "step": 325
    },
    {
      "epoch": 0.4928193499622071,
      "grad_norm": 10.984821319580078,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.035642631351947784,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 3.012016670162977e-07,
      "logits/chosen": -1.5135228633880615,
      "logits/rejected": -1.453247308731079,
      "logps/chosen": -113.7421875,
      "logps/ref_chosen": -91.65318298339844,
      "logps/ref_rejected": -90.64222717285156,
      "logps/rejected": -124.34673309326172,
      "loss": 1.2212,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.7893346548080444,
      "rewards/margins": 0.40609291195869446,
      "rewards/rejected": -1.195427656173706,
      "step": 326
    },
    {
      "epoch": 0.4943310657596372,
      "grad_norm": 13.8237886428833,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.035542670637369156,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 2.99906765620341e-07,
      "logits/chosen": -1.5450892448425293,
      "logits/rejected": -1.5019280910491943,
      "logps/chosen": -112.25006103515625,
      "logps/ref_chosen": -89.97216796875,
      "logps/ref_rejected": -97.54869079589844,
      "logps/rejected": -133.211181640625,
      "loss": 1.2382,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7921844720840454,
      "rewards/margins": 0.4665413796901703,
      "rewards/rejected": -1.258725881576538,
      "step": 327
    },
    {
      "epoch": 0.4958427815570673,
      "grad_norm": 9.504220008850098,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.035442985594272614,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.9861046822486766e-07,
      "logits/chosen": -1.2479305267333984,
      "logits/rejected": -1.1524548530578613,
      "logps/chosen": -99.87271118164062,
      "logps/ref_chosen": -80.27335357666016,
      "logps/ref_rejected": -99.04093933105469,
      "logps/rejected": -132.95858764648438,
      "loss": 1.135,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6942028999328613,
      "rewards/margins": 0.4996661841869354,
      "rewards/rejected": -1.1938691139221191,
      "step": 328
    },
    {
      "epoch": 0.4973544973544973,
      "grad_norm": 11.80764389038086,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.035288598388433456,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.9731281109010253e-07,
      "logits/chosen": -1.4124984741210938,
      "logits/rejected": -1.446164846420288,
      "logps/chosen": -100.15596008300781,
      "logps/ref_chosen": -79.75892639160156,
      "logps/ref_rejected": -102.06265258789062,
      "logps/rejected": -139.7376708984375,
      "loss": 1.0659,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7199461460113525,
      "rewards/margins": 0.601253867149353,
      "rewards/rejected": -1.3212000131607056,
      "step": 329
    },
    {
      "epoch": 0.4988662131519274,
      "grad_norm": 9.457327842712402,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.03514581918716431,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.9601383051430505e-07,
      "logits/chosen": -1.3547937870025635,
      "logits/rejected": -1.2515625953674316,
      "logps/chosen": -85.88477325439453,
      "logps/ref_chosen": -70.55734252929688,
      "logps/ref_rejected": -94.53077697753906,
      "logps/rejected": -131.9217529296875,
      "loss": 0.9911,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5381969213485718,
      "rewards/margins": 0.7649656534194946,
      "rewards/rejected": -1.3031624555587769,
      "step": 330
    },
    {
      "epoch": 0.5003779289493575,
      "grad_norm": 9.513692855834961,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.03496009111404419,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.947135628327544e-07,
      "logits/chosen": -1.3662301301956177,
      "logits/rejected": -1.1261534690856934,
      "logps/chosen": -97.10047149658203,
      "logps/ref_chosen": -75.46063995361328,
      "logps/ref_rejected": -84.78495788574219,
      "logps/rejected": -129.4873046875,
      "loss": 0.9935,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.7566953897476196,
      "rewards/margins": 0.7958719730377197,
      "rewards/rejected": -1.5525672435760498,
      "step": 331
    },
    {
      "epoch": 0.5018896447467877,
      "grad_norm": 11.208243370056152,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.0347861610352993,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.934120444167326e-07,
      "logits/chosen": -1.6225413084030151,
      "logits/rejected": -1.5228370428085327,
      "logps/chosen": -105.17870330810547,
      "logps/ref_chosen": -84.32807922363281,
      "logps/ref_rejected": -95.63302612304688,
      "logps/rejected": -136.43002319335938,
      "loss": 1.0095,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.7220751643180847,
      "rewards/margins": 0.6849751472473145,
      "rewards/rejected": -1.4070502519607544,
      "step": 332
    },
    {
      "epoch": 0.5034013605442177,
      "grad_norm": 10.370922088623047,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.03459158539772034,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.921093116725076e-07,
      "logits/chosen": -1.5618640184402466,
      "logits/rejected": -1.3652313947677612,
      "logps/chosen": -102.90045166015625,
      "logps/ref_chosen": -78.21322631835938,
      "logps/ref_rejected": -103.82716369628906,
      "logps/rejected": -152.17138671875,
      "loss": 0.9057,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8525649905204773,
      "rewards/margins": 0.8088828325271606,
      "rewards/rejected": -1.6614477634429932,
      "step": 333
    },
    {
      "epoch": 0.5049130763416477,
      "grad_norm": 11.503582000732422,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.03440878912806511,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.9080540104031484e-07,
      "logits/chosen": -1.3274097442626953,
      "logits/rejected": -1.448803424835205,
      "logps/chosen": -109.16493225097656,
      "logps/ref_chosen": -85.0171127319336,
      "logps/ref_rejected": -106.79039764404297,
      "logps/rejected": -147.31170654296875,
      "loss": 1.2069,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8311678171157837,
      "rewards/margins": 0.5532088279724121,
      "rewards/rejected": -1.3843765258789062,
      "step": 334
    },
    {
      "epoch": 0.5064247921390779,
      "grad_norm": 19.717958450317383,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.034280236810445786,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.895003489933375e-07,
      "logits/chosen": -1.1505557298660278,
      "logits/rejected": -1.2536959648132324,
      "logps/chosen": -106.75822448730469,
      "logps/ref_chosen": -78.56512451171875,
      "logps/ref_rejected": -92.68515014648438,
      "logps/rejected": -138.57762145996094,
      "loss": 1.2316,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.9674444198608398,
      "rewards/margins": 0.5958920121192932,
      "rewards/rejected": -1.5633363723754883,
      "step": 335
    },
    {
      "epoch": 0.5079365079365079,
      "grad_norm": 14.143689155578613,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.03417344391345978,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.8819419203668675e-07,
      "logits/chosen": -1.2581086158752441,
      "logits/rejected": -1.164086103439331,
      "logps/chosen": -122.58998107910156,
      "logps/ref_chosen": -88.63243103027344,
      "logps/ref_rejected": -107.89385223388672,
      "logps/rejected": -161.6552734375,
      "loss": 1.0498,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.1570062637329102,
      "rewards/margins": 0.6672290563583374,
      "rewards/rejected": -1.824235439300537,
      "step": 336
    },
    {
      "epoch": 0.509448223733938,
      "grad_norm": 11.089780807495117,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.03400342911481857,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.8688696670638053e-07,
      "logits/chosen": -1.247017502784729,
      "logits/rejected": -1.4651862382888794,
      "logps/chosen": -127.10426330566406,
      "logps/ref_chosen": -93.25018310546875,
      "logps/ref_rejected": -103.8592529296875,
      "logps/rejected": -151.89874267578125,
      "loss": 1.203,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1513574123382568,
      "rewards/margins": 0.47332531213760376,
      "rewards/rejected": -1.624682903289795,
      "step": 337
    },
    {
      "epoch": 0.5109599395313681,
      "grad_norm": 12.785868644714355,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.03388693928718567,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.8557870956832133e-07,
      "logits/chosen": -0.8946573138237,
      "logits/rejected": -0.9418930411338806,
      "logps/chosen": -117.18191528320312,
      "logps/ref_chosen": -81.79462432861328,
      "logps/ref_rejected": -90.98943328857422,
      "logps/rejected": -141.94183349609375,
      "loss": 1.1434,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.197884202003479,
      "rewards/margins": 0.5193731784820557,
      "rewards/rejected": -1.7172573804855347,
      "step": 338
    },
    {
      "epoch": 0.5124716553287982,
      "grad_norm": 10.959848403930664,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.033770851790905,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 2.842694572172736e-07,
      "logits/chosen": -1.2875313758850098,
      "logits/rejected": -1.4024841785430908,
      "logps/chosen": -85.72515869140625,
      "logps/ref_chosen": -61.803558349609375,
      "logps/ref_rejected": -85.16979217529297,
      "logps/rejected": -131.8690948486328,
      "loss": 0.9512,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8063684701919556,
      "rewards/margins": 0.7594180107116699,
      "rewards/rejected": -1.565786361694336,
      "step": 339
    },
    {
      "epoch": 0.5139833711262283,
      "grad_norm": 9.914621353149414,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.0335715226829052,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.8295924627584004e-07,
      "logits/chosen": -1.2250884771347046,
      "logits/rejected": -0.937452495098114,
      "logps/chosen": -104.68017578125,
      "logps/ref_chosen": -72.48607635498047,
      "logps/ref_rejected": -79.86129760742188,
      "logps/rejected": -131.80706787109375,
      "loss": 1.1311,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.0805590152740479,
      "rewards/margins": 0.6526724696159363,
      "rewards/rejected": -1.7332314252853394,
      "step": 340
    },
    {
      "epoch": 0.5154950869236583,
      "grad_norm": 11.14721393585205,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.03343569114804268,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.816481133934373e-07,
      "logits/chosen": -1.2342188358306885,
      "logits/rejected": -1.3302650451660156,
      "logps/chosen": -108.94258117675781,
      "logps/ref_chosen": -77.36830139160156,
      "logps/ref_rejected": -94.64933013916016,
      "logps/rejected": -147.7334442138672,
      "loss": 1.0595,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.0553505420684814,
      "rewards/margins": 0.7088862657546997,
      "rewards/rejected": -1.7642368078231812,
      "step": 341
    },
    {
      "epoch": 0.5170068027210885,
      "grad_norm": 9.97767162322998,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.033310774713754654,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.8033609524527046e-07,
      "logits/chosen": -1.2789771556854248,
      "logits/rejected": -1.195197582244873,
      "logps/chosen": -99.66441345214844,
      "logps/ref_chosen": -71.0083236694336,
      "logps/ref_rejected": -84.22953796386719,
      "logps/rejected": -134.83029174804688,
      "loss": 1.0397,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9530007243156433,
      "rewards/margins": 0.7210956811904907,
      "rewards/rejected": -1.6740963459014893,
      "step": 342
    },
    {
      "epoch": 0.5185185185185185,
      "grad_norm": 12.255571365356445,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.03316567465662956,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.7902322853130753e-07,
      "logits/chosen": -1.649799108505249,
      "logits/rejected": -1.6315422058105469,
      "logps/chosen": -120.76673889160156,
      "logps/ref_chosen": -91.44624328613281,
      "logps/ref_rejected": -99.0604476928711,
      "logps/rejected": -140.72201538085938,
      "loss": 1.2552,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9723671078681946,
      "rewards/margins": 0.4006619453430176,
      "rewards/rejected": -1.3730289936065674,
      "step": 343
    },
    {
      "epoch": 0.5200302343159486,
      "grad_norm": 10.119938850402832,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.03306235373020172,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.7770954997525274e-07,
      "logits/chosen": -1.1954832077026367,
      "logits/rejected": -1.3106439113616943,
      "logps/chosen": -108.83035278320312,
      "logps/ref_chosen": -73.43608093261719,
      "logps/ref_rejected": -100.76569366455078,
      "logps/rejected": -157.56735229492188,
      "loss": 1.0125,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.1673158407211304,
      "rewards/margins": 0.6984189748764038,
      "rewards/rejected": -1.8657348155975342,
      "step": 344
    },
    {
      "epoch": 0.5215419501133787,
      "grad_norm": 12.316679954528809,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.032877419143915176,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.7639509632351927e-07,
      "logits/chosen": -1.2922027111053467,
      "logits/rejected": -1.3272606134414673,
      "logps/chosen": -100.18463134765625,
      "logps/ref_chosen": -75.79296875,
      "logps/ref_rejected": -94.34156799316406,
      "logps/rejected": -137.67491149902344,
      "loss": 1.1094,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8025590181350708,
      "rewards/margins": 0.6133404970169067,
      "rewards/rejected": -1.4158995151519775,
      "step": 345
    },
    {
      "epoch": 0.5230536659108088,
      "grad_norm": 12.418992042541504,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.03275458887219429,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.7507990434420123e-07,
      "logits/chosen": -1.4952889680862427,
      "logits/rejected": -1.435889482498169,
      "logps/chosen": -98.2471694946289,
      "logps/ref_chosen": -72.26289367675781,
      "logps/ref_rejected": -106.36925506591797,
      "logps/rejected": -154.2978515625,
      "loss": 1.1168,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.8507057428359985,
      "rewards/margins": 0.7072672843933105,
      "rewards/rejected": -1.5579731464385986,
      "step": 346
    },
    {
      "epoch": 0.5245653817082389,
      "grad_norm": 12.831889152526855,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.03263222053647041,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 2.737640108260456e-07,
      "logits/chosen": -1.141088604927063,
      "logits/rejected": -1.2003611326217651,
      "logps/chosen": -102.23765563964844,
      "logps/ref_chosen": -71.19871520996094,
      "logps/ref_rejected": -91.543212890625,
      "logps/rejected": -139.07275390625,
      "loss": 1.1903,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0140210390090942,
      "rewards/margins": 0.5289695858955383,
      "rewards/rejected": -1.5429906845092773,
      "step": 347
    },
    {
      "epoch": 0.5260770975056689,
      "grad_norm": 9.99068546295166,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.03255084156990051,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 2.724474525774229e-07,
      "logits/chosen": -1.159302830696106,
      "logits/rejected": -1.177544355392456,
      "logps/chosen": -96.44154357910156,
      "logps/ref_chosen": -69.95603942871094,
      "logps/ref_rejected": -83.64309692382812,
      "logps/rejected": -132.4287109375,
      "loss": 1.0353,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.8636122941970825,
      "rewards/margins": 0.7161925435066223,
      "rewards/rejected": -1.57980477809906,
      "step": 348
    },
    {
      "epoch": 0.527588813303099,
      "grad_norm": 10.788860321044922,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.032459549605846405,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.711302664252973e-07,
      "logits/chosen": -1.4271414279937744,
      "logits/rejected": -1.271451473236084,
      "logps/chosen": -96.35053253173828,
      "logps/ref_chosen": -70.71857452392578,
      "logps/ref_rejected": -99.93263244628906,
      "logps/rejected": -146.58157348632812,
      "loss": 1.07,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8293413519859314,
      "rewards/margins": 0.6725594401359558,
      "rewards/rejected": -1.5019009113311768,
      "step": 349
    },
    {
      "epoch": 0.5291005291005291,
      "grad_norm": 10.467818260192871,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.03231313079595566,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.698124892141971e-07,
      "logits/chosen": -1.440155267715454,
      "logits/rejected": -1.2903451919555664,
      "logps/chosen": -106.1199722290039,
      "logps/ref_chosen": -78.16873168945312,
      "logps/ref_rejected": -104.84309387207031,
      "logps/rejected": -159.8768310546875,
      "loss": 0.904,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9024109840393066,
      "rewards/margins": 0.8648760318756104,
      "rewards/rejected": -1.767287015914917,
      "step": 350
    },
    {
      "epoch": 0.5306122448979592,
      "grad_norm": 9.843159675598145,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.03218238800764084,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 2.6849415780518357e-07,
      "logits/chosen": -1.2251023054122925,
      "logits/rejected": -1.3334765434265137,
      "logps/chosen": -101.87110900878906,
      "logps/ref_chosen": -71.79151916503906,
      "logps/ref_rejected": -97.04634094238281,
      "logps/rejected": -147.25169372558594,
      "loss": 1.1072,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.9649567604064941,
      "rewards/margins": 0.6374739408493042,
      "rewards/rejected": -1.602430820465088,
      "step": 351
    },
    {
      "epoch": 0.5321239606953893,
      "grad_norm": 11.369997024536133,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.031992435455322266,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.6717530907482024e-07,
      "logits/chosen": -1.220479130744934,
      "logits/rejected": -1.3581123352050781,
      "logps/chosen": -112.46723937988281,
      "logps/ref_chosen": -80.86544799804688,
      "logps/ref_rejected": -102.02128601074219,
      "logps/rejected": -151.5338592529297,
      "loss": 1.1227,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.0110710859298706,
      "rewards/margins": 0.5643045902252197,
      "rewards/rejected": -1.5753756761550903,
      "step": 352
    },
    {
      "epoch": 0.5336356764928194,
      "grad_norm": 10.501523971557617,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.031882837414741516,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.658559799141411e-07,
      "logits/chosen": -1.4000976085662842,
      "logits/rejected": -1.3244915008544922,
      "logps/chosen": -108.99580383300781,
      "logps/ref_chosen": -84.77235412597656,
      "logps/ref_rejected": -86.77130126953125,
      "logps/rejected": -132.26800537109375,
      "loss": 1.0936,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.7711913585662842,
      "rewards/margins": 0.6679372787475586,
      "rewards/rejected": -1.4391286373138428,
      "step": 353
    },
    {
      "epoch": 0.5351473922902494,
      "grad_norm": 9.845510482788086,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.031743958592414856,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.6453620722761895e-07,
      "logits/chosen": -1.3057725429534912,
      "logits/rejected": -1.5508718490600586,
      "logps/chosen": -80.18466186523438,
      "logps/ref_chosen": -54.33562088012695,
      "logps/ref_rejected": -92.4120101928711,
      "logps/rejected": -140.17367553710938,
      "loss": 1.0502,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8189990520477295,
      "rewards/margins": 0.6852681636810303,
      "rewards/rejected": -1.5042672157287598,
      "step": 354
    },
    {
      "epoch": 0.5366591080876795,
      "grad_norm": 9.737431526184082,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.03158602863550186,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.632160279321328e-07,
      "logits/chosen": -1.1256142854690552,
      "logits/rejected": -1.3838088512420654,
      "logps/chosen": -90.2604751586914,
      "logps/ref_chosen": -61.8388671875,
      "logps/ref_rejected": -98.65571594238281,
      "logps/rejected": -151.04173278808594,
      "loss": 1.0026,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8956200480461121,
      "rewards/margins": 0.746633768081665,
      "rewards/rejected": -1.6422538757324219,
      "step": 355
    },
    {
      "epoch": 0.5381708238851096,
      "grad_norm": 11.063801765441895,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.03141911327838898,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.618954789559356e-07,
      "logits/chosen": -1.2441139221191406,
      "logits/rejected": -1.296750783920288,
      "logps/chosen": -86.55245971679688,
      "logps/ref_chosen": -63.92546463012695,
      "logps/ref_rejected": -89.682861328125,
      "logps/rejected": -133.89390563964844,
      "loss": 1.1363,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7118154764175415,
      "rewards/margins": 0.6670767664909363,
      "rewards/rejected": -1.378892183303833,
      "step": 356
    },
    {
      "epoch": 0.5396825396825397,
      "grad_norm": 9.868560791015625,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.031262800097465515,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.6057459723762076e-07,
      "logits/chosen": -1.2802687883377075,
      "logits/rejected": -1.6558847427368164,
      "logps/chosen": -110.63601684570312,
      "logps/ref_chosen": -81.07589721679688,
      "logps/ref_rejected": -85.06967163085938,
      "logps/rejected": -132.71707153320312,
      "loss": 1.1316,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9249534010887146,
      "rewards/margins": 0.5564345121383667,
      "rewards/rejected": -1.4813878536224365,
      "step": 357
    },
    {
      "epoch": 0.5411942554799698,
      "grad_norm": 9.289213180541992,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.03114600107073784,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.5925341972508954e-07,
      "logits/chosen": -1.5175108909606934,
      "logits/rejected": -1.3619422912597656,
      "logps/chosen": -109.77130126953125,
      "logps/ref_chosen": -84.09109497070312,
      "logps/ref_rejected": -85.07244110107422,
      "logps/rejected": -135.58029174804688,
      "loss": 0.9844,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.799083948135376,
      "rewards/margins": 0.7630775570869446,
      "rewards/rejected": -1.5621614456176758,
      "step": 358
    },
    {
      "epoch": 0.5427059712773998,
      "grad_norm": 12.027239799499512,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.03102963976562023,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 2.579319833745169e-07,
      "logits/chosen": -1.8530116081237793,
      "logits/rejected": -1.6771414279937744,
      "logps/chosen": -108.74325561523438,
      "logps/ref_chosen": -80.7490234375,
      "logps/ref_rejected": -94.92912292480469,
      "logps/rejected": -136.96435546875,
      "loss": 1.2222,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.8691080808639526,
      "rewards/margins": 0.42722088098526,
      "rewards/rejected": -1.2963290214538574,
      "step": 359
    },
    {
      "epoch": 0.54421768707483,
      "grad_norm": 8.521217346191406,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.030942615121603012,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.5661032514931834e-07,
      "logits/chosen": -1.2853665351867676,
      "logits/rejected": -1.4590386152267456,
      "logps/chosen": -107.85910034179688,
      "logps/ref_chosen": -78.38680267333984,
      "logps/ref_rejected": -109.6893310546875,
      "logps/rejected": -161.26239013671875,
      "loss": 1.038,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9122262001037598,
      "rewards/margins": 0.6750233173370361,
      "rewards/rejected": -1.587249517440796,
      "step": 360
    },
    {
      "epoch": 0.54572940287226,
      "grad_norm": 9.228687286376953,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.030836613848805428,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.552884820191154e-07,
      "logits/chosen": -1.172134280204773,
      "logits/rejected": -1.341686725616455,
      "logps/chosen": -102.47615814208984,
      "logps/ref_chosen": -73.9055404663086,
      "logps/ref_rejected": -89.8489990234375,
      "logps/rejected": -143.1851806640625,
      "loss": 0.9892,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8791087865829468,
      "rewards/margins": 0.7538092136383057,
      "rewards/rejected": -1.632917881011963,
      "step": 361
    },
    {
      "epoch": 0.54724111866969,
      "grad_norm": 10.797682762145996,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.03068319708108902,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.53966490958702e-07,
      "logits/chosen": -1.5308431386947632,
      "logits/rejected": -1.8336783647537231,
      "logps/chosen": -116.3746337890625,
      "logps/ref_chosen": -82.32565307617188,
      "logps/ref_rejected": -123.14100646972656,
      "logps/rejected": -175.5985565185547,
      "loss": 1.1361,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.0415807962417603,
      "rewards/margins": 0.5558252334594727,
      "rewards/rejected": -1.5974061489105225,
      "step": 362
    },
    {
      "epoch": 0.5487528344671202,
      "grad_norm": 10.823992729187012,
      "kl/avg_steps": 0.78125,
      "kl/beta": 0.030544791370630264,
      "kl/n_epsilon_steps": 0.109375,
      "kl/p_epsilon_steps": 0.890625,
      "learning_rate": 2.526443889470099e-07,
      "logits/chosen": -1.1187574863433838,
      "logits/rejected": -1.4465909004211426,
      "logps/chosen": -98.7936782836914,
      "logps/ref_chosen": -66.05493927001953,
      "logps/ref_rejected": -106.79598999023438,
      "logps/rejected": -164.49728393554688,
      "loss": 0.9382,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9929654598236084,
      "rewards/margins": 0.7533215284347534,
      "rewards/rejected": -1.7462869882583618,
      "step": 363
    },
    {
      "epoch": 0.5502645502645502,
      "grad_norm": 8.624401092529297,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.030308010056614876,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.513222129660744e-07,
      "logits/chosen": -1.4273147583007812,
      "logits/rejected": -1.2582423686981201,
      "logps/chosen": -104.58074951171875,
      "logps/ref_chosen": -76.38365173339844,
      "logps/ref_rejected": -100.22221374511719,
      "logps/rejected": -153.84181213378906,
      "loss": 1.0464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8530045747756958,
      "rewards/margins": 0.7596189975738525,
      "rewards/rejected": -1.6126235723495483,
      "step": 364
    },
    {
      "epoch": 0.5517762660619804,
      "grad_norm": 10.569746017456055,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.03016660362482071,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.4147711992263794,
      "logits/rejected": -1.129202961921692,
      "logps/chosen": -104.73855590820312,
      "logps/ref_chosen": -81.83399963378906,
      "logps/ref_rejected": -89.06932830810547,
      "logps/rejected": -133.12460327148438,
      "loss": 0.995,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6893056631088257,
      "rewards/margins": 0.6299935579299927,
      "rewards/rejected": -1.3192992210388184,
      "step": 365
    },
    {
      "epoch": 0.5532879818594104,
      "grad_norm": 10.754393577575684,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.029997866600751877,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.486777870339255e-07,
      "logits/chosen": -1.580064296722412,
      "logits/rejected": -1.4120087623596191,
      "logps/chosen": -98.59486389160156,
      "logps/ref_chosen": -72.03398895263672,
      "logps/ref_rejected": -83.65354919433594,
      "logps/rejected": -129.49868774414062,
      "loss": 1.1278,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7969068288803101,
      "rewards/margins": 0.5694468021392822,
      "rewards/rejected": -1.3663535118103027,
      "step": 366
    },
    {
      "epoch": 0.5547996976568406,
      "grad_norm": 9.318233489990234,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.029895102605223656,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.4735561105299014e-07,
      "logits/chosen": -1.5403746366500854,
      "logits/rejected": -1.6996790170669556,
      "logps/chosen": -104.44827270507812,
      "logps/ref_chosen": -72.39827728271484,
      "logps/ref_rejected": -95.58364868164062,
      "logps/rejected": -145.46209716796875,
      "loss": 1.1044,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9578984975814819,
      "rewards/margins": 0.5246537327766418,
      "rewards/rejected": -1.4825522899627686,
      "step": 367
    },
    {
      "epoch": 0.5563114134542706,
      "grad_norm": 10.34378433227539,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.029774144291877747,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.46033509041298e-07,
      "logits/chosen": -1.5399154424667358,
      "logits/rejected": -1.1814687252044678,
      "logps/chosen": -126.15576171875,
      "logps/ref_chosen": -90.12812042236328,
      "logps/ref_rejected": -91.6636962890625,
      "logps/rejected": -148.67593383789062,
      "loss": 1.0811,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0728702545166016,
      "rewards/margins": 0.6154052019119263,
      "rewards/rejected": -1.6882755756378174,
      "step": 368
    },
    {
      "epoch": 0.5578231292517006,
      "grad_norm": 17.82675552368164,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.02967214584350586,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.447115179808846e-07,
      "logits/chosen": -1.8711051940917969,
      "logits/rejected": -1.640881061553955,
      "logps/chosen": -102.46626281738281,
      "logps/ref_chosen": -71.29417419433594,
      "logps/ref_rejected": -99.03875732421875,
      "logps/rejected": -147.19813537597656,
      "loss": 1.1811,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9218083620071411,
      "rewards/margins": 0.49515968561172485,
      "rewards/rejected": -1.4169681072235107,
      "step": 369
    },
    {
      "epoch": 0.5593348450491308,
      "grad_norm": 9.260988235473633,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.02952452376484871,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.4338967485068164e-07,
      "logits/chosen": -1.1396031379699707,
      "logits/rejected": -1.0747642517089844,
      "logps/chosen": -98.35624694824219,
      "logps/ref_chosen": -69.14627075195312,
      "logps/ref_rejected": -93.58651733398438,
      "logps/rejected": -149.8125457763672,
      "loss": 1.0041,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8626432418823242,
      "rewards/margins": 0.786967933177948,
      "rewards/rejected": -1.649611234664917,
      "step": 370
    },
    {
      "epoch": 0.5608465608465608,
      "grad_norm": 10.98085880279541,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.029386773705482483,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.420680166254831e-07,
      "logits/chosen": -1.3450419902801514,
      "logits/rejected": -1.1297814846038818,
      "logps/chosen": -95.94750213623047,
      "logps/ref_chosen": -65.76728820800781,
      "logps/ref_rejected": -79.9320068359375,
      "logps/rejected": -134.09359741210938,
      "loss": 1.0643,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8851276636123657,
      "rewards/margins": 0.6940947771072388,
      "rewards/rejected": -1.5792224407196045,
      "step": 371
    },
    {
      "epoch": 0.562358276643991,
      "grad_norm": 11.20559310913086,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.029240570962429047,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.4074658027491044e-07,
      "logits/chosen": -1.160583734512329,
      "logits/rejected": -1.33573579788208,
      "logps/chosen": -96.75958251953125,
      "logps/ref_chosen": -69.97252655029297,
      "logps/ref_rejected": -92.38317108154297,
      "logps/rejected": -138.94244384765625,
      "loss": 1.2057,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7855249643325806,
      "rewards/margins": 0.5673984885215759,
      "rewards/rejected": -1.3529233932495117,
      "step": 372
    },
    {
      "epoch": 0.563869992441421,
      "grad_norm": 12.406031608581543,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.029149478301405907,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.394254027623792e-07,
      "logits/chosen": -1.3365615606307983,
      "logits/rejected": -1.7029169797897339,
      "logps/chosen": -114.59992980957031,
      "logps/ref_chosen": -79.34700012207031,
      "logps/ref_rejected": -95.69737243652344,
      "logps/rejected": -154.38079833984375,
      "loss": 1.0857,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.0270100831985474,
      "rewards/margins": 0.6728541851043701,
      "rewards/rejected": -1.699864387512207,
      "step": 373
    },
    {
      "epoch": 0.5653817082388511,
      "grad_norm": 11.336803436279297,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.02903153747320175,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": -1.3897533416748047,
      "logits/rejected": -1.2900179624557495,
      "logps/chosen": -124.2972412109375,
      "logps/ref_chosen": -93.45108032226562,
      "logps/ref_rejected": -93.575927734375,
      "logps/rejected": -156.41477966308594,
      "loss": 0.8913,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8952039480209351,
      "rewards/margins": 0.917777419090271,
      "rewards/rejected": -1.812981367111206,
      "step": 374
    },
    {
      "epoch": 0.5668934240362812,
      "grad_norm": 9.698143005371094,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.028878122568130493,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.3678397206786715e-07,
      "logits/chosen": -1.5062309503555298,
      "logits/rejected": -1.3306735754013062,
      "logps/chosen": -100.72938537597656,
      "logps/ref_chosen": -77.37176513671875,
      "logps/ref_rejected": -98.59054565429688,
      "logps/rejected": -143.8408203125,
      "loss": 1.0636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6728950142860413,
      "rewards/margins": 0.6229599118232727,
      "rewards/rejected": -1.295854926109314,
      "step": 375
    },
    {
      "epoch": 0.5684051398337112,
      "grad_norm": 9.342375755310059,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.02872551791369915,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.3546379277238103e-07,
      "logits/chosen": -1.0259218215942383,
      "logits/rejected": -1.408305048942566,
      "logps/chosen": -99.64701080322266,
      "logps/ref_chosen": -68.99790954589844,
      "logps/ref_rejected": -90.37117004394531,
      "logps/rejected": -149.88748168945312,
      "loss": 1.0369,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8794429302215576,
      "rewards/margins": 0.8176605105400085,
      "rewards/rejected": -1.697103500366211,
      "step": 376
    },
    {
      "epoch": 0.5699168556311414,
      "grad_norm": 9.864453315734863,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.028582604601979256,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.3414402008585886e-07,
      "logits/chosen": -1.0788323879241943,
      "logits/rejected": -0.9475747346878052,
      "logps/chosen": -95.7227554321289,
      "logps/ref_chosen": -64.22705841064453,
      "logps/ref_rejected": -73.10292053222656,
      "logps/rejected": -123.53993225097656,
      "loss": 1.105,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8982629179954529,
      "rewards/margins": 0.532755970954895,
      "rewards/rejected": -1.4310189485549927,
      "step": 377
    },
    {
      "epoch": 0.5714285714285714,
      "grad_norm": 10.464985847473145,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.028466958552598953,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.3282469092517977e-07,
      "logits/chosen": -1.3532541990280151,
      "logits/rejected": -1.3627448081970215,
      "logps/chosen": -108.75630187988281,
      "logps/ref_chosen": -76.90864562988281,
      "logps/ref_rejected": -90.53459930419922,
      "logps/rejected": -139.50833129882812,
      "loss": 1.1656,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.9033360481262207,
      "rewards/margins": 0.479516863822937,
      "rewards/rejected": -1.3828529119491577,
      "step": 378
    },
    {
      "epoch": 0.5729402872260015,
      "grad_norm": 15.603812217712402,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.028307726606726646,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 2.3150584219481643e-07,
      "logits/chosen": -1.583913803100586,
      "logits/rejected": -1.7535749673843384,
      "logps/chosen": -120.983154296875,
      "logps/ref_chosen": -91.2371597290039,
      "logps/ref_rejected": -120.1969985961914,
      "logps/rejected": -173.04830932617188,
      "loss": 1.0491,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8407952785491943,
      "rewards/margins": 0.6449382305145264,
      "rewards/rejected": -1.4857335090637207,
      "step": 379
    },
    {
      "epoch": 0.5744520030234316,
      "grad_norm": 8.755367279052734,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.028140641748905182,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 2.3018751078580283e-07,
      "logits/chosen": -1.3944610357284546,
      "logits/rejected": -1.2365813255310059,
      "logps/chosen": -99.56253051757812,
      "logps/ref_chosen": -77.78315734863281,
      "logps/ref_rejected": -92.56083679199219,
      "logps/rejected": -143.6357421875,
      "loss": 0.9618,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6120294332504272,
      "rewards/margins": 0.813266932964325,
      "rewards/rejected": -1.4252963066101074,
      "step": 380
    },
    {
      "epoch": 0.5759637188208617,
      "grad_norm": 12.221405982971191,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.02797454409301281,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.288697335747027e-07,
      "logits/chosen": -1.468564510345459,
      "logits/rejected": -1.4412519931793213,
      "logps/chosen": -109.62928771972656,
      "logps/ref_chosen": -75.28189086914062,
      "logps/ref_rejected": -81.1995849609375,
      "logps/rejected": -131.20687866210938,
      "loss": 1.2431,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.9610272645950317,
      "rewards/margins": 0.4295161962509155,
      "rewards/rejected": -1.3905434608459473,
      "step": 381
    },
    {
      "epoch": 0.5774754346182918,
      "grad_norm": 9.942258834838867,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.027887394651770592,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.2755254742257706e-07,
      "logits/chosen": -1.4594696760177612,
      "logits/rejected": -1.5281555652618408,
      "logps/chosen": -113.67582702636719,
      "logps/ref_chosen": -78.74870300292969,
      "logps/ref_rejected": -99.77484130859375,
      "logps/rejected": -154.184814453125,
      "loss": 1.0931,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9716081023216248,
      "rewards/margins": 0.535379946231842,
      "rewards/rejected": -1.5069879293441772,
      "step": 382
    },
    {
      "epoch": 0.5789871504157218,
      "grad_norm": 13.791141510009766,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.027765920385718346,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.2623598917395436e-07,
      "logits/chosen": -1.6387135982513428,
      "logits/rejected": -1.3257139921188354,
      "logps/chosen": -126.18170166015625,
      "logps/ref_chosen": -95.92772674560547,
      "logps/ref_rejected": -92.13604736328125,
      "logps/rejected": -146.85276794433594,
      "loss": 1.0978,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8407931923866272,
      "rewards/margins": 0.6689916849136353,
      "rewards/rejected": -1.5097849369049072,
      "step": 383
    },
    {
      "epoch": 0.5804988662131519,
      "grad_norm": 9.773907661437988,
      "kl/avg_steps": 0.546875,
      "kl/beta": 0.027644973248243332,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.2492009565579875e-07,
      "logits/chosen": -1.1795517206192017,
      "logits/rejected": -1.1752303838729858,
      "logps/chosen": -112.3473129272461,
      "logps/ref_chosen": -80.20899200439453,
      "logps/ref_rejected": -94.39380645751953,
      "logps/rejected": -150.83253479003906,
      "loss": 1.0315,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.885909378528595,
      "rewards/margins": 0.6628715991973877,
      "rewards/rejected": -1.548780918121338,
      "step": 384
    },
    {
      "epoch": 0.582010582010582,
      "grad_norm": 9.207103729248047,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.027494611218571663,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.2360490367648084e-07,
      "logits/chosen": -1.5247611999511719,
      "logits/rejected": -1.5652906894683838,
      "logps/chosen": -121.40055084228516,
      "logps/ref_chosen": -85.26632690429688,
      "logps/ref_rejected": -102.1983413696289,
      "logps/rejected": -163.07257080078125,
      "loss": 1.0272,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9921504259109497,
      "rewards/margins": 0.6710430979728699,
      "rewards/rejected": -1.6631935834884644,
      "step": 385
    },
    {
      "epoch": 0.5835222978080121,
      "grad_norm": 11.019001007080078,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.027383366599678993,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.2229045002474724e-07,
      "logits/chosen": -1.2989556789398193,
      "logits/rejected": -1.536360502243042,
      "logps/chosen": -135.63621520996094,
      "logps/ref_chosen": -93.19975280761719,
      "logps/ref_rejected": -112.98829650878906,
      "logps/rejected": -170.507568359375,
      "loss": 1.2507,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.1580798625946045,
      "rewards/margins": 0.40500038862228394,
      "rewards/rejected": -1.5630803108215332,
      "step": 386
    },
    {
      "epoch": 0.5850340136054422,
      "grad_norm": 8.650019645690918,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.02726408652961254,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.209767714686924e-07,
      "logits/chosen": -1.4123413562774658,
      "logits/rejected": -1.272484540939331,
      "logps/chosen": -100.4596939086914,
      "logps/ref_chosen": -66.32860565185547,
      "logps/ref_rejected": -100.56486511230469,
      "logps/rejected": -163.15472412109375,
      "loss": 0.9735,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9282127022743225,
      "rewards/margins": 0.7661646604537964,
      "rewards/rejected": -1.6943774223327637,
      "step": 387
    },
    {
      "epoch": 0.5865457294028723,
      "grad_norm": 17.062511444091797,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.027128444984555244,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.1966390475472954e-07,
      "logits/chosen": -1.4685077667236328,
      "logits/rejected": -1.4835941791534424,
      "logps/chosen": -132.53494262695312,
      "logps/ref_chosen": -92.95967864990234,
      "logps/ref_rejected": -97.9437255859375,
      "logps/rejected": -153.61680603027344,
      "loss": 1.2439,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.0734572410583496,
      "rewards/margins": 0.4278140664100647,
      "rewards/rejected": -1.5012712478637695,
      "step": 388
    },
    {
      "epoch": 0.5880574452003023,
      "grad_norm": 8.395482063293457,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.027043931186199188,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.1835188660656265e-07,
      "logits/chosen": -1.2796024084091187,
      "logits/rejected": -1.412411093711853,
      "logps/chosen": -111.56185913085938,
      "logps/ref_chosen": -76.89031982421875,
      "logps/ref_rejected": -93.79212951660156,
      "logps/rejected": -154.0435028076172,
      "loss": 1.02,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9355847835540771,
      "rewards/margins": 0.6820655465126038,
      "rewards/rejected": -1.6176502704620361,
      "step": 389
    },
    {
      "epoch": 0.5895691609977324,
      "grad_norm": 8.924866676330566,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.026901019737124443,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.170407537241599e-07,
      "logits/chosen": -1.3276596069335938,
      "logits/rejected": -1.1806645393371582,
      "logps/chosen": -88.36257934570312,
      "logps/ref_chosen": -61.05881881713867,
      "logps/ref_rejected": -79.55152893066406,
      "logps/rejected": -131.11795043945312,
      "loss": 1.0329,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.7331553101539612,
      "rewards/margins": 0.6438949108123779,
      "rewards/rejected": -1.3770502805709839,
      "step": 390
    },
    {
      "epoch": 0.5910808767951625,
      "grad_norm": 9.212340354919434,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.02675054781138897,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.1573054278272636e-07,
      "logits/chosen": -1.4783949851989746,
      "logits/rejected": -1.3130332231521606,
      "logps/chosen": -112.14136505126953,
      "logps/ref_chosen": -78.60820770263672,
      "logps/ref_rejected": -103.33676147460938,
      "logps/rejected": -168.4642333984375,
      "loss": 1.0115,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8957010507583618,
      "rewards/margins": 0.8333579301834106,
      "rewards/rejected": -1.729059100151062,
      "step": 391
    },
    {
      "epoch": 0.5925925925925926,
      "grad_norm": 8.47270679473877,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.0266257394105196,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.1442129043167873e-07,
      "logits/chosen": -1.5991801023483276,
      "logits/rejected": -1.2611522674560547,
      "logps/chosen": -114.23375701904297,
      "logps/ref_chosen": -86.99469757080078,
      "logps/ref_rejected": -112.73616790771484,
      "logps/rejected": -174.4199676513672,
      "loss": 0.9338,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.7235299348831177,
      "rewards/margins": 0.9055388569831848,
      "rewards/rejected": -1.6290687322616577,
      "step": 392
    },
    {
      "epoch": 0.5941043083900227,
      "grad_norm": 8.3270902633667,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.026485038921236992,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.131130332936195e-07,
      "logits/chosen": -1.2096143960952759,
      "logits/rejected": -1.2805571556091309,
      "logps/chosen": -108.41553497314453,
      "logps/ref_chosen": -71.26398468017578,
      "logps/ref_rejected": -88.99722290039062,
      "logps/rejected": -152.43466186523438,
      "loss": 1.0012,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9834790229797363,
      "rewards/margins": 0.6872824430465698,
      "rewards/rejected": -1.6707613468170166,
      "step": 393
    },
    {
      "epoch": 0.5956160241874527,
      "grad_norm": 9.936538696289062,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.026377879083156586,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.1180580796331323e-07,
      "logits/chosen": -1.1447997093200684,
      "logits/rejected": -1.3278568983078003,
      "logps/chosen": -113.02339935302734,
      "logps/ref_chosen": -78.70564270019531,
      "logps/ref_rejected": -87.01431274414062,
      "logps/rejected": -145.6602020263672,
      "loss": 1.0376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9038447141647339,
      "rewards/margins": 0.6327704191207886,
      "rewards/rejected": -1.5366151332855225,
      "step": 394
    },
    {
      "epoch": 0.5971277399848829,
      "grad_norm": 9.601051330566406,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.026238486170768738,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.104996510066625e-07,
      "logits/chosen": -1.2977375984191895,
      "logits/rejected": -1.5407524108886719,
      "logps/chosen": -100.53248596191406,
      "logps/ref_chosen": -65.30274963378906,
      "logps/ref_rejected": -93.22492980957031,
      "logps/rejected": -150.34323120117188,
      "loss": 1.0998,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9225190281867981,
      "rewards/margins": 0.5657308101654053,
      "rewards/rejected": -1.4882497787475586,
      "step": 395
    },
    {
      "epoch": 0.5986394557823129,
      "grad_norm": 9.81004810333252,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.02614045888185501,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 2.0919459895968517e-07,
      "logits/chosen": -1.2869203090667725,
      "logits/rejected": -1.2694096565246582,
      "logps/chosen": -102.52066040039062,
      "logps/ref_chosen": -67.33502197265625,
      "logps/ref_rejected": -98.8193359375,
      "logps/rejected": -159.61468505859375,
      "loss": 0.9813,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.915213406085968,
      "rewards/margins": 0.6612294912338257,
      "rewards/rejected": -1.576442837715149,
      "step": 396
    },
    {
      "epoch": 0.600151171579743,
      "grad_norm": 12.078638076782227,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.025978095829486847,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.078906883274924e-07,
      "logits/chosen": -1.2485475540161133,
      "logits/rejected": -1.1371123790740967,
      "logps/chosen": -126.35214233398438,
      "logps/ref_chosen": -89.6042251586914,
      "logps/ref_rejected": -104.9779052734375,
      "logps/rejected": -159.32118225097656,
      "loss": 1.2242,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9524945616722107,
      "rewards/margins": 0.4481835961341858,
      "rewards/rejected": -1.4006781578063965,
      "step": 397
    },
    {
      "epoch": 0.6016628873771731,
      "grad_norm": 8.951614379882812,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.025864936411380768,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.065879555832674e-07,
      "logits/chosen": -1.4257827997207642,
      "logits/rejected": -1.3792155981063843,
      "logps/chosen": -99.28257751464844,
      "logps/ref_chosen": -66.43465423583984,
      "logps/ref_rejected": -90.90376281738281,
      "logps/rejected": -153.3382568359375,
      "loss": 0.9944,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.847280740737915,
      "rewards/margins": 0.7553883790969849,
      "rewards/rejected": -1.6026690006256104,
      "step": 398
    },
    {
      "epoch": 0.6031746031746031,
      "grad_norm": 10.219222068786621,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.025760285556316376,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.052864371672457e-07,
      "logits/chosen": -1.0542405843734741,
      "logits/rejected": -1.3199812173843384,
      "logps/chosen": -131.15020751953125,
      "logps/ref_chosen": -87.22315979003906,
      "logps/ref_rejected": -136.32411193847656,
      "logps/rejected": -211.09661865234375,
      "loss": 0.9769,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.130582332611084,
      "rewards/margins": 0.7845978736877441,
      "rewards/rejected": -1.9151802062988281,
      "step": 399
    },
    {
      "epoch": 0.6046863189720333,
      "grad_norm": 13.402877807617188,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.02562415786087513,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 2.0398616948569493e-07,
      "logits/chosen": -1.2406623363494873,
      "logits/rejected": -1.7748053073883057,
      "logps/chosen": -139.27703857421875,
      "logps/ref_chosen": -91.1212158203125,
      "logps/ref_rejected": -108.19235229492188,
      "logps/rejected": -173.50042724609375,
      "loss": 1.2239,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.2333451509475708,
      "rewards/margins": 0.431148886680603,
      "rewards/rejected": -1.6644940376281738,
      "step": 400
    },
    {
      "epoch": 0.6046863189720333,
      "eval_kl/n_epsilon_steps": 0.2614436745643616,
      "eval_kl/p_epsilon_steps": 0.7372359037399292,
      "eval_logits/chosen": -1.249387264251709,
      "eval_logits/rejected": -1.3749488592147827,
      "eval_logps/chosen": -124.13003540039062,
      "eval_logps/ref_chosen": -86.90177917480469,
      "eval_logps/ref_rejected": -96.69639587402344,
      "eval_logps/rejected": -159.53257751464844,
      "eval_loss": 0.5372243523597717,
      "eval_rewards/accuracies": 0.7429577708244324,
      "eval_rewards/chosen": -0.950084388256073,
      "eval_rewards/margins": 0.6450637578964233,
      "eval_rewards/rejected": -1.5951480865478516,
      "eval_runtime": 46.8655,
      "eval_samples_per_second": 49.141,
      "eval_steps_per_second": 1.536,
      "step": 400
    },
    {
      "epoch": 0.6061980347694633,
      "grad_norm": 8.485547065734863,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.025560257956385612,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.0268718890989752e-07,
      "logits/chosen": -1.0869808197021484,
      "logits/rejected": -1.0903010368347168,
      "logps/chosen": -98.18634033203125,
      "logps/ref_chosen": -67.54151153564453,
      "logps/ref_rejected": -98.06488800048828,
      "logps/rejected": -162.6023406982422,
      "loss": 0.9048,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7803738117218018,
      "rewards/margins": 0.8559675216674805,
      "rewards/rejected": -1.6363413333892822,
      "step": 401
    },
    {
      "epoch": 0.6077097505668935,
      "grad_norm": 11.565892219543457,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.025417285040020943,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.013895317751323e-07,
      "logits/chosen": -1.3836373090744019,
      "logits/rejected": -1.0982781648635864,
      "logps/chosen": -110.84662628173828,
      "logps/ref_chosen": -77.44487762451172,
      "logps/ref_rejected": -83.1333236694336,
      "logps/rejected": -144.36251831054688,
      "loss": 0.993,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8493508100509644,
      "rewards/margins": 0.6979364156723022,
      "rewards/rejected": -1.5472872257232666,
      "step": 402
    },
    {
      "epoch": 0.6092214663643235,
      "grad_norm": 9.853551864624023,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.025306569412350655,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.0009323437965898e-07,
      "logits/chosen": -1.2379374504089355,
      "logits/rejected": -1.3333361148834229,
      "logps/chosen": -106.8541030883789,
      "logps/ref_chosen": -68.8230972290039,
      "logps/ref_rejected": -99.82356262207031,
      "logps/rejected": -171.64031982421875,
      "loss": 0.9869,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.9604059457778931,
      "rewards/margins": 0.8437454700469971,
      "rewards/rejected": -1.8041512966156006,
      "step": 403
    },
    {
      "epoch": 0.6107331821617535,
      "grad_norm": 9.483668327331543,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.02517283894121647,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.9879833298370237e-07,
      "logits/chosen": -1.5257608890533447,
      "logits/rejected": -1.4390606880187988,
      "logps/chosen": -115.48838806152344,
      "logps/ref_chosen": -80.26783752441406,
      "logps/ref_rejected": -111.60258483886719,
      "logps/rejected": -178.8803253173828,
      "loss": 0.9802,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8856651186943054,
      "rewards/margins": 0.7965131998062134,
      "rewards/rejected": -1.6821783781051636,
      "step": 404
    },
    {
      "epoch": 0.6122448979591837,
      "grad_norm": 10.482511520385742,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.02505928836762905,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.975048638084379e-07,
      "logits/chosen": -1.202235460281372,
      "logits/rejected": -1.3670843839645386,
      "logps/chosen": -107.26358032226562,
      "logps/ref_chosen": -68.31065368652344,
      "logps/ref_rejected": -81.56044006347656,
      "logps/rejected": -144.6019287109375,
      "loss": 1.0878,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.9755101203918457,
      "rewards/margins": 0.5946328043937683,
      "rewards/rejected": -1.5701429843902588,
      "step": 405
    },
    {
      "epoch": 0.6137566137566137,
      "grad_norm": 9.058980941772461,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.024973442777991295,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 1.9621286303497914e-07,
      "logits/chosen": -1.4929070472717285,
      "logits/rejected": -1.4613780975341797,
      "logps/chosen": -98.95768737792969,
      "logps/ref_chosen": -64.86714172363281,
      "logps/ref_rejected": -110.06051635742188,
      "logps/rejected": -177.30059814453125,
      "loss": 0.9677,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.8477808833122253,
      "rewards/margins": 0.817082405090332,
      "rewards/rejected": -1.6648633480072021,
      "step": 406
    },
    {
      "epoch": 0.6152683295540439,
      "grad_norm": 16.868221282958984,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.02481832727789879,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.9492236680336483e-07,
      "logits/chosen": -1.4997903108596802,
      "logits/rejected": -1.9023494720458984,
      "logps/chosen": -155.658447265625,
      "logps/ref_chosen": -102.01712799072266,
      "logps/ref_rejected": -121.53548431396484,
      "logps/rejected": -199.111572265625,
      "loss": 1.0753,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.3289562463760376,
      "rewards/margins": 0.5858536958694458,
      "rewards/rejected": -1.9148099422454834,
      "step": 407
    },
    {
      "epoch": 0.6167800453514739,
      "grad_norm": 8.977225303649902,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.024714065715670586,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.9363341121154895e-07,
      "logits/chosen": -1.303640604019165,
      "logits/rejected": -1.3134398460388184,
      "logps/chosen": -109.69934844970703,
      "logps/ref_chosen": -72.77989959716797,
      "logps/ref_rejected": -92.01815795898438,
      "logps/rejected": -163.4852294921875,
      "loss": 0.9017,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9086741805076599,
      "rewards/margins": 0.8438174724578857,
      "rewards/rejected": -1.7524917125701904,
      "step": 408
    },
    {
      "epoch": 0.618291761148904,
      "grad_norm": 11.181131362915039,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.02459110878407955,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 1.9234603231438994e-07,
      "logits/chosen": -1.167852520942688,
      "logits/rejected": -1.0533185005187988,
      "logps/chosen": -126.26255798339844,
      "logps/ref_chosen": -77.7901611328125,
      "logps/ref_rejected": -79.2997055053711,
      "logps/rejected": -147.17803955078125,
      "loss": 1.2168,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -1.1924008131027222,
      "rewards/margins": 0.46864524483680725,
      "rewards/rejected": -1.661046028137207,
      "step": 409
    },
    {
      "epoch": 0.6198034769463341,
      "grad_norm": 11.200785636901855,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.02452978491783142,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -1.6598620414733887,
      "logits/rejected": -1.5288690328598022,
      "logps/chosen": -122.84921264648438,
      "logps/ref_chosen": -80.35844421386719,
      "logps/ref_rejected": -92.19056701660156,
      "logps/rejected": -167.47018432617188,
      "loss": 0.9488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0397820472717285,
      "rewards/margins": 0.7941303849220276,
      "rewards/rejected": -1.8339124917984009,
      "step": 410
    },
    {
      "epoch": 0.6213151927437641,
      "grad_norm": 18.039907455444336,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.024407746270298958,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.8977614860195296e-07,
      "logits/chosen": -1.147395133972168,
      "logits/rejected": -1.2293355464935303,
      "logps/chosen": -113.1170654296875,
      "logps/ref_chosen": -70.72857666015625,
      "logps/ref_rejected": -93.19205474853516,
      "logps/rejected": -168.95037841796875,
      "loss": 0.963,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.0353784561157227,
      "rewards/margins": 0.8042628765106201,
      "rewards/rejected": -1.8396413326263428,
      "step": 411
    },
    {
      "epoch": 0.6228269085411943,
      "grad_norm": 11.914444923400879,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.024293867871165276,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.8849371567184662e-07,
      "logits/chosen": -0.9596520662307739,
      "logits/rejected": -0.8119238615036011,
      "logps/chosen": -122.37300872802734,
      "logps/ref_chosen": -72.87568664550781,
      "logps/ref_rejected": -88.21068572998047,
      "logps/rejected": -167.08526611328125,
      "loss": 0.9993,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.200660228729248,
      "rewards/margins": 0.7046034336090088,
      "rewards/rejected": -1.9052636623382568,
      "step": 412
    },
    {
      "epoch": 0.6243386243386243,
      "grad_norm": 12.280434608459473,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.02416548877954483,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.872130032047302e-07,
      "logits/chosen": -1.359757900238037,
      "logits/rejected": -1.4349701404571533,
      "logps/chosen": -136.6390838623047,
      "logps/ref_chosen": -84.70051574707031,
      "logps/ref_rejected": -92.06742095947266,
      "logps/rejected": -169.40145874023438,
      "loss": 1.1385,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.253061294555664,
      "rewards/margins": 0.6037291288375854,
      "rewards/rejected": -1.8567904233932495,
      "step": 413
    },
    {
      "epoch": 0.6258503401360545,
      "grad_norm": 9.417985916137695,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.02405274286866188,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.8593404702488436e-07,
      "logits/chosen": -0.892124354839325,
      "logits/rejected": -0.8772637844085693,
      "logps/chosen": -122.20513916015625,
      "logps/ref_chosen": -70.97660827636719,
      "logps/ref_rejected": -92.90523529052734,
      "logps/rejected": -174.98516845703125,
      "loss": 1.0183,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.2277061939239502,
      "rewards/margins": 0.73204505443573,
      "rewards/rejected": -1.9597512483596802,
      "step": 414
    },
    {
      "epoch": 0.6273620559334845,
      "grad_norm": 12.158763885498047,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.02392563782632351,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.846568829074628e-07,
      "logits/chosen": -1.1391077041625977,
      "logits/rejected": -1.0729453563690186,
      "logps/chosen": -122.39847564697266,
      "logps/ref_chosen": -71.7189712524414,
      "logps/ref_rejected": -74.54219818115234,
      "logps/rejected": -152.57752990722656,
      "loss": 1.1305,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.2116117477416992,
      "rewards/margins": 0.6440663933753967,
      "rewards/rejected": -1.8556783199310303,
      "step": 415
    },
    {
      "epoch": 0.6288737717309146,
      "grad_norm": 10.729681015014648,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.023836251348257065,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.8338154657749128e-07,
      "logits/chosen": -1.4660162925720215,
      "logits/rejected": -1.5791254043579102,
      "logps/chosen": -126.84781646728516,
      "logps/ref_chosen": -72.88249206542969,
      "logps/ref_rejected": -85.30692291259766,
      "logps/rejected": -160.66006469726562,
      "loss": 1.2088,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.285239338874817,
      "rewards/margins": 0.5007971525192261,
      "rewards/rejected": -1.786036491394043,
      "step": 416
    },
    {
      "epoch": 0.6303854875283447,
      "grad_norm": 10.104597091674805,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.023739809170365334,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 1.8210807370886849e-07,
      "logits/chosen": -1.3048408031463623,
      "logits/rejected": -1.5203348398208618,
      "logps/chosen": -124.10682678222656,
      "logps/ref_chosen": -72.49703216552734,
      "logps/ref_rejected": -89.38966369628906,
      "logps/rejected": -176.7666015625,
      "loss": 0.9937,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.2212988138198853,
      "rewards/margins": 0.837921142578125,
      "rewards/rejected": -2.0592198371887207,
      "step": 417
    },
    {
      "epoch": 0.6318972033257747,
      "grad_norm": 11.858148574829102,
      "kl/avg_steps": 0.265625,
      "kl/beta": 0.02360701933503151,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 1.8083649992336825e-07,
      "logits/chosen": -1.4539004564285278,
      "logits/rejected": -1.4051744937896729,
      "logps/chosen": -150.91690063476562,
      "logps/ref_chosen": -89.70926666259766,
      "logps/ref_rejected": -90.98756408691406,
      "logps/rejected": -169.56109619140625,
      "loss": 1.2647,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.445763111114502,
      "rewards/margins": 0.4013369083404541,
      "rewards/rejected": -1.8470999002456665,
      "step": 418
    },
    {
      "epoch": 0.6334089191232048,
      "grad_norm": 10.206515312194824,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.02354447916150093,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 1.7956686078964255e-07,
      "logits/chosen": -1.3517265319824219,
      "logits/rejected": -1.5228190422058105,
      "logps/chosen": -115.05638122558594,
      "logps/ref_chosen": -75.65210723876953,
      "logps/ref_rejected": -91.00135040283203,
      "logps/rejected": -171.54263305664062,
      "loss": 0.8518,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.9251964092254639,
      "rewards/margins": 0.9574536085128784,
      "rewards/rejected": -1.8826498985290527,
      "step": 419
    },
    {
      "epoch": 0.6349206349206349,
      "grad_norm": 12.082380294799805,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.02339823916554451,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.782991918222275e-07,
      "logits/chosen": -1.1388458013534546,
      "logits/rejected": -1.2260863780975342,
      "logps/chosen": -128.3040771484375,
      "logps/ref_chosen": -72.58028411865234,
      "logps/ref_rejected": -79.90303039550781,
      "logps/rejected": -158.22250366210938,
      "loss": 1.2252,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3038926124572754,
      "rewards/margins": 0.5187857151031494,
      "rewards/rejected": -1.8226780891418457,
      "step": 420
    },
    {
      "epoch": 0.636432350718065,
      "grad_norm": 11.983504295349121,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.023318083956837654,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 1.7703352848054887e-07,
      "logits/chosen": -1.274233341217041,
      "logits/rejected": -1.7651447057724,
      "logps/chosen": -131.89462280273438,
      "logps/ref_chosen": -78.71546936035156,
      "logps/ref_rejected": -90.82321166992188,
      "logps/rejected": -167.80934143066406,
      "loss": 1.2296,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.2419317960739136,
      "rewards/margins": 0.5444414615631104,
      "rewards/rejected": -1.7863733768463135,
      "step": 421
    },
    {
      "epoch": 0.6379440665154951,
      "grad_norm": 11.744466781616211,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.023252686485648155,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.7576990616793137e-07,
      "logits/chosen": -1.4543811082839966,
      "logits/rejected": -1.436232566833496,
      "logps/chosen": -134.90113830566406,
      "logps/ref_chosen": -86.74519348144531,
      "logps/ref_rejected": -94.02015686035156,
      "logps/rejected": -169.43231201171875,
      "loss": 1.0689,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1186872720718384,
      "rewards/margins": 0.6243889331817627,
      "rewards/rejected": -1.7430763244628906,
      "step": 422
    },
    {
      "epoch": 0.6394557823129252,
      "grad_norm": 9.857087135314941,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.0231586042791605,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.745083602306071e-07,
      "logits/chosen": -1.21421217918396,
      "logits/rejected": -1.77446448802948,
      "logps/chosen": -122.7453384399414,
      "logps/ref_chosen": -72.02232360839844,
      "logps/ref_rejected": -93.269775390625,
      "logps/rejected": -175.97735595703125,
      "loss": 1.0005,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.1713683605194092,
      "rewards/margins": 0.7311166524887085,
      "rewards/rejected": -1.9024851322174072,
      "step": 423
    },
    {
      "epoch": 0.6409674981103552,
      "grad_norm": 10.320049285888672,
      "kl/avg_steps": 0.6875,
      "kl/beta": 0.023036224767565727,
      "kl/n_epsilon_steps": 0.15625,
      "kl/p_epsilon_steps": 0.84375,
      "learning_rate": 1.7324892595672804e-07,
      "logits/chosen": -1.5990121364593506,
      "logits/rejected": -1.411468505859375,
      "logps/chosen": -121.66751098632812,
      "logps/ref_chosen": -68.22148132324219,
      "logps/ref_rejected": -94.12411499023438,
      "logps/rejected": -182.5116424560547,
      "loss": 0.9414,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.224773645401001,
      "rewards/margins": 0.7947477698326111,
      "rewards/rejected": -2.019521474838257,
      "step": 424
    },
    {
      "epoch": 0.6424792139077853,
      "grad_norm": 11.145761489868164,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.02287893183529377,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.7199163857537824e-07,
      "logits/chosen": -0.9588738679885864,
      "logits/rejected": -0.9888179898262024,
      "logps/chosen": -124.32681274414062,
      "logps/ref_chosen": -75.90104675292969,
      "logps/ref_rejected": -86.08673095703125,
      "logps/rejected": -165.00479125976562,
      "loss": 0.9915,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.1062071323394775,
      "rewards/margins": 0.6885690689086914,
      "rewards/rejected": -1.794776201248169,
      "step": 425
    },
    {
      "epoch": 0.6439909297052154,
      "grad_norm": 13.805582046508789,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.022765105590224266,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 1.7073653325558828e-07,
      "logits/chosen": -1.2274202108383179,
      "logits/rejected": -1.1886839866638184,
      "logps/chosen": -151.05612182617188,
      "logps/ref_chosen": -89.93118286132812,
      "logps/ref_rejected": -91.04658508300781,
      "logps/rejected": -170.2470703125,
      "loss": 1.2932,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3905560970306396,
      "rewards/margins": 0.4026485085487366,
      "rewards/rejected": -1.7932045459747314,
      "step": 426
    },
    {
      "epoch": 0.6455026455026455,
      "grad_norm": 10.216066360473633,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.022701257839798927,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.6948364510535218e-07,
      "logits/chosen": -1.3483772277832031,
      "logits/rejected": -1.4812953472137451,
      "logps/chosen": -135.0600128173828,
      "logps/ref_chosen": -77.83393859863281,
      "logps/ref_rejected": -98.69865417480469,
      "logps/rejected": -183.77285766601562,
      "loss": 1.0638,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2972296476364136,
      "rewards/margins": 0.6236185431480408,
      "rewards/rejected": -1.9208481311798096,
      "step": 427
    },
    {
      "epoch": 0.6470143613000756,
      "grad_norm": 10.091684341430664,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.022602373734116554,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.6823300917064458e-07,
      "logits/chosen": -1.261292576789856,
      "logits/rejected": -1.0671913623809814,
      "logps/chosen": -145.90863037109375,
      "logps/ref_chosen": -90.3450927734375,
      "logps/ref_rejected": -100.24185180664062,
      "logps/rejected": -189.25491333007812,
      "loss": 1.0105,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.2532129287719727,
      "rewards/margins": 0.7458755970001221,
      "rewards/rejected": -1.9990885257720947,
      "step": 428
    },
    {
      "epoch": 0.6485260770975056,
      "grad_norm": 11.277990341186523,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.022503918036818504,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.669846604344412e-07,
      "logits/chosen": -1.235177993774414,
      "logits/rejected": -1.189939022064209,
      "logps/chosen": -137.588134765625,
      "logps/ref_chosen": -78.24811553955078,
      "logps/ref_rejected": -75.24494934082031,
      "logps/rejected": -165.09100341796875,
      "loss": 1.1433,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3354589939117432,
      "rewards/margins": 0.6751536726951599,
      "rewards/rejected": -2.0106124877929688,
      "step": 429
    },
    {
      "epoch": 0.6500377928949358,
      "grad_norm": 8.702858924865723,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.022405892610549927,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.6573863381573954e-07,
      "logits/chosen": -1.2727080583572388,
      "logits/rejected": -0.9296888113021851,
      "logps/chosen": -130.50823974609375,
      "logps/ref_chosen": -76.08027648925781,
      "logps/ref_rejected": -84.09554290771484,
      "logps/rejected": -175.60972595214844,
      "loss": 0.9741,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.2184827327728271,
      "rewards/margins": 0.8200922012329102,
      "rewards/rejected": -2.0385749340057373,
      "step": 430
    },
    {
      "epoch": 0.6515495086923658,
      "grad_norm": 11.595307350158691,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.02229442074894905,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.6449496416858282e-07,
      "logits/chosen": -1.2344250679016113,
      "logits/rejected": -1.259427547454834,
      "logps/chosen": -116.51612854003906,
      "logps/ref_chosen": -66.88581085205078,
      "logps/ref_rejected": -89.56040954589844,
      "logps/rejected": -169.1386260986328,
      "loss": 1.0887,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.1038061380386353,
      "rewards/margins": 0.6578041315078735,
      "rewards/rejected": -1.7616102695465088,
      "step": 431
    },
    {
      "epoch": 0.6530612244897959,
      "grad_norm": 10.99130630493164,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.022190403193235397,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 1.632536862810844e-07,
      "logits/chosen": -1.6262152194976807,
      "logits/rejected": -1.450348973274231,
      "logps/chosen": -132.7317352294922,
      "logps/ref_chosen": -79.65066528320312,
      "logps/ref_rejected": -103.92634582519531,
      "logps/rejected": -187.55831909179688,
      "loss": 1.1065,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.1787724494934082,
      "rewards/margins": 0.6679859161376953,
      "rewards/rejected": -1.846758246421814,
      "step": 432
    },
    {
      "epoch": 0.654572940287226,
      "grad_norm": 8.838223457336426,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.022128168493509293,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.6201483487445515e-07,
      "logits/chosen": -1.1491540670394897,
      "logits/rejected": -1.1288440227508545,
      "logps/chosen": -128.4044952392578,
      "logps/ref_chosen": -77.30774688720703,
      "logps/ref_rejected": -81.65180206298828,
      "logps/rejected": -174.46002197265625,
      "loss": 0.9816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1291104555130005,
      "rewards/margins": 0.9108319282531738,
      "rewards/rejected": -2.039942502975464,
      "step": 433
    },
    {
      "epoch": 0.656084656084656,
      "grad_norm": 8.635628700256348,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.022018076851963997,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.6077844460203204e-07,
      "logits/chosen": -1.2930514812469482,
      "logits/rejected": -1.4299508333206177,
      "logps/chosen": -105.981689453125,
      "logps/ref_chosen": -63.31850051879883,
      "logps/ref_rejected": -89.15093994140625,
      "logps/rejected": -168.12149047851562,
      "loss": 1.0291,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9386637806892395,
      "rewards/margins": 0.7881932854652405,
      "rewards/rejected": -1.7268571853637695,
      "step": 434
    },
    {
      "epoch": 0.6575963718820862,
      "grad_norm": 10.655919075012207,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.02192899025976658,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 1.5954455004830878e-07,
      "logits/chosen": -1.2391023635864258,
      "logits/rejected": -1.1685447692871094,
      "logps/chosen": -127.31586456298828,
      "logps/ref_chosen": -71.1719741821289,
      "logps/ref_rejected": -86.42095184326172,
      "logps/rejected": -169.24099731445312,
      "loss": 1.1213,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.2311064004898071,
      "rewards/margins": 0.575607419013977,
      "rewards/rejected": -1.8067138195037842,
      "step": 435
    },
    {
      "epoch": 0.6591080876795162,
      "grad_norm": 10.871426582336426,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.02186748757958412,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.5831318572796847e-07,
      "logits/chosen": -1.319658637046814,
      "logits/rejected": -1.2668474912643433,
      "logps/chosen": -123.50926208496094,
      "logps/ref_chosen": -74.45087432861328,
      "logps/ref_rejected": -86.01708984375,
      "logps/rejected": -165.40402221679688,
      "loss": 1.0927,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.072027564048767,
      "rewards/margins": 0.6532148718833923,
      "rewards/rejected": -1.7252424955368042,
      "step": 436
    },
    {
      "epoch": 0.6606198034769464,
      "grad_norm": 11.484429359436035,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.021785791963338852,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 1.5708438608491815e-07,
      "logits/chosen": -0.9691067337989807,
      "logits/rejected": -1.2407987117767334,
      "logps/chosen": -133.60284423828125,
      "logps/ref_chosen": -72.38908386230469,
      "logps/ref_rejected": -111.03279876708984,
      "logps/rejected": -195.80502319335938,
      "loss": 1.2222,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.33245050907135,
      "rewards/margins": 0.5034982562065125,
      "rewards/rejected": -1.8359487056732178,
      "step": 437
    },
    {
      "epoch": 0.6621315192743764,
      "grad_norm": 10.047585487365723,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.02172469161450863,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.558581854913253e-07,
      "logits/chosen": -1.2113059759140015,
      "logits/rejected": -1.1109917163848877,
      "logps/chosen": -108.18560791015625,
      "logps/ref_chosen": -57.27682876586914,
      "logps/ref_rejected": -83.07940673828125,
      "logps/rejected": -171.33041381835938,
      "loss": 0.9888,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.106644630432129,
      "rewards/margins": 0.8007879257202148,
      "rewards/rejected": -1.9074325561523438,
      "step": 438
    },
    {
      "epoch": 0.6636432350718064,
      "grad_norm": 8.809502601623535,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.02163005992770195,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 1.5463461824665658e-07,
      "logits/chosen": -1.3769385814666748,
      "logits/rejected": -1.4923155307769775,
      "logps/chosen": -153.1571044921875,
      "logps/ref_chosen": -98.35890197753906,
      "logps/ref_rejected": -112.69817352294922,
      "logps/rejected": -200.78346252441406,
      "loss": 1.0063,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.1812466382980347,
      "rewards/margins": 0.7106728553771973,
      "rewards/rejected": -1.8919193744659424,
      "step": 439
    },
    {
      "epoch": 0.6651549508692366,
      "grad_norm": 9.849800109863281,
      "kl/avg_steps": 0.65625,
      "kl/beta": 0.021509071812033653,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.828125,
      "learning_rate": 1.534137185767178e-07,
      "logits/chosen": -1.2204618453979492,
      "logits/rejected": -1.4051882028579712,
      "logps/chosen": -104.81169128417969,
      "logps/ref_chosen": -61.662452697753906,
      "logps/ref_rejected": -86.81646728515625,
      "logps/rejected": -172.91555786132812,
      "loss": 0.8625,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.923372745513916,
      "rewards/margins": 0.9133882522583008,
      "rewards/rejected": -1.8367609977722168,
      "step": 440
    },
    {
      "epoch": 0.6666666666666666,
      "grad_norm": 10.69460678100586,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.021368838846683502,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 1.521955206326976e-07,
      "logits/chosen": -1.348102331161499,
      "logits/rejected": -1.6415989398956299,
      "logps/chosen": -119.5263671875,
      "logps/ref_chosen": -74.33235168457031,
      "logps/ref_rejected": -99.654541015625,
      "logps/rejected": -179.17562866210938,
      "loss": 0.9737,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.9617807865142822,
      "rewards/margins": 0.7241696119308472,
      "rewards/rejected": -1.6859502792358398,
      "step": 441
    },
    {
      "epoch": 0.6681783824640968,
      "grad_norm": 11.308244705200195,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.021242709830403328,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.5098005849021078e-07,
      "logits/chosen": -1.4728314876556396,
      "logits/rejected": -1.384155035018921,
      "logps/chosen": -144.53326416015625,
      "logps/ref_chosen": -82.42591857910156,
      "logps/ref_rejected": -106.71090698242188,
      "logps/rejected": -197.6875,
      "loss": 1.0805,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.3178826570510864,
      "rewards/margins": 0.6041462421417236,
      "rewards/rejected": -1.9220290184020996,
      "step": 442
    },
    {
      "epoch": 0.6696900982615268,
      "grad_norm": 11.918970108032227,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.021156759932637215,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.4976736614834662e-07,
      "logits/chosen": -1.001205563545227,
      "logits/rejected": -1.202185869216919,
      "logps/chosen": -124.41624450683594,
      "logps/ref_chosen": -72.87019348144531,
      "logps/ref_rejected": -94.48143005371094,
      "logps/rejected": -187.20260620117188,
      "loss": 0.9738,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.0882420539855957,
      "rewards/margins": 0.8598648905754089,
      "rewards/rejected": -1.9481067657470703,
      "step": 443
    },
    {
      "epoch": 0.671201814058957,
      "grad_norm": 14.023918151855469,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.021044958382844925,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.4855747752871654e-07,
      "logits/chosen": -1.3784263134002686,
      "logits/rejected": -1.6147680282592773,
      "logps/chosen": -137.09361267089844,
      "logps/ref_chosen": -74.65039825439453,
      "logps/ref_rejected": -106.89204406738281,
      "logps/rejected": -185.17578125,
      "loss": 1.2926,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3123301267623901,
      "rewards/margins": 0.32617539167404175,
      "rewards/rejected": -1.638505458831787,
      "step": 444
    },
    {
      "epoch": 0.672713529856387,
      "grad_norm": 13.924370765686035,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.02097286470234394,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.473504264745062e-07,
      "logits/chosen": -1.3273162841796875,
      "logits/rejected": -1.1322157382965088,
      "logps/chosen": -133.45980834960938,
      "logps/ref_chosen": -76.26957702636719,
      "logps/ref_rejected": -89.84994506835938,
      "logps/rejected": -186.78306579589844,
      "loss": 0.973,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.1988749504089355,
      "rewards/margins": 0.8226101398468018,
      "rewards/rejected": -2.021484851837158,
      "step": 445
    },
    {
      "epoch": 0.674225245653817,
      "grad_norm": 9.73694896697998,
      "kl/avg_steps": 0.65625,
      "kl/beta": 0.02087501250207424,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.828125,
      "learning_rate": 1.461462467495284e-07,
      "logits/chosen": -1.1069025993347168,
      "logits/rejected": -1.1735448837280273,
      "logps/chosen": -109.88125610351562,
      "logps/ref_chosen": -62.74647521972656,
      "logps/ref_rejected": -86.395751953125,
      "logps/rejected": -176.1666259765625,
      "loss": 0.8619,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.9798285365104675,
      "rewards/margins": 0.8798956274986267,
      "rewards/rejected": -1.8597241640090942,
      "step": 446
    },
    {
      "epoch": 0.6757369614512472,
      "grad_norm": 10.825597763061523,
      "kl/avg_steps": 0.546875,
      "kl/beta": 0.020738914608955383,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.4494497203727843e-07,
      "logits/chosen": -1.481667399406433,
      "logits/rejected": -1.7828559875488281,
      "logps/chosen": -115.90046691894531,
      "logps/ref_chosen": -71.06666564941406,
      "logps/ref_rejected": -103.57110595703125,
      "logps/rejected": -184.4591064453125,
      "loss": 1.0198,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9274003505706787,
      "rewards/margins": 0.7373617887496948,
      "rewards/rejected": -1.6647621393203735,
      "step": 447
    },
    {
      "epoch": 0.6772486772486772,
      "grad_norm": 9.556791305541992,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.020626114681363106,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.4374663593999256e-07,
      "logits/chosen": -1.3543277978897095,
      "logits/rejected": -1.264389991760254,
      "logps/chosen": -129.90399169921875,
      "logps/ref_chosen": -73.400146484375,
      "logps/ref_rejected": -96.34330749511719,
      "logps/rejected": -183.10574340820312,
      "loss": 1.0692,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.161041498184204,
      "rewards/margins": 0.6152825355529785,
      "rewards/rejected": -1.776323914527893,
      "step": 448
    },
    {
      "epoch": 0.6787603930461074,
      "grad_norm": 14.48768424987793,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.02052988111972809,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 1.4255127197770707e-07,
      "logits/chosen": -1.723862886428833,
      "logits/rejected": -1.4851081371307373,
      "logps/chosen": -156.7047576904297,
      "logps/ref_chosen": -93.66099548339844,
      "logps/ref_rejected": -102.53019714355469,
      "logps/rejected": -185.07656860351562,
      "loss": 1.2583,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.292679786682129,
      "rewards/margins": 0.39184582233428955,
      "rewards/rejected": -1.6845256090164185,
      "step": 449
    },
    {
      "epoch": 0.6802721088435374,
      "grad_norm": 8.63284969329834,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.020478684455156326,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.4135891358732205e-07,
      "logits/chosen": -1.5146286487579346,
      "logits/rejected": -1.8276360034942627,
      "logps/chosen": -109.09320831298828,
      "logps/ref_chosen": -62.52460479736328,
      "logps/ref_rejected": -94.04987335205078,
      "logps/rejected": -168.84970092773438,
      "loss": 1.0938,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.952759861946106,
      "rewards/margins": 0.569326639175415,
      "rewards/rejected": -1.522086501121521,
      "step": 450
    },
    {
      "epoch": 0.6817838246409675,
      "grad_norm": 8.414955139160156,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.02040852978825569,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.4016959412166437e-07,
      "logits/chosen": -1.3371424674987793,
      "logits/rejected": -1.300230622291565,
      "logps/chosen": -128.77247619628906,
      "logps/ref_chosen": -79.14009094238281,
      "logps/ref_rejected": -93.23920440673828,
      "logps/rejected": -169.08416748046875,
      "loss": 1.1148,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.0107604265213013,
      "rewards/margins": 0.5268110036849976,
      "rewards/rejected": -1.5375714302062988,
      "step": 451
    },
    {
      "epoch": 0.6832955404383976,
      "grad_norm": 8.979512214660645,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.02033228427171707,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.3898334684855645e-07,
      "logits/chosen": -1.4430395364761353,
      "logits/rejected": -1.396399736404419,
      "logps/chosen": -122.95205688476562,
      "logps/ref_chosen": -70.38827514648438,
      "logps/ref_rejected": -95.47691345214844,
      "logps/rejected": -180.23406982421875,
      "loss": 1.0688,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0676555633544922,
      "rewards/margins": 0.644894540309906,
      "rewards/rejected": -1.7125499248504639,
      "step": 452
    },
    {
      "epoch": 0.6848072562358276,
      "grad_norm": 9.879496574401855,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.020250018686056137,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.3780020494988445e-07,
      "logits/chosen": -1.1745672225952148,
      "logits/rejected": -1.3190906047821045,
      "logps/chosen": -132.80862426757812,
      "logps/ref_chosen": -79.9207763671875,
      "logps/ref_rejected": -90.20779418945312,
      "logps/rejected": -173.9451904296875,
      "loss": 1.1077,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0687193870544434,
      "rewards/margins": 0.6148155927658081,
      "rewards/rejected": -1.683534860610962,
      "step": 453
    },
    {
      "epoch": 0.6863189720332578,
      "grad_norm": 9.429729461669922,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.020155539736151695,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.366202015206706e-07,
      "logits/chosen": -1.3146743774414062,
      "logits/rejected": -1.2765040397644043,
      "logps/chosen": -113.04515075683594,
      "logps/ref_chosen": -69.71887969970703,
      "logps/ref_rejected": -82.86952209472656,
      "logps/rejected": -163.2214813232422,
      "loss": 1.0098,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8697865605354309,
      "rewards/margins": 0.7362134456634521,
      "rewards/rejected": -1.6060001850128174,
      "step": 454
    },
    {
      "epoch": 0.6878306878306878,
      "grad_norm": 8.902891159057617,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.020055262371897697,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 1.354433695681474e-07,
      "logits/chosen": -1.547599196434021,
      "logits/rejected": -1.477367877960205,
      "logps/chosen": -142.42257690429688,
      "logps/ref_chosen": -89.51481628417969,
      "logps/ref_rejected": -97.93235778808594,
      "logps/rejected": -190.3402099609375,
      "loss": 0.9365,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0567333698272705,
      "rewards/margins": 0.7826640605926514,
      "rewards/rejected": -1.8393973112106323,
      "step": 455
    },
    {
      "epoch": 0.6893424036281179,
      "grad_norm": 9.697734832763672,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.01993688754737377,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 1.3426974201083439e-07,
      "logits/chosen": -1.1608035564422607,
      "logits/rejected": -1.2271933555603027,
      "logps/chosen": -125.051513671875,
      "logps/ref_chosen": -74.60526275634766,
      "logps/ref_rejected": -97.98377227783203,
      "logps/rejected": -183.13381958007812,
      "loss": 0.9867,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.0037286281585693,
      "rewards/margins": 0.6831802129745483,
      "rewards/rejected": -1.6869087219238281,
      "step": 456
    },
    {
      "epoch": 0.690854119425548,
      "grad_norm": 9.47767448425293,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.019825369119644165,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.3309935167761717e-07,
      "logits/chosen": -1.1805062294006348,
      "logits/rejected": -1.66481614112854,
      "logps/chosen": -122.52874755859375,
      "logps/ref_chosen": -63.927032470703125,
      "logps/ref_rejected": -83.15243530273438,
      "logps/rejected": -170.2135772705078,
      "loss": 1.0579,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.1584389209747314,
      "rewards/margins": 0.5565627217292786,
      "rewards/rejected": -1.7150015830993652,
      "step": 457
    },
    {
      "epoch": 0.6923658352229781,
      "grad_norm": 10.947969436645508,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.019726736471056938,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 1.3193223130682936e-07,
      "logits/chosen": -1.3556028604507446,
      "logits/rejected": -1.7164582014083862,
      "logps/chosen": -115.37059020996094,
      "logps/ref_chosen": -67.68869018554688,
      "logps/ref_rejected": -104.40899658203125,
      "logps/rejected": -187.53775024414062,
      "loss": 0.9928,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.9373903274536133,
      "rewards/margins": 0.6901355981826782,
      "rewards/rejected": -1.627525806427002,
      "step": 458
    },
    {
      "epoch": 0.6938775510204082,
      "grad_norm": 10.526549339294434,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.019610300660133362,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.3076841354533658e-07,
      "logits/chosen": -1.671908974647522,
      "logits/rejected": -1.4922375679016113,
      "logps/chosen": -131.16941833496094,
      "logps/ref_chosen": -83.82363891601562,
      "logps/ref_rejected": -103.7593765258789,
      "logps/rejected": -188.73544311523438,
      "loss": 0.984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9256250262260437,
      "rewards/margins": 0.7286085486412048,
      "rewards/rejected": -1.654233694076538,
      "step": 459
    },
    {
      "epoch": 0.6953892668178382,
      "grad_norm": 9.452888488769531,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.01950667053461075,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 1.2960793094762345e-07,
      "logits/chosen": -1.1332123279571533,
      "logits/rejected": -1.6192915439605713,
      "logps/chosen": -136.13206481933594,
      "logps/ref_chosen": -79.4836654663086,
      "logps/ref_rejected": -112.31745910644531,
      "logps/rejected": -207.57310485839844,
      "loss": 0.9451,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.1002308130264282,
      "rewards/margins": 0.744149386882782,
      "rewards/rejected": -1.844380259513855,
      "step": 460
    },
    {
      "epoch": 0.6969009826152683,
      "grad_norm": 9.566939353942871,
      "kl/avg_steps": 0.515625,
      "kl/beta": 0.019385511055588722,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.2845081597488286e-07,
      "logits/chosen": -1.3776705265045166,
      "logits/rejected": -1.5931401252746582,
      "logps/chosen": -109.80137634277344,
      "logps/ref_chosen": -64.28482055664062,
      "logps/ref_rejected": -93.73818969726562,
      "logps/rejected": -176.29751586914062,
      "loss": 0.9552,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.8800297975540161,
      "rewards/margins": 0.7092906832695007,
      "rewards/rejected": -1.5893205404281616,
      "step": 461
    },
    {
      "epoch": 0.6984126984126984,
      "grad_norm": 9.090144157409668,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.01928606815636158,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 1.27297100994108e-07,
      "logits/chosen": -0.8449782133102417,
      "logits/rejected": -1.0140793323516846,
      "logps/chosen": -128.93240356445312,
      "logps/ref_chosen": -77.15335083007812,
      "logps/ref_rejected": -91.12923431396484,
      "logps/rejected": -184.95663452148438,
      "loss": 0.9597,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9942206144332886,
      "rewards/margins": 0.8003889918327332,
      "rewards/rejected": -1.794609546661377,
      "step": 462
    },
    {
      "epoch": 0.6999244142101285,
      "grad_norm": 11.438972473144531,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.01917223259806633,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 1.2614681827718695e-07,
      "logits/chosen": -1.5319080352783203,
      "logits/rejected": -1.303840160369873,
      "logps/chosen": -144.3251953125,
      "logps/ref_chosen": -87.58760070800781,
      "logps/ref_rejected": -87.97022247314453,
      "logps/rejected": -171.95938110351562,
      "loss": 1.1095,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.0876948833465576,
      "rewards/margins": 0.5146704912185669,
      "rewards/rejected": -1.602365493774414,
      "step": 463
    },
    {
      "epoch": 0.7014361300075586,
      "grad_norm": 9.427376747131348,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.019130384549498558,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.2500000000000005e-07,
      "logits/chosen": -1.2840232849121094,
      "logits/rejected": -1.3823883533477783,
      "logps/chosen": -134.31118774414062,
      "logps/ref_chosen": -75.83175659179688,
      "logps/ref_rejected": -84.4811019897461,
      "logps/rejected": -181.33981323242188,
      "loss": 1.0668,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.116503357887268,
      "rewards/margins": 0.7235656380653381,
      "rewards/rejected": -1.840069055557251,
      "step": 464
    },
    {
      "epoch": 0.7029478458049887,
      "grad_norm": 11.430929183959961,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.01904112845659256,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.238566782415197e-07,
      "logits/chosen": -1.1909315586090088,
      "logits/rejected": -1.3115463256835938,
      "logps/chosen": -136.56283569335938,
      "logps/ref_chosen": -77.057861328125,
      "logps/ref_rejected": -102.75727844238281,
      "logps/rejected": -197.5943603515625,
      "loss": 1.0598,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.1309858560562134,
      "rewards/margins": 0.6631190776824951,
      "rewards/rejected": -1.794105052947998,
      "step": 465
    },
    {
      "epoch": 0.7044595616024187,
      "grad_norm": 15.998496055603027,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.018958186730742455,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 1.2271688498291334e-07,
      "logits/chosen": -1.4902276992797852,
      "logits/rejected": -1.216938853263855,
      "logps/chosen": -155.81655883789062,
      "logps/ref_chosen": -91.7751693725586,
      "logps/ref_rejected": -90.2679443359375,
      "logps/rejected": -177.01776123046875,
      "loss": 1.1964,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.215767741203308,
      "rewards/margins": 0.422646701335907,
      "rewards/rejected": -1.6384143829345703,
      "step": 466
    },
    {
      "epoch": 0.7059712773998488,
      "grad_norm": 10.740083694458008,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.01889912784099579,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 1.2158065210664848e-07,
      "logits/chosen": -1.1391918659210205,
      "logits/rejected": -1.6300973892211914,
      "logps/chosen": -118.06198120117188,
      "logps/ref_chosen": -64.77557373046875,
      "logps/ref_rejected": -102.58863830566406,
      "logps/rejected": -190.88726806640625,
      "loss": 0.9962,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.0028396844863892,
      "rewards/margins": 0.6537683010101318,
      "rewards/rejected": -1.6566078662872314,
      "step": 467
    },
    {
      "epoch": 0.7074829931972789,
      "grad_norm": 11.700135231018066,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.018793415278196335,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.204480113956011e-07,
      "logits/chosen": -1.5250056982040405,
      "logits/rejected": -1.129932165145874,
      "logps/chosen": -136.98934936523438,
      "logps/ref_chosen": -82.22445678710938,
      "logps/ref_rejected": -92.99041748046875,
      "logps/rejected": -191.02359008789062,
      "loss": 0.9621,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.0257847309112549,
      "rewards/margins": 0.8028579950332642,
      "rewards/rejected": -1.8286426067352295,
      "step": 468
    },
    {
      "epoch": 0.708994708994709,
      "grad_norm": 9.69684886932373,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.01869991421699524,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.1931899453216697e-07,
      "logits/chosen": -1.6485295295715332,
      "logits/rejected": -1.3863415718078613,
      "logps/chosen": -131.82183837890625,
      "logps/ref_chosen": -75.93031311035156,
      "logps/ref_rejected": -92.26559448242188,
      "logps/rejected": -180.31640625,
      "loss": 1.0334,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -1.0436447858810425,
      "rewards/margins": 0.5939034819602966,
      "rewards/rejected": -1.6375482082366943,
      "step": 469
    },
    {
      "epoch": 0.7105064247921391,
      "grad_norm": 8.640731811523438,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.018621355295181274,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.1819363309737438e-07,
      "logits/chosen": -1.0694873332977295,
      "logits/rejected": -1.247729778289795,
      "logps/chosen": -122.56364440917969,
      "logps/ref_chosen": -65.86345672607422,
      "logps/ref_rejected": -85.89833068847656,
      "logps/rejected": -177.00157165527344,
      "loss": 1.0405,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.054185152053833,
      "rewards/margins": 0.6317137479782104,
      "rewards/rejected": -1.685899019241333,
      "step": 470
    },
    {
      "epoch": 0.7120181405895691,
      "grad_norm": 12.291424751281738,
      "kl/avg_steps": 0.703125,
      "kl/beta": 0.01854601316154003,
      "kl/n_epsilon_steps": 0.140625,
      "kl/p_epsilon_steps": 0.84375,
      "learning_rate": 1.1707195857000215e-07,
      "logits/chosen": -1.5064555406570435,
      "logits/rejected": -1.5125834941864014,
      "logps/chosen": -124.05204772949219,
      "logps/ref_chosen": -74.3460922241211,
      "logps/ref_rejected": -93.43672943115234,
      "logps/rejected": -188.60585021972656,
      "loss": 0.9318,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.9173819422721863,
      "rewards/margins": 0.8328464031219482,
      "rewards/rejected": -1.7502284049987793,
      "step": 471
    },
    {
      "epoch": 0.7135298563869993,
      "grad_norm": 9.380815505981445,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.018416522070765495,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.1595400232569768e-07,
      "logits/chosen": -1.3893640041351318,
      "logits/rejected": -1.1879550218582153,
      "logps/chosen": -125.4832763671875,
      "logps/ref_chosen": -74.75674438476562,
      "logps/ref_rejected": -95.18183135986328,
      "logps/rejected": -183.03448486328125,
      "loss": 1.0654,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9331221580505371,
      "rewards/margins": 0.6739637851715088,
      "rewards/rejected": -1.607085943222046,
      "step": 472
    },
    {
      "epoch": 0.7150415721844293,
      "grad_norm": 10.035146713256836,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.018336299806833267,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 1.1483979563610069e-07,
      "logits/chosen": -1.2917176485061646,
      "logits/rejected": -1.6067132949829102,
      "logps/chosen": -117.21717071533203,
      "logps/ref_chosen": -71.65933227539062,
      "logps/ref_rejected": -109.99200439453125,
      "logps/rejected": -192.41232299804688,
      "loss": 1.0543,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8327977657318115,
      "rewards/margins": 0.6665883660316467,
      "rewards/rejected": -1.4993860721588135,
      "step": 473
    },
    {
      "epoch": 0.7165532879818595,
      "grad_norm": 11.2937593460083,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.018233735114336014,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.1372936966796709e-07,
      "logits/chosen": -1.3110771179199219,
      "logits/rejected": -1.6031813621520996,
      "logps/chosen": -124.39402770996094,
      "logps/ref_chosen": -65.91990661621094,
      "logps/ref_rejected": -89.09432983398438,
      "logps/rejected": -178.44122314453125,
      "loss": 1.121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0649316310882568,
      "rewards/margins": 0.5543969869613647,
      "rewards/rejected": -1.619328498840332,
      "step": 474
    },
    {
      "epoch": 0.7180650037792895,
      "grad_norm": 9.53951644897461,
      "kl/avg_steps": 0.6875,
      "kl/beta": 0.018148664385080338,
      "kl/n_epsilon_steps": 0.15625,
      "kl/p_epsilon_steps": 0.84375,
      "learning_rate": 1.126227554822985e-07,
      "logits/chosen": -1.5834732055664062,
      "logits/rejected": -1.2278413772583008,
      "logps/chosen": -133.9127197265625,
      "logps/ref_chosen": -79.02459716796875,
      "logps/ref_rejected": -107.33058166503906,
      "logps/rejected": -212.11447143554688,
      "loss": 0.856,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.9914153814315796,
      "rewards/margins": 0.8952299952507019,
      "rewards/rejected": -1.8866453170776367,
      "step": 475
    },
    {
      "epoch": 0.7195767195767195,
      "grad_norm": 9.075240135192871,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.01802474446594715,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.1151998403347243e-07,
      "logits/chosen": -1.4091227054595947,
      "logits/rejected": -1.4494162797927856,
      "logps/chosen": -156.70401000976562,
      "logps/ref_chosen": -93.72602844238281,
      "logps/ref_rejected": -94.390625,
      "logps/rejected": -191.04698181152344,
      "loss": 1.0586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.132453441619873,
      "rewards/margins": 0.5988330245018005,
      "rewards/rejected": -1.7312864065170288,
      "step": 476
    },
    {
      "epoch": 0.7210884353741497,
      "grad_norm": 10.94466781616211,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.017949020490050316,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.1042108616837692e-07,
      "logits/chosen": -1.2909693717956543,
      "logits/rejected": -1.2547205686569214,
      "logps/chosen": -147.51918029785156,
      "logps/ref_chosen": -76.51399993896484,
      "logps/ref_rejected": -99.14356231689453,
      "logps/rejected": -197.98812866210938,
      "loss": 1.203,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.2738592624664307,
      "rewards/margins": 0.49053436517715454,
      "rewards/rejected": -1.7643935680389404,
      "step": 477
    },
    {
      "epoch": 0.7226001511715797,
      "grad_norm": 14.016806602478027,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.017887532711029053,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 1.0932609262554746e-07,
      "logits/chosen": -1.0645103454589844,
      "logits/rejected": -1.1659934520721436,
      "logps/chosen": -135.13836669921875,
      "logps/ref_chosen": -77.95185852050781,
      "logps/ref_rejected": -69.77754211425781,
      "logps/rejected": -157.57369995117188,
      "loss": 1.2105,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0235099792480469,
      "rewards/margins": 0.537567138671875,
      "rewards/rejected": -1.5610769987106323,
      "step": 478
    },
    {
      "epoch": 0.7241118669690099,
      "grad_norm": 9.91667366027832,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.01784292608499527,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.0823503403430734e-07,
      "logits/chosen": -1.2003769874572754,
      "logits/rejected": -1.3509670495986938,
      "logps/chosen": -132.9344482421875,
      "logps/ref_chosen": -76.56551361083984,
      "logps/ref_rejected": -84.33758544921875,
      "logps/rejected": -169.57940673828125,
      "loss": 1.1758,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.003492832183838,
      "rewards/margins": 0.5064426064491272,
      "rewards/rejected": -1.5099353790283203,
      "step": 479
    },
    {
      "epoch": 0.7256235827664399,
      "grad_norm": 15.995817184448242,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.01776520349085331,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.0714794091391072e-07,
      "logits/chosen": -1.562372088432312,
      "logits/rejected": -1.1541080474853516,
      "logps/chosen": -133.58810424804688,
      "logps/ref_chosen": -80.15884399414062,
      "logps/ref_rejected": -84.88697814941406,
      "logps/rejected": -176.2301788330078,
      "loss": 1.0802,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9479645490646362,
      "rewards/margins": 0.6631921529769897,
      "rewards/rejected": -1.611156702041626,
      "step": 480
    },
    {
      "epoch": 0.72713529856387,
      "grad_norm": 11.84682559967041,
      "kl/avg_steps": 0.484375,
      "kl/beta": 0.01768231764435768,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.0606484367268906e-07,
      "logits/chosen": -1.604417085647583,
      "logits/rejected": -1.5034980773925781,
      "logps/chosen": -142.07493591308594,
      "logps/ref_chosen": -84.56254577636719,
      "logps/ref_rejected": -90.06451416015625,
      "logps/rejected": -183.5068359375,
      "loss": 1.067,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.0142407417297363,
      "rewards/margins": 0.6266486644744873,
      "rewards/rejected": -1.6408894062042236,
      "step": 481
    },
    {
      "epoch": 0.7286470143613001,
      "grad_norm": 13.854905128479004,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.017597081139683723,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.0498577260720048e-07,
      "logits/chosen": -1.169127106666565,
      "logits/rejected": -1.413175344467163,
      "logps/chosen": -147.22222900390625,
      "logps/ref_chosen": -78.88141632080078,
      "logps/ref_rejected": -125.41990661621094,
      "logps/rejected": -226.0455322265625,
      "loss": 1.1832,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.2037124633789062,
      "rewards/margins": 0.5583795309066772,
      "rewards/rejected": -1.762091875076294,
      "step": 482
    },
    {
      "epoch": 0.7301587301587301,
      "grad_norm": 8.845148086547852,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.017536798492074013,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.0391075790138232e-07,
      "logits/chosen": -1.3627674579620361,
      "logits/rejected": -1.4596325159072876,
      "logps/chosen": -127.99396514892578,
      "logps/ref_chosen": -72.690185546875,
      "logps/ref_rejected": -98.37237548828125,
      "logps/rejected": -193.22244262695312,
      "loss": 1.0312,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.968032956123352,
      "rewards/margins": 0.6844265460968018,
      "rewards/rejected": -1.6524595022201538,
      "step": 483
    },
    {
      "epoch": 0.7316704459561603,
      "grad_norm": 10.676874160766602,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.017457693815231323,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.0283982962570681e-07,
      "logits/chosen": -1.6720235347747803,
      "logits/rejected": -1.4070327281951904,
      "logps/chosen": -128.504150390625,
      "logps/ref_chosen": -73.98435974121094,
      "logps/ref_rejected": -89.99177551269531,
      "logps/rejected": -174.51548767089844,
      "loss": 1.0856,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9500585794448853,
      "rewards/margins": 0.5166537761688232,
      "rewards/rejected": -1.466712474822998,
      "step": 484
    },
    {
      "epoch": 0.7331821617535903,
      "grad_norm": 9.896965026855469,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.017376242205500603,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.0177301773633992e-07,
      "logits/chosen": -1.1052076816558838,
      "logits/rejected": -1.263450026512146,
      "logps/chosen": -133.36056518554688,
      "logps/ref_chosen": -78.0927963256836,
      "logps/ref_rejected": -89.14010620117188,
      "logps/rejected": -179.0596923828125,
      "loss": 1.0581,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.9572244882583618,
      "rewards/margins": 0.5939412713050842,
      "rewards/rejected": -1.5511658191680908,
      "step": 485
    },
    {
      "epoch": 0.7346938775510204,
      "grad_norm": 8.270903587341309,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.017284419387578964,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.007103520743035e-07,
      "logits/chosen": -0.9377778172492981,
      "logits/rejected": -1.2564281225204468,
      "logps/chosen": -140.08824157714844,
      "logps/ref_chosen": -73.74685668945312,
      "logps/ref_rejected": -107.752685546875,
      "logps/rejected": -210.189453125,
      "loss": 1.1214,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.144246220588684,
      "rewards/margins": 0.6144800186157227,
      "rewards/rejected": -1.7587262392044067,
      "step": 486
    },
    {
      "epoch": 0.7362055933484505,
      "grad_norm": 9.55226993560791,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.017211806029081345,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 9.965186236464046e-08,
      "logits/chosen": -1.361416220664978,
      "logits/rejected": -1.6992472410202026,
      "logps/chosen": -143.29461669921875,
      "logps/ref_chosen": -79.57780456542969,
      "logps/ref_rejected": -102.29163360595703,
      "logps/rejected": -202.83920288085938,
      "loss": 1.0364,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0940316915512085,
      "rewards/margins": 0.6254779100418091,
      "rewards/rejected": -1.7195096015930176,
      "step": 487
    },
    {
      "epoch": 0.7377173091458806,
      "grad_norm": 13.117379188537598,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.01713150180876255,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 9.859757821558337e-08,
      "logits/chosen": -1.5597364902496338,
      "logits/rejected": -1.713277816772461,
      "logps/chosen": -137.8337860107422,
      "logps/ref_chosen": -80.62767791748047,
      "logps/ref_rejected": -100.45410919189453,
      "logps/rejected": -197.331787109375,
      "loss": 1.0627,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9779952764511108,
      "rewards/margins": 0.6699644327163696,
      "rewards/rejected": -1.6479597091674805,
      "step": 488
    },
    {
      "epoch": 0.7392290249433107,
      "grad_norm": 10.174800872802734,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.017067499458789825,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 9.754752911772615e-08,
      "logits/chosen": -1.595609188079834,
      "logits/rejected": -1.548938274383545,
      "logps/chosen": -150.69955444335938,
      "logps/ref_chosen": -85.39521026611328,
      "logps/ref_rejected": -101.97309875488281,
      "logps/rejected": -184.8128204345703,
      "loss": 1.2807,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -1.1136564016342163,
      "rewards/margins": 0.2926303744316101,
      "rewards/rejected": -1.4062868356704712,
      "step": 489
    },
    {
      "epoch": 0.7407407407407407,
      "grad_norm": 10.684184074401855,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.017014330253005028,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 9.650174444319956e-08,
      "logits/chosen": -1.3730194568634033,
      "logits/rejected": -1.2876062393188477,
      "logps/chosen": -136.75741577148438,
      "logps/ref_chosen": -77.75589752197266,
      "logps/ref_rejected": -88.98885345458984,
      "logps/rejected": -183.05950927734375,
      "loss": 1.1311,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.0055129528045654,
      "rewards/margins": 0.5872576236724854,
      "rewards/rejected": -1.5927705764770508,
      "step": 490
    },
    {
      "epoch": 0.7422524565381708,
      "grad_norm": 8.054734230041504,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.01696661114692688,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 9.546025344484868e-08,
      "logits/chosen": -1.355436086654663,
      "logits/rejected": -1.3936882019042969,
      "logps/chosen": -132.57339477539062,
      "logps/ref_chosen": -74.33360290527344,
      "logps/ref_rejected": -91.4105224609375,
      "logps/rejected": -186.3638916015625,
      "loss": 1.0384,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9841344356536865,
      "rewards/margins": 0.6149394512176514,
      "rewards/rejected": -1.599073886871338,
      "step": 491
    },
    {
      "epoch": 0.7437641723356009,
      "grad_norm": 10.10815715789795,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.01688219979405403,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 9.442308525541589e-08,
      "logits/chosen": -1.2477145195007324,
      "logits/rejected": -1.6418591737747192,
      "logps/chosen": -160.24461364746094,
      "logps/ref_chosen": -85.14178466796875,
      "logps/ref_rejected": -103.44204711914062,
      "logps/rejected": -204.75823974609375,
      "loss": 1.1862,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -1.2674788236618042,
      "rewards/margins": 0.4344896674156189,
      "rewards/rejected": -1.7019684314727783,
      "step": 492
    },
    {
      "epoch": 0.745275888133031,
      "grad_norm": 8.398894309997559,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.016840100288391113,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 9.339026888672468e-08,
      "logits/chosen": -1.0465887784957886,
      "logits/rejected": -1.2313003540039062,
      "logps/chosen": -134.85989379882812,
      "logps/ref_chosen": -75.81439971923828,
      "logps/ref_rejected": -95.30766296386719,
      "logps/rejected": -191.33767700195312,
      "loss": 1.0973,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.9912848472595215,
      "rewards/margins": 0.6130992770195007,
      "rewards/rejected": -1.604384183883667,
      "step": 493
    },
    {
      "epoch": 0.7467876039304611,
      "grad_norm": 10.89151382446289,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.016751108691096306,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 9.236183322886945e-08,
      "logits/chosen": -1.4720494747161865,
      "logits/rejected": -1.2916717529296875,
      "logps/chosen": -153.79876708984375,
      "logps/ref_chosen": -93.83562469482422,
      "logps/ref_rejected": -112.21142578125,
      "logps/rejected": -202.45025634765625,
      "loss": 1.1706,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0018885135650635,
      "rewards/margins": 0.4987478256225586,
      "rewards/rejected": -1.500636339187622,
      "step": 494
    },
    {
      "epoch": 0.7482993197278912,
      "grad_norm": 10.691729545593262,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.01668333262205124,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 9.133780704940594e-08,
      "logits/chosen": -1.0881528854370117,
      "logits/rejected": -1.3684172630310059,
      "logps/chosen": -129.056884765625,
      "logps/ref_chosen": -68.52467346191406,
      "logps/ref_rejected": -89.65379333496094,
      "logps/rejected": -179.65640258789062,
      "loss": 1.1726,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -1.006978154182434,
      "rewards/margins": 0.483287513256073,
      "rewards/rejected": -1.4902657270431519,
      "step": 495
    },
    {
      "epoch": 0.7498110355253212,
      "grad_norm": 8.830723762512207,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.01661583222448826,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 9.031821899254797e-08,
      "logits/chosen": -1.227933406829834,
      "logits/rejected": -1.4093971252441406,
      "logps/chosen": -136.625732421875,
      "logps/ref_chosen": -73.13617706298828,
      "logps/ref_rejected": -111.5093002319336,
      "logps/rejected": -209.126220703125,
      "loss": 1.1085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0537065267562866,
      "rewards/margins": 0.5583138465881348,
      "rewards/rejected": -1.6120203733444214,
      "step": 496
    },
    {
      "epoch": 0.7513227513227513,
      "grad_norm": 11.715909004211426,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.016540881246328354,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 8.930309757836516e-08,
      "logits/chosen": -1.3785967826843262,
      "logits/rejected": -1.4464075565338135,
      "logps/chosen": -156.37203979492188,
      "logps/ref_chosen": -88.71475219726562,
      "logps/ref_rejected": -105.74935913085938,
      "logps/rejected": -214.29281616210938,
      "loss": 0.9959,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.1154170036315918,
      "rewards/margins": 0.6677843332290649,
      "rewards/rejected": -1.7832013368606567,
      "step": 497
    },
    {
      "epoch": 0.7528344671201814,
      "grad_norm": 8.241373062133789,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.01645858772099018,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 8.829247120198563e-08,
      "logits/chosen": -1.360666036605835,
      "logits/rejected": -1.3529174327850342,
      "logps/chosen": -139.6077880859375,
      "logps/ref_chosen": -83.3353271484375,
      "logps/ref_rejected": -89.34942626953125,
      "logps/rejected": -183.6534423828125,
      "loss": 1.0478,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9240692853927612,
      "rewards/margins": 0.617653489112854,
      "rewards/rejected": -1.5417227745056152,
      "step": 498
    },
    {
      "epoch": 0.7543461829176115,
      "grad_norm": 11.564859390258789,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.01638689450919628,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 8.728636813280163e-08,
      "logits/chosen": -1.5715296268463135,
      "logits/rejected": -1.7359554767608643,
      "logps/chosen": -143.91238403320312,
      "logps/ref_chosen": -79.373779296875,
      "logps/ref_rejected": -104.62533569335938,
      "logps/rejected": -201.68991088867188,
      "loss": 1.1696,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.0568264722824097,
      "rewards/margins": 0.5236354470252991,
      "rewards/rejected": -1.580461859703064,
      "step": 499
    },
    {
      "epoch": 0.7558578987150416,
      "grad_norm": 9.394903182983398,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.01632567308843136,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -1.5726468563079834,
      "logits/rejected": -1.5170438289642334,
      "logps/chosen": -148.0132293701172,
      "logps/ref_chosen": -85.953857421875,
      "logps/ref_rejected": -90.40995788574219,
      "logps/rejected": -189.79977416992188,
      "loss": 1.0454,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0117685794830322,
      "rewards/margins": 0.6012208461761475,
      "rewards/rejected": -1.6129894256591797,
      "step": 500
    },
    {
      "epoch": 0.7558578987150416,
      "eval_kl/n_epsilon_steps": 0.2680457830429077,
      "eval_kl/p_epsilon_steps": 0.7293133735656738,
      "eval_logits/chosen": -1.2671034336090088,
      "eval_logits/rejected": -1.397099256515503,
      "eval_logps/chosen": -145.2611083984375,
      "eval_logps/ref_chosen": -86.90177917480469,
      "eval_logps/ref_rejected": -96.69639587402344,
      "eval_logps/rejected": -190.7860870361328,
      "eval_loss": 0.5415008068084717,
      "eval_rewards/accuracies": 0.735035240650177,
      "eval_rewards/chosen": -0.9467713236808777,
      "eval_rewards/margins": 0.5722161531448364,
      "eval_rewards/rejected": -1.5189874172210693,
      "eval_runtime": 46.9472,
      "eval_samples_per_second": 49.055,
      "eval_steps_per_second": 1.534,
      "step": 500
    },
    {
      "epoch": 0.7573696145124716,
      "grad_norm": 7.951656818389893,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.016249503940343857,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 8.528784436016878e-08,
      "logits/chosen": -1.4893511533737183,
      "logits/rejected": -1.2975850105285645,
      "logps/chosen": -142.8973846435547,
      "logps/ref_chosen": -81.2226791381836,
      "logps/ref_rejected": -86.97892761230469,
      "logps/rejected": -178.79690551757812,
      "loss": 1.0854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.999392569065094,
      "rewards/margins": 0.48362892866134644,
      "rewards/rejected": -1.4830214977264404,
      "step": 501
    },
    {
      "epoch": 0.7588813303099018,
      "grad_norm": 10.511534690856934,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.01617872156202793,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 8.4295479559726e-08,
      "logits/chosen": -1.8129953145980835,
      "logits/rejected": -1.8130979537963867,
      "logps/chosen": -143.57913208007812,
      "logps/ref_chosen": -83.15675354003906,
      "logps/ref_rejected": -106.74440002441406,
      "logps/rejected": -194.28616333007812,
      "loss": 1.1175,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9750958681106567,
      "rewards/margins": 0.4325859248638153,
      "rewards/rejected": -1.4076817035675049,
      "step": 502
    },
    {
      "epoch": 0.7603930461073318,
      "grad_norm": 8.138505935668945,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.016103237867355347,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 8.330774987092712e-08,
      "logits/chosen": -1.2757725715637207,
      "logits/rejected": -0.8529649972915649,
      "logps/chosen": -123.6448745727539,
      "logps/ref_chosen": -68.51583862304688,
      "logps/ref_rejected": -75.02178955078125,
      "logps/rejected": -169.09393310546875,
      "loss": 1.0351,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.88608318567276,
      "rewards/margins": 0.618719220161438,
      "rewards/rejected": -1.5048024654388428,
      "step": 503
    },
    {
      "epoch": 0.7619047619047619,
      "grad_norm": 22.84149742126465,
      "kl/avg_steps": 0.609375,
      "kl/beta": 0.016013164073228836,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 8.232468292269479e-08,
      "logits/chosen": -1.4639678001403809,
      "logits/rejected": -1.071527123451233,
      "logps/chosen": -139.56570434570312,
      "logps/ref_chosen": -85.15829467773438,
      "logps/ref_rejected": -96.16879272460938,
      "logps/rejected": -197.9119110107422,
      "loss": 0.917,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.8674378991127014,
      "rewards/margins": 0.7494259476661682,
      "rewards/rejected": -1.6168639659881592,
      "step": 504
    },
    {
      "epoch": 0.763416477702192,
      "grad_norm": 12.432364463806152,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.01591617427766323,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 8.134630621352483e-08,
      "logits/chosen": -1.513573408126831,
      "logits/rejected": -1.8110153675079346,
      "logps/chosen": -138.6746826171875,
      "logps/ref_chosen": -79.26185607910156,
      "logps/ref_rejected": -96.34947967529297,
      "logps/rejected": -183.6324920654297,
      "loss": 1.2128,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9458433389663696,
      "rewards/margins": 0.435134619474411,
      "rewards/rejected": -1.380977988243103,
      "step": 505
    },
    {
      "epoch": 0.764928193499622,
      "grad_norm": 10.85437297821045,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.01585671305656433,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 8.037264711071698e-08,
      "logits/chosen": -1.711536169052124,
      "logits/rejected": -1.5470472574234009,
      "logps/chosen": -148.20022583007812,
      "logps/ref_chosen": -88.192626953125,
      "logps/ref_rejected": -100.86880493164062,
      "logps/rejected": -192.3444061279297,
      "loss": 1.1906,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.950194776058197,
      "rewards/margins": 0.48996031284332275,
      "rewards/rejected": -1.440155029296875,
      "step": 506
    },
    {
      "epoch": 0.7664399092970522,
      "grad_norm": 8.822526931762695,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.015777822583913803,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 7.940373284960933e-08,
      "logits/chosen": -1.528835654258728,
      "logits/rejected": -1.691847801208496,
      "logps/chosen": -153.1142578125,
      "logps/ref_chosen": -86.04632568359375,
      "logps/ref_rejected": -111.44412994384766,
      "logps/rejected": -212.93182373046875,
      "loss": 1.1061,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.054894208908081,
      "rewards/margins": 0.5349427461624146,
      "rewards/rejected": -1.5898369550704956,
      "step": 507
    },
    {
      "epoch": 0.7679516250944822,
      "grad_norm": 9.772814750671387,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.015699326992034912,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 7.843959053281663e-08,
      "logits/chosen": -1.3107966184616089,
      "logits/rejected": -1.7589651346206665,
      "logps/chosen": -139.20782470703125,
      "logps/ref_chosen": -79.25038146972656,
      "logps/ref_rejected": -118.49089813232422,
      "logps/rejected": -216.39041137695312,
      "loss": 1.0755,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9390788674354553,
      "rewards/margins": 0.5872691869735718,
      "rewards/rejected": -1.5263481140136719,
      "step": 508
    },
    {
      "epoch": 0.7694633408919124,
      "grad_norm": 8.684977531433105,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.015635807067155838,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 7.748024712947204e-08,
      "logits/chosen": -1.1436741352081299,
      "logits/rejected": -1.225794792175293,
      "logps/chosen": -136.03652954101562,
      "logps/ref_chosen": -80.7039566040039,
      "logps/ref_rejected": -90.50444793701172,
      "logps/rejected": -183.49346923828125,
      "loss": 1.0166,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.862238883972168,
      "rewards/margins": 0.5809536576271057,
      "rewards/rejected": -1.443192481994629,
      "step": 509
    },
    {
      "epoch": 0.7709750566893424,
      "grad_norm": 9.093579292297363,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.015553180128335953,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 7.652572947447272e-08,
      "logits/chosen": -1.6221449375152588,
      "logits/rejected": -1.680511474609375,
      "logps/chosen": -123.47549438476562,
      "logps/ref_chosen": -67.64491271972656,
      "logps/ref_rejected": -108.92274475097656,
      "logps/rejected": -207.5115966796875,
      "loss": 1.0332,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8659316301345825,
      "rewards/margins": 0.6559004187583923,
      "rewards/rejected": -1.52183198928833,
      "step": 510
    },
    {
      "epoch": 0.7724867724867724,
      "grad_norm": 9.894843101501465,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.015475801192224026,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 7.557606426772961e-08,
      "logits/chosen": -1.6408867835998535,
      "logits/rejected": -1.515726089477539,
      "logps/chosen": -135.91790771484375,
      "logps/ref_chosen": -75.66263580322266,
      "logps/ref_rejected": -104.26296997070312,
      "logps/rejected": -210.71661376953125,
      "loss": 1.0095,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.9304167032241821,
      "rewards/margins": 0.7054550647735596,
      "rewards/rejected": -1.6358717679977417,
      "step": 511
    },
    {
      "epoch": 0.7739984882842026,
      "grad_norm": 9.156575202941895,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.015403596684336662,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 7.463127807341966e-08,
      "logits/chosen": -1.653001070022583,
      "logits/rejected": -1.8152376413345337,
      "logps/chosen": -139.13134765625,
      "logps/ref_chosen": -79.31925964355469,
      "logps/ref_rejected": -82.22052001953125,
      "logps/rejected": -171.9754180908203,
      "loss": 1.1527,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9223538637161255,
      "rewards/margins": 0.4540788233280182,
      "rewards/rejected": -1.3764326572418213,
      "step": 512
    },
    {
      "epoch": 0.7755102040816326,
      "grad_norm": 7.260792255401611,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.01535082794725895,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 7.369139731924401e-08,
      "logits/chosen": -0.9878244400024414,
      "logits/rejected": -1.1484776735305786,
      "logps/chosen": -127.03314971923828,
      "logps/ref_chosen": -72.02534484863281,
      "logps/ref_rejected": -86.56224060058594,
      "logps/rejected": -179.06666564941406,
      "loss": 1.0315,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.8418123722076416,
      "rewards/margins": 0.568528413772583,
      "rewards/rejected": -1.4103407859802246,
      "step": 513
    },
    {
      "epoch": 0.7770219198790628,
      "grad_norm": 9.606550216674805,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.015264962799847126,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 7.275644829568747e-08,
      "logits/chosen": -1.5925896167755127,
      "logits/rejected": -1.3574076890945435,
      "logps/chosen": -144.4132537841797,
      "logps/ref_chosen": -84.94092559814453,
      "logps/ref_rejected": -102.44367218017578,
      "logps/rejected": -203.36727905273438,
      "loss": 1.0128,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.906743049621582,
      "rewards/margins": 0.6244295835494995,
      "rewards/rejected": -1.5311726331710815,
      "step": 514
    },
    {
      "epoch": 0.7785336356764928,
      "grad_norm": 10.59394359588623,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.01519610546529293,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 7.182645715528435e-08,
      "logits/chosen": -0.9610180854797363,
      "logits/rejected": -1.0923925638198853,
      "logps/chosen": -137.7475128173828,
      "logps/ref_chosen": -72.9662094116211,
      "logps/ref_rejected": -102.53651428222656,
      "logps/rejected": -201.09156799316406,
      "loss": 1.1114,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9821167588233948,
      "rewards/margins": 0.5052266120910645,
      "rewards/rejected": -1.4873433113098145,
      "step": 515
    },
    {
      "epoch": 0.780045351473923,
      "grad_norm": 11.120777130126953,
      "kl/avg_steps": 0.6875,
      "kl/beta": 0.015125205740332603,
      "kl/n_epsilon_steps": 0.15625,
      "kl/p_epsilon_steps": 0.84375,
      "learning_rate": 7.090144991188568e-08,
      "logits/chosen": -0.9688495397567749,
      "logits/rejected": -1.2150192260742188,
      "logps/chosen": -126.80594635009766,
      "logps/ref_chosen": -76.63414001464844,
      "logps/ref_rejected": -91.01750183105469,
      "logps/rejected": -173.46884155273438,
      "loss": 1.1035,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.7560686469078064,
      "rewards/margins": 0.48095256090164185,
      "rewards/rejected": -1.2370212078094482,
      "step": 516
    },
    {
      "epoch": 0.781557067271353,
      "grad_norm": 9.966470718383789,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.015021930448710918,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 6.998145243993284e-08,
      "logits/chosen": -1.5444862842559814,
      "logits/rejected": -1.267151117324829,
      "logps/chosen": -144.53884887695312,
      "logps/ref_chosen": -77.06816864013672,
      "logps/ref_rejected": -80.048583984375,
      "logps/rejected": -176.18679809570312,
      "loss": 1.2028,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.0134719610214233,
      "rewards/margins": 0.42320364713668823,
      "rewards/rejected": -1.4366756677627563,
      "step": 517
    },
    {
      "epoch": 0.783068783068783,
      "grad_norm": 7.715234279632568,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.014975132420659065,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 6.906649047373245e-08,
      "logits/chosen": -1.4431686401367188,
      "logits/rejected": -1.650618314743042,
      "logps/chosen": -130.3668212890625,
      "logps/ref_chosen": -78.69026184082031,
      "logps/ref_rejected": -97.58125305175781,
      "logps/rejected": -183.90213012695312,
      "loss": 1.1056,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7733626365661621,
      "rewards/margins": 0.5110686421394348,
      "rewards/rejected": -1.2844313383102417,
      "step": 518
    },
    {
      "epoch": 0.7845804988662132,
      "grad_norm": 9.29223346710205,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.014919186010956764,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 6.815658960673781e-08,
      "logits/chosen": -1.1009117364883423,
      "logits/rejected": -1.2734102010726929,
      "logps/chosen": -145.77960205078125,
      "logps/ref_chosen": -78.35087585449219,
      "logps/ref_rejected": -95.79212188720703,
      "logps/rejected": -184.5224151611328,
      "loss": 1.3496,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0064210891723633,
      "rewards/margins": 0.30923837423324585,
      "rewards/rejected": -1.3156594038009644,
      "step": 519
    },
    {
      "epoch": 0.7860922146636432,
      "grad_norm": 9.650991439819336,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.014872708357870579,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 6.725177529083209e-08,
      "logits/chosen": -1.399254560470581,
      "logits/rejected": -1.952781319618225,
      "logps/chosen": -138.1239471435547,
      "logps/ref_chosen": -80.40513610839844,
      "logps/ref_rejected": -93.02791595458984,
      "logps/rejected": -183.82025146484375,
      "loss": 1.0674,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8555145859718323,
      "rewards/margins": 0.48555320501327515,
      "rewards/rejected": -1.3410677909851074,
      "step": 520
    },
    {
      "epoch": 0.7876039304610734,
      "grad_norm": 10.00867748260498,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.014789517968893051,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 6.63520728356167e-08,
      "logits/chosen": -1.0487563610076904,
      "logits/rejected": -1.4563612937927246,
      "logps/chosen": -147.1343994140625,
      "logps/ref_chosen": -86.5218276977539,
      "logps/ref_rejected": -109.20257568359375,
      "logps/rejected": -205.20623779296875,
      "loss": 1.0869,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8942147493362427,
      "rewards/margins": 0.5161318778991699,
      "rewards/rejected": -1.4103467464447021,
      "step": 521
    },
    {
      "epoch": 0.7891156462585034,
      "grad_norm": 8.67531681060791,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.01471593789756298,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 6.545750740770336e-08,
      "logits/chosen": -1.360521674156189,
      "logits/rejected": -0.9759971499443054,
      "logps/chosen": -138.8271484375,
      "logps/ref_chosen": -78.2425537109375,
      "logps/ref_rejected": -85.23554992675781,
      "logps/rejected": -174.7383575439453,
      "loss": 1.1936,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.8904162049293518,
      "rewards/margins": 0.4182692766189575,
      "rewards/rejected": -1.3086854219436646,
      "step": 522
    },
    {
      "epoch": 0.7906273620559335,
      "grad_norm": 8.602282524108887,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.014665525406599045,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 6.456810403001012e-08,
      "logits/chosen": -1.4247796535491943,
      "logits/rejected": -1.5719811916351318,
      "logps/chosen": -149.7041778564453,
      "logps/ref_chosen": -83.50096893310547,
      "logps/ref_rejected": -117.4521713256836,
      "logps/rejected": -217.1758575439453,
      "loss": 1.1016,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9677379131317139,
      "rewards/margins": 0.4840565621852875,
      "rewards/rejected": -1.4517945051193237,
      "step": 523
    },
    {
      "epoch": 0.7921390778533636,
      "grad_norm": 7.558107376098633,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.014588026329874992,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 6.368388758106134e-08,
      "logits/chosen": -1.659937858581543,
      "logits/rejected": -1.5439331531524658,
      "logps/chosen": -143.3587646484375,
      "logps/ref_chosen": -93.22590637207031,
      "logps/ref_rejected": -108.17863464355469,
      "logps/rejected": -189.51800537109375,
      "loss": 1.1301,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7300490140914917,
      "rewards/margins": 0.4482584297657013,
      "rewards/rejected": -1.1783075332641602,
      "step": 524
    },
    {
      "epoch": 0.7936507936507936,
      "grad_norm": 9.238025665283203,
      "kl/avg_steps": 0.265625,
      "kl/beta": 0.014524482190608978,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 6.280488279429185e-08,
      "logits/chosen": -1.7793774604797363,
      "logits/rejected": -1.6026604175567627,
      "logps/chosen": -159.54000854492188,
      "logps/ref_chosen": -94.08831787109375,
      "logps/ref_rejected": -100.682373046875,
      "logps/rejected": -191.29754638671875,
      "loss": 1.199,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.9504117965698242,
      "rewards/margins": 0.35912391543388367,
      "rewards/rejected": -1.3095357418060303,
      "step": 525
    },
    {
      "epoch": 0.7951625094482238,
      "grad_norm": 8.711167335510254,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.014486003667116165,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 6.193111425735515e-08,
      "logits/chosen": -1.275062918663025,
      "logits/rejected": -1.5753190517425537,
      "logps/chosen": -144.21517944335938,
      "logps/ref_chosen": -77.78373718261719,
      "logps/ref_rejected": -100.29583740234375,
      "logps/rejected": -194.74301147460938,
      "loss": 1.1563,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.9615944623947144,
      "rewards/margins": 0.39926403760910034,
      "rewards/rejected": -1.36085844039917,
      "step": 526
    },
    {
      "epoch": 0.7966742252456538,
      "grad_norm": 10.081496238708496,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.014431883580982685,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 6.106260641143546e-08,
      "logits/chosen": -1.1058743000030518,
      "logits/rejected": -1.5538671016693115,
      "logps/chosen": -149.97344970703125,
      "logps/ref_chosen": -76.695068359375,
      "logps/ref_rejected": -107.68281555175781,
      "logps/rejected": -203.9995574951172,
      "loss": 1.2175,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0548813343048096,
      "rewards/margins": 0.3263258934020996,
      "rewards/rejected": -1.3812072277069092,
      "step": 527
    },
    {
      "epoch": 0.7981859410430839,
      "grad_norm": 8.379530906677246,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.01436454989016056,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 6.019938355056422e-08,
      "logits/chosen": -1.0639445781707764,
      "logits/rejected": -1.2222740650177002,
      "logps/chosen": -134.2487030029297,
      "logps/ref_chosen": -75.0361328125,
      "logps/ref_rejected": -94.67579650878906,
      "logps/rejected": -180.45919799804688,
      "loss": 1.2042,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.8484035730361938,
      "rewards/margins": 0.37475496530532837,
      "rewards/rejected": -1.2231584787368774,
      "step": 528
    },
    {
      "epoch": 0.799697656840514,
      "grad_norm": 7.623414993286133,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.01430197898298502,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 5.934146982094049e-08,
      "logits/chosen": -1.2689933776855469,
      "logits/rejected": -1.3251078128814697,
      "logps/chosen": -130.27891540527344,
      "logps/ref_chosen": -72.84869384765625,
      "logps/ref_rejected": -93.25855255126953,
      "logps/rejected": -204.9322052001953,
      "loss": 0.9281,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.8188842535018921,
      "rewards/margins": 0.7665591835975647,
      "rewards/rejected": -1.5854434967041016,
      "step": 529
    },
    {
      "epoch": 0.8012093726379441,
      "grad_norm": 8.185614585876465,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.0142219802364707,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 5.848888922025552e-08,
      "logits/chosen": -1.2606332302093506,
      "logits/rejected": -1.3375990390777588,
      "logps/chosen": -137.87921142578125,
      "logps/ref_chosen": -79.49717712402344,
      "logps/ref_rejected": -93.59564208984375,
      "logps/rejected": -189.87530517578125,
      "loss": 1.0558,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.8295097947120667,
      "rewards/margins": 0.5320351123809814,
      "rewards/rejected": -1.3615449666976929,
      "step": 530
    },
    {
      "epoch": 0.8027210884353742,
      "grad_norm": 10.98009967803955,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.01417325995862484,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 5.7641665597021435e-08,
      "logits/chosen": -1.1490346193313599,
      "logits/rejected": -1.1379203796386719,
      "logps/chosen": -130.52809143066406,
      "logps/ref_chosen": -69.45396423339844,
      "logps/ref_rejected": -96.30017852783203,
      "logps/rejected": -187.3482666015625,
      "loss": 1.1267,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8644498586654663,
      "rewards/margins": 0.41859444975852966,
      "rewards/rejected": -1.2830442190170288,
      "step": 531
    },
    {
      "epoch": 0.8042328042328042,
      "grad_norm": 7.920943260192871,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.014120308682322502,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 5.679982264990424e-08,
      "logits/chosen": -1.4232215881347656,
      "logits/rejected": -1.4004766941070557,
      "logps/chosen": -149.61624145507812,
      "logps/ref_chosen": -76.52011108398438,
      "logps/ref_rejected": -94.79593658447266,
      "logps/rejected": -202.38143920898438,
      "loss": 1.1168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0281970500946045,
      "rewards/margins": 0.47959816455841064,
      "rewards/rejected": -1.5077950954437256,
      "step": 532
    },
    {
      "epoch": 0.8057445200302343,
      "grad_norm": 10.504121780395508,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.014050058089196682,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 5.596338392706076e-08,
      "logits/chosen": -1.232681393623352,
      "logits/rejected": -1.6070609092712402,
      "logps/chosen": -123.11308288574219,
      "logps/ref_chosen": -72.31800842285156,
      "logps/ref_rejected": -89.26652526855469,
      "logps/rejected": -177.32489013671875,
      "loss": 1.0956,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7114356756210327,
      "rewards/margins": 0.5157041549682617,
      "rewards/rejected": -1.227139949798584,
      "step": 533
    },
    {
      "epoch": 0.8072562358276644,
      "grad_norm": 9.713748931884766,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.013980157673358917,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 5.513237282548033e-08,
      "logits/chosen": -1.711059331893921,
      "logits/rejected": -1.5232676267623901,
      "logps/chosen": -129.87283325195312,
      "logps/ref_chosen": -77.87559509277344,
      "logps/ref_rejected": -92.21171569824219,
      "logps/rejected": -180.92959594726562,
      "loss": 1.0767,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7261112928390503,
      "rewards/margins": 0.5059158205986023,
      "rewards/rejected": -1.2320270538330078,
      "step": 534
    },
    {
      "epoch": 0.8087679516250945,
      "grad_norm": 6.756545543670654,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.013923592865467072,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 5.430681259032957e-08,
      "logits/chosen": -1.16239333152771,
      "logits/rejected": -1.4077019691467285,
      "logps/chosen": -142.74710083007812,
      "logps/ref_chosen": -78.16358184814453,
      "logps/ref_rejected": -97.78164672851562,
      "logps/rejected": -195.040771484375,
      "loss": 1.1583,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.898932695388794,
      "rewards/margins": 0.4476352632045746,
      "rewards/rejected": -1.3465681076049805,
      "step": 535
    },
    {
      "epoch": 0.8102796674225246,
      "grad_norm": 6.646007537841797,
      "kl/avg_steps": 0.6875,
      "kl/beta": 0.013862942345440388,
      "kl/n_epsilon_steps": 0.15625,
      "kl/p_epsilon_steps": 0.84375,
      "learning_rate": 5.3486726314303175e-08,
      "logits/chosen": -1.2160377502441406,
      "logits/rejected": -1.4614768028259277,
      "logps/chosen": -123.78688049316406,
      "logps/ref_chosen": -66.65623474121094,
      "logps/ref_rejected": -89.49085998535156,
      "logps/rejected": -190.17738342285156,
      "loss": 0.973,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.7878190875053406,
      "rewards/margins": 0.5973426103591919,
      "rewards/rejected": -1.3851616382598877,
      "step": 536
    },
    {
      "epoch": 0.8117913832199547,
      "grad_norm": 9.00659465789795,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.013768285512924194,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 5.267213693697695e-08,
      "logits/chosen": -1.0336707830429077,
      "logits/rejected": -1.3107452392578125,
      "logps/chosen": -148.0464324951172,
      "logps/ref_chosen": -74.99390411376953,
      "logps/ref_rejected": -110.6627197265625,
      "logps/rejected": -216.4197235107422,
      "loss": 1.14,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0053305625915527,
      "rewards/margins": 0.44309288263320923,
      "rewards/rejected": -1.4484233856201172,
      "step": 537
    },
    {
      "epoch": 0.8133030990173847,
      "grad_norm": 7.129688739776611,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.013725393451750278,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 5.1863067244167144e-08,
      "logits/chosen": -1.4286093711853027,
      "logits/rejected": -1.3614468574523926,
      "logps/chosen": -154.15213012695312,
      "logps/ref_chosen": -87.61151885986328,
      "logps/ref_rejected": -98.1150131225586,
      "logps/rejected": -204.0767059326172,
      "loss": 1.0504,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.9111753106117249,
      "rewards/margins": 0.533942699432373,
      "rewards/rejected": -1.4451179504394531,
      "step": 538
    },
    {
      "epoch": 0.8148148148148148,
      "grad_norm": 9.4677152633667,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.0136528629809618,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 5.105953986729195e-08,
      "logits/chosen": -0.8625047206878662,
      "logits/rejected": -1.7717041969299316,
      "logps/chosen": -143.09307861328125,
      "logps/ref_chosen": -78.86481475830078,
      "logps/ref_rejected": -100.84349822998047,
      "logps/rejected": -197.5748291015625,
      "loss": 1.1097,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8741194009780884,
      "rewards/margins": 0.4376823902130127,
      "rewards/rejected": -1.3118019104003906,
      "step": 539
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 8.074810981750488,
      "kl/avg_steps": 0.578125,
      "kl/beta": 0.013584937900304794,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 5.026157728273966e-08,
      "logits/chosen": -1.2832505702972412,
      "logits/rejected": -1.7057878971099854,
      "logps/chosen": -145.83828735351562,
      "logps/ref_chosen": -83.66409301757812,
      "logps/ref_rejected": -114.8860092163086,
      "logps/rejected": -219.39007568359375,
      "loss": 1.0522,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8425248265266418,
      "rewards/margins": 0.5671800374984741,
      "rewards/rejected": -1.4097049236297607,
      "step": 540
    },
    {
      "epoch": 0.817838246409675,
      "grad_norm": 11.797026634216309,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.01350685115903616,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.9469201811239035e-08,
      "logits/chosen": -1.5112102031707764,
      "logits/rejected": -1.2197654247283936,
      "logps/chosen": -140.15115356445312,
      "logps/ref_chosen": -83.12225341796875,
      "logps/ref_rejected": -74.80526733398438,
      "logps/rejected": -176.5388946533203,
      "loss": 1.0053,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7699480056762695,
      "rewards/margins": 0.5961588621139526,
      "rewards/rejected": -1.3661069869995117,
      "step": 541
    },
    {
      "epoch": 0.8193499622071051,
      "grad_norm": 9.535528182983398,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.013448016718029976,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 4.868243561723534e-08,
      "logits/chosen": -1.2761826515197754,
      "logits/rejected": -1.1277570724487305,
      "logps/chosen": -116.5569076538086,
      "logps/ref_chosen": -66.3132553100586,
      "logps/ref_rejected": -83.24588012695312,
      "logps/rejected": -182.03997802734375,
      "loss": 0.9986,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.675069272518158,
      "rewards/margins": 0.644609272480011,
      "rewards/rejected": -1.319678544998169,
      "step": 542
    },
    {
      "epoch": 0.8208616780045351,
      "grad_norm": 6.756430625915527,
      "kl/avg_steps": 0.65625,
      "kl/beta": 0.0133811105042696,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.828125,
      "learning_rate": 4.790130070827028e-08,
      "logits/chosen": -1.245871663093567,
      "logits/rejected": -1.3569632768630981,
      "logps/chosen": -128.3046417236328,
      "logps/ref_chosen": -68.11430358886719,
      "logps/ref_rejected": -94.62380981445312,
      "logps/rejected": -196.82461547851562,
      "loss": 1.0586,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8011859059333801,
      "rewards/margins": 0.5547268390655518,
      "rewards/rejected": -1.355912685394287,
      "step": 543
    },
    {
      "epoch": 0.8223733938019653,
      "grad_norm": 8.612573623657227,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.01329386979341507,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.7125818934366454e-08,
      "logits/chosen": -1.2979918718338013,
      "logits/rejected": -1.4303150177001953,
      "logps/chosen": -141.66336059570312,
      "logps/ref_chosen": -81.187255859375,
      "logps/ref_rejected": -105.84722900390625,
      "logps/rejected": -211.17544555664062,
      "loss": 1.0335,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8036271333694458,
      "rewards/margins": 0.588148832321167,
      "rewards/rejected": -1.3917759656906128,
      "step": 544
    },
    {
      "epoch": 0.8238851095993953,
      "grad_norm": 8.018584251403809,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.013248329050838947,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.635601198741607e-08,
      "logits/chosen": -1.3559893369674683,
      "logits/rejected": -1.618308186531067,
      "logps/chosen": -144.0803985595703,
      "logps/ref_chosen": -78.81717681884766,
      "logps/ref_rejected": -98.65876770019531,
      "logps/rejected": -194.8354034423828,
      "loss": 1.1542,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8626161813735962,
      "rewards/margins": 0.40304312109947205,
      "rewards/rejected": -1.265659213066101,
      "step": 545
    },
    {
      "epoch": 0.8253968253968254,
      "grad_norm": 8.517661094665527,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.013198832981288433,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.559190140057428e-08,
      "logits/chosen": -1.4798083305358887,
      "logits/rejected": -1.1381564140319824,
      "logps/chosen": -131.03048706054688,
      "logps/ref_chosen": -74.2529296875,
      "logps/ref_rejected": -80.32308959960938,
      "logps/rejected": -172.29904174804688,
      "loss": 1.0965,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.7476749420166016,
      "rewards/margins": 0.45806336402893066,
      "rewards/rejected": -1.2057383060455322,
      "step": 546
    },
    {
      "epoch": 0.8269085411942555,
      "grad_norm": 9.102763175964355,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.013141339644789696,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 4.483350854765672e-08,
      "logits/chosen": -1.624086856842041,
      "logits/rejected": -1.6054657697677612,
      "logps/chosen": -124.8319320678711,
      "logps/ref_chosen": -69.9368896484375,
      "logps/ref_rejected": -90.25672912597656,
      "logps/rejected": -191.79208374023438,
      "loss": 0.9961,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7193492650985718,
      "rewards/margins": 0.6049911975860596,
      "rewards/rejected": -1.324340581893921,
      "step": 547
    },
    {
      "epoch": 0.8284202569916855,
      "grad_norm": 7.866860389709473,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.013059716671705246,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.4080854642541826e-08,
      "logits/chosen": -1.3938990831375122,
      "logits/rejected": -1.667206883430481,
      "logps/chosen": -150.2476806640625,
      "logps/ref_chosen": -81.16053009033203,
      "logps/ref_rejected": -99.7246322631836,
      "logps/rejected": -199.9046630859375,
      "loss": 1.1613,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9006539583206177,
      "rewards/margins": 0.39945849776268005,
      "rewards/rejected": -1.3001124858856201,
      "step": 548
    },
    {
      "epoch": 0.8299319727891157,
      "grad_norm": 10.859885215759277,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.013002828694880009,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.333396073857723e-08,
      "logits/chosen": -1.2998642921447754,
      "logits/rejected": -1.5371828079223633,
      "logps/chosen": -145.29257202148438,
      "logps/ref_chosen": -80.49800872802734,
      "logps/ref_rejected": -113.20750427246094,
      "logps/rejected": -204.1650390625,
      "loss": 1.2344,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8418738842010498,
      "rewards/margins": 0.3334413170814514,
      "rewards/rejected": -1.1753151416778564,
      "step": 549
    },
    {
      "epoch": 0.8314436885865457,
      "grad_norm": 13.323412895202637,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.012954250909388065,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 4.259284772799099e-08,
      "logits/chosen": -1.330683946609497,
      "logits/rejected": -1.3588637113571167,
      "logps/chosen": -143.0060272216797,
      "logps/ref_chosen": -75.13760375976562,
      "logps/ref_rejected": -79.04876708984375,
      "logps/rejected": -175.20448303222656,
      "loss": 1.206,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.8783211708068848,
      "rewards/margins": 0.3595704436302185,
      "rewards/rejected": -1.237891674041748,
      "step": 550
    },
    {
      "epoch": 0.8329554043839759,
      "grad_norm": 10.381247520446777,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.012907862663269043,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 4.1857536341307176e-08,
      "logits/chosen": -1.2748838663101196,
      "logits/rejected": -1.6231095790863037,
      "logps/chosen": -150.6908721923828,
      "logps/ref_chosen": -85.44961547851562,
      "logps/ref_rejected": -103.48530578613281,
      "logps/rejected": -202.43276977539062,
      "loss": 1.1017,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8378610014915466,
      "rewards/margins": 0.42936253547668457,
      "rewards/rejected": -1.267223596572876,
      "step": 551
    },
    {
      "epoch": 0.8344671201814059,
      "grad_norm": 8.95328140258789,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.012851636856794357,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 4.112804714676593e-08,
      "logits/chosen": -1.331233263015747,
      "logits/rejected": -1.4290517568588257,
      "logps/chosen": -144.78970336914062,
      "logps/ref_chosen": -82.01036071777344,
      "logps/ref_rejected": -101.61883544921875,
      "logps/rejected": -194.92196655273438,
      "loss": 1.1654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8052812814712524,
      "rewards/margins": 0.38555577397346497,
      "rewards/rejected": -1.190837025642395,
      "step": 552
    },
    {
      "epoch": 0.8359788359788359,
      "grad_norm": 14.167531967163086,
      "kl/avg_steps": 0.296875,
      "kl/beta": 0.012791676446795464,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.0404400549748144e-08,
      "logits/chosen": -1.1294629573822021,
      "logits/rejected": -1.3467081785202026,
      "logps/chosen": -143.1853485107422,
      "logps/ref_chosen": -73.81416320800781,
      "logps/ref_rejected": -104.27049255371094,
      "logps/rejected": -211.3553466796875,
      "loss": 1.1377,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.8864268660545349,
      "rewards/margins": 0.47457796335220337,
      "rewards/rejected": -1.3610048294067383,
      "step": 553
    },
    {
      "epoch": 0.8374905517762661,
      "grad_norm": 8.690716743469238,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.012753813527524471,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": -1.6826214790344238,
      "logits/rejected": -1.6105924844741821,
      "logps/chosen": -148.13287353515625,
      "logps/ref_chosen": -81.43980407714844,
      "logps/ref_rejected": -89.32518005371094,
      "logps/rejected": -194.12750244140625,
      "loss": 1.102,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8492714762687683,
      "rewards/margins": 0.4789280295372009,
      "rewards/rejected": -1.3281995058059692,
      "step": 554
    },
    {
      "epoch": 0.8390022675736961,
      "grad_norm": 7.373947620391846,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.012690361589193344,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 3.89747159520904e-08,
      "logits/chosen": -1.260491967201233,
      "logits/rejected": -1.153106451034546,
      "logps/chosen": -151.04708862304688,
      "logps/ref_chosen": -81.66071319580078,
      "logps/ref_rejected": -87.20857238769531,
      "logps/rejected": -195.9105224609375,
      "loss": 1.1195,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.8781550526618958,
      "rewards/margins": 0.49082207679748535,
      "rewards/rejected": -1.3689770698547363,
      "step": 555
    },
    {
      "epoch": 0.8405139833711263,
      "grad_norm": 7.1971821784973145,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.01261545717716217,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.826871794280192e-08,
      "logits/chosen": -1.067447543144226,
      "logits/rejected": -1.1644517183303833,
      "logps/chosen": -138.2876739501953,
      "logps/ref_chosen": -66.02448272705078,
      "logps/ref_rejected": -82.7474594116211,
      "logps/rejected": -190.40380859375,
      "loss": 1.125,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9105790853500366,
      "rewards/margins": 0.4397725760936737,
      "rewards/rejected": -1.3503515720367432,
      "step": 556
    },
    {
      "epoch": 0.8420256991685563,
      "grad_norm": 7.1572651863098145,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.012568325735628605,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.756864251262143e-08,
      "logits/chosen": -1.3210291862487793,
      "logits/rejected": -1.946714162826538,
      "logps/chosen": -140.40304565429688,
      "logps/ref_chosen": -73.08985900878906,
      "logps/ref_rejected": -97.43034362792969,
      "logps/rejected": -209.40664672851562,
      "loss": 1.0402,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8453344106674194,
      "rewards/margins": 0.5539640188217163,
      "rewards/rejected": -1.3992985486984253,
      "step": 557
    },
    {
      "epoch": 0.8435374149659864,
      "grad_norm": 8.183286666870117,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.01250968687236309,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.687450924416341e-08,
      "logits/chosen": -1.392795443534851,
      "logits/rejected": -1.3525338172912598,
      "logps/chosen": -148.27149963378906,
      "logps/ref_chosen": -80.1357192993164,
      "logps/ref_rejected": -106.65797424316406,
      "logps/rejected": -216.39918518066406,
      "loss": 1.0724,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8508350253105164,
      "rewards/margins": 0.5131612420082092,
      "rewards/rejected": -1.3639963865280151,
      "step": 558
    },
    {
      "epoch": 0.8450491307634165,
      "grad_norm": 8.342129707336426,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.012451320886611938,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.6186337553827743e-08,
      "logits/chosen": -1.437471866607666,
      "logits/rejected": -1.7483036518096924,
      "logps/chosen": -144.3199462890625,
      "logps/ref_chosen": -79.42267608642578,
      "logps/ref_rejected": -98.59402465820312,
      "logps/rejected": -200.57098388671875,
      "loss": 1.1613,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.8071417808532715,
      "rewards/margins": 0.4540879726409912,
      "rewards/rejected": -1.2612297534942627,
      "step": 559
    },
    {
      "epoch": 0.8465608465608465,
      "grad_norm": 9.320462226867676,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.012400942854583263,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.550414669125573e-08,
      "logits/chosen": -1.2708487510681152,
      "logits/rejected": -1.1666545867919922,
      "logps/chosen": -143.94570922851562,
      "logps/ref_chosen": -77.49559020996094,
      "logps/ref_rejected": -92.61347961425781,
      "logps/rejected": -200.3899383544922,
      "loss": 1.074,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8230661749839783,
      "rewards/margins": 0.5055770874023438,
      "rewards/rejected": -1.3286433219909668,
      "step": 560
    },
    {
      "epoch": 0.8480725623582767,
      "grad_norm": 6.0093607902526855,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.012350767850875854,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.482795573879241e-08,
      "logits/chosen": -1.5282151699066162,
      "logits/rejected": -1.2929655313491821,
      "logps/chosen": -144.45321655273438,
      "logps/ref_chosen": -79.20771789550781,
      "logps/ref_rejected": -93.46514892578125,
      "logps/rejected": -201.19638061523438,
      "loss": 1.0516,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8044455647468567,
      "rewards/margins": 0.518153965473175,
      "rewards/rejected": -1.3225995302200317,
      "step": 561
    },
    {
      "epoch": 0.8495842781557067,
      "grad_norm": 7.105699062347412,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.012296968139708042,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.415778361095226e-08,
      "logits/chosen": -1.346250057220459,
      "logits/rejected": -1.459496021270752,
      "logps/chosen": -162.78598022460938,
      "logps/ref_chosen": -94.88652801513672,
      "logps/ref_rejected": -109.33815002441406,
      "logps/rejected": -219.45779418945312,
      "loss": 1.0508,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8341754078865051,
      "rewards/margins": 0.5127763152122498,
      "rewards/rejected": -1.3469517230987549,
      "step": 562
    },
    {
      "epoch": 0.8510959939531368,
      "grad_norm": 8.515876770019531,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.012245308607816696,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.349364905389032e-08,
      "logits/chosen": -1.0890107154846191,
      "logits/rejected": -1.182828664779663,
      "logps/chosen": -123.77273559570312,
      "logps/ref_chosen": -65.90719604492188,
      "logps/ref_rejected": -84.07121276855469,
      "logps/rejected": -183.04513549804688,
      "loss": 1.1116,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.7075780630111694,
      "rewards/margins": 0.4956972599029541,
      "rewards/rejected": -1.2032753229141235,
      "step": 563
    },
    {
      "epoch": 0.8526077097505669,
      "grad_norm": 7.4643330574035645,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.01218817662447691,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.283557064487785e-08,
      "logits/chosen": -1.354698657989502,
      "logits/rejected": -1.3166128396987915,
      "logps/chosen": -131.1250457763672,
      "logps/ref_chosen": -72.32070922851562,
      "logps/ref_rejected": -88.05013275146484,
      "logps/rejected": -188.62979125976562,
      "loss": 1.1037,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.7161024212837219,
      "rewards/margins": 0.501507580280304,
      "rewards/rejected": -1.2176098823547363,
      "step": 564
    },
    {
      "epoch": 0.854119425547997,
      "grad_norm": 6.4226274490356445,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.012131310999393463,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.218356679178252e-08,
      "logits/chosen": -1.488287329673767,
      "logits/rejected": -1.343492031097412,
      "logps/chosen": -151.63772583007812,
      "logps/ref_chosen": -80.18453979492188,
      "logps/ref_rejected": -99.55126953125,
      "logps/rejected": -208.01727294921875,
      "loss": 1.1099,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8652534484863281,
      "rewards/margins": 0.44262397289276123,
      "rewards/rejected": -1.3078773021697998,
      "step": 565
    },
    {
      "epoch": 0.8556311413454271,
      "grad_norm": 10.024022102355957,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.012085988186299801,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.1537655732553764e-08,
      "logits/chosen": -1.485852599143982,
      "logits/rejected": -1.6031005382537842,
      "logps/chosen": -150.26303100585938,
      "logps/ref_chosen": -88.0877914428711,
      "logps/ref_rejected": -87.7589111328125,
      "logps/rejected": -187.7733154296875,
      "loss": 1.1223,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7501992583274841,
      "rewards/margins": 0.45009276270866394,
      "rewards/rejected": -1.2002918720245361,
      "step": 566
    },
    {
      "epoch": 0.8571428571428571,
      "grad_norm": 8.078023910522461,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.012033342383801937,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.089785553471233e-08,
      "logits/chosen": -1.1604113578796387,
      "logits/rejected": -1.6355693340301514,
      "logps/chosen": -135.47256469726562,
      "logps/ref_chosen": -69.93267822265625,
      "logps/ref_rejected": -95.71786499023438,
      "logps/rejected": -205.11859130859375,
      "loss": 1.0358,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.7865634560585022,
      "rewards/margins": 0.5210832953453064,
      "rewards/rejected": -1.3076467514038086,
      "step": 567
    },
    {
      "epoch": 0.8586545729402872,
      "grad_norm": 6.185924053192139,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.011969752609729767,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 3.026418409484513e-08,
      "logits/chosen": -1.6622331142425537,
      "logits/rejected": -1.738410472869873,
      "logps/chosen": -126.7143325805664,
      "logps/ref_chosen": -70.33343505859375,
      "logps/ref_rejected": -108.86270904541016,
      "logps/rejected": -210.11373901367188,
      "loss": 1.0413,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6726874113082886,
      "rewards/margins": 0.5302541255950928,
      "rewards/rejected": -1.2029415369033813,
      "step": 568
    },
    {
      "epoch": 0.8601662887377173,
      "grad_norm": 8.796441078186035,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.011913906782865524,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.963665913810451e-08,
      "logits/chosen": -1.6774063110351562,
      "logits/rejected": -1.4297425746917725,
      "logps/chosen": -146.59378051757812,
      "logps/ref_chosen": -80.85043334960938,
      "logps/ref_rejected": -92.77810668945312,
      "logps/rejected": -190.6685791015625,
      "loss": 1.1798,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7816181778907776,
      "rewards/margins": 0.3761640191078186,
      "rewards/rejected": -1.1577821969985962,
      "step": 569
    },
    {
      "epoch": 0.8616780045351474,
      "grad_norm": 5.814436912536621,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.0118583207949996,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 2.9015298217712453e-08,
      "logits/chosen": -1.6274659633636475,
      "logits/rejected": -1.604426383972168,
      "logps/chosen": -124.55775451660156,
      "logps/ref_chosen": -69.94769287109375,
      "logps/ref_rejected": -97.37059020996094,
      "logps/rejected": -204.11428833007812,
      "loss": 0.9832,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.6452935338020325,
      "rewards/margins": 0.6108759641647339,
      "rewards/rejected": -1.2561695575714111,
      "step": 570
    },
    {
      "epoch": 0.8631897203325775,
      "grad_norm": 11.918949127197266,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.01178466621786356,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.840011871446962e-08,
      "logits/chosen": -1.4016327857971191,
      "logits/rejected": -1.4089144468307495,
      "logps/chosen": -135.0509796142578,
      "logps/ref_chosen": -72.28555297851562,
      "logps/ref_rejected": -84.57748413085938,
      "logps/rejected": -176.27073669433594,
      "loss": 1.2017,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7387359142303467,
      "rewards/margins": 0.3344320058822632,
      "rewards/rejected": -1.0731679201126099,
      "step": 571
    },
    {
      "epoch": 0.8647014361300076,
      "grad_norm": 7.906334400177002,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.011733333580195904,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 2.7791137836269158e-08,
      "logits/chosen": -1.5348182916641235,
      "logits/rejected": -1.1779015064239502,
      "logps/chosen": -158.45730590820312,
      "logps/ref_chosen": -91.49070739746094,
      "logps/ref_rejected": -80.44602966308594,
      "logps/rejected": -184.7002716064453,
      "loss": 1.0925,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7837091684341431,
      "rewards/margins": 0.4318312406539917,
      "rewards/rejected": -1.2155404090881348,
      "step": 572
    },
    {
      "epoch": 0.8662131519274376,
      "grad_norm": 8.190794944763184,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.011674958281219006,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.718837261761528e-08,
      "logits/chosen": -1.6026732921600342,
      "logits/rejected": -1.552854061126709,
      "logps/chosen": -156.527587890625,
      "logps/ref_chosen": -87.54232788085938,
      "logps/ref_rejected": -104.32984924316406,
      "logps/rejected": -210.606689453125,
      "loss": 1.1659,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8060653209686279,
      "rewards/margins": 0.4282020926475525,
      "rewards/rejected": -1.2342674732208252,
      "step": 573
    },
    {
      "epoch": 0.8677248677248677,
      "grad_norm": 6.598454475402832,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.011638588272035122,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.659183991914696e-08,
      "logits/chosen": -1.359856367111206,
      "logits/rejected": -1.107245683670044,
      "logps/chosen": -137.58914184570312,
      "logps/ref_chosen": -75.36632537841797,
      "logps/ref_rejected": -103.27328491210938,
      "logps/rejected": -213.42083740234375,
      "loss": 1.0219,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.722530722618103,
      "rewards/margins": 0.5507799386978149,
      "rewards/rejected": -1.273310661315918,
      "step": 574
    },
    {
      "epoch": 0.8692365835222978,
      "grad_norm": 6.816895008087158,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.011584286577999592,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.600155642716606e-08,
      "logits/chosen": -1.1317976713180542,
      "logits/rejected": -1.4034576416015625,
      "logps/chosen": -150.38589477539062,
      "logps/ref_chosen": -81.678466796875,
      "logps/ref_rejected": -112.84233093261719,
      "logps/rejected": -212.930419921875,
      "loss": 1.193,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7960365414619446,
      "rewards/margins": 0.3575257360935211,
      "rewards/rejected": -1.153562307357788,
      "step": 575
    },
    {
      "epoch": 0.8707482993197279,
      "grad_norm": 6.988912105560303,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.011544601991772652,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 2.5417538653170754e-08,
      "logits/chosen": -1.3333091735839844,
      "logits/rejected": -1.4861361980438232,
      "logps/chosen": -125.4324722290039,
      "logps/ref_chosen": -68.78944396972656,
      "logps/ref_rejected": -102.79037475585938,
      "logps/rejected": -207.86514282226562,
      "loss": 1.0285,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.6519675254821777,
      "rewards/margins": 0.5519933700561523,
      "rewards/rejected": -1.20396089553833,
      "step": 576
    },
    {
      "epoch": 0.872260015117158,
      "grad_norm": 7.1114277839660645,
      "kl/avg_steps": 0.390625,
      "kl/beta": 0.011480026878416538,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.4839802933393607e-08,
      "logits/chosen": -1.477907419204712,
      "logits/rejected": -1.5636188983917236,
      "logps/chosen": -141.4581298828125,
      "logps/ref_chosen": -79.84674835205078,
      "logps/ref_rejected": -84.08309936523438,
      "logps/rejected": -173.64480590820312,
      "loss": 1.2078,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7065240144729614,
      "rewards/margins": 0.3152022957801819,
      "rewards/rejected": -1.021726369857788,
      "step": 577
    },
    {
      "epoch": 0.873771730914588,
      "grad_norm": 6.939554691314697,
      "kl/avg_steps": 0.21875,
      "kl/beta": 0.011435357853770256,
      "kl/n_epsilon_steps": 0.390625,
      "kl/p_epsilon_steps": 0.609375,
      "learning_rate": 2.4268365428344733e-08,
      "logits/chosen": -1.0849415063858032,
      "logits/rejected": -1.3800511360168457,
      "logps/chosen": -138.2547149658203,
      "logps/ref_chosen": -74.91357421875,
      "logps/ref_rejected": -83.64881896972656,
      "logps/rejected": -173.4388885498047,
      "loss": 1.2029,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7240015268325806,
      "rewards/margins": 0.2974473834037781,
      "rewards/rejected": -1.0214489698410034,
      "step": 578
    },
    {
      "epoch": 0.8752834467120182,
      "grad_norm": 6.664099216461182,
      "kl/avg_steps": 0.65625,
      "kl/beta": 0.011410397477447987,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.828125,
      "learning_rate": 2.3703242122359357e-08,
      "logits/chosen": -1.2816755771636963,
      "logits/rejected": -1.155468225479126,
      "logps/chosen": -143.69326782226562,
      "logps/ref_chosen": -75.51022338867188,
      "logps/ref_rejected": -84.83192443847656,
      "logps/rejected": -195.86459350585938,
      "loss": 1.059,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.7744640707969666,
      "rewards/margins": 0.48263052105903625,
      "rewards/rejected": -1.2570946216583252,
      "step": 579
    },
    {
      "epoch": 0.8767951625094482,
      "grad_norm": 7.74460506439209,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.011336004361510277,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 2.3144448823151392e-08,
      "logits/chosen": -1.5838890075683594,
      "logits/rejected": -1.6980791091918945,
      "logps/chosen": -133.63670349121094,
      "logps/ref_chosen": -76.61564636230469,
      "logps/ref_rejected": -97.09959411621094,
      "logps/rejected": -192.4454345703125,
      "loss": 1.1247,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6440767645835876,
      "rewards/margins": 0.4278578460216522,
      "rewards/rejected": -1.0719345808029175,
      "step": 580
    },
    {
      "epoch": 0.8783068783068783,
      "grad_norm": 6.301844120025635,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.011276100762188435,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.259200116137039e-08,
      "logits/chosen": -1.5977568626403809,
      "logits/rejected": -1.468361735343933,
      "logps/chosen": -145.87472534179688,
      "logps/ref_chosen": -74.8531265258789,
      "logps/ref_rejected": -101.5344009399414,
      "logps/rejected": -210.183349609375,
      "loss": 1.1315,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.8000357151031494,
      "rewards/margins": 0.41744327545166016,
      "rewards/rejected": -1.2174789905548096,
      "step": 581
    },
    {
      "epoch": 0.8798185941043084,
      "grad_norm": 6.888444423675537,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.011225235648453236,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.204591459016525e-08,
      "logits/chosen": -1.4427552223205566,
      "logits/rejected": -1.0628981590270996,
      "logps/chosen": -148.77474975585938,
      "logps/ref_chosen": -81.07638549804688,
      "logps/ref_rejected": -72.83570861816406,
      "logps/rejected": -178.07257080078125,
      "loss": 1.1383,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.7592304348945618,
      "rewards/margins": 0.41485366225242615,
      "rewards/rejected": -1.1740840673446655,
      "step": 582
    },
    {
      "epoch": 0.8813303099017384,
      "grad_norm": 8.803990364074707,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.011179817840456963,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 2.1506204384751064e-08,
      "logits/chosen": -1.0734798908233643,
      "logits/rejected": -1.4834539890289307,
      "logps/chosen": -133.71963500976562,
      "logps/ref_chosen": -66.78465270996094,
      "logps/ref_rejected": -106.45826721191406,
      "logps/rejected": -207.7127685546875,
      "loss": 1.1892,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7471137046813965,
      "rewards/margins": 0.3767395615577698,
      "rewards/rejected": -1.1238532066345215,
      "step": 583
    },
    {
      "epoch": 0.8828420256991686,
      "grad_norm": 7.770105361938477,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.011138049885630608,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 2.09728856419826e-08,
      "logits/chosen": -0.9404406547546387,
      "logits/rejected": -1.262127161026001,
      "logps/chosen": -116.41361236572266,
      "logps/ref_chosen": -60.80291748046875,
      "logps/ref_rejected": -99.45012664794922,
      "logps/rejected": -191.92051696777344,
      "loss": 1.1544,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.6201804876327515,
      "rewards/margins": 0.4037685990333557,
      "rewards/rejected": -1.023949146270752,
      "step": 584
    },
    {
      "epoch": 0.8843537414965986,
      "grad_norm": 5.805047988891602,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.011103352531790733,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.044597327993153e-08,
      "logits/chosen": -1.5974360704421997,
      "logits/rejected": -1.2341102361679077,
      "logps/chosen": -140.76992797851562,
      "logps/ref_chosen": -75.92616271972656,
      "logps/ref_rejected": -94.47601318359375,
      "logps/rejected": -186.14279174804688,
      "loss": 1.2217,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.7173092365264893,
      "rewards/margins": 0.2924202084541321,
      "rewards/rejected": -1.0097295045852661,
      "step": 585
    },
    {
      "epoch": 0.8858654572940288,
      "grad_norm": 7.581094264984131,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.011051548644900322,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 1.9925482037469187e-08,
      "logits/chosen": -1.2922358512878418,
      "logits/rejected": -1.3954906463623047,
      "logps/chosen": -128.05166625976562,
      "logps/ref_chosen": -68.62062072753906,
      "logps/ref_rejected": -81.98324584960938,
      "logps/rejected": -186.36367797851562,
      "loss": 1.0506,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6543450355529785,
      "rewards/margins": 0.4905773401260376,
      "rewards/rejected": -1.1449224948883057,
      "step": 586
    },
    {
      "epoch": 0.8873771730914588,
      "grad_norm": 10.84594440460205,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.010986316949129105,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.9411426473854687e-08,
      "logits/chosen": -1.5820139646530151,
      "logits/rejected": -1.4005577564239502,
      "logps/chosen": -133.91355895996094,
      "logps/ref_chosen": -77.67031860351562,
      "logps/ref_rejected": -79.35327911376953,
      "logps/rejected": -183.68438720703125,
      "loss": 1.0622,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.6174869537353516,
      "rewards/margins": 0.5209751129150391,
      "rewards/rejected": -1.1384620666503906,
      "step": 587
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 12.146368026733398,
      "kl/avg_steps": 0.546875,
      "kl/beta": 0.010935058817267418,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.890382096832699e-08,
      "logits/chosen": -1.804541826248169,
      "logits/rejected": -1.8112810850143433,
      "logps/chosen": -143.16195678710938,
      "logps/ref_chosen": -77.94320678710938,
      "logps/ref_rejected": -98.41210174560547,
      "logps/rejected": -204.6304168701172,
      "loss": 1.0801,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.7118812203407288,
      "rewards/margins": 0.44240012764930725,
      "rewards/rejected": -1.1542813777923584,
      "step": 588
    },
    {
      "epoch": 0.890400604686319,
      "grad_norm": 7.836277008056641,
      "kl/avg_steps": 0.6875,
      "kl/beta": 0.010875582695007324,
      "kl/n_epsilon_steps": 0.15625,
      "kl/p_epsilon_steps": 0.84375,
      "learning_rate": 1.840267971970344e-08,
      "logits/chosen": -1.6829819679260254,
      "logits/rejected": -1.4134776592254639,
      "logps/chosen": -132.46250915527344,
      "logps/ref_chosen": -75.18646240234375,
      "logps/ref_rejected": -93.35910034179688,
      "logps/rejected": -196.56468200683594,
      "loss": 1.0411,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.6206521987915039,
      "rewards/margins": 0.493114709854126,
      "rewards/rejected": -1.1137669086456299,
      "step": 589
    },
    {
      "epoch": 0.891912320483749,
      "grad_norm": 8.583623886108398,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.010801323689520359,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -1.4794654846191406,
      "logits/rejected": -1.2498228549957275,
      "logps/chosen": -152.60191345214844,
      "logps/ref_chosen": -86.9908447265625,
      "logps/ref_rejected": -100.61723327636719,
      "logps/rejected": -209.6766357421875,
      "loss": 1.0804,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7086925506591797,
      "rewards/margins": 0.463258296251297,
      "rewards/rejected": -1.1719508171081543,
      "step": 590
    },
    {
      "epoch": 0.8934240362811792,
      "grad_norm": 8.543137550354004,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.01075427420437336,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.7419845883949098e-08,
      "logits/chosen": -1.5658700466156006,
      "logits/rejected": -1.5437417030334473,
      "logps/chosen": -126.18582153320312,
      "logps/ref_chosen": -74.85809326171875,
      "logps/ref_rejected": -102.75840759277344,
      "logps/rejected": -199.054931640625,
      "loss": 1.0779,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5521831512451172,
      "rewards/margins": 0.476978600025177,
      "rewards/rejected": -1.029161810874939,
      "step": 591
    },
    {
      "epoch": 0.8949357520786092,
      "grad_norm": 6.509529113769531,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.010709094814956188,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.6938180788793556e-08,
      "logits/chosen": -1.451214075088501,
      "logits/rejected": -1.452272653579712,
      "logps/chosen": -123.59173583984375,
      "logps/ref_chosen": -67.90579223632812,
      "logps/ref_rejected": -100.35234069824219,
      "logps/rejected": -196.5985107421875,
      "loss": 1.0975,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5943005084991455,
      "rewards/margins": 0.42885279655456543,
      "rewards/rejected": -1.023153305053711,
      "step": 592
    },
    {
      "epoch": 0.8964474678760394,
      "grad_norm": 7.256924152374268,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.010652503930032253,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.6463034933723336e-08,
      "logits/chosen": -1.1163625717163086,
      "logits/rejected": -1.4272210597991943,
      "logps/chosen": -113.16133117675781,
      "logps/ref_chosen": -59.29489517211914,
      "logps/ref_rejected": -85.31307983398438,
      "logps/rejected": -180.05154418945312,
      "loss": 1.1152,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5734096765518188,
      "rewards/margins": 0.4289560914039612,
      "rewards/rejected": -1.0023657083511353,
      "step": 593
    },
    {
      "epoch": 0.8979591836734694,
      "grad_norm": 8.194808959960938,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.010599506087601185,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.5994421609589385e-08,
      "logits/chosen": -1.1510372161865234,
      "logits/rejected": -1.1432057619094849,
      "logps/chosen": -146.3094482421875,
      "logps/ref_chosen": -83.14643859863281,
      "logps/ref_rejected": -88.201904296875,
      "logps/rejected": -188.5896453857422,
      "loss": 1.1219,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6681286096572876,
      "rewards/margins": 0.3892432451248169,
      "rewards/rejected": -1.0573718547821045,
      "step": 594
    },
    {
      "epoch": 0.8994708994708994,
      "grad_norm": 9.508260726928711,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.010559906251728535,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.553235392451377e-08,
      "logits/chosen": -1.1011279821395874,
      "logits/rejected": -1.4126760959625244,
      "logps/chosen": -129.28802490234375,
      "logps/ref_chosen": -70.40016174316406,
      "logps/ref_rejected": -103.95550537109375,
      "logps/rejected": -212.98629760742188,
      "loss": 1.0528,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.6204409599304199,
      "rewards/margins": 0.5221980810165405,
      "rewards/rejected": -1.14263916015625,
      "step": 595
    },
    {
      "epoch": 0.9009826152683296,
      "grad_norm": 7.493839740753174,
      "kl/avg_steps": 0.09375,
      "kl/beta": 0.010504103265702724,
      "kl/n_epsilon_steps": 0.453125,
      "kl/p_epsilon_steps": 0.546875,
      "learning_rate": 1.507684480352292e-08,
      "logits/chosen": -1.470296859741211,
      "logits/rejected": -1.4461066722869873,
      "logps/chosen": -156.56211853027344,
      "logps/ref_chosen": -86.083740234375,
      "logps/ref_rejected": -78.41990661621094,
      "logps/rejected": -165.70452880859375,
      "loss": 1.3176,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7421023845672607,
      "rewards/margins": 0.17188970744609833,
      "rewards/rejected": -0.9139920473098755,
      "step": 596
    },
    {
      "epoch": 0.9024943310657596,
      "grad_norm": 6.163330554962158,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.010494264774024487,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.4627906988186111e-08,
      "logits/chosen": -1.3851312398910522,
      "logits/rejected": -1.1142826080322266,
      "logps/chosen": -121.51483154296875,
      "logps/ref_chosen": -67.8086166381836,
      "logps/ref_rejected": -71.09245300292969,
      "logps/rejected": -162.0400390625,
      "loss": 1.1369,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5630039572715759,
      "rewards/margins": 0.3849661350250244,
      "rewards/rejected": -0.9479700922966003,
      "step": 597
    },
    {
      "epoch": 0.9040060468631897,
      "grad_norm": 6.907870292663574,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.010451804846525192,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.4185553036259095e-08,
      "logits/chosen": -1.369914174079895,
      "logits/rejected": -1.562995195388794,
      "logps/chosen": -149.61148071289062,
      "logps/ref_chosen": -74.31095123291016,
      "logps/ref_rejected": -98.08122253417969,
      "logps/rejected": -198.58651733398438,
      "loss": 1.2303,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7870241403579712,
      "rewards/margins": 0.2586115598678589,
      "rewards/rejected": -1.04563570022583,
      "step": 598
    },
    {
      "epoch": 0.9055177626606198,
      "grad_norm": 7.62575101852417,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.01041437778621912,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 1.3749795321332885e-08,
      "logits/chosen": -1.333286166191101,
      "logits/rejected": -1.314929723739624,
      "logps/chosen": -143.8874969482422,
      "logps/ref_chosen": -74.21861267089844,
      "logps/ref_rejected": -90.1492919921875,
      "logps/rejected": -192.62564086914062,
      "loss": 1.1717,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.7245805263519287,
      "rewards/margins": 0.3361985683441162,
      "rewards/rejected": -1.060779094696045,
      "step": 599
    },
    {
      "epoch": 0.9070294784580499,
      "grad_norm": 7.58573579788208,
      "kl/avg_steps": 0.3125,
      "kl/beta": 0.01037870068103075,
      "kl/n_epsilon_steps": 0.34375,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 1.3320646032487393e-08,
      "logits/chosen": -1.4237275123596191,
      "logits/rejected": -1.5688844919204712,
      "logps/chosen": -144.674560546875,
      "logps/ref_chosen": -79.34190368652344,
      "logps/ref_rejected": -97.0519790649414,
      "logps/rejected": -193.67965698242188,
      "loss": 1.1764,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6775575876235962,
      "rewards/margins": 0.3196982741355896,
      "rewards/rejected": -0.997255802154541,
      "step": 600
    },
    {
      "epoch": 0.9070294784580499,
      "eval_kl/n_epsilon_steps": 0.26892605423927307,
      "eval_kl/p_epsilon_steps": 0.7288732528686523,
      "eval_logits/chosen": -1.305195689201355,
      "eval_logits/rejected": -1.431878685951233,
      "eval_logps/chosen": -149.94447326660156,
      "eval_logps/ref_chosen": -86.90177917480469,
      "eval_logps/ref_rejected": -96.69639587402344,
      "eval_logps/rejected": -198.72705078125,
      "eval_loss": 0.563292920589447,
      "eval_rewards/accuracies": 0.73283451795578,
      "eval_rewards/chosen": -0.6511958837509155,
      "eval_rewards/margins": 0.3975641429424286,
      "eval_rewards/rejected": -1.0487600564956665,
      "eval_runtime": 46.8107,
      "eval_samples_per_second": 49.198,
      "eval_steps_per_second": 1.538,
      "step": 600
    },
    {
      "epoch": 0.90854119425548,
      "grad_norm": 8.043039321899414,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.0103463688865304,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.2898117173950868e-08,
      "logits/chosen": -1.517151117324829,
      "logits/rejected": -1.5460131168365479,
      "logps/chosen": -125.53500366210938,
      "logps/ref_chosen": -72.06497192382812,
      "logps/ref_rejected": -97.60928344726562,
      "logps/rejected": -198.99093627929688,
      "loss": 1.0588,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5525596141815186,
      "rewards/margins": 0.48932531476020813,
      "rewards/rejected": -1.0418848991394043,
      "step": 601
    },
    {
      "epoch": 0.91005291005291,
      "grad_norm": 8.432663917541504,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.010294893756508827,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.2482220564763667e-08,
      "logits/chosen": -1.4632465839385986,
      "logits/rejected": -1.6077954769134521,
      "logps/chosen": -126.64736938476562,
      "logps/ref_chosen": -77.80416870117188,
      "logps/ref_rejected": -89.05025482177734,
      "logps/rejected": -179.86968994140625,
      "loss": 1.0903,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5022462606430054,
      "rewards/margins": 0.4265432357788086,
      "rewards/rejected": -0.9287895560264587,
      "step": 602
    },
    {
      "epoch": 0.9115646258503401,
      "grad_norm": 7.695731163024902,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.010246861726045609,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 1.2072967838448051e-08,
      "logits/chosen": -1.5648958683013916,
      "logits/rejected": -1.3041167259216309,
      "logps/chosen": -133.23721313476562,
      "logps/ref_chosen": -68.30155944824219,
      "logps/ref_rejected": -90.542724609375,
      "logps/rejected": -192.48605346679688,
      "loss": 1.1478,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6637769937515259,
      "rewards/margins": 0.3735578656196594,
      "rewards/rejected": -1.0373347997665405,
      "step": 603
    },
    {
      "epoch": 0.9130763416477702,
      "grad_norm": 5.707242012023926,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.010199054144322872,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 1.1670370442682459e-08,
      "logits/chosen": -1.4136242866516113,
      "logits/rejected": -1.5028131008148193,
      "logps/chosen": -142.22543334960938,
      "logps/ref_chosen": -90.55952453613281,
      "logps/ref_rejected": -84.6327133178711,
      "logps/rejected": -173.35304260253906,
      "loss": 1.1714,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5263998508453369,
      "rewards/margins": 0.37176942825317383,
      "rewards/rejected": -0.8981692790985107,
      "step": 604
    },
    {
      "epoch": 0.9145880574452003,
      "grad_norm": 7.411596298217773,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.010157788172364235,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.1274439638981532e-08,
      "logits/chosen": -1.4782711267471313,
      "logits/rejected": -1.6140611171722412,
      "logps/chosen": -147.58096313476562,
      "logps/ref_chosen": -80.26661682128906,
      "logps/ref_rejected": -100.26485443115234,
      "logps/rejected": -201.37017822265625,
      "loss": 1.1592,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6824374794960022,
      "rewards/margins": 0.33790159225463867,
      "rewards/rejected": -1.0203391313552856,
      "step": 605
    },
    {
      "epoch": 0.9160997732426304,
      "grad_norm": 6.872825622558594,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.010113541036844254,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.0885186502381016e-08,
      "logits/chosen": -1.3783996105194092,
      "logits/rejected": -1.3939440250396729,
      "logps/chosen": -126.33575439453125,
      "logps/ref_chosen": -70.73554992675781,
      "logps/ref_rejected": -95.9410400390625,
      "logps/rejected": -190.52963256835938,
      "loss": 1.1098,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5604730844497681,
      "rewards/margins": 0.38919681310653687,
      "rewards/rejected": -0.9496699571609497,
      "step": 606
    },
    {
      "epoch": 0.9176114890400605,
      "grad_norm": 6.968571186065674,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.010075757279992104,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 1.0502621921127774e-08,
      "logits/chosen": -1.3670051097869873,
      "logits/rejected": -1.3761980533599854,
      "logps/chosen": -146.70367431640625,
      "logps/ref_chosen": -81.26203918457031,
      "logps/ref_rejected": -92.71575927734375,
      "logps/rejected": -199.63870239257812,
      "loss": 1.0874,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6559546589851379,
      "rewards/margins": 0.4126221537590027,
      "rewards/rejected": -1.0685768127441406,
      "step": 607
    },
    {
      "epoch": 0.9191232048374905,
      "grad_norm": 8.588226318359375,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.010019398294389248,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.0126756596375685e-08,
      "logits/chosen": -1.1897143125534058,
      "logits/rejected": -1.706724762916565,
      "logps/chosen": -152.38760375976562,
      "logps/ref_chosen": -82.65309143066406,
      "logps/ref_rejected": -110.64334106445312,
      "logps/rejected": -209.7471923828125,
      "loss": 1.1965,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6966760754585266,
      "rewards/margins": 0.28950873017311096,
      "rewards/rejected": -0.9861847758293152,
      "step": 608
    },
    {
      "epoch": 0.9206349206349206,
      "grad_norm": 7.8772454261779785,
      "kl/avg_steps": 0.65625,
      "kl/beta": 0.009969550184905529,
      "kl/n_epsilon_steps": 0.171875,
      "kl/p_epsilon_steps": 0.828125,
      "learning_rate": 9.757601041885694e-09,
      "logits/chosen": -1.3571372032165527,
      "logits/rejected": -1.4184623956680298,
      "logps/chosen": -123.52580261230469,
      "logps/ref_chosen": -68.20231628417969,
      "logps/ref_rejected": -81.90515899658203,
      "logps/rejected": -178.79531860351562,
      "loss": 1.0748,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.5487229228019714,
      "rewards/margins": 0.4092875123023987,
      "rewards/rejected": -0.9580105543136597,
      "step": 609
    },
    {
      "epoch": 0.9221466364323507,
      "grad_norm": 9.354720115661621,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.009904551319777966,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 9.395165583732379e-09,
      "logits/chosen": -1.4947376251220703,
      "logits/rejected": -1.5565390586853027,
      "logps/chosen": -162.8250732421875,
      "logps/ref_chosen": -99.01324462890625,
      "logps/ref_rejected": -102.26054382324219,
      "logps/rejected": -206.55862426757812,
      "loss": 1.1376,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.633036732673645,
      "rewards/margins": 0.39495325088500977,
      "rewards/rejected": -1.0279901027679443,
      "step": 610
    },
    {
      "epoch": 0.9236583522297808,
      "grad_norm": 6.941373348236084,
      "kl/avg_steps": 0.328125,
      "kl/beta": 0.009867548011243343,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.65625,
      "learning_rate": 9.03946036001449e-09,
      "logits/chosen": -1.7924981117248535,
      "logits/rejected": -1.8070666790008545,
      "logps/chosen": -124.20469665527344,
      "logps/ref_chosen": -66.36254119873047,
      "logps/ref_rejected": -88.74557495117188,
      "logps/rejected": -175.82785034179688,
      "loss": 1.192,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.5700873136520386,
      "rewards/margins": 0.28417256474494934,
      "rewards/rejected": -0.8542598485946655,
      "step": 611
    },
    {
      "epoch": 0.9251700680272109,
      "grad_norm": 6.236355781555176,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.009835276752710342,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 8.690495320571839e-09,
      "logits/chosen": -1.3240669965744019,
      "logits/rejected": -1.4752863645553589,
      "logps/chosen": -145.45907592773438,
      "logps/ref_chosen": -78.6339111328125,
      "logps/ref_rejected": -108.34970092773438,
      "logps/rejected": -221.23175048828125,
      "loss": 1.0902,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.6558361053466797,
      "rewards/margins": 0.4467426538467407,
      "rewards/rejected": -1.10257887840271,
      "step": 612
    },
    {
      "epoch": 0.926681783824641,
      "grad_norm": 6.665154933929443,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.00978634413331747,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 8.348280226706722e-09,
      "logits/chosen": -1.4203985929489136,
      "logits/rejected": -0.9556962251663208,
      "logps/chosen": -125.75762939453125,
      "logps/ref_chosen": -73.3539047241211,
      "logps/ref_rejected": -76.91837310791016,
      "logps/rejected": -180.04534912109375,
      "loss": 1.0513,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5114285945892334,
      "rewards/margins": 0.4900427460670471,
      "rewards/rejected": -1.0014714002609253,
      "step": 613
    },
    {
      "epoch": 0.9281934996220711,
      "grad_norm": 7.1324639320373535,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.009728580713272095,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 8.012824650910937e-09,
      "logits/chosen": -1.2377476692199707,
      "logits/rejected": -0.9786287546157837,
      "logps/chosen": -145.08615112304688,
      "logps/ref_chosen": -77.80007934570312,
      "logps/ref_rejected": -89.05572509765625,
      "logps/rejected": -194.45004272460938,
      "loss": 1.1136,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.6522784233093262,
      "rewards/margins": 0.36581069231033325,
      "rewards/rejected": -1.0180891752243042,
      "step": 614
    },
    {
      "epoch": 0.9297052154195011,
      "grad_norm": 6.5117716789245605,
      "kl/avg_steps": 0.546875,
      "kl/beta": 0.009674163535237312,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 7.684137976598088e-09,
      "logits/chosen": -1.635801911354065,
      "logits/rejected": -1.423606038093567,
      "logps/chosen": -156.5076446533203,
      "logps/ref_chosen": -90.06971740722656,
      "logps/ref_rejected": -118.7764892578125,
      "logps/rejected": -231.023681640625,
      "loss": 1.0939,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.641755223274231,
      "rewards/margins": 0.43708616495132446,
      "rewards/rejected": -1.0788414478302002,
      "step": 615
    },
    {
      "epoch": 0.9312169312169312,
      "grad_norm": 6.9720258712768555,
      "kl/avg_steps": 0.390625,
      "kl/beta": 0.009621545672416687,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 7.36222939784098e-09,
      "logits/chosen": -1.3394547700881958,
      "logits/rejected": -1.4010214805603027,
      "logps/chosen": -138.58969116210938,
      "logps/ref_chosen": -74.62954711914062,
      "logps/ref_rejected": -93.655029296875,
      "logps/rejected": -195.61346435546875,
      "loss": 1.1321,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.6138941049575806,
      "rewards/margins": 0.360870897769928,
      "rewards/rejected": -0.9747650623321533,
      "step": 616
    },
    {
      "epoch": 0.9327286470143613,
      "grad_norm": 7.865924835205078,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.009584108367562294,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 7.047107919114586e-09,
      "logits/chosen": -1.1265982389450073,
      "logits/rejected": -1.2221885919570923,
      "logps/chosen": -147.52825927734375,
      "logps/ref_chosen": -75.98182678222656,
      "logps/ref_rejected": -97.1640625,
      "logps/rejected": -204.99545288085938,
      "loss": 1.1393,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.6835160255432129,
      "rewards/margins": 0.34294775128364563,
      "rewards/rejected": -1.0264636278152466,
      "step": 617
    },
    {
      "epoch": 0.9342403628117913,
      "grad_norm": 13.488085746765137,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.00953346211463213,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 6.738782355044048e-09,
      "logits/chosen": -1.4303864240646362,
      "logits/rejected": -1.5783617496490479,
      "logps/chosen": -134.54786682128906,
      "logps/ref_chosen": -74.47208404541016,
      "logps/ref_rejected": -107.09980010986328,
      "logps/rejected": -201.03564453125,
      "loss": 1.1718,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5725345611572266,
      "rewards/margins": 0.31801384687423706,
      "rewards/rejected": -0.8905483484268188,
      "step": 618
    },
    {
      "epoch": 0.9357520786092215,
      "grad_norm": 6.205716133117676,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.009497844614088535,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 6.437261330158206e-09,
      "logits/chosen": -1.154737949371338,
      "logits/rejected": -1.3640652894973755,
      "logps/chosen": -131.15756225585938,
      "logps/ref_chosen": -70.84220886230469,
      "logps/ref_rejected": -98.07801818847656,
      "logps/rejected": -203.77041625976562,
      "loss": 1.0922,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.5716289281845093,
      "rewards/margins": 0.425299733877182,
      "rewards/rejected": -0.9969286918640137,
      "step": 619
    },
    {
      "epoch": 0.9372637944066515,
      "grad_norm": 6.4136128425598145,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.00944471824914217,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 6.142553278648238e-09,
      "logits/chosen": -1.5454761981964111,
      "logits/rejected": -1.2460343837738037,
      "logps/chosen": -134.32882690429688,
      "logps/ref_chosen": -76.93606567382812,
      "logps/ref_rejected": -81.28453063964844,
      "logps/rejected": -172.30657958984375,
      "loss": 1.1796,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.540812611579895,
      "rewards/margins": 0.31257152557373047,
      "rewards/rejected": -0.8533841371536255,
      "step": 620
    },
    {
      "epoch": 0.9387755102040817,
      "grad_norm": 6.1778082847595215,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.009394808672368526,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 5.854666444131934e-09,
      "logits/chosen": -1.2180217504501343,
      "logits/rejected": -1.4678857326507568,
      "logps/chosen": -135.63153076171875,
      "logps/ref_chosen": -69.87464904785156,
      "logps/ref_rejected": -105.61328887939453,
      "logps/rejected": -203.57086181640625,
      "loss": 1.1843,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.6170968413352966,
      "rewards/margins": 0.29768118262290955,
      "rewards/rejected": -0.9147779941558838,
      "step": 621
    },
    {
      "epoch": 0.9402872260015117,
      "grad_norm": 6.1825947761535645,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.009356796741485596,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 5.573608879422875e-09,
      "logits/chosen": -1.7117747068405151,
      "logits/rejected": -1.6740427017211914,
      "logps/chosen": -144.62969970703125,
      "logps/ref_chosen": -78.9598388671875,
      "logps/ref_rejected": -97.906494140625,
      "logps/rejected": -199.75100708007812,
      "loss": 1.1453,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.6132454872131348,
      "rewards/margins": 0.3336794972419739,
      "rewards/rejected": -0.9469249844551086,
      "step": 622
    },
    {
      "epoch": 0.9417989417989417,
      "grad_norm": 5.886653423309326,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.00930735096335411,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 5.299388446305342e-09,
      "logits/chosen": -1.468321442604065,
      "logits/rejected": -1.4235600233078003,
      "logps/chosen": -155.97503662109375,
      "logps/ref_chosen": -83.22647094726562,
      "logps/ref_rejected": -105.13624572753906,
      "logps/rejected": -216.748046875,
      "loss": 1.1266,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.6760995388031006,
      "rewards/margins": 0.3567197918891907,
      "rewards/rejected": -1.0328192710876465,
      "step": 623
    },
    {
      "epoch": 0.9433106575963719,
      "grad_norm": 6.489195346832275,
      "kl/avg_steps": 0.484375,
      "kl/beta": 0.009263926185667515,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 5.03201281531429e-09,
      "logits/chosen": -1.2762702703475952,
      "logits/rejected": -1.4331917762756348,
      "logps/chosen": -121.82546997070312,
      "logps/ref_chosen": -66.10560607910156,
      "logps/ref_rejected": -91.66778564453125,
      "logps/rejected": -195.05531311035156,
      "loss": 1.0694,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5147716999053955,
      "rewards/margins": 0.4362095594406128,
      "rewards/rejected": -0.9509812593460083,
      "step": 624
    },
    {
      "epoch": 0.9448223733938019,
      "grad_norm": 6.928590297698975,
      "kl/avg_steps": 0.28125,
      "kl/beta": 0.009219270199537277,
      "kl/n_epsilon_steps": 0.359375,
      "kl/p_epsilon_steps": 0.640625,
      "learning_rate": 4.7714894655209174e-09,
      "logits/chosen": -1.1287927627563477,
      "logits/rejected": -1.2948472499847412,
      "logps/chosen": -138.32406616210938,
      "logps/ref_chosen": -73.20295715332031,
      "logps/ref_rejected": -105.31025695800781,
      "logps/rejected": -197.34921264648438,
      "loss": 1.2353,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.6002695560455322,
      "rewards/margins": 0.24321919679641724,
      "rewards/rejected": -0.8434886932373047,
      "step": 625
    },
    {
      "epoch": 0.9463340891912321,
      "grad_norm": 6.270933628082275,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.009193413890898228,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 4.517825684323323e-09,
      "logits/chosen": -1.0171754360198975,
      "logits/rejected": -1.2670618295669556,
      "logps/chosen": -121.7349853515625,
      "logps/ref_chosen": -62.181278228759766,
      "logps/ref_rejected": -108.17747497558594,
      "logps/rejected": -211.99676513671875,
      "loss": 1.1301,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.5475019216537476,
      "rewards/margins": 0.40066835284233093,
      "rewards/rejected": -0.9481702446937561,
      "step": 626
    },
    {
      "epoch": 0.9478458049886621,
      "grad_norm": 6.459384918212891,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.009159067645668983,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 4.271028567242818e-09,
      "logits/chosen": -1.36244797706604,
      "logits/rejected": -1.6176857948303223,
      "logps/chosen": -138.19891357421875,
      "logps/ref_chosen": -77.72123718261719,
      "logps/ref_rejected": -114.40547180175781,
      "logps/rejected": -227.02944946289062,
      "loss": 1.0443,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.551235556602478,
      "rewards/margins": 0.4718520939350128,
      "rewards/rejected": -1.023087501525879,
      "step": 627
    },
    {
      "epoch": 0.9493575207860923,
      "grad_norm": 6.517147541046143,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.009105006232857704,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 4.0311050177251895e-09,
      "logits/chosen": -1.5668630599975586,
      "logits/rejected": -1.0957720279693604,
      "logps/chosen": -128.68606567382812,
      "logps/ref_chosen": -70.71195983886719,
      "logps/ref_rejected": -93.85910034179688,
      "logps/rejected": -200.63211059570312,
      "loss": 1.0998,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5267102718353271,
      "rewards/margins": 0.4377779960632324,
      "rewards/rejected": -0.9644882678985596,
      "step": 628
    },
    {
      "epoch": 0.9508692365835223,
      "grad_norm": 7.322593688964844,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.00905689224600792,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 3.798061746947995e-09,
      "logits/chosen": -1.5906280279159546,
      "logits/rejected": -1.5747921466827393,
      "logps/chosen": -145.18203735351562,
      "logps/ref_chosen": -88.66283416748047,
      "logps/ref_rejected": -94.67845153808594,
      "logps/rejected": -190.83462524414062,
      "loss": 1.1164,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5112582445144653,
      "rewards/margins": 0.35449928045272827,
      "rewards/rejected": -0.8657574653625488,
      "step": 629
    },
    {
      "epoch": 0.9523809523809523,
      "grad_norm": 4.681629657745361,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.009017440490424633,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.5719052736323806e-09,
      "logits/chosen": -1.400179386138916,
      "logits/rejected": -1.4847885370254517,
      "logps/chosen": -132.58355712890625,
      "logps/ref_chosen": -72.94979858398438,
      "logps/ref_rejected": -92.7632827758789,
      "logps/rejected": -196.56716918945312,
      "loss": 1.0964,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.535965085029602,
      "rewards/margins": 0.39343059062957764,
      "rewards/rejected": -0.9293956160545349,
      "step": 630
    },
    {
      "epoch": 0.9538926681783825,
      "grad_norm": 6.4820966720581055,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.00897257775068283,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 3.352641923861144e-09,
      "logits/chosen": -1.6833112239837646,
      "logits/rejected": -1.9373281002044678,
      "logps/chosen": -134.91073608398438,
      "logps/ref_chosen": -78.58656311035156,
      "logps/ref_rejected": -115.38685607910156,
      "logps/rejected": -221.26397705078125,
      "loss": 1.0794,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.504284143447876,
      "rewards/margins": 0.43889355659484863,
      "rewards/rejected": -0.9431777000427246,
      "step": 631
    },
    {
      "epoch": 0.9554043839758125,
      "grad_norm": 6.496461868286133,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.008922388777136803,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 3.140277830901428e-09,
      "logits/chosen": -1.3603768348693848,
      "logits/rejected": -1.4694833755493164,
      "logps/chosen": -133.49276733398438,
      "logps/ref_chosen": -75.24861907958984,
      "logps/ref_rejected": -82.98665618896484,
      "logps/rejected": -185.62823486328125,
      "loss": 1.0996,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5186818242073059,
      "rewards/margins": 0.3908722400665283,
      "rewards/rejected": -0.9095540642738342,
      "step": 632
    },
    {
      "epoch": 0.9569160997732427,
      "grad_norm": 8.019292831420898,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.008877999149262905,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.9348189350335007e-09,
      "logits/chosen": -1.3611078262329102,
      "logits/rejected": -1.2718690633773804,
      "logps/chosen": -115.50188446044922,
      "logps/ref_chosen": -68.8402099609375,
      "logps/ref_rejected": -84.64610290527344,
      "logps/rejected": -178.7694091796875,
      "loss": 1.0889,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41344064474105835,
      "rewards/margins": 0.4160441756248474,
      "rewards/rejected": -0.8294848203659058,
      "step": 633
    },
    {
      "epoch": 0.9584278155706727,
      "grad_norm": 7.965339183807373,
      "kl/avg_steps": 0.25,
      "kl/beta": 0.00883795227855444,
      "kl/n_epsilon_steps": 0.375,
      "kl/p_epsilon_steps": 0.625,
      "learning_rate": 2.736270983384276e-09,
      "logits/chosen": -1.409487009048462,
      "logits/rejected": -1.4064576625823975,
      "logps/chosen": -145.90066528320312,
      "logps/ref_chosen": -77.0589599609375,
      "logps/ref_rejected": -74.37579345703125,
      "logps/rejected": -159.47303771972656,
      "loss": 1.3238,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6090530753135681,
      "rewards/margins": 0.13959276676177979,
      "rewards/rejected": -0.7486459016799927,
      "step": 634
    },
    {
      "epoch": 0.9599395313681028,
      "grad_norm": 5.619506359100342,
      "kl/avg_steps": 0.359375,
      "kl/beta": 0.008815912529826164,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 2.5446395297668287e-09,
      "logits/chosen": -1.5071735382080078,
      "logits/rejected": -1.6686877012252808,
      "logps/chosen": -162.4663543701172,
      "logps/ref_chosen": -85.60243225097656,
      "logps/ref_rejected": -104.29497528076172,
      "logps/rejected": -209.23851013183594,
      "loss": 1.2379,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6778690218925476,
      "rewards/margins": 0.24298283457756042,
      "rewards/rejected": -0.9208518266677856,
      "step": 635
    },
    {
      "epoch": 0.9614512471655329,
      "grad_norm": 6.538994312286377,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.008784343488514423,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 2.359929934524829e-09,
      "logits/chosen": -1.1326661109924316,
      "logits/rejected": -1.557888388633728,
      "logps/chosen": -122.93463134765625,
      "logps/ref_chosen": -68.72154235839844,
      "logps/ref_rejected": -97.44863891601562,
      "logps/rejected": -203.28976440429688,
      "loss": 1.0504,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.47411519289016724,
      "rewards/margins": 0.4482189416885376,
      "rewards/rejected": -0.9223341345787048,
      "step": 636
    },
    {
      "epoch": 0.9629629629629629,
      "grad_norm": 5.887253284454346,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.00873249489814043,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 2.1821473643827137e-09,
      "logits/chosen": -1.506112813949585,
      "logits/rejected": -1.5703227519989014,
      "logps/chosen": -167.31407165527344,
      "logps/ref_chosen": -92.38919067382812,
      "logps/ref_rejected": -103.70460510253906,
      "logps/rejected": -220.37106323242188,
      "loss": 1.1261,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.652953565120697,
      "rewards/margins": 0.35979682207107544,
      "rewards/rejected": -1.0127503871917725,
      "step": 637
    },
    {
      "epoch": 0.9644746787603931,
      "grad_norm": 5.912980556488037,
      "kl/avg_steps": 0.453125,
      "kl/beta": 0.008697162382304668,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 2.0112967923011646e-09,
      "logits/chosen": -1.3402650356292725,
      "logits/rejected": -1.5520743131637573,
      "logps/chosen": -152.41412353515625,
      "logps/ref_chosen": -83.36921691894531,
      "logps/ref_rejected": -103.04508209228516,
      "logps/rejected": -209.3965301513672,
      "loss": 1.1568,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5985315442085266,
      "rewards/margins": 0.31978681683540344,
      "rewards/rejected": -0.9183183312416077,
      "step": 638
    },
    {
      "epoch": 0.9659863945578231,
      "grad_norm": 5.9383931159973145,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.008657931350171566,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.847382997337943e-09,
      "logits/chosen": -1.5128107070922852,
      "logits/rejected": -1.6139025688171387,
      "logps/chosen": -128.01541137695312,
      "logps/ref_chosen": -70.45248413085938,
      "logps/ref_rejected": -93.77748107910156,
      "logps/rejected": -196.63209533691406,
      "loss": 1.1036,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.49751636385917664,
      "rewards/margins": 0.3872116506099701,
      "rewards/rejected": -0.8847280144691467,
      "step": 639
    },
    {
      "epoch": 0.9674981103552532,
      "grad_norm": 6.486922264099121,
      "kl/avg_steps": 0.53125,
      "kl/beta": 0.008620217442512512,
      "kl/n_epsilon_steps": 0.234375,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.690410564514244e-09,
      "logits/chosen": -1.3309577703475952,
      "logits/rejected": -1.6244860887527466,
      "logps/chosen": -132.64532470703125,
      "logps/ref_chosen": -68.51570129394531,
      "logps/ref_rejected": -92.35081481933594,
      "logps/rejected": -189.81338500976562,
      "loss": 1.2017,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.5512025356292725,
      "rewards/margins": 0.2822040319442749,
      "rewards/rejected": -0.8334065675735474,
      "step": 640
    },
    {
      "epoch": 0.9690098261526833,
      "grad_norm": 6.680337905883789,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.00857466459274292,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.5403838846864692e-09,
      "logits/chosen": -1.489738941192627,
      "logits/rejected": -1.2841875553131104,
      "logps/chosen": -161.3524169921875,
      "logps/ref_chosen": -92.35102844238281,
      "logps/ref_rejected": -102.4269790649414,
      "logps/rejected": -206.33120727539062,
      "loss": 1.1733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5910571813583374,
      "rewards/margins": 0.29499414563179016,
      "rewards/rejected": -0.8860512971878052,
      "step": 641
    },
    {
      "epoch": 0.9705215419501134,
      "grad_norm": 6.556896686553955,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.008537313900887966,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.3973071544233218e-09,
      "logits/chosen": -1.4611645936965942,
      "logits/rejected": -1.2641469240188599,
      "logps/chosen": -156.3619384765625,
      "logps/ref_chosen": -88.39617919921875,
      "logps/ref_rejected": -88.73035430908203,
      "logps/rejected": -188.46615600585938,
      "loss": 1.1993,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5795704126358032,
      "rewards/margins": 0.2666812539100647,
      "rewards/rejected": -0.8462516069412231,
      "step": 642
    },
    {
      "epoch": 0.9720332577475435,
      "grad_norm": 8.884759902954102,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.00850012619048357,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.261184375888541e-09,
      "logits/chosen": -1.7491729259490967,
      "logits/rejected": -2.072495222091675,
      "logps/chosen": -149.3084716796875,
      "logps/ref_chosen": -84.83087921142578,
      "logps/ref_rejected": -105.31499481201172,
      "logps/rejected": -205.96807861328125,
      "loss": 1.1847,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5469406247138977,
      "rewards/margins": 0.3024827241897583,
      "rewards/rejected": -0.849423348903656,
      "step": 643
    },
    {
      "epoch": 0.9735449735449735,
      "grad_norm": 5.750825881958008,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.008463099598884583,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 1.1320193567288527e-09,
      "logits/chosen": -1.1256132125854492,
      "logits/rejected": -1.4251909255981445,
      "logps/chosen": -128.88160705566406,
      "logps/ref_chosen": -65.11122131347656,
      "logps/ref_rejected": -80.4027328491211,
      "logps/rejected": -177.0889892578125,
      "loss": 1.2123,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.5400751233100891,
      "rewards/margins": 0.2735128402709961,
      "rewards/rejected": -0.81358802318573,
      "step": 644
    },
    {
      "epoch": 0.9750566893424036,
      "grad_norm": 6.494144439697266,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.008431482128798962,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 1.0098157099674987e-09,
      "logits/chosen": -1.3750994205474854,
      "logits/rejected": -1.037635087966919,
      "logps/chosen": -139.2389373779297,
      "logps/ref_chosen": -76.93634033203125,
      "logps/ref_rejected": -89.14311981201172,
      "logps/rejected": -194.06703186035156,
      "loss": 1.1195,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5241885185241699,
      "rewards/margins": 0.35475391149520874,
      "rewards/rejected": -0.8789424300193787,
      "step": 645
    },
    {
      "epoch": 0.9765684051398337,
      "grad_norm": 6.124319076538086,
      "kl/avg_steps": 0.5625,
      "kl/beta": 0.008389534428715706,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.78125,
      "learning_rate": 8.945768539031783e-10,
      "logits/chosen": -1.310829758644104,
      "logits/rejected": -1.2478933334350586,
      "logps/chosen": -150.88714599609375,
      "logps/ref_chosen": -77.69122314453125,
      "logps/ref_rejected": -98.14374542236328,
      "logps/rejected": -213.10272216796875,
      "loss": 1.1296,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.6121255159378052,
      "rewards/margins": 0.34585699439048767,
      "rewards/rejected": -0.9579824209213257,
      "step": 646
    },
    {
      "epoch": 0.9780801209372638,
      "grad_norm": 7.085869789123535,
      "kl/avg_steps": 0.71875,
      "kl/beta": 0.008342606946825981,
      "kl/n_epsilon_steps": 0.140625,
      "kl/p_epsilon_steps": 0.859375,
      "learning_rate": 7.863060120144316e-10,
      "logits/chosen": -1.622401475906372,
      "logits/rejected": -1.438711404800415,
      "logps/chosen": -154.6905059814453,
      "logps/ref_chosen": -83.79997253417969,
      "logps/ref_rejected": -116.81964874267578,
      "logps/rejected": -240.5399932861328,
      "loss": 1.0493,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.5890640616416931,
      "rewards/margins": 0.4357026517391205,
      "rewards/rejected": -1.0247666835784912,
      "step": 647
    },
    {
      "epoch": 0.9795918367346939,
      "grad_norm": 5.251621723175049,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.008283072151243687,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 6.850062128694045e-10,
      "logits/chosen": -1.2734894752502441,
      "logits/rejected": -1.5064399242401123,
      "logps/chosen": -159.29747009277344,
      "logps/ref_chosen": -85.9629898071289,
      "logps/ref_rejected": -101.36550903320312,
      "logps/rejected": -210.07115173339844,
      "loss": 1.1738,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.6054705381393433,
      "rewards/margins": 0.2887694835662842,
      "rewards/rejected": -0.8942400217056274,
      "step": 648
    },
    {
      "epoch": 0.981103552532124,
      "grad_norm": 6.989095211029053,
      "kl/avg_steps": 0.5,
      "kl/beta": 0.008241862989962101,
      "kl/n_epsilon_steps": 0.25,
      "kl/p_epsilon_steps": 0.75,
      "learning_rate": 5.906802900412788e-10,
      "logits/chosen": -1.115774393081665,
      "logits/rejected": -1.133882999420166,
      "logps/chosen": -133.21107482910156,
      "logps/ref_chosen": -68.64892578125,
      "logps/ref_rejected": -89.84898376464844,
      "logps/rejected": -193.7150421142578,
      "loss": 1.1629,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5316280722618103,
      "rewards/margins": 0.31877434253692627,
      "rewards/rejected": -0.8504023551940918,
      "step": 649
    },
    {
      "epoch": 0.982615268329554,
      "grad_norm": 6.10765266418457,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.00820085871964693,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 5.033308820289184e-10,
      "logits/chosen": -0.9832993745803833,
      "logits/rejected": -1.3950834274291992,
      "logps/chosen": -128.94815063476562,
      "logps/ref_chosen": -72.97265625,
      "logps/ref_rejected": -93.0461654663086,
      "logps/rejected": -192.0283203125,
      "loss": 1.1354,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.45911887288093567,
      "rewards/margins": 0.3477191925048828,
      "rewards/rejected": -0.8068380355834961,
      "step": 650
    },
    {
      "epoch": 0.9841269841269841,
      "grad_norm": 9.630555152893066,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.008162596262991428,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 4.2296043218295606e-10,
      "logits/chosen": -1.4361473321914673,
      "logits/rejected": -1.7182174921035767,
      "logps/chosen": -134.17694091796875,
      "logps/ref_chosen": -71.05281066894531,
      "logps/ref_rejected": -94.23469543457031,
      "logps/rejected": -193.00314331054688,
      "loss": 1.1756,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.5147565603256226,
      "rewards/margins": 0.2868105173110962,
      "rewards/rejected": -0.8015670776367188,
      "step": 651
    },
    {
      "epoch": 0.9856386999244142,
      "grad_norm": 7.7754902839660645,
      "kl/avg_steps": 0.375,
      "kl/beta": 0.008129570633172989,
      "kl/n_epsilon_steps": 0.3125,
      "kl/p_epsilon_steps": 0.6875,
      "learning_rate": 3.4957118863768176e-10,
      "logits/chosen": -1.7230968475341797,
      "logits/rejected": -1.4568369388580322,
      "logps/chosen": -147.6585693359375,
      "logps/ref_chosen": -80.06941223144531,
      "logps/ref_rejected": -99.22327423095703,
      "logps/rejected": -207.12478637695312,
      "loss": 1.1528,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.54934161901474,
      "rewards/margins": 0.32308200001716614,
      "rewards/rejected": -0.8724236488342285,
      "step": 652
    },
    {
      "epoch": 0.9871504157218443,
      "grad_norm": 7.7242231369018555,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.008099198341369629,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.831652042480093e-10,
      "logits/chosen": -1.3579241037368774,
      "logits/rejected": -1.3024935722351074,
      "logps/chosen": -139.64584350585938,
      "logps/ref_chosen": -80.35701751708984,
      "logps/ref_rejected": -92.1295394897461,
      "logps/rejected": -197.7828369140625,
      "loss": 1.1113,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4791703522205353,
      "rewards/margins": 0.3704642653465271,
      "rewards/rejected": -0.8496346473693848,
      "step": 653
    },
    {
      "epoch": 0.9886621315192744,
      "grad_norm": 6.726657390594482,
      "kl/avg_steps": 0.46875,
      "kl/beta": 0.008061409927904606,
      "kl/n_epsilon_steps": 0.265625,
      "kl/p_epsilon_steps": 0.734375,
      "learning_rate": 2.2374433653205016e-10,
      "logits/chosen": -1.325247883796692,
      "logits/rejected": -1.6915186643600464,
      "logps/chosen": -146.48092651367188,
      "logps/ref_chosen": -78.06475830078125,
      "logps/ref_rejected": -106.05763244628906,
      "logps/rejected": -208.97283935546875,
      "loss": 1.1938,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.5504453778266907,
      "rewards/margins": 0.2738248109817505,
      "rewards/rejected": -0.8242701888084412,
      "step": 654
    },
    {
      "epoch": 0.9901738473167044,
      "grad_norm": 6.092385768890381,
      "kl/avg_steps": 0.4375,
      "kl/beta": 0.00802379846572876,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.71875,
      "learning_rate": 1.7131024761923852e-10,
      "logits/chosen": -1.2895491123199463,
      "logits/rejected": -1.8892626762390137,
      "logps/chosen": -125.02395629882812,
      "logps/ref_chosen": -67.03407287597656,
      "logps/ref_rejected": -97.57197570800781,
      "logps/rejected": -196.07412719726562,
      "loss": 1.1371,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.4643661677837372,
      "rewards/margins": 0.32108306884765625,
      "rewards/rejected": -0.7854492664337158,
      "step": 655
    },
    {
      "epoch": 0.9916855631141346,
      "grad_norm": 4.83301305770874,
      "kl/avg_steps": 0.59375,
      "kl/beta": 0.00798884779214859,
      "kl/n_epsilon_steps": 0.203125,
      "kl/p_epsilon_steps": 0.796875,
      "learning_rate": 1.2586440420372934e-10,
      "logits/chosen": -1.4277193546295166,
      "logits/rejected": -1.4072365760803223,
      "logps/chosen": -158.8524627685547,
      "logps/ref_chosen": -89.31462860107422,
      "logps/ref_rejected": -105.14315795898438,
      "logps/rejected": -217.38514709472656,
      "loss": 1.1364,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5535690188407898,
      "rewards/margins": 0.33648404479026794,
      "rewards/rejected": -0.8900530338287354,
      "step": 656
    },
    {
      "epoch": 0.9931972789115646,
      "grad_norm": 7.831459999084473,
      "kl/avg_steps": 0.625,
      "kl/beta": 0.007941693998873234,
      "kl/n_epsilon_steps": 0.1875,
      "kl/p_epsilon_steps": 0.8125,
      "learning_rate": 8.740807750345913e-11,
      "logits/chosen": -1.1185672283172607,
      "logits/rejected": -1.3250389099121094,
      "logps/chosen": -120.76776123046875,
      "logps/ref_chosen": -64.89747619628906,
      "logps/ref_rejected": -94.21998596191406,
      "logps/rejected": -206.50807189941406,
      "loss": 1.0521,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.44156578183174133,
      "rewards/margins": 0.44281652569770813,
      "rewards/rejected": -0.8843823671340942,
      "step": 657
    },
    {
      "epoch": 0.9947089947089947,
      "grad_norm": 8.166825294494629,
      "kl/avg_steps": 0.40625,
      "kl/beta": 0.007892366498708725,
      "kl/n_epsilon_steps": 0.296875,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 5.594234322453539e-11,
      "logits/chosen": -1.4240777492523193,
      "logits/rejected": -1.3959312438964844,
      "logps/chosen": -142.5401611328125,
      "logps/ref_chosen": -81.16606140136719,
      "logps/ref_rejected": -97.72825622558594,
      "logps/rejected": -198.3736572265625,
      "loss": 1.1763,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.48336413502693176,
      "rewards/margins": 0.305160254240036,
      "rewards/rejected": -0.7885243892669678,
      "step": 658
    },
    {
      "epoch": 0.9962207105064248,
      "grad_norm": 5.3906121253967285,
      "kl/avg_steps": 0.421875,
      "kl/beta": 0.00786043331027031,
      "kl/n_epsilon_steps": 0.28125,
      "kl/p_epsilon_steps": 0.703125,
      "learning_rate": 3.146808153123293e-11,
      "logits/chosen": -1.1394093036651611,
      "logits/rejected": -1.665954828262329,
      "logps/chosen": -145.08837890625,
      "logps/ref_chosen": -74.42193603515625,
      "logps/ref_rejected": -87.81561279296875,
      "logps/rejected": -184.11553955078125,
      "loss": 1.2645,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.5546475052833557,
      "rewards/margins": 0.19737288355827332,
      "rewards/rejected": -0.7520203590393066,
      "step": 659
    },
    {
      "epoch": 0.9977324263038548,
      "grad_norm": 6.83611536026001,
      "kl/avg_steps": 0.546875,
      "kl/beta": 0.007827411405742168,
      "kl/n_epsilon_steps": 0.21875,
      "kl/p_epsilon_steps": 0.765625,
      "learning_rate": 1.3985977021235829e-11,
      "logits/chosen": -1.5256366729736328,
      "logits/rejected": -1.4637858867645264,
      "logps/chosen": -136.3591766357422,
      "logps/ref_chosen": -71.68512725830078,
      "logps/ref_rejected": -98.01472473144531,
      "logps/rejected": -211.43475341796875,
      "loss": 1.0893,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.5045329332351685,
      "rewards/margins": 0.3771495819091797,
      "rewards/rejected": -0.8816825151443481,
      "step": 660
    },
    {
      "epoch": 0.999244142101285,
      "grad_norm": 5.742647647857666,
      "kl/avg_steps": 0.34375,
      "kl/beta": 0.007784838322550058,
      "kl/n_epsilon_steps": 0.328125,
      "kl/p_epsilon_steps": 0.671875,
      "learning_rate": 3.4965187065971735e-12,
      "logits/chosen": -1.083855152130127,
      "logits/rejected": -1.362886667251587,
      "logps/chosen": -159.15509033203125,
      "logps/ref_chosen": -78.35111999511719,
      "logps/ref_rejected": -99.47113037109375,
      "logps/rejected": -208.73788452148438,
      "loss": 1.2429,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.6288719177246094,
      "rewards/margins": 0.21748466789722443,
      "rewards/rejected": -0.8463565111160278,
      "step": 661
    },
    {
      "epoch": 0.999244142101285,
      "step": 661,
      "total_flos": 0.0,
      "train_loss": 1.1575256359739492,
      "train_runtime": 3253.0214,
      "train_samples_per_second": 13.014,
      "train_steps_per_second": 0.203
    }
  ],
  "logging_steps": 1,
  "max_steps": 661,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}