{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 5.7006369426751595,
  "eval_steps": 500,
  "global_step": 450,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 247.125,
      "epoch": 0.050955414012738856,
      "grad_norm": 5.409891128540039,
      "kl": 2.317987572401762,
      "learning_rate": 8.57142857142857e-08,
      "loss": 0.0023,
      "reward": 17.20423835515976,
      "reward_std": 1.1507948003709316,
      "rewards/concensus_correctness_reward_func": 12.706375002861023,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.7556760907173157,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.023437537252903,
      "step": 4
    },
    {
      "completion_length": 228.625,
      "epoch": 0.10191082802547771,
      "grad_norm": 4.3909173011779785,
      "kl": 1.6623684950172901,
      "learning_rate": 1.714285714285714e-07,
      "loss": 0.0017,
      "reward": 12.55974566936493,
      "reward_std": 6.854554038494825,
      "rewards/concensus_correctness_reward_func": 8.75,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5599956843070686,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.1247500106692314,
      "step": 8
    },
    {
      "completion_length": 194.375,
      "epoch": 0.15286624203821655,
      "grad_norm": 2.985093832015991,
      "kl": 6.860304161906242,
      "learning_rate": 2.571428571428571e-07,
      "loss": 0.0069,
      "reward": 16.054461300373077,
      "reward_std": 9.667749413754791,
      "rewards/concensus_correctness_reward_func": 11.670499965548515,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.6383362491615117,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.2143750190734863,
      "step": 12
    },
    {
      "completion_length": 269.75,
      "epoch": 0.20382165605095542,
      "grad_norm": 3.6396210193634033,
      "kl": 5.6482887379825115,
      "learning_rate": 2.99984424553496e-07,
      "loss": 0.0056,
      "reward": 12.981057226657867,
      "reward_std": 6.896631011739373,
      "rewards/concensus_correctness_reward_func": 9.086749993264675,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.597869697958231,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.15625,
      "rewards/xmlcount_reward_func": 0.8901875242590904,
      "step": 16
    },
    {
      "completion_length": 213.6875,
      "epoch": 0.25477707006369427,
      "grad_norm": 2.7366411685943604,
      "kl": 8.909522596746683,
      "learning_rate": 2.998598403883547e-07,
      "loss": 0.0089,
      "reward": 11.70977608859539,
      "reward_std": 4.304964765789919,
      "rewards/concensus_correctness_reward_func": 7.7207499742507935,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.46571373101323843,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.179562509059906,
      "step": 20
    },
    {
      "completion_length": 250.625,
      "epoch": 0.3057324840764331,
      "grad_norm": 4.964949607849121,
      "kl": 5.63778755068779,
      "learning_rate": 2.9961077554357913e-07,
      "loss": 0.0056,
      "reward": 10.035876393318176,
      "reward_std": 6.212635695934296,
      "rewards/concensus_correctness_reward_func": 6.3186250030994415,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.7127514779567719,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.0670000314712524,
      "step": 24
    },
    {
      "completion_length": 279.0,
      "epoch": 0.35668789808917195,
      "grad_norm": 2.4961581230163574,
      "kl": 67.01326366513968,
      "learning_rate": 2.9923743690422306e-07,
      "loss": 0.067,
      "reward": 16.109780073165894,
      "reward_std": 5.918968662619591,
      "rewards/concensus_correctness_reward_func": 11.880625024437904,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.6405299380421638,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.1198750287294388,
      "step": 28
    },
    {
      "completion_length": 273.6875,
      "epoch": 0.40764331210191085,
      "grad_norm": 5.211795330047607,
      "kl": 5.373722583055496,
      "learning_rate": 2.9874013458303877e-07,
      "loss": 0.0054,
      "reward": 17.488071978092194,
      "reward_std": 4.414429775904864,
      "rewards/concensus_correctness_reward_func": 12.954499989748001,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.5903223338536918,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.15625,
      "rewards/xmlcount_reward_func": 1.162000022828579,
      "step": 32
    },
    {
      "completion_length": 241.6875,
      "epoch": 0.4585987261146497,
      "grad_norm": 1.7989726066589355,
      "kl": 5.3555001355707645,
      "learning_rate": 2.9811928166288275e-07,
      "loss": 0.0053,
      "reward": 15.544344544410706,
      "reward_std": 2.0546061204513535,
      "rewards/concensus_correctness_reward_func": 11.492375005036592,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.5364694236777723,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.1717500165104866,
      "step": 36
    },
    {
      "completion_length": 250.4375,
      "epoch": 0.5095541401273885,
      "grad_norm": 55.185237884521484,
      "kl": 47.606556087732315,
      "learning_rate": 2.973753938535892e-07,
      "loss": 0.0476,
      "reward": 12.839633375406265,
      "reward_std": 2.927978876978159,
      "rewards/concensus_correctness_reward_func": 8.834124997258186,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.5,
      "rewards/question_recreation_reward_func": 0.6650712452828884,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.1529375314712524,
      "step": 40
    },
    {
      "completion_length": 263.125,
      "epoch": 0.5605095541401274,
      "grad_norm": 3.3247878551483154,
      "kl": 9.563120489940047,
      "learning_rate": 2.9650908906359733e-07,
      "loss": 0.0096,
      "reward": 17.1003201007843,
      "reward_std": 4.631968731526285,
      "rewards/concensus_correctness_reward_func": 12.573749996721745,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.6720699109137058,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.104500025510788,
      "step": 44
    },
    {
      "completion_length": 290.75,
      "epoch": 0.6114649681528662,
      "grad_norm": 3.066596508026123,
      "kl": 12.11058185622096,
      "learning_rate": 2.955210868866872e-07,
      "loss": 0.0121,
      "reward": 19.856602907180786,
      "reward_std": 8.727525525609963,
      "rewards/concensus_correctness_reward_func": 15.0,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.625,
      "rewards/question_recreation_reward_func": 0.700602900236845,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 0.8122500255703926,
      "step": 48
    },
    {
      "completion_length": 229.4375,
      "epoch": 0.6624203821656051,
      "grad_norm": 2.433725118637085,
      "kl": 1.3885855339467525,
      "learning_rate": 2.944122080042512e-07,
      "loss": 0.0014,
      "reward": 14.267784029245377,
      "reward_std": 5.394140258431435,
      "rewards/concensus_correctness_reward_func": 10.306124992668629,
      "rewards/consensus_reward_func": 0.75,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.6182841658592224,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.46875,
      "rewards/xmlcount_reward_func": 1.2496250122785568,
      "step": 52
    },
    {
      "completion_length": 265.3125,
      "epoch": 0.7133757961783439,
      "grad_norm": 1.840900182723999,
      "kl": 6.538037680089474,
      "learning_rate": 2.93183373503597e-07,
      "loss": 0.0065,
      "reward": 16.03761088848114,
      "reward_std": 10.041991762816906,
      "rewards/concensus_correctness_reward_func": 11.560124978423119,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.6827985420823097,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1071875169873238,
      "step": 56
    },
    {
      "completion_length": 211.0,
      "epoch": 0.7643312101910829,
      "grad_norm": 1.3512978553771973,
      "kl": 7.67945366166532,
      "learning_rate": 2.91835604112849e-07,
      "loss": 0.0077,
      "reward": 19.18588936328888,
      "reward_std": 6.0852950513362885,
      "rewards/concensus_correctness_reward_func": 14.20750005543232,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6377642604056746,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.2468750029802322,
      "step": 60
    },
    {
      "completion_length": 209.0625,
      "epoch": 0.8152866242038217,
      "grad_norm": 2.5626749992370605,
      "kl": 2.9191525876522064,
      "learning_rate": 2.9037001935308313e-07,
      "loss": 0.0029,
      "reward": 14.56528091430664,
      "reward_std": 4.982788998633623,
      "rewards/concensus_correctness_reward_func": 10.215499997138977,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.7710309810936451,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.2037499845027924,
      "step": 64
    },
    {
      "completion_length": 215.3125,
      "epoch": 0.8662420382165605,
      "grad_norm": 3.383816719055176,
      "kl": 3.1076866053044796,
      "learning_rate": 2.8878783660839964e-07,
      "loss": 0.0031,
      "reward": 20.26084053516388,
      "reward_std": 4.580408729612827,
      "rewards/concensus_correctness_reward_func": 15.626375034451485,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.41534046828746796,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.1253750175237656,
      "step": 68
    },
    {
      "completion_length": 245.3125,
      "epoch": 0.9171974522292994,
      "grad_norm": 11.53940200805664,
      "kl": 36.37679213285446,
      "learning_rate": 2.8709037011470624e-07,
      "loss": 0.0364,
      "reward": 12.166111320257187,
      "reward_std": 7.469847652129829,
      "rewards/concensus_correctness_reward_func": 8.189249992370605,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6527361150365323,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.0428750179708004,
      "step": 72
    },
    {
      "completion_length": 252.6875,
      "epoch": 0.9681528662420382,
      "grad_norm": 1.9474271535873413,
      "kl": 5.756619825959206,
      "learning_rate": 2.852790298680516e-07,
      "loss": 0.0058,
      "reward": 18.147495985031128,
      "reward_std": 5.955903281457722,
      "rewards/concensus_correctness_reward_func": 13.75,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5002461355179548,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.084750011563301,
      "step": 76
    },
    {
      "completion_length": 231.14285714285714,
      "epoch": 1.0127388535031847,
      "grad_norm": 2.1662113666534424,
      "kl": 4.730595575911658,
      "learning_rate": 2.833553204534156e-07,
      "loss": 0.0041,
      "reward": 19.707179682595388,
      "reward_std": 4.576129041612148,
      "rewards/concensus_correctness_reward_func": 14.77628571646554,
      "rewards/consensus_reward_func": 1.5714285714285714,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.1428571428571428,
      "rewards/question_recreation_reward_func": 0.6548936026436942,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.32142857142857145,
      "rewards/xmlcount_reward_func": 1.2402857201439994,
      "step": 80
    },
    {
      "completion_length": 181.9375,
      "epoch": 1.0636942675159236,
      "grad_norm": 3.710115909576416,
      "kl": 7.103449460119009,
      "learning_rate": 2.8132083979493016e-07,
      "loss": 0.0071,
      "reward": 15.854183077812195,
      "reward_std": 8.889904107898474,
      "rewards/concensus_correctness_reward_func": 11.673750042915344,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5401829741895199,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.40625,
      "rewards/xmlcount_reward_func": 1.234000027179718,
      "step": 84
    },
    {
      "completion_length": 250.4375,
      "epoch": 1.1146496815286624,
      "grad_norm": 4.344290256500244,
      "kl": 3.607717391103506,
      "learning_rate": 2.7917727782856764e-07,
      "loss": 0.0036,
      "reward": 13.101991266012192,
      "reward_std": 6.616666306043044,
      "rewards/concensus_correctness_reward_func": 8.970250010490417,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.5953665748238564,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.1926250606775284,
      "step": 88
    },
    {
      "completion_length": 224.5625,
      "epoch": 1.1656050955414012,
      "grad_norm": 1.8617960214614868,
      "kl": 2.941370815038681,
      "learning_rate": 2.769264150983997e-07,
      "loss": 0.0029,
      "reward": 18.346871256828308,
      "reward_std": 3.0000894721597433,
      "rewards/concensus_correctness_reward_func": 13.75,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.7451213523745537,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.1017500162124634,
      "step": 92
    },
    {
      "completion_length": 275.3125,
      "epoch": 1.21656050955414,
      "grad_norm": 2.1597092151641846,
      "kl": 7.501682840287685,
      "learning_rate": 2.745701212775937e-07,
      "loss": 0.0075,
      "reward": 15.922978341579437,
      "reward_std": 10.099833406507969,
      "rewards/concensus_correctness_reward_func": 11.25,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6603539576753974,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.1063750311732292,
      "step": 96
    },
    {
      "completion_length": 232.375,
      "epoch": 1.267515923566879,
      "grad_norm": 2.969134569168091,
      "kl": 7.7300273440778255,
      "learning_rate": 2.7211035361537307e-07,
      "loss": 0.0077,
      "reward": 11.717751204967499,
      "reward_std": 7.577772298827767,
      "rewards/concensus_correctness_reward_func": 7.94712507724762,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.5,
      "rewards/question_recreation_reward_func": 0.5368763818405569,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.1400000303983688,
      "step": 100
    },
    {
      "completion_length": 226.0625,
      "epoch": 1.3184713375796178,
      "grad_norm": 1.5485366582870483,
      "kl": 2.664130464196205,
      "learning_rate": 2.695491553112336e-07,
      "loss": 0.0027,
      "reward": 20.976629972457886,
      "reward_std": 6.513700008392334,
      "rewards/concensus_correctness_reward_func": 16.460624933242798,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.4631927441805601,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.1778125315904617,
      "step": 104
    },
    {
      "completion_length": 202.1875,
      "epoch": 1.3694267515923566,
      "grad_norm": 5.035473346710205,
      "kl": 19.53648843243718,
      "learning_rate": 2.6688865381776554e-07,
      "loss": 0.0195,
      "reward": 16.44457733631134,
      "reward_std": 6.439986160025001,
      "rewards/concensus_correctness_reward_func": 11.878250032663345,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.6380774034187198,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.40625,
      "rewards/xmlcount_reward_func": 1.1470000073313713,
      "step": 108
    },
    {
      "completion_length": 238.375,
      "epoch": 1.4203821656050954,
      "grad_norm": 2.5546350479125977,
      "kl": 4.930681709200144,
      "learning_rate": 2.6413105907349006e-07,
      "loss": 0.0049,
      "reward": 19.794534921646118,
      "reward_std": 4.685703623574227,
      "rewards/concensus_correctness_reward_func": 15.0,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.7159727662801743,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.1723125129938126,
      "step": 112
    },
    {
      "completion_length": 213.0625,
      "epoch": 1.4713375796178343,
      "grad_norm": 2.9883830547332764,
      "kl": 3.1261193864047527,
      "learning_rate": 2.612786616671803e-07,
      "loss": 0.0031,
      "reward": 13.908211499452591,
      "reward_std": 4.823537867516279,
      "rewards/concensus_correctness_reward_func": 10.0,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.689586691558361,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1561250165104866,
      "step": 116
    },
    {
      "completion_length": 229.25,
      "epoch": 1.5222929936305731,
      "grad_norm": 2.6060760021209717,
      "kl": 93.09971876814961,
      "learning_rate": 2.583338309351894e-07,
      "loss": 0.0931,
      "reward": 14.722533643245697,
      "reward_std": 3.7422131042694673,
      "rewards/concensus_correctness_reward_func": 10.464500008150935,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.5823462381958961,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.14443751424551,
      "step": 120
    },
    {
      "completion_length": 241.9375,
      "epoch": 1.573248407643312,
      "grad_norm": 3.838078260421753,
      "kl": 6.129395470023155,
      "learning_rate": 2.55299012993368e-07,
      "loss": 0.0061,
      "reward": 18.527520298957825,
      "reward_std": 6.345299224369228,
      "rewards/concensus_correctness_reward_func": 14.059624962508678,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.5597703522071242,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 0.8456250205636024,
      "step": 124
    },
    {
      "completion_length": 228.1875,
      "epoch": 1.6242038216560508,
      "grad_norm": 2.3569176197052,
      "kl": 190.0281139537692,
      "learning_rate": 2.5217672870520407e-07,
      "loss": 0.19,
      "reward": 19.29222685098648,
      "reward_std": 0.9815351580036804,
      "rewards/concensus_correctness_reward_func": 15.0,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5401642136275768,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.0958125218749046,
      "step": 128
    },
    {
      "completion_length": 216.75,
      "epoch": 1.6751592356687897,
      "grad_norm": 1.5537621974945068,
      "kl": 2.767712377011776,
      "learning_rate": 2.489695715878747e-07,
      "loss": 0.0028,
      "reward": 18.718332588672638,
      "reward_std": 6.182364942971617,
      "rewards/concensus_correctness_reward_func": 13.75,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.625,
      "rewards/question_recreation_reward_func": 0.5509578062337823,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.1673750281333923,
      "step": 132
    },
    {
      "completion_length": 257.9375,
      "epoch": 1.7261146496815285,
      "grad_norm": 3.8648273944854736,
      "kl": 2.600251194089651,
      "learning_rate": 2.4568020565794784e-07,
      "loss": 0.0026,
      "reward": 10.30246677994728,
      "reward_std": 6.988724898546934,
      "rewards/concensus_correctness_reward_func": 6.4593749940395355,
      "rewards/consensus_reward_func": 0.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6528416611254215,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.0965000167489052,
      "step": 136
    },
    {
      "completion_length": 242.6875,
      "epoch": 1.7770700636942676,
      "grad_norm": 54.2701530456543,
      "kl": 31.665705926716328,
      "learning_rate": 2.4231136321852394e-07,
      "loss": 0.0317,
      "reward": 20.147151827812195,
      "reward_std": 7.880714102386264,
      "rewards/concensus_correctness_reward_func": 15.221750020980835,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6395266838371754,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.0983750149607658,
      "step": 140
    },
    {
      "completion_length": 229.6875,
      "epoch": 1.8280254777070064,
      "grad_norm": 2.66843581199646,
      "kl": 3.5122765973210335,
      "learning_rate": 2.3886584258965555e-07,
      "loss": 0.0035,
      "reward": 15.746762156486511,
      "reward_std": 5.882234873250127,
      "rewards/concensus_correctness_reward_func": 11.25,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.5443250723183155,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.1711875051259995,
      "step": 144
    },
    {
      "completion_length": 270.125,
      "epoch": 1.8789808917197452,
      "grad_norm": 1.622960090637207,
      "kl": 7.444879077374935,
      "learning_rate": 2.3534650578393037e-07,
      "loss": 0.0074,
      "reward": 13.791082382202148,
      "reward_std": 6.4896623231470585,
      "rewards/concensus_correctness_reward_func": 9.334750011563301,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.698457557708025,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.03912503272295,
      "step": 148
    },
    {
      "completion_length": 199.625,
      "epoch": 1.929936305732484,
      "grad_norm": 2.772538900375366,
      "kl": 1.7244556173682213,
      "learning_rate": 2.3175627612914788e-07,
      "loss": 0.0017,
      "reward": 18.972005277872086,
      "reward_std": 2.4147312035784125,
      "rewards/concensus_correctness_reward_func": 13.971750006079674,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.5098803527653217,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.40625,
      "rewards/xmlcount_reward_func": 1.2091249972581863,
      "step": 152
    },
    {
      "completion_length": 244.0625,
      "epoch": 1.980891719745223,
      "grad_norm": 2.382359504699707,
      "kl": 6.68058829754591,
      "learning_rate": 2.2809813584006504e-07,
      "loss": 0.0067,
      "reward": 18.733210682868958,
      "reward_std": 5.422195242252201,
      "rewards/concensus_correctness_reward_func": 14.069375038146973,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.5,
      "rewards/question_recreation_reward_func": 0.4836479816585779,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.0864375233650208,
      "step": 156
    },
    {
      "completion_length": 189.71428571428572,
      "epoch": 2.0254777070063694,
      "grad_norm": 4.063220977783203,
      "kl": 2.7896384043352946,
      "learning_rate": 2.243751235412272e-07,
      "loss": 0.0024,
      "reward": 19.006362915039062,
      "reward_std": 5.089762266193118,
      "rewards/concensus_correctness_reward_func": 14.781285745756966,
      "rewards/consensus_reward_func": 1.5714285714285714,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.8571428571428571,
      "rewards/question_recreation_reward_func": 0.515291694019522,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.2857142857142857,
      "rewards/xmlcount_reward_func": 0.9955000025885445,
      "step": 160
    },
    {
      "completion_length": 213.4375,
      "epoch": 2.0764331210191083,
      "grad_norm": 45.27822494506836,
      "kl": 18.627060689032078,
      "learning_rate": 2.2059033174294308e-07,
      "loss": 0.0186,
      "reward": 17.683844089508057,
      "reward_std": 10.830251511186361,
      "rewards/concensus_correctness_reward_func": 13.190500020980835,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.5491568585857749,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.162937492132187,
      "step": 164
    },
    {
      "completion_length": 233.25,
      "epoch": 2.127388535031847,
      "grad_norm": 10.763078689575195,
      "kl": 8.058194138109684,
      "learning_rate": 2.167469042724989e-07,
      "loss": 0.0081,
      "reward": 16.62453094124794,
      "reward_std": 2.8090486722067,
      "rewards/concensus_correctness_reward_func": 11.670500010251999,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.5,
      "rewards/question_recreation_reward_func": 0.6700308993458748,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.065250001847744,
      "step": 168
    },
    {
      "completion_length": 239.9375,
      "epoch": 2.178343949044586,
      "grad_norm": 2.4544873237609863,
      "kl": 4.6812648214399815,
      "learning_rate": 2.1284803366274665e-07,
      "loss": 0.0047,
      "reward": 10.69625610113144,
      "reward_std": 2.8211628943681717,
      "rewards/concensus_correctness_reward_func": 6.55375000089407,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.5454436782747507,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.2220625132322311,
      "step": 172
    },
    {
      "completion_length": 243.1875,
      "epoch": 2.229299363057325,
      "grad_norm": 2.8351073265075684,
      "kl": 11.15944654121995,
      "learning_rate": 2.0889695850023532e-07,
      "loss": 0.0111,
      "reward": 14.30355390906334,
      "reward_std": 3.1945501037698705,
      "rewards/concensus_correctness_reward_func": 10.470749974250793,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.5688038393855095,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.0452500134706497,
      "step": 176
    },
    {
      "completion_length": 227.25,
      "epoch": 2.2802547770700636,
      "grad_norm": 2.5390284061431885,
      "kl": 4.675673048943281,
      "learning_rate": 2.048969607350874e-07,
      "loss": 0.0047,
      "reward": 13.208377301692963,
      "reward_std": 6.394375938922167,
      "rewards/concensus_correctness_reward_func": 9.178874969482422,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5128772370517254,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.0478749945759773,
      "step": 180
    },
    {
      "completion_length": 241.5,
      "epoch": 2.3312101910828025,
      "grad_norm": 339.2472839355469,
      "kl": 120.68484226241708,
      "learning_rate": 2.008513629548557e-07,
      "loss": 0.1207,
      "reward": 12.759812444448471,
      "reward_std": 5.974757697433233,
      "rewards/concensus_correctness_reward_func": 9.136750057339668,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.5173125192523003,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.0745000019669533,
      "step": 184
    },
    {
      "completion_length": 266.1875,
      "epoch": 2.3821656050955413,
      "grad_norm": 9.494641304016113,
      "kl": 6.892231393605471,
      "learning_rate": 1.967635256246248e-07,
      "loss": 0.0069,
      "reward": 18.39091458916664,
      "reward_std": 3.120848624035716,
      "rewards/concensus_correctness_reward_func": 13.973124995827675,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.630602179095149,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.03718750923872,
      "step": 188
    },
    {
      "completion_length": 256.1875,
      "epoch": 2.43312101910828,
      "grad_norm": 3.3058788776397705,
      "kl": 3.385168194770813,
      "learning_rate": 1.9263684429564973e-07,
      "loss": 0.0034,
      "reward": 15.78951108455658,
      "reward_std": 9.866211914690211,
      "rewards/concensus_correctness_reward_func": 11.25,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.7607610449194908,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.0912500247359276,
      "step": 192
    },
    {
      "completion_length": 266.625,
      "epoch": 2.484076433121019,
      "grad_norm": 3.944892644882202,
      "kl": 8.661903837695718,
      "learning_rate": 1.8847474678485014e-07,
      "loss": 0.0087,
      "reward": 17.91079044342041,
      "reward_std": 7.754661478335038,
      "rewards/concensus_correctness_reward_func": 12.71512496471405,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.5,
      "rewards/question_recreation_reward_func": 0.666103033348918,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.2170625180006027,
      "step": 196
    },
    {
      "completion_length": 225.8125,
      "epoch": 2.535031847133758,
      "grad_norm": 2.1859211921691895,
      "kl": 4.23666650429368,
      "learning_rate": 1.8428069032750338e-07,
      "loss": 0.0042,
      "reward": 19.735592007637024,
      "reward_std": 7.996539369225502,
      "rewards/concensus_correctness_reward_func": 15.209874987602234,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.5920920763164759,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.1523750126361847,
      "step": 200
    },
    {
      "completion_length": 256.375,
      "epoch": 2.5859872611464967,
      "grad_norm": 6.963754653930664,
      "kl": 56.80678379908204,
      "learning_rate": 1.8005815870550113e-07,
      "loss": 0.0568,
      "reward": 16.95860594511032,
      "reward_std": 7.698275171220303,
      "rewards/concensus_correctness_reward_func": 12.922000050544739,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5784185063093901,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 0.9894375093281269,
      "step": 204
    },
    {
      "completion_length": 235.6875,
      "epoch": 2.6369426751592355,
      "grad_norm": 56.92842102050781,
      "kl": 21.92546533420682,
      "learning_rate": 1.7581065935355522e-07,
      "loss": 0.0219,
      "reward": 18.0313401222229,
      "reward_std": 5.557751599699259,
      "rewards/concensus_correctness_reward_func": 13.97724997997284,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.42977757239714265,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.15625,
      "rewards/xmlcount_reward_func": 1.0930624902248383,
      "step": 208
    },
    {
      "completion_length": 239.125,
      "epoch": 2.6878980891719744,
      "grad_norm": 4.275964736938477,
      "kl": 2.6714395098388195,
      "learning_rate": 1.715417204457562e-07,
      "loss": 0.0027,
      "reward": 18.481036722660065,
      "reward_std": 5.910495655843988,
      "rewards/concensus_correctness_reward_func": 13.918125033378601,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6785994963720441,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1968125253915787,
      "step": 212
    },
    {
      "completion_length": 238.1875,
      "epoch": 2.738853503184713,
      "grad_norm": 2.9757699966430664,
      "kl": 4.989309325814247,
      "learning_rate": 1.6725488796490503e-07,
      "loss": 0.0051,
      "reward": 16.237927079200745,
      "reward_std": 5.6823064510535914,
      "rewards/concensus_correctness_reward_func": 11.685875058174133,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.57005206332542,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.2320000380277634,
      "step": 216
    },
    {
      "completion_length": 210.75,
      "epoch": 2.789808917197452,
      "grad_norm": 6.644160270690918,
      "kl": 28.66787153482437,
      "learning_rate": 1.6295372275705162e-07,
      "loss": 0.0287,
      "reward": 15.902063801884651,
      "reward_std": 2.8661335185170174,
      "rewards/concensus_correctness_reward_func": 11.25,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.5894390102475882,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 0.9688750207424164,
      "step": 220
    },
    {
      "completion_length": 225.875,
      "epoch": 2.840764331210191,
      "grad_norm": 2.6777331829071045,
      "kl": 139.06324845552444,
      "learning_rate": 1.5864179757368764e-07,
      "loss": 0.1391,
      "reward": 17.266418665647507,
      "reward_std": 7.936541106551886,
      "rewards/concensus_correctness_reward_func": 12.5,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.7135435082018375,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.1778749972581863,
      "step": 224
    },
    {
      "completion_length": 223.4375,
      "epoch": 2.8917197452229297,
      "grad_norm": 5.439139366149902,
      "kl": 3.8189243748784065,
      "learning_rate": 1.5432269410404998e-07,
      "loss": 0.0038,
      "reward": 13.274927824735641,
      "reward_std": 2.9763048992026597,
      "rewards/concensus_correctness_reward_func": 9.352000012993813,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.6428653616458178,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.15625,
      "rewards/xmlcount_reward_func": 1.123812511563301,
      "step": 228
    },
    {
      "completion_length": 247.5,
      "epoch": 2.9426751592356686,
      "grad_norm": 170.2266845703125,
      "kl": 167.23719503730536,
      "learning_rate": 1.5e-07,
      "loss": 0.1672,
      "reward": 14.749447345733643,
      "reward_std": 7.275530036538839,
      "rewards/concensus_correctness_reward_func": 10.215250015258789,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.5158222373574972,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.2058750241994858,
      "step": 232
    },
    {
      "completion_length": 268.0,
      "epoch": 2.9936305732484074,
      "grad_norm": 1.7273995876312256,
      "kl": 5.3727107755839825,
      "learning_rate": 1.4567730589595006e-07,
      "loss": 0.0054,
      "reward": 19.68195617198944,
      "reward_std": 0.35719050047919154,
      "rewards/concensus_correctness_reward_func": 15.112999998033047,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6851438917219639,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.0713125243782997,
      "step": 236
    },
    {
      "completion_length": 256.64285714285717,
      "epoch": 3.038216560509554,
      "grad_norm": 110.74983978271484,
      "kl": 41.563892917973654,
      "learning_rate": 1.4135820242631235e-07,
      "loss": 0.0364,
      "reward": 15.650528294699532,
      "reward_std": 4.540684355156762,
      "rewards/concensus_correctness_reward_func": 11.661857196262904,
      "rewards/consensus_reward_func": 1.5714285714285714,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.7142857142857143,
      "rewards/question_recreation_reward_func": 0.49124259342040333,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.17857142857142858,
      "rewards/xmlcount_reward_func": 1.0331428689616067,
      "step": 240
    },
    {
      "completion_length": 241.625,
      "epoch": 3.089171974522293,
      "grad_norm": 2.8471550941467285,
      "kl": 26.10317723825574,
      "learning_rate": 1.3704627724294842e-07,
      "loss": 0.0261,
      "reward": 15.622052431106567,
      "reward_std": 6.783383652102202,
      "rewards/concensus_correctness_reward_func": 11.25,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6173020545393229,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.0985000133514404,
      "step": 244
    },
    {
      "completion_length": 237.625,
      "epoch": 3.140127388535032,
      "grad_norm": 4.9774088859558105,
      "kl": 2.7739184722304344,
      "learning_rate": 1.3274511203509498e-07,
      "loss": 0.0028,
      "reward": 16.195281475782394,
      "reward_std": 5.6568857207894325,
      "rewards/concensus_correctness_reward_func": 11.760749965906143,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6031564082950354,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.2063750326633453,
      "step": 248
    },
    {
      "completion_length": 232.25,
      "epoch": 3.1910828025477707,
      "grad_norm": 3.6411895751953125,
      "kl": 8.415857467800379,
      "learning_rate": 1.2845827955424378e-07,
      "loss": 0.0084,
      "reward": 21.606714069843292,
      "reward_std": 2.034520313842222,
      "rewards/concensus_correctness_reward_func": 16.477874994277954,
      "rewards/consensus_reward_func": 1.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.5895260944962502,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.1330625265836716,
      "step": 252
    },
    {
      "completion_length": 219.5625,
      "epoch": 3.2420382165605095,
      "grad_norm": 1.8258823156356812,
      "kl": 16.24698005989194,
      "learning_rate": 1.241893406464448e-07,
      "loss": 0.0162,
      "reward": 18.671132683753967,
      "reward_std": 5.8178422772325575,
      "rewards/concensus_correctness_reward_func": 13.984624981880188,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6623825784772635,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.2116250097751617,
      "step": 256
    },
    {
      "completion_length": 207.75,
      "epoch": 3.2929936305732483,
      "grad_norm": 4.799376010894775,
      "kl": 5.706813037395477,
      "learning_rate": 1.199418412944989e-07,
      "loss": 0.0058,
      "reward": 17.392418444156647,
      "reward_std": 4.783364934177371,
      "rewards/concensus_correctness_reward_func": 12.703624963760376,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.6976060681045055,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1786875426769257,
      "step": 260
    },
    {
      "completion_length": 222.8125,
      "epoch": 3.343949044585987,
      "grad_norm": 2.5605955123901367,
      "kl": 9.123412758111954,
      "learning_rate": 1.1571930967249665e-07,
      "loss": 0.0091,
      "reward": 14.08166092634201,
      "reward_std": 6.08101740386337,
      "rewards/concensus_correctness_reward_func": 9.220499992370605,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.6586607862263918,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.2337500303983688,
      "step": 264
    },
    {
      "completion_length": 289.375,
      "epoch": 3.394904458598726,
      "grad_norm": 1.0773470401763916,
      "kl": 7.33378242328763,
      "learning_rate": 1.1152525321514992e-07,
      "loss": 0.0073,
      "reward": 15.252871230244637,
      "reward_std": 2.67856452986598,
      "rewards/concensus_correctness_reward_func": 11.25,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5149340042844415,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.050437517464161,
      "step": 268
    },
    {
      "completion_length": 248.8125,
      "epoch": 3.445859872611465,
      "grad_norm": 3.0849928855895996,
      "kl": 4.085891917347908,
      "learning_rate": 1.0736315570435027e-07,
      "loss": 0.0041,
      "reward": 12.67966365814209,
      "reward_std": 6.439146310091019,
      "rewards/concensus_correctness_reward_func": 9.08587496727705,
      "rewards/consensus_reward_func": 0.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5481634996831417,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 0.9831250309944153,
      "step": 272
    },
    {
      "completion_length": 250.5,
      "epoch": 3.4968152866242037,
      "grad_norm": 2.536210536956787,
      "kl": 2.399098828434944,
      "learning_rate": 1.0323647437537522e-07,
      "loss": 0.0024,
      "reward": 14.057019233703613,
      "reward_std": 11.72206101194024,
      "rewards/concensus_correctness_reward_func": 10.0,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5792695693671703,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1652500182390213,
      "step": 276
    },
    {
      "completion_length": 223.6875,
      "epoch": 3.5477707006369426,
      "grad_norm": 8.720197677612305,
      "kl": 10.37247185409069,
      "learning_rate": 9.914863704514432e-08,
      "loss": 0.0104,
      "reward": 18.15606963634491,
      "reward_std": 7.864195089787245,
      "rewards/concensus_correctness_reward_func": 13.134375005960464,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.5,
      "rewards/question_recreation_reward_func": 0.6000700099393725,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.375,
      "rewards/xmlcount_reward_func": 1.171625018119812,
      "step": 280
    },
    {
      "completion_length": 241.5,
      "epoch": 3.5987261146496814,
      "grad_norm": 2.412242889404297,
      "kl": 5.458444844931364,
      "learning_rate": 9.510303926491262e-08,
      "loss": 0.0055,
      "reward": 15.639052152633667,
      "reward_std": 8.874945568852127,
      "rewards/concensus_correctness_reward_func": 11.855250000953674,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.5041148196905851,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.1859375089406967,
      "step": 284
    },
    {
      "completion_length": 262.0,
      "epoch": 3.6496815286624202,
      "grad_norm": 2.1715633869171143,
      "kl": 3.581681564450264,
      "learning_rate": 9.11030414997647e-08,
      "loss": 0.0036,
      "reward": 15.71645438671112,
      "reward_std": 5.47196826711297,
      "rewards/concensus_correctness_reward_func": 11.672625064849854,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.6106421127915382,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.2144375145435333,
      "step": 288
    },
    {
      "completion_length": 287.875,
      "epoch": 3.700636942675159,
      "grad_norm": 1.9456199407577515,
      "kl": 37.41057540290058,
      "learning_rate": 8.715196633725341e-08,
      "loss": 0.0374,
      "reward": 15.976769387722015,
      "reward_std": 6.066280534490943,
      "rewards/concensus_correctness_reward_func": 11.459874987602234,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.7050191890448332,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.15625,
      "rewards/xmlcount_reward_func": 1.1556250378489494,
      "step": 292
    },
    {
      "completion_length": 189.875,
      "epoch": 3.7515923566878984,
      "grad_norm": 4.28585147857666,
      "kl": 12.30507379770279,
      "learning_rate": 8.325309572750114e-08,
      "loss": 0.0123,
      "reward": 18.18193393945694,
      "reward_std": 6.495595270884223,
      "rewards/concensus_correctness_reward_func": 13.848375000059605,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.6111842412501574,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1598750203847885,
      "step": 296
    },
    {
      "completion_length": 209.5625,
      "epoch": 3.802547770700637,
      "grad_norm": 2.9255335330963135,
      "kl": 2.6841611191630363,
      "learning_rate": 7.940966825705694e-08,
      "loss": 0.0028,
      "reward": 18.69688767194748,
      "reward_std": 7.003051041887375,
      "rewards/concensus_correctness_reward_func": 13.75,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.6851999312639236,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1991875022649765,
      "step": 300
    },
    {
      "completion_length": 239.4375,
      "epoch": 3.853503184713376,
      "grad_norm": 1089.72265625,
      "kl": 959.4794948212802,
      "learning_rate": 7.562487645877278e-08,
      "loss": 0.9595,
      "reward": 15.485539972782135,
      "reward_std": 0.4287999947555363,
      "rewards/concensus_correctness_reward_func": 10.538875006139278,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.7834777049720287,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.1944375187158585,
      "step": 304
    },
    {
      "completion_length": 183.5,
      "epoch": 3.904458598726115,
      "grad_norm": 2.360800266265869,
      "kl": 7.300635326653719,
      "learning_rate": 7.1901864159935e-08,
      "loss": 0.0073,
      "reward": 20.999223351478577,
      "reward_std": 5.513432171428576,
      "rewards/concensus_correctness_reward_func": 16.687874913215637,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.4483482171781361,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.206750027835369,
      "step": 308
    },
    {
      "completion_length": 219.875,
      "epoch": 3.9554140127388537,
      "grad_norm": 1.964698314666748,
      "kl": 1.7256745472550392,
      "learning_rate": 6.824372387085212e-08,
      "loss": 0.0017,
      "reward": 13.001979649066925,
      "reward_std": 9.3370072809048,
      "rewards/concensus_correctness_reward_func": 9.080874972045422,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.5092292642220855,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.224374994635582,
      "step": 312
    },
    {
      "completion_length": 235.14285714285714,
      "epoch": 4.0,
      "grad_norm": 0.010738754644989967,
      "kl": 12.37380084821156,
      "learning_rate": 6.465349421606967e-08,
      "loss": 0.0109,
      "reward": 16.182638917650497,
      "reward_std": 4.973704278768439,
      "rewards/concensus_correctness_reward_func": 11.428571428571429,
      "rewards/consensus_reward_func": 1.7142857142857142,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.7142857142857143,
      "rewards/question_recreation_reward_func": 0.760781603200095,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.32142857142857145,
      "rewards/xmlcount_reward_func": 1.2432857240949358,
      "step": 316
    },
    {
      "completion_length": 222.625,
      "epoch": 4.050955414012739,
      "grad_norm": 3.1885955333709717,
      "kl": 10.466495990753174,
      "learning_rate": 6.113415741034446e-08,
      "loss": 0.0105,
      "reward": 18.242493152618408,
      "reward_std": 2.5054425531998277,
      "rewards/concensus_correctness_reward_func": 13.963750004768372,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.5736186504364014,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.1738750040531158,
      "step": 320
    },
    {
      "completion_length": 211.25,
      "epoch": 4.101910828025478,
      "grad_norm": 2.3097496032714844,
      "kl": 10.439931187778711,
      "learning_rate": 5.768863678147609e-08,
      "loss": 0.0104,
      "reward": 15.326348930597305,
      "reward_std": 4.878300238400698,
      "rewards/concensus_correctness_reward_func": 10.434500008821487,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.7187239453196526,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.0793750137090683,
      "step": 324
    },
    {
      "completion_length": 242.8125,
      "epoch": 4.1528662420382165,
      "grad_norm": 2.6428279876708984,
      "kl": 31.37735340744257,
      "learning_rate": 5.4319794342052186e-08,
      "loss": 0.0314,
      "reward": 13.825124025344849,
      "reward_std": 9.31145294936141,
      "rewards/concensus_correctness_reward_func": 9.380625009536743,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.5849994292948395,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.2032500058412552,
      "step": 328
    },
    {
      "completion_length": 185.875,
      "epoch": 4.203821656050955,
      "grad_norm": 8.287538528442383,
      "kl": 4.1289902329444885,
      "learning_rate": 5.103042841212535e-08,
      "loss": 0.0041,
      "reward": 23.004461526870728,
      "reward_std": 3.9006739165633917,
      "rewards/concensus_correctness_reward_func": 17.7224999666214,
      "rewards/consensus_reward_func": 1.75,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6572116315364838,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.218500018119812,
      "step": 332
    },
    {
      "completion_length": 221.75,
      "epoch": 4.254777070063694,
      "grad_norm": 148.15139770507812,
      "kl": 78.43285623937845,
      "learning_rate": 4.782327129479596e-08,
      "loss": 0.0784,
      "reward": 17.33521395921707,
      "reward_std": 4.419136416167021,
      "rewards/concensus_correctness_reward_func": 12.714375019073486,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.875,
      "rewards/question_recreation_reward_func": 0.6673390492796898,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.1097500286996365,
      "step": 336
    },
    {
      "completion_length": 229.9375,
      "epoch": 4.305732484076433,
      "grad_norm": 1.7480417490005493,
      "kl": 28.588999539613724,
      "learning_rate": 4.4700987006632047e-08,
      "loss": 0.0286,
      "reward": 22.500885009765625,
      "reward_std": 4.617068930529058,
      "rewards/concensus_correctness_reward_func": 17.5,
      "rewards/consensus_reward_func": 1.75,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.5405731201171875,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.2415625154972076,
      "step": 340
    },
    {
      "completion_length": 241.875,
      "epoch": 4.356687898089172,
      "grad_norm": 2.9678404331207275,
      "kl": 11.135519992560148,
      "learning_rate": 4.166616906481058e-08,
      "loss": 0.0111,
      "reward": 22.390071988105774,
      "reward_std": 4.402062423527241,
      "rewards/concensus_correctness_reward_func": 17.70799994468689,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.6001966744661331,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.0818749964237213,
      "step": 344
    },
    {
      "completion_length": 237.4375,
      "epoch": 4.407643312101911,
      "grad_norm": 4.497431755065918,
      "kl": 3.662317145615816,
      "learning_rate": 3.872133833281973e-08,
      "loss": 0.0037,
      "reward": 18.673893690109253,
      "reward_std": 5.760366526897997,
      "rewards/concensus_correctness_reward_func": 14.171375036239624,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.5415192926302552,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.086000021547079,
      "step": 348
    },
    {
      "completion_length": 243.5625,
      "epoch": 4.45859872611465,
      "grad_norm": 3.805427312850952,
      "kl": 9.271330177783966,
      "learning_rate": 3.586894092650992e-08,
      "loss": 0.0093,
      "reward": 9.116474032402039,
      "reward_std": 7.954511460848153,
      "rewards/concensus_correctness_reward_func": 5.309750065207481,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.5349741466343403,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.0842500105500221,
      "step": 352
    },
    {
      "completion_length": 216.0,
      "epoch": 4.509554140127388,
      "grad_norm": 1.2827595472335815,
      "kl": 586.344656933099,
      "learning_rate": 3.311134618223448e-08,
      "loss": 0.5863,
      "reward": 18.384202480316162,
      "reward_std": 6.0147824686719105,
      "rewards/concensus_correctness_reward_func": 13.971250057220459,
      "rewards/consensus_reward_func": 1.125,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.6950150849297643,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.375,
      "rewards/xmlcount_reward_func": 1.2179375290870667,
      "step": 356
    },
    {
      "completion_length": 238.25,
      "epoch": 4.560509554140127,
      "grad_norm": 2.5090322494506836,
      "kl": 14.409381460398436,
      "learning_rate": 3.045084468876636e-08,
      "loss": 0.0144,
      "reward": 11.096249029040337,
      "reward_std": 5.002050343900919,
      "rewards/concensus_correctness_reward_func": 7.652750000357628,
      "rewards/consensus_reward_func": 1.0,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.6521867662668228,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 0.9788125269114971,
      "step": 360
    },
    {
      "completion_length": 231.6875,
      "epoch": 4.611464968152866,
      "grad_norm": 5.940495014190674,
      "kl": 5.444742456078529,
      "learning_rate": 2.7889646384626974e-08,
      "loss": 0.0054,
      "reward": 16.203705072402954,
      "reward_std": 6.409488521516323,
      "rewards/concensus_correctness_reward_func": 11.46025002002716,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.7280802987515926,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.1716250255703926,
      "step": 364
    },
    {
      "completion_length": 261.9375,
      "epoch": 4.662420382165605,
      "grad_norm": 4.413754463195801,
      "kl": 40.07878900319338,
      "learning_rate": 2.5429878722406294e-08,
      "loss": 0.0401,
      "reward": 12.807837784290314,
      "reward_std": 5.907793074846268,
      "rewards/concensus_correctness_reward_func": 9.157750010490417,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.5562750669196248,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.0313125103712082,
      "step": 368
    },
    {
      "completion_length": 246.5625,
      "epoch": 4.713375796178344,
      "grad_norm": 2.676518201828003,
      "kl": 193.69527792930603,
      "learning_rate": 2.3073584901600297e-08,
      "loss": 0.1937,
      "reward": 15.810337275266647,
      "reward_std": 7.08709304407239,
      "rewards/concensus_correctness_reward_func": 11.445250004529953,
      "rewards/consensus_reward_func": 0.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.5,
      "rewards/question_recreation_reward_func": 0.7263994300737977,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.0761875212192535,
      "step": 372
    },
    {
      "completion_length": 220.375,
      "epoch": 4.764331210191083,
      "grad_norm": 15.561013221740723,
      "kl": 10.431547999382019,
      "learning_rate": 2.0822722171432373e-08,
      "loss": 0.0104,
      "reward": 16.888904064893723,
      "reward_std": 4.636547790840268,
      "rewards/concensus_correctness_reward_func": 12.990749955177307,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.42490432411432266,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.125,
      "rewards/xmlcount_reward_func": 1.0982500240206718,
      "step": 376
    },
    {
      "completion_length": 284.375,
      "epoch": 4.8152866242038215,
      "grad_norm": 2.221562385559082,
      "kl": 28.070011030882597,
      "learning_rate": 1.867916020506981e-08,
      "loss": 0.0281,
      "reward": 17.162397533655167,
      "reward_std": 4.522519879043102,
      "rewards/concensus_correctness_reward_func": 12.688500046730042,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.0,
      "rewards/question_recreation_reward_func": 0.600835133343935,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.0605625286698341,
      "step": 380
    },
    {
      "completion_length": 255.9375,
      "epoch": 4.86624203821656,
      "grad_norm": 3.4525911808013916,
      "kl": 13.200175400823355,
      "learning_rate": 1.6644679546584393e-08,
      "loss": 0.0132,
      "reward": 20.264204382896423,
      "reward_std": 3.887154497206211,
      "rewards/concensus_correctness_reward_func": 15.187625050544739,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.729454442858696,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.159624993801117,
      "step": 384
    },
    {
      "completion_length": 250.0625,
      "epoch": 4.917197452229299,
      "grad_norm": 1.8867841958999634,
      "kl": 8.001008473336697,
      "learning_rate": 1.4720970131948435e-08,
      "loss": 0.008,
      "reward": 18.74275043606758,
      "reward_std": 2.8640489764511585,
      "rewards/concensus_correctness_reward_func": 13.75,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6746251694858074,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.0681250244379044,
      "step": 388
    },
    {
      "completion_length": 244.5,
      "epoch": 4.968152866242038,
      "grad_norm": 3.219937801361084,
      "kl": 2.38198983669281,
      "learning_rate": 1.290962988529376e-08,
      "loss": 0.0024,
      "reward": 15.04402369260788,
      "reward_std": 4.118689671158791,
      "rewards/concensus_correctness_reward_func": 10.858500003814697,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.6415863204747438,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.2001875191926956,
      "step": 392
    },
    {
      "completion_length": 240.0,
      "epoch": 5.012738853503185,
      "grad_norm": 3.246076822280884,
      "kl": 16.680181052003586,
      "learning_rate": 1.1212163391600377e-08,
      "loss": 0.0146,
      "reward": 14.724290439060756,
      "reward_std": 6.1574646064213345,
      "rewards/concensus_correctness_reward_func": 10.47371428353446,
      "rewards/consensus_reward_func": 1.2857142857142858,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.2857142857142858,
      "rewards/question_recreation_reward_func": 0.5896476145301547,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.17857142857142858,
      "rewards/xmlcount_reward_func": 0.9109285984720502,
      "step": 396
    },
    {
      "completion_length": 235.25,
      "epoch": 5.063694267515924,
      "grad_norm": 2.46883487701416,
      "kl": 2.3545443788170815,
      "learning_rate": 9.629980646916868e-09,
      "loss": 0.0024,
      "reward": 16.084708213806152,
      "reward_std": 8.817421786487103,
      "rewards/concensus_correctness_reward_func": 11.687125086784363,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.4455205723643303,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.2020625323057175,
      "step": 400
    },
    {
      "completion_length": 247.125,
      "epoch": 5.114649681528663,
      "grad_norm": 2.4745118618011475,
      "kl": 2.9937894009053707,
      "learning_rate": 8.164395887150988e-09,
      "loss": 0.003,
      "reward": 14.124228060245514,
      "reward_std": 9.18977701291442,
      "rewards/concensus_correctness_reward_func": 9.378499984741211,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.720603097230196,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.2126250267028809,
      "step": 404
    },
    {
      "completion_length": 250.1875,
      "epoch": 5.165605095541402,
      "grad_norm": 2.0114660263061523,
      "kl": 21.8758534938097,
      "learning_rate": 6.816626496402977e-09,
      "loss": 0.0219,
      "reward": 13.668376445770264,
      "reward_std": 2.5443186461925507,
      "rewards/concensus_correctness_reward_func": 9.617874965071678,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.5688141155987978,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.0754375234246254,
      "step": 408
    },
    {
      "completion_length": 241.625,
      "epoch": 5.2165605095541405,
      "grad_norm": 2.3374829292297363,
      "kl": 3.71017612144351,
      "learning_rate": 5.5877919957487805e-09,
      "loss": 0.0037,
      "reward": 19.984151780605316,
      "reward_std": 4.235110512934625,
      "rewards/concensus_correctness_reward_func": 15.0,
      "rewards/consensus_reward_func": 1.75,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.8057144843041897,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.21875,
      "rewards/xmlcount_reward_func": 1.0846875384449959,
      "step": 412
    },
    {
      "completion_length": 218.5,
      "epoch": 5.267515923566879,
      "grad_norm": 1151.6732177734375,
      "kl": 297.7076738625765,
      "learning_rate": 4.478913113312771e-09,
      "loss": 0.2977,
      "reward": 13.16609039902687,
      "reward_std": 9.443016417324543,
      "rewards/concensus_correctness_reward_func": 8.948874950408936,
      "rewards/consensus_reward_func": 0.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.6226529218256474,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.40625,
      "rewards/xmlcount_reward_func": 1.1883125305175781,
      "step": 416
    },
    {
      "completion_length": 222.9375,
      "epoch": 5.318471337579618,
      "grad_norm": 2.7844438552856445,
      "kl": 9.570182114839554,
      "learning_rate": 3.490910936402691e-09,
      "loss": 0.0096,
      "reward": 18.053339332342148,
      "reward_std": 6.633316494524479,
      "rewards/concensus_correctness_reward_func": 13.75,
      "rewards/consensus_reward_func": 1.5,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.6550895310938358,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.1875,
      "rewards/xmlcount_reward_func": 1.2107500284910202,
      "step": 420
    },
    {
      "completion_length": 213.75,
      "epoch": 5.369426751592357,
      "grad_norm": 11.249397277832031,
      "kl": 17.067561771720648,
      "learning_rate": 2.6246061464108014e-09,
      "loss": 0.0171,
      "reward": 17.524891823530197,
      "reward_std": 4.37577617354691,
      "rewards/concensus_correctness_reward_func": 12.710250005126,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6043295841664076,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 1.2103125154972076,
      "step": 424
    },
    {
      "completion_length": 209.3125,
      "epoch": 5.420382165605096,
      "grad_norm": 3.837719202041626,
      "kl": 2.012119587510824,
      "learning_rate": 1.8807183371172484e-09,
      "loss": 0.002,
      "reward": 17.180835396051407,
      "reward_std": 4.515378940850496,
      "rewards/concensus_correctness_reward_func": 12.949124962091446,
      "rewards/consensus_reward_func": 0.75,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.5834604157134891,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.210750013589859,
      "step": 428
    },
    {
      "completion_length": 205.6875,
      "epoch": 5.471337579617835,
      "grad_norm": 2.5619990825653076,
      "kl": 13.116049729287624,
      "learning_rate": 1.2598654169611977e-09,
      "loss": 0.0131,
      "reward": 18.689254820346832,
      "reward_std": 5.8718895125202835,
      "rewards/concensus_correctness_reward_func": 13.973749995231628,
      "rewards/consensus_reward_func": 1.625,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.4517549993470311,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.28125,
      "rewards/xmlcount_reward_func": 1.2325000166893005,
      "step": 432
    },
    {
      "completion_length": 221.4375,
      "epoch": 5.522292993630574,
      "grad_norm": 4.5784711837768555,
      "kl": 13.171938437968493,
      "learning_rate": 7.625630957769357e-10,
      "loss": 0.0132,
      "reward": 12.938094705343246,
      "reward_std": 9.368400201201439,
      "rewards/concensus_correctness_reward_func": 9.259250115603209,
      "rewards/consensus_reward_func": 0.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.625,
      "rewards/question_recreation_reward_func": 0.6295947674661875,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.34375,
      "rewards/xmlcount_reward_func": 1.2055000215768814,
      "step": 436
    },
    {
      "completion_length": 235.4375,
      "epoch": 5.573248407643312,
      "grad_norm": 16.68247413635254,
      "kl": 12.497772630304098,
      "learning_rate": 3.8922445642086687e-10,
      "loss": 0.0125,
      "reward": 15.425481706857681,
      "reward_std": 2.6193624921143055,
      "rewards/concensus_correctness_reward_func": 11.479375004768372,
      "rewards/consensus_reward_func": 1.375,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 0.75,
      "rewards/question_recreation_reward_func": 0.4780444288626313,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.15625,
      "rewards/xmlcount_reward_func": 1.1868125051259995,
      "step": 440
    },
    {
      "completion_length": 259.8125,
      "epoch": 5.624203821656051,
      "grad_norm": 2.506514549255371,
      "kl": 2.1330077573657036,
      "learning_rate": 1.4015961164528522e-10,
      "loss": 0.0023,
      "reward": 20.30971536040306,
      "reward_std": 0.7802602792944526,
      "rewards/concensus_correctness_reward_func": 15.0,
      "rewards/consensus_reward_func": 1.875,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.375,
      "rewards/question_recreation_reward_func": 0.6319650914520025,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.1152500286698341,
      "step": 444
    },
    {
      "completion_length": 213.75,
      "epoch": 5.67515923566879,
      "grad_norm": 1.8979109525680542,
      "kl": 5.903007443994284,
      "learning_rate": 1.5575446504029198e-11,
      "loss": 0.0059,
      "reward": 15.859139263629913,
      "reward_std": 3.203619036823511,
      "rewards/concensus_correctness_reward_func": 11.463249996304512,
      "rewards/consensus_reward_func": 1.25,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.125,
      "rewards/question_recreation_reward_func": 0.637013990432024,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.3125,
      "rewards/xmlcount_reward_func": 1.0713750198483467,
      "step": 448
    },
    {
      "completion_length": 231.25,
      "epoch": 5.7006369426751595,
      "kl": 1.2166772335767746,
      "reward": 22.533211946487427,
      "reward_std": 5.739101491868496,
      "rewards/concensus_correctness_reward_func": 17.5,
      "rewards/consensus_reward_func": 1.75,
      "rewards/cumulative_reward_2": 0.0,
      "rewards/final_correctness_reward_func": 1.25,
      "rewards/question_recreation_reward_func": 0.7859619110822678,
      "rewards/soft_format_reward_func": 0.0,
      "rewards/strict_format_reward_func": 0.25,
      "rewards/xmlcount_reward_func": 0.9972500205039978,
      "step": 450,
      "total_flos": 0.0,
      "train_loss": 0.03514481741314133,
      "train_runtime": 2046.9514,
      "train_samples_per_second": 0.879,
      "train_steps_per_second": 0.22
    }
  ],
  "logging_steps": 4,
  "max_steps": 450,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 6,
  "save_steps": 40,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}