{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 565,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 193.0,
      "completions/max_terminated_length": 193.0,
      "completions/mean_length": 71.78125,
      "completions/mean_terminated_length": 71.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.43730998039245605,
      "epoch": 0.0017699115044247787,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 14.733362336222088,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.2291,
      "num_tokens": 14258.0,
      "reward": 0.75,
      "reward_std": 0.25819888710975647,
      "rewards/decision_reward_func/mean": 0.75,
      "rewards/decision_reward_func/std": 0.6666666865348816,
      "sampling/importance_sampling_ratio/max": 1.748260736465454,
      "sampling/importance_sampling_ratio/mean": 1.0003552436828613,
      "sampling/importance_sampling_ratio/min": 0.3887093663215637,
      "sampling/sampling_logp_difference/max": 0.9449234008789062,
      "sampling/sampling_logp_difference/mean": 0.020397081971168518,
      "step": 1
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 181.0,
      "completions/max_terminated_length": 181.0,
      "completions/mean_length": 61.546875,
      "completions/mean_terminated_length": 61.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2651497721672058,
      "epoch": 0.0035398230088495575,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0,
      "kl": 0.0,
      "learning_rate": 4.4247787610619464e-09,
      "loss": 0.0,
      "num_tokens": 28357.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2627105712890625,
      "sampling/importance_sampling_ratio/mean": 0.9985069036483765,
      "sampling/importance_sampling_ratio/min": 0.6752722263336182,
      "sampling/sampling_logp_difference/max": 0.3926393985748291,
      "sampling/sampling_logp_difference/mean": 0.01767190359532833,
      "step": 2
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 155.0,
      "completions/max_terminated_length": 155.0,
      "completions/mean_length": 32.0,
      "completions/mean_terminated_length": 32.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2806015908718109,
      "epoch": 0.005309734513274336,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 23.26227259495331,
      "kl": 0.0005717705935239792,
      "learning_rate": 8.849557522123893e-09,
      "loss": 0.1264,
      "num_tokens": 44085.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9994414448738098,
      "sampling/importance_sampling_ratio/min": 0.6796828508377075,
      "sampling/sampling_logp_difference/max": 1.3504528999328613,
      "sampling/sampling_logp_difference/mean": 0.020524393767118454,
      "step": 3
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 198.0,
      "completions/max_terminated_length": 198.0,
      "completions/mean_length": 35.875,
      "completions/mean_terminated_length": 35.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.23058325052261353,
      "epoch": 0.007079646017699115,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.797167113972659,
      "kl": 0.0005771875730715692,
      "learning_rate": 1.327433628318584e-08,
      "loss": -0.0717,
      "num_tokens": 58621.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3210742473602295,
      "sampling/importance_sampling_ratio/mean": 1.002717137336731,
      "sampling/importance_sampling_ratio/min": 0.5016065239906311,
      "sampling/sampling_logp_difference/max": 0.6899392604827881,
      "sampling/sampling_logp_difference/mean": 0.01839931309223175,
      "step": 4
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 309.0,
      "completions/max_terminated_length": 309.0,
      "completions/mean_length": 89.03125,
      "completions/mean_terminated_length": 89.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4322381019592285,
      "epoch": 0.008849557522123894,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.9613059543793785,
      "kl": 0.0008884783601388335,
      "learning_rate": 1.7699115044247786e-08,
      "loss": 0.0106,
      "num_tokens": 75759.0,
      "reward": 0.125,
      "reward_std": 0.3265564441680908,
      "rewards/decision_reward_func/mean": 0.125,
      "rewards/decision_reward_func/std": 1.0,
      "sampling/importance_sampling_ratio/max": 1.5849732160568237,
      "sampling/importance_sampling_ratio/mean": 0.9992538690567017,
      "sampling/importance_sampling_ratio/min": 0.6169490814208984,
      "sampling/sampling_logp_difference/max": 0.482968807220459,
      "sampling/sampling_logp_difference/mean": 0.01715037226676941,
      "step": 5
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 240.0,
      "completions/max_terminated_length": 240.0,
      "completions/mean_length": 91.265625,
      "completions/mean_terminated_length": 91.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.45085543394088745,
      "epoch": 0.010619469026548672,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.013659497090174135,
      "kl": 0.00046727299923077226,
      "learning_rate": 2.2123893805309735e-08,
      "loss": 0.0,
      "num_tokens": 92496.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3823795318603516,
      "sampling/importance_sampling_ratio/mean": 0.9996471405029297,
      "sampling/importance_sampling_ratio/min": 0.6376644372940063,
      "sampling/sampling_logp_difference/max": 0.44994306564331055,
      "sampling/sampling_logp_difference/mean": 0.01948455721139908,
      "step": 6
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 248.0,
      "completions/max_terminated_length": 248.0,
      "completions/mean_length": 91.75,
      "completions/mean_terminated_length": 91.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5193246006965637,
      "epoch": 0.012389380530973451,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.340318994776948,
      "kl": 0.0008482749690301716,
      "learning_rate": 2.654867256637168e-08,
      "loss": 0.1265,
      "num_tokens": 112880.0,
      "reward": 0.65625,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.9475336074829102,
      "sampling/importance_sampling_ratio/mean": 1.0007236003875732,
      "sampling/importance_sampling_ratio/min": 0.4580051898956299,
      "sampling/sampling_logp_difference/max": 0.7808747291564941,
      "sampling/sampling_logp_difference/mean": 0.023239202797412872,
      "step": 7
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 132.0,
      "completions/max_terminated_length": 132.0,
      "completions/mean_length": 36.015625,
      "completions/mean_terminated_length": 36.015625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.20200984179973602,
      "epoch": 0.01415929203539823,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.4503832105678285,
      "kl": 0.0009719593799673021,
      "learning_rate": 3.0973451327433626e-08,
      "loss": 0.0025,
      "num_tokens": 126161.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.3440406322479248,
      "sampling/importance_sampling_ratio/mean": 1.001471757888794,
      "sampling/importance_sampling_ratio/min": 0.7208306193351746,
      "sampling/sampling_logp_difference/max": 0.32735109329223633,
      "sampling/sampling_logp_difference/mean": 0.012839527800679207,
      "step": 8
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 253.0,
      "completions/max_terminated_length": 253.0,
      "completions/mean_length": 117.671875,
      "completions/mean_terminated_length": 117.671875,
      "completions/min_length": 48.0,
      "completions/min_terminated_length": 48.0,
      "entropy": 0.5152678489685059,
      "epoch": 0.01592920353982301,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.00915880506969509,
      "kl": 0.0004841999616473913,
      "learning_rate": 3.539823008849557e-08,
      "loss": 0.0,
      "num_tokens": 143500.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.999586820602417,
      "sampling/importance_sampling_ratio/min": 0.6153971552848816,
      "sampling/sampling_logp_difference/max": 0.7832646369934082,
      "sampling/sampling_logp_difference/mean": 0.018428195267915726,
      "step": 9
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 245.0,
      "completions/max_terminated_length": 245.0,
      "completions/mean_length": 67.609375,
      "completions/mean_terminated_length": 67.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2896856963634491,
      "epoch": 0.017699115044247787,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.022535488141927375,
      "kl": 0.0009633470908738673,
      "learning_rate": 3.982300884955752e-08,
      "loss": 0.0,
      "num_tokens": 159507.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4800368547439575,
      "sampling/importance_sampling_ratio/mean": 0.9995597004890442,
      "sampling/importance_sampling_ratio/min": 0.7342386245727539,
      "sampling/sampling_logp_difference/max": 0.39206695556640625,
      "sampling/sampling_logp_difference/mean": 0.012184200808405876,
      "step": 10
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 225.0,
      "completions/max_terminated_length": 225.0,
      "completions/mean_length": 65.53125,
      "completions/mean_terminated_length": 65.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19404293596744537,
      "epoch": 0.019469026548672566,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 6.824992842595387,
      "kl": 0.0015200059860944748,
      "learning_rate": 4.424778761061947e-08,
      "loss": -0.0771,
      "num_tokens": 173685.0,
      "reward": 0.90625,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.90625,
      "rewards/decision_reward_func/std": 0.42608407139778137,
      "sampling/importance_sampling_ratio/max": 1.4212366342544556,
      "sampling/importance_sampling_ratio/mean": 1.0023908615112305,
      "sampling/importance_sampling_ratio/min": 0.6726571917533875,
      "sampling/sampling_logp_difference/max": 0.3965195417404175,
      "sampling/sampling_logp_difference/mean": 0.014005091972649097,
      "step": 11
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 323.0,
      "completions/max_terminated_length": 323.0,
      "completions/mean_length": 100.984375,
      "completions/mean_terminated_length": 100.984375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.365769624710083,
      "epoch": 0.021238938053097345,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014960286603652508,
      "kl": 0.0005460747634060681,
      "learning_rate": 4.8672566371681415e-08,
      "loss": 0.0,
      "num_tokens": 190452.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9997556209564209,
      "sampling/importance_sampling_ratio/min": 0.4474831819534302,
      "sampling/sampling_logp_difference/max": 0.8911995887756348,
      "sampling/sampling_logp_difference/mean": 0.013871857896447182,
      "step": 12
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 463.0,
      "completions/max_terminated_length": 463.0,
      "completions/mean_length": 106.375,
      "completions/mean_terminated_length": 106.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.303292840719223,
      "epoch": 0.023008849557522124,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.5792693910632183,
      "kl": 0.0007246644818224013,
      "learning_rate": 5.309734513274336e-08,
      "loss": -0.1241,
      "num_tokens": 208620.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.6080149412155151,
      "sampling/importance_sampling_ratio/mean": 1.0023037195205688,
      "sampling/importance_sampling_ratio/min": 0.5335606932640076,
      "sampling/sampling_logp_difference/max": 0.6281824111938477,
      "sampling/sampling_logp_difference/mean": 0.017998535186052322,
      "step": 13
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 143.0,
      "completions/max_terminated_length": 143.0,
      "completions/mean_length": 34.828125,
      "completions/mean_terminated_length": 34.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17239561676979065,
      "epoch": 0.024778761061946902,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014602460582552898,
      "kl": 0.000353145704139024,
      "learning_rate": 5.7522123893805306e-08,
      "loss": 0.0,
      "num_tokens": 224305.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5867887735366821,
      "sampling/importance_sampling_ratio/mean": 1.0006211996078491,
      "sampling/importance_sampling_ratio/min": 0.39414942264556885,
      "sampling/sampling_logp_difference/max": 0.9310252666473389,
      "sampling/sampling_logp_difference/mean": 0.013991497457027435,
      "step": 14
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 307.0,
      "completions/max_terminated_length": 307.0,
      "completions/mean_length": 86.9375,
      "completions/mean_terminated_length": 86.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3107295036315918,
      "epoch": 0.02654867256637168,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.007229437086416403,
      "kl": 0.0005225496715866029,
      "learning_rate": 6.194690265486725e-08,
      "loss": 0.0,
      "num_tokens": 243069.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4709138870239258,
      "sampling/importance_sampling_ratio/mean": 0.9995893239974976,
      "sampling/importance_sampling_ratio/min": 0.65098637342453,
      "sampling/sampling_logp_difference/max": 0.4292665719985962,
      "sampling/sampling_logp_difference/mean": 0.01991843990981579,
      "step": 15
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 144.0,
      "completions/max_terminated_length": 144.0,
      "completions/mean_length": 46.296875,
      "completions/mean_terminated_length": 46.296875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.32953619956970215,
      "epoch": 0.02831858407079646,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02767166983547206,
      "kl": 0.00113023747690022,
      "learning_rate": 6.63716814159292e-08,
      "loss": 0.0,
      "num_tokens": 257248.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6294353008270264,
      "sampling/importance_sampling_ratio/mean": 0.9990890026092529,
      "sampling/importance_sampling_ratio/min": 0.6623630523681641,
      "sampling/sampling_logp_difference/max": 0.4882335662841797,
      "sampling/sampling_logp_difference/mean": 0.0199204720556736,
      "step": 16
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 183.0,
      "completions/max_terminated_length": 183.0,
      "completions/mean_length": 44.796875,
      "completions/mean_terminated_length": 44.796875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17392346262931824,
      "epoch": 0.03008849557522124,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01600255624211106,
      "kl": 0.0004199473187327385,
      "learning_rate": 7.079646017699114e-08,
      "loss": 0.0,
      "num_tokens": 270659.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.328696370124817,
      "sampling/importance_sampling_ratio/mean": 0.9982379674911499,
      "sampling/importance_sampling_ratio/min": 0.6624902486801147,
      "sampling/sampling_logp_difference/max": 0.4117494821548462,
      "sampling/sampling_logp_difference/mean": 0.01237030141055584,
      "step": 17
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 126.0,
      "completions/max_terminated_length": 126.0,
      "completions/mean_length": 28.84375,
      "completions/mean_terminated_length": 28.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.159395232796669,
      "epoch": 0.03185840707964602,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.513435214167038,
      "kl": 0.0023553031496703625,
      "learning_rate": 7.52212389380531e-08,
      "loss": -0.0075,
      "num_tokens": 282889.0,
      "reward": -0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": -0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.9696966409683228,
      "sampling/importance_sampling_ratio/mean": 0.9974421262741089,
      "sampling/importance_sampling_ratio/min": 0.5129315257072449,
      "sampling/sampling_logp_difference/max": 0.6778795719146729,
      "sampling/sampling_logp_difference/mean": 0.022663142532110214,
      "step": 18
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 327.0,
      "completions/max_terminated_length": 327.0,
      "completions/mean_length": 73.859375,
      "completions/mean_terminated_length": 73.859375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2566125690937042,
      "epoch": 0.033628318584070796,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01530068562910834,
      "kl": 0.0003858095151372254,
      "learning_rate": 7.964601769911503e-08,
      "loss": 0.0,
      "num_tokens": 299264.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.795694351196289,
      "sampling/importance_sampling_ratio/mean": 1.0008893013000488,
      "sampling/importance_sampling_ratio/min": 0.5062554478645325,
      "sampling/sampling_logp_difference/max": 0.6807138919830322,
      "sampling/sampling_logp_difference/mean": 0.014813372865319252,
      "step": 19
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 276.0,
      "completions/max_terminated_length": 276.0,
      "completions/mean_length": 89.984375,
      "completions/mean_terminated_length": 89.984375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4346693754196167,
      "epoch": 0.035398230088495575,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.7359841983795175,
      "kl": 0.0004782738978974521,
      "learning_rate": 8.4070796460177e-08,
      "loss": 0.0061,
      "num_tokens": 314959.0,
      "reward": 0.9375,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.620740532875061,
      "sampling/importance_sampling_ratio/mean": 0.9991461038589478,
      "sampling/importance_sampling_ratio/min": 0.5147078633308411,
      "sampling/sampling_logp_difference/max": 0.6641558408737183,
      "sampling/sampling_logp_difference/mean": 0.01632934808731079,
      "step": 20
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 286.0,
      "completions/max_terminated_length": 286.0,
      "completions/mean_length": 69.453125,
      "completions/mean_terminated_length": 69.453125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3627976179122925,
      "epoch": 0.03716814159292035,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.59221061699448,
      "kl": 0.0004590965108945966,
      "learning_rate": 8.849557522123894e-08,
      "loss": 0.0508,
      "num_tokens": 329756.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.8821483850479126,
      "sampling/importance_sampling_ratio/mean": 1.0001276731491089,
      "sampling/importance_sampling_ratio/min": 0.4833468794822693,
      "sampling/sampling_logp_difference/max": 0.7270207405090332,
      "sampling/sampling_logp_difference/mean": 0.02273380383849144,
      "step": 21
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 187.0,
      "completions/max_terminated_length": 187.0,
      "completions/mean_length": 39.90625,
      "completions/mean_terminated_length": 39.90625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24342747032642365,
      "epoch": 0.03893805309734513,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.7038806022357496,
      "kl": 0.0023908826988190413,
      "learning_rate": 9.292035398230089e-08,
      "loss": 0.0698,
      "num_tokens": 344998.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.872116208076477,
      "sampling/importance_sampling_ratio/mean": 0.9997944235801697,
      "sampling/importance_sampling_ratio/min": 0.19888915121555328,
      "sampling/sampling_logp_difference/max": 1.6150076389312744,
      "sampling/sampling_logp_difference/mean": 0.019733227789402008,
      "step": 22
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 171.0,
      "completions/max_terminated_length": 171.0,
      "completions/mean_length": 39.5,
      "completions/mean_terminated_length": 39.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.16345232725143433,
      "epoch": 0.04070796460176991,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04997415895244329,
      "kl": 0.0014484870480373502,
      "learning_rate": 9.734513274336283e-08,
      "loss": 0.0,
      "num_tokens": 357894.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5752054452896118,
      "sampling/importance_sampling_ratio/mean": 0.999841570854187,
      "sampling/importance_sampling_ratio/min": 0.6862316131591797,
      "sampling/sampling_logp_difference/max": 0.45438575744628906,
      "sampling/sampling_logp_difference/mean": 0.019694484770298004,
      "step": 23
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 52.890625,
      "completions/mean_terminated_length": 52.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.37000778317451477,
      "epoch": 0.04247787610619469,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.338317286703484,
      "kl": 0.000826447329018265,
      "learning_rate": 1.0176991150442478e-07,
      "loss": -0.1221,
      "num_tokens": 372159.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0006983280181885,
      "sampling/importance_sampling_ratio/min": 0.5474603176116943,
      "sampling/sampling_logp_difference/max": 0.7353544235229492,
      "sampling/sampling_logp_difference/mean": 0.018441716209053993,
      "step": 24
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 319.0,
      "completions/max_terminated_length": 319.0,
      "completions/mean_length": 65.078125,
      "completions/mean_terminated_length": 65.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.33777666091918945,
      "epoch": 0.04424778761061947,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.659760033795731,
      "kl": 0.0006184541853144765,
      "learning_rate": 1.0619469026548672e-07,
      "loss": -0.1307,
      "num_tokens": 388756.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.999577522277832,
      "sampling/importance_sampling_ratio/min": 0.46445992588996887,
      "sampling/sampling_logp_difference/max": 1.3790621757507324,
      "sampling/sampling_logp_difference/mean": 0.019229542464017868,
      "step": 25
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 299.0,
      "completions/max_terminated_length": 299.0,
      "completions/mean_length": 69.046875,
      "completions/mean_terminated_length": 69.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25738662481307983,
      "epoch": 0.04601769911504425,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.487071516705344,
      "kl": 0.0003331995103508234,
      "learning_rate": 1.1061946902654867e-07,
      "loss": -0.2253,
      "num_tokens": 404391.0,
      "reward": 0.6875,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.6875,
      "rewards/decision_reward_func/std": 0.7319250702857971,
      "sampling/importance_sampling_ratio/max": 1.2855082750320435,
      "sampling/importance_sampling_ratio/mean": 0.9986451864242554,
      "sampling/importance_sampling_ratio/min": 0.6865718364715576,
      "sampling/sampling_logp_difference/max": 0.3760443925857544,
      "sampling/sampling_logp_difference/mean": 0.018185608088970184,
      "step": 26
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 259.0,
      "completions/max_terminated_length": 259.0,
      "completions/mean_length": 78.109375,
      "completions/mean_terminated_length": 78.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.44814449548721313,
      "epoch": 0.047787610619469026,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.756923111271524,
      "kl": 0.00046124885557219386,
      "learning_rate": 1.1504424778761061e-07,
      "loss": -0.1409,
      "num_tokens": 419470.0,
      "reward": 0.28125,
      "reward_std": 0.2561737596988678,
      "rewards/decision_reward_func/mean": 0.28125,
      "rewards/decision_reward_func/std": 0.9672207236289978,
      "sampling/importance_sampling_ratio/max": 1.4353742599487305,
      "sampling/importance_sampling_ratio/mean": 0.9993364214897156,
      "sampling/importance_sampling_ratio/min": 0.5488451719284058,
      "sampling/sampling_logp_difference/max": 0.5999388694763184,
      "sampling/sampling_logp_difference/mean": 0.018295522779226303,
      "step": 27
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 324.0,
      "completions/max_terminated_length": 324.0,
      "completions/mean_length": 76.953125,
      "completions/mean_terminated_length": 76.953125,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.31486934423446655,
      "epoch": 0.049557522123893805,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.009386442378514733,
      "kl": 0.00041278538992628455,
      "learning_rate": 1.1946902654867256e-07,
      "loss": 0.0,
      "num_tokens": 435595.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5819377899169922,
      "sampling/importance_sampling_ratio/mean": 1.0006744861602783,
      "sampling/importance_sampling_ratio/min": 0.4626096785068512,
      "sampling/sampling_logp_difference/max": 0.770871639251709,
      "sampling/sampling_logp_difference/mean": 0.01879153959453106,
      "step": 28
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 236.0,
      "completions/max_terminated_length": 236.0,
      "completions/mean_length": 98.671875,
      "completions/mean_terminated_length": 98.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4051646590232849,
      "epoch": 0.05132743362831858,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014069863796196987,
      "kl": 0.0004532830498646945,
      "learning_rate": 1.238938053097345e-07,
      "loss": 0.0,
      "num_tokens": 452342.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.7859727144241333,
      "sampling/importance_sampling_ratio/mean": 0.9995170831680298,
      "sampling/importance_sampling_ratio/min": 0.5645666718482971,
      "sampling/sampling_logp_difference/max": 0.579963207244873,
      "sampling/sampling_logp_difference/mean": 0.017181504517793655,
      "step": 29
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 231.0,
      "completions/max_terminated_length": 231.0,
      "completions/mean_length": 41.71875,
      "completions/mean_terminated_length": 41.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22401276230812073,
      "epoch": 0.05309734513274336,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.61094425527448,
      "kl": 0.00048795485054142773,
      "learning_rate": 1.2831858407079647e-07,
      "loss": 0.0822,
      "num_tokens": 467092.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.5831127166748047,
      "sampling/importance_sampling_ratio/mean": 0.9997624158859253,
      "sampling/importance_sampling_ratio/min": 0.5481081604957581,
      "sampling/sampling_logp_difference/max": 0.6012825965881348,
      "sampling/sampling_logp_difference/mean": 0.013244973495602608,
      "step": 30
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.71875,
      "completions/mean_terminated_length": 15.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07598844170570374,
      "epoch": 0.05486725663716814,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.874575477437448,
      "kl": 0.00039827151340432465,
      "learning_rate": 1.327433628318584e-07,
      "loss": -0.0032,
      "num_tokens": 480098.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.4121800661087036,
      "sampling/importance_sampling_ratio/mean": 1.0020051002502441,
      "sampling/importance_sampling_ratio/min": 0.5527469515800476,
      "sampling/sampling_logp_difference/max": 0.5928549766540527,
      "sampling/sampling_logp_difference/mean": 0.013948867097496986,
      "step": 31
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 164.0,
      "completions/max_terminated_length": 164.0,
      "completions/mean_length": 39.421875,
      "completions/mean_terminated_length": 39.421875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1963411271572113,
      "epoch": 0.05663716814159292,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.856088448360933,
      "kl": 0.0015413790242746472,
      "learning_rate": 1.3716814159292035e-07,
      "loss": -0.0009,
      "num_tokens": 494525.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.797372579574585,
      "sampling/importance_sampling_ratio/mean": 1.001387596130371,
      "sampling/importance_sampling_ratio/min": 0.6285156011581421,
      "sampling/sampling_logp_difference/max": 0.5863258838653564,
      "sampling/sampling_logp_difference/mean": 0.01898869127035141,
      "step": 32
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 397.0,
      "completions/max_terminated_length": 397.0,
      "completions/mean_length": 89.921875,
      "completions/mean_terminated_length": 89.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.40926527976989746,
      "epoch": 0.0584070796460177,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.86902251988677,
      "kl": 0.0004499046190176159,
      "learning_rate": 1.4159292035398229e-07,
      "loss": -0.3134,
      "num_tokens": 511432.0,
      "reward": 0.25,
      "reward_std": 0.25819888710975647,
      "rewards/decision_reward_func/mean": 0.25,
      "rewards/decision_reward_func/std": 0.9759001135826111,
      "sampling/importance_sampling_ratio/max": 1.9430650472640991,
      "sampling/importance_sampling_ratio/mean": 1.0010493993759155,
      "sampling/importance_sampling_ratio/min": 0.62600177526474,
      "sampling/sampling_logp_difference/max": 0.6642667055130005,
      "sampling/sampling_logp_difference/mean": 0.016596786677837372,
      "step": 33
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 184.0,
      "completions/max_terminated_length": 184.0,
      "completions/mean_length": 56.71875,
      "completions/mean_terminated_length": 56.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2863810658454895,
      "epoch": 0.06017699115044248,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.021468556262928137,
      "kl": 0.0007147843716666102,
      "learning_rate": 1.4601769911504425e-07,
      "loss": 0.0,
      "num_tokens": 526534.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.8784087896347046,
      "sampling/importance_sampling_ratio/mean": 0.9997020363807678,
      "sampling/importance_sampling_ratio/min": 0.6882390379905701,
      "sampling/sampling_logp_difference/max": 0.630424976348877,
      "sampling/sampling_logp_difference/mean": 0.017147017642855644,
      "step": 34
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 207.0,
      "completions/max_terminated_length": 207.0,
      "completions/mean_length": 85.953125,
      "completions/mean_terminated_length": 85.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4597724378108978,
      "epoch": 0.061946902654867256,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.7681090870036456,
      "kl": 0.0005873649497516453,
      "learning_rate": 1.504424778761062e-07,
      "loss": 0.0506,
      "num_tokens": 545475.0,
      "reward": 0.4375,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.5615980625152588,
      "sampling/importance_sampling_ratio/mean": 0.9998810291290283,
      "sampling/importance_sampling_ratio/min": 0.6170110106468201,
      "sampling/sampling_logp_difference/max": 0.4828684329986572,
      "sampling/sampling_logp_difference/mean": 0.016421593725681305,
      "step": 35
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 213.0,
      "completions/max_terminated_length": 213.0,
      "completions/mean_length": 99.140625,
      "completions/mean_terminated_length": 99.140625,
      "completions/min_length": 48.0,
      "completions/min_terminated_length": 48.0,
      "entropy": 0.5353041887283325,
      "epoch": 0.06371681415929203,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.6580441755876607,
      "kl": 0.0004714885726571083,
      "learning_rate": 1.5486725663716813e-07,
      "loss": 0.004,
      "num_tokens": 561196.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.4393292665481567,
      "sampling/importance_sampling_ratio/mean": 0.9998955726623535,
      "sampling/importance_sampling_ratio/min": 0.6063184142112732,
      "sampling/sampling_logp_difference/max": 0.5003499984741211,
      "sampling/sampling_logp_difference/mean": 0.01866915076971054,
      "step": 36
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 242.0,
      "completions/max_terminated_length": 242.0,
      "completions/mean_length": 52.53125,
      "completions/mean_terminated_length": 52.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3300973176956177,
      "epoch": 0.06548672566371681,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 20.36985643572742,
      "kl": 0.0006205074023455381,
      "learning_rate": 1.5929203539823007e-07,
      "loss": 0.4553,
      "num_tokens": 579022.0,
      "reward": 0.84375,
      "reward_std": 0.34860679507255554,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9995156526565552,
      "sampling/importance_sampling_ratio/min": 0.47040438652038574,
      "sampling/sampling_logp_difference/max": 1.002622365951538,
      "sampling/sampling_logp_difference/mean": 0.021546784788370132,
      "step": 37
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 213.0,
      "completions/max_terminated_length": 213.0,
      "completions/mean_length": 81.078125,
      "completions/mean_terminated_length": 81.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3800387382507324,
      "epoch": 0.06725663716814159,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 10.339012444363068,
      "kl": 0.0006738771917298436,
      "learning_rate": 1.6371681415929203e-07,
      "loss": 0.1046,
      "num_tokens": 594019.0,
      "reward": 0.0625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.0625,
      "rewards/decision_reward_func/std": 1.0059348344802856,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0005218982696533,
      "sampling/importance_sampling_ratio/min": 0.5071792602539062,
      "sampling/sampling_logp_difference/max": 0.8020846843719482,
      "sampling/sampling_logp_difference/mean": 0.01670372113585472,
      "step": 38
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 259.0,
      "completions/max_terminated_length": 259.0,
      "completions/mean_length": 93.484375,
      "completions/mean_terminated_length": 93.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4449535310268402,
      "epoch": 0.06902654867256637,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.317212480269803,
      "kl": 0.0006473226239904761,
      "learning_rate": 1.68141592920354e-07,
      "loss": -0.1538,
      "num_tokens": 609394.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.8720818758010864,
      "sampling/importance_sampling_ratio/mean": 0.9996934533119202,
      "sampling/importance_sampling_ratio/min": 0.6724444627761841,
      "sampling/sampling_logp_difference/max": 0.6270511150360107,
      "sampling/sampling_logp_difference/mean": 0.017083389684557915,
      "step": 39
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 343.0,
      "completions/max_terminated_length": 343.0,
      "completions/mean_length": 81.90625,
      "completions/mean_terminated_length": 81.90625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.33130159974098206,
      "epoch": 0.07079646017699115,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.038053707633894,
      "kl": 0.0009376220987178385,
      "learning_rate": 1.725663716814159e-07,
      "loss": 0.0787,
      "num_tokens": 625884.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.4165065288543701,
      "sampling/importance_sampling_ratio/mean": 1.0004032850265503,
      "sampling/importance_sampling_ratio/min": 0.5999572277069092,
      "sampling/sampling_logp_difference/max": 0.5108969211578369,
      "sampling/sampling_logp_difference/mean": 0.019599106162786484,
      "step": 40
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 228.0,
      "completions/max_terminated_length": 228.0,
      "completions/mean_length": 47.71875,
      "completions/mean_terminated_length": 47.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1945371925830841,
      "epoch": 0.07256637168141593,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.3978063035136974,
      "kl": 0.0002761489013209939,
      "learning_rate": 1.7699115044247788e-07,
      "loss": 0.0003,
      "num_tokens": 642314.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.3843897581100464,
      "sampling/importance_sampling_ratio/mean": 1.0020694732666016,
      "sampling/importance_sampling_ratio/min": 0.5751755833625793,
      "sampling/sampling_logp_difference/max": 0.5530799627304077,
      "sampling/sampling_logp_difference/mean": 0.01604965329170227,
      "step": 41
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 272.0,
      "completions/max_terminated_length": 272.0,
      "completions/mean_length": 75.296875,
      "completions/mean_terminated_length": 75.296875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.35504159331321716,
      "epoch": 0.0743362831858407,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.922965398733,
      "kl": 0.0010760982986539602,
      "learning_rate": 1.8141592920353982e-07,
      "loss": 0.1741,
      "num_tokens": 658733.0,
      "reward": 0.25,
      "reward_std": 0.42078250646591187,
      "rewards/decision_reward_func/mean": 0.25,
      "rewards/decision_reward_func/std": 0.9759001135826111,
      "sampling/importance_sampling_ratio/max": 1.5612423419952393,
      "sampling/importance_sampling_ratio/mean": 0.9998565912246704,
      "sampling/importance_sampling_ratio/min": 0.6919164061546326,
      "sampling/sampling_logp_difference/max": 0.44548189640045166,
      "sampling/sampling_logp_difference/mean": 0.017055584117770195,
      "step": 42
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 457.0,
      "completions/max_terminated_length": 457.0,
      "completions/mean_length": 76.390625,
      "completions/mean_terminated_length": 76.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3077981173992157,
      "epoch": 0.07610619469026549,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.929794139932175,
      "kl": 0.000669544271659106,
      "learning_rate": 1.8584070796460178e-07,
      "loss": 0.1094,
      "num_tokens": 674374.0,
      "reward": 0.0625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.0625,
      "rewards/decision_reward_func/std": 1.0059348344802856,
      "sampling/importance_sampling_ratio/max": 1.8034026622772217,
      "sampling/importance_sampling_ratio/mean": 1.0019267797470093,
      "sampling/importance_sampling_ratio/min": 0.48356494307518005,
      "sampling/sampling_logp_difference/max": 0.726569652557373,
      "sampling/sampling_logp_difference/mean": 0.028325039893388748,
      "step": 43
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 427.0,
      "completions/max_terminated_length": 427.0,
      "completions/mean_length": 122.9375,
      "completions/mean_terminated_length": 122.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.399412602186203,
      "epoch": 0.07787610619469026,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.011838867228157263,
      "kl": 0.0004714487586170435,
      "learning_rate": 1.902654867256637e-07,
      "loss": 0.0,
      "num_tokens": 694210.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4246737957000732,
      "sampling/importance_sampling_ratio/mean": 1.0002517700195312,
      "sampling/importance_sampling_ratio/min": 0.6519432663917542,
      "sampling/sampling_logp_difference/max": 0.427797794342041,
      "sampling/sampling_logp_difference/mean": 0.014120910316705704,
      "step": 44
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 479.0,
      "completions/max_terminated_length": 479.0,
      "completions/mean_length": 144.4375,
      "completions/mean_terminated_length": 144.4375,
      "completions/min_length": 14.0,
      "completions/min_terminated_length": 14.0,
      "entropy": 0.5294415354728699,
      "epoch": 0.07964601769911504,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.00685115726939914,
      "kl": 0.0005526579916477203,
      "learning_rate": 1.9469026548672566e-07,
      "loss": 0.0,
      "num_tokens": 712510.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6863336563110352,
      "sampling/importance_sampling_ratio/mean": 0.9996277093887329,
      "sampling/importance_sampling_ratio/min": 0.5524927973747253,
      "sampling/sampling_logp_difference/max": 0.5933148860931396,
      "sampling/sampling_logp_difference/mean": 0.019174396991729736,
      "step": 45
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 170.0,
      "completions/max_terminated_length": 170.0,
      "completions/mean_length": 56.796875,
      "completions/mean_terminated_length": 56.796875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.33027297258377075,
      "epoch": 0.08141592920353982,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03531126435999156,
      "kl": 0.001525461906567216,
      "learning_rate": 1.991150442477876e-07,
      "loss": 0.0,
      "num_tokens": 726929.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4253754615783691,
      "sampling/importance_sampling_ratio/mean": 1.002415418624878,
      "sampling/importance_sampling_ratio/min": 0.6252660751342773,
      "sampling/sampling_logp_difference/max": 0.46957799792289734,
      "sampling/sampling_logp_difference/mean": 0.018032699823379517,
      "step": 46
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 175.0,
      "completions/max_terminated_length": 175.0,
      "completions/mean_length": 55.4375,
      "completions/mean_terminated_length": 55.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3531503677368164,
      "epoch": 0.0831858407079646,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03232207870409841,
      "kl": 0.0017060365062206984,
      "learning_rate": 2.0353982300884956e-07,
      "loss": 0.0,
      "num_tokens": 741069.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.9697322845458984,
      "sampling/importance_sampling_ratio/mean": 1.0025861263275146,
      "sampling/importance_sampling_ratio/min": 0.6927888989448547,
      "sampling/sampling_logp_difference/max": 0.6778976917266846,
      "sampling/sampling_logp_difference/mean": 0.024530623108148575,
      "step": 47
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1186.0,
      "completions/max_terminated_length": 1186.0,
      "completions/mean_length": 136.765625,
      "completions/mean_terminated_length": 136.765625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3937437832355499,
      "epoch": 0.08495575221238938,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 5.142458555957129,
      "kl": 0.0008791740983724594,
      "learning_rate": 2.0796460176991148e-07,
      "loss": 0.3402,
      "num_tokens": 759774.0,
      "reward": -0.09375,
      "reward_std": 0.4515564441680908,
      "rewards/decision_reward_func/mean": -0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.628279685974121,
      "sampling/importance_sampling_ratio/mean": 1.0001052618026733,
      "sampling/importance_sampling_ratio/min": 0.49396607279777527,
      "sampling/sampling_logp_difference/max": 0.7052884101867676,
      "sampling/sampling_logp_difference/mean": 0.015893086791038513,
      "step": 48
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 187.0,
      "completions/max_terminated_length": 187.0,
      "completions/mean_length": 55.25,
      "completions/mean_terminated_length": 55.25,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.3387606143951416,
      "epoch": 0.08672566371681416,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03927816701742412,
      "kl": 0.001997170504182577,
      "learning_rate": 2.1238938053097344e-07,
      "loss": 0.0,
      "num_tokens": 773310.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4214093685150146,
      "sampling/importance_sampling_ratio/mean": 0.9992483854293823,
      "sampling/importance_sampling_ratio/min": 0.44950494170188904,
      "sampling/sampling_logp_difference/max": 0.7996084690093994,
      "sampling/sampling_logp_difference/mean": 0.022078003734350204,
      "step": 49
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 427.0,
      "completions/max_terminated_length": 427.0,
      "completions/mean_length": 103.703125,
      "completions/mean_terminated_length": 103.703125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5467600226402283,
      "epoch": 0.08849557522123894,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 19.329756091300126,
      "kl": 0.00122804322745651,
      "learning_rate": 2.1681415929203538e-07,
      "loss": 0.3504,
      "num_tokens": 790427.0,
      "reward": -0.4375,
      "reward_std": 0.3943893015384674,
      "rewards/decision_reward_func/mean": -0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.999853253364563,
      "sampling/importance_sampling_ratio/min": 0.6109908223152161,
      "sampling/sampling_logp_difference/max": 1.131711483001709,
      "sampling/sampling_logp_difference/mean": 0.02128685638308525,
      "step": 50
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 287.0,
      "completions/max_terminated_length": 287.0,
      "completions/mean_length": 92.84375,
      "completions/mean_terminated_length": 92.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3790997266769409,
      "epoch": 0.09026548672566372,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.3632322784690682,
      "kl": 0.00037776207318529487,
      "learning_rate": 2.2123893805309735e-07,
      "loss": 0.0385,
      "num_tokens": 806721.0,
      "reward": 0.4375,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.6228275299072266,
      "sampling/importance_sampling_ratio/mean": 0.9994162321090698,
      "sampling/importance_sampling_ratio/min": 0.5379316806793213,
      "sampling/sampling_logp_difference/max": 0.6200237274169922,
      "sampling/sampling_logp_difference/mean": 0.0147955771535635,
      "step": 51
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 190.0,
      "completions/max_terminated_length": 190.0,
      "completions/mean_length": 66.234375,
      "completions/mean_terminated_length": 66.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2744632959365845,
      "epoch": 0.0920353982300885,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014767897890408175,
      "kl": 0.0005663610063493252,
      "learning_rate": 2.2566371681415928e-07,
      "loss": 0.0,
      "num_tokens": 825280.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3668099641799927,
      "sampling/importance_sampling_ratio/mean": 0.9997496604919434,
      "sampling/importance_sampling_ratio/min": 0.628419041633606,
      "sampling/sampling_logp_difference/max": 0.46454811096191406,
      "sampling/sampling_logp_difference/mean": 0.012168833054602146,
      "step": 52
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 249.0,
      "completions/max_terminated_length": 249.0,
      "completions/mean_length": 51.671875,
      "completions/mean_terminated_length": 51.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22427326440811157,
      "epoch": 0.09380530973451327,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.227366331101068,
      "kl": 0.002840688219293952,
      "learning_rate": 2.3008849557522122e-07,
      "loss": -0.1688,
      "num_tokens": 840923.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9992762804031372,
      "sampling/importance_sampling_ratio/min": 0.5337795615196228,
      "sampling/sampling_logp_difference/max": 0.7441730499267578,
      "sampling/sampling_logp_difference/mean": 0.021255411207675934,
      "step": 53
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 364.0,
      "completions/max_terminated_length": 364.0,
      "completions/mean_length": 89.21875,
      "completions/mean_terminated_length": 89.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2630472183227539,
      "epoch": 0.09557522123893805,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.008579094624632907,
      "kl": 0.00046598073095083237,
      "learning_rate": 2.345132743362832e-07,
      "loss": 0.0,
      "num_tokens": 858249.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.662152647972107,
      "sampling/importance_sampling_ratio/mean": 1.0007331371307373,
      "sampling/importance_sampling_ratio/min": 0.5581602454185486,
      "sampling/sampling_logp_difference/max": 0.5831091403961182,
      "sampling/sampling_logp_difference/mean": 0.016348931938409805,
      "step": 54
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 304.0,
      "completions/max_terminated_length": 304.0,
      "completions/mean_length": 96.078125,
      "completions/mean_terminated_length": 96.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.47166889905929565,
      "epoch": 0.09734513274336283,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.010793334962958628,
      "kl": 0.0005595291731879115,
      "learning_rate": 2.3893805309734513e-07,
      "loss": 0.0,
      "num_tokens": 875118.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.7272074222564697,
      "sampling/importance_sampling_ratio/mean": 0.9997779130935669,
      "sampling/importance_sampling_ratio/min": 0.21025174856185913,
      "sampling/sampling_logp_difference/max": 1.5594496726989746,
      "sampling/sampling_logp_difference/mean": 0.018348487094044685,
      "step": 55
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 167.0,
      "completions/max_terminated_length": 167.0,
      "completions/mean_length": 54.390625,
      "completions/mean_terminated_length": 54.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.32978034019470215,
      "epoch": 0.09911504424778761,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03360661011059917,
      "kl": 0.0016041224589571357,
      "learning_rate": 2.4336283185840704e-07,
      "loss": 0.0,
      "num_tokens": 891415.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.544557809829712,
      "sampling/importance_sampling_ratio/mean": 0.9991042613983154,
      "sampling/importance_sampling_ratio/min": 0.6957619786262512,
      "sampling/sampling_logp_difference/max": 0.4347376823425293,
      "sampling/sampling_logp_difference/mean": 0.01648845709860325,
      "step": 56
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 330.0,
      "completions/max_terminated_length": 330.0,
      "completions/mean_length": 98.546875,
      "completions/mean_terminated_length": 98.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.40308648347854614,
      "epoch": 0.10088495575221239,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.015656436625466928,
      "kl": 0.00042458786629140377,
      "learning_rate": 2.47787610619469e-07,
      "loss": 0.0,
      "num_tokens": 907450.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5252330303192139,
      "sampling/importance_sampling_ratio/mean": 0.9999468922615051,
      "sampling/importance_sampling_ratio/min": 0.6792412996292114,
      "sampling/sampling_logp_difference/max": 0.4221472144126892,
      "sampling/sampling_logp_difference/mean": 0.017018482089042664,
      "step": 57
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 503.0,
      "completions/max_terminated_length": 503.0,
      "completions/mean_length": 64.71875,
      "completions/mean_terminated_length": 64.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3652716875076294,
      "epoch": 0.10265486725663717,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.7572585149403777,
      "kl": 0.000706555787473917,
      "learning_rate": 2.5221238938053097e-07,
      "loss": 0.0038,
      "num_tokens": 922536.0,
      "reward": -0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.5869511365890503,
      "sampling/importance_sampling_ratio/mean": 1.0007580518722534,
      "sampling/importance_sampling_ratio/min": 0.4306277632713318,
      "sampling/sampling_logp_difference/max": 0.8425111770629883,
      "sampling/sampling_logp_difference/mean": 0.013503305613994598,
      "step": 58
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.765625,
      "completions/mean_terminated_length": 15.765625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.08323395252227783,
      "epoch": 0.10442477876106195,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10431399096983622,
      "kl": 0.0004693373921327293,
      "learning_rate": 2.5663716814159294e-07,
      "loss": 0.0,
      "num_tokens": 937513.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3673667907714844,
      "sampling/importance_sampling_ratio/mean": 0.9999486207962036,
      "sampling/importance_sampling_ratio/min": 0.8102474212646484,
      "sampling/sampling_logp_difference/max": 0.3128868341445923,
      "sampling/sampling_logp_difference/mean": 0.012339731678366661,
      "step": 59
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 103.0,
      "completions/max_terminated_length": 103.0,
      "completions/mean_length": 28.484375,
      "completions/mean_terminated_length": 28.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.15878352522850037,
      "epoch": 0.10619469026548672,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08825490514685248,
      "kl": 0.0016191593604162335,
      "learning_rate": 2.6106194690265485e-07,
      "loss": 0.0,
      "num_tokens": 950072.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3832296133041382,
      "sampling/importance_sampling_ratio/mean": 0.9970600605010986,
      "sampling/importance_sampling_ratio/min": 0.6773175597190857,
      "sampling/sampling_logp_difference/max": 0.3896150588989258,
      "sampling/sampling_logp_difference/mean": 0.021082065999507904,
      "step": 60
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 535.0,
      "completions/max_terminated_length": 535.0,
      "completions/mean_length": 87.203125,
      "completions/mean_terminated_length": 87.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2835385203361511,
      "epoch": 0.1079646017699115,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.039394391202979,
      "kl": 0.000507982331328094,
      "learning_rate": 2.654867256637168e-07,
      "loss": 0.3259,
      "num_tokens": 969429.0,
      "reward": 0.625,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.4433850049972534,
      "sampling/importance_sampling_ratio/mean": 0.9994461536407471,
      "sampling/importance_sampling_ratio/min": 0.6672996878623962,
      "sampling/sampling_logp_difference/max": 0.4045161008834839,
      "sampling/sampling_logp_difference/mean": 0.014681078493595123,
      "step": 61
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 185.0,
      "completions/max_terminated_length": 185.0,
      "completions/mean_length": 80.875,
      "completions/mean_terminated_length": 80.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.42828965187072754,
      "epoch": 0.10973451327433628,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.219259006737941,
      "kl": 0.0006035867263562977,
      "learning_rate": 2.6991150442477873e-07,
      "loss": 0.0008,
      "num_tokens": 984573.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.999299168586731,
      "sampling/importance_sampling_ratio/min": 0.5546641945838928,
      "sampling/sampling_logp_difference/max": 1.1937751770019531,
      "sampling/sampling_logp_difference/mean": 0.019111908972263336,
      "step": 62
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 141.0,
      "completions/max_terminated_length": 141.0,
      "completions/mean_length": 38.890625,
      "completions/mean_terminated_length": 38.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2435263991355896,
      "epoch": 0.11150442477876106,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 20.082345347768168,
      "kl": 0.002106260508298874,
      "learning_rate": 2.743362831858407e-07,
      "loss": 0.1822,
      "num_tokens": 998118.0,
      "reward": 0.34375,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.34375,
      "rewards/decision_reward_func/std": 0.9464847445487976,
      "sampling/importance_sampling_ratio/max": 1.9094908237457275,
      "sampling/importance_sampling_ratio/mean": 1.0004956722259521,
      "sampling/importance_sampling_ratio/min": 0.7219632267951965,
      "sampling/sampling_logp_difference/max": 0.6468366384506226,
      "sampling/sampling_logp_difference/mean": 0.019784800708293915,
      "step": 63
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 156.0,
      "completions/max_terminated_length": 156.0,
      "completions/mean_length": 34.28125,
      "completions/mean_terminated_length": 34.28125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17724953591823578,
      "epoch": 0.11327433628318584,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.055329392007934075,
      "kl": 0.0014039300149306655,
      "learning_rate": 2.787610619469026e-07,
      "loss": 0.0,
      "num_tokens": 1012360.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3268667459487915,
      "sampling/importance_sampling_ratio/mean": 0.9999594688415527,
      "sampling/importance_sampling_ratio/min": 0.5984827280044556,
      "sampling/sampling_logp_difference/max": 0.5133576393127441,
      "sampling/sampling_logp_difference/mean": 0.019908219575881958,
      "step": 64
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 303.0,
      "completions/max_terminated_length": 303.0,
      "completions/mean_length": 74.0,
      "completions/mean_terminated_length": 74.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25674378871917725,
      "epoch": 0.11504424778761062,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.358973041717826,
      "kl": 0.0008139506680890918,
      "learning_rate": 2.8318584070796457e-07,
      "loss": -0.0235,
      "num_tokens": 1027176.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.5614769458770752,
      "sampling/importance_sampling_ratio/mean": 0.9981610774993896,
      "sampling/importance_sampling_ratio/min": 0.6101521253585815,
      "sampling/sampling_logp_difference/max": 0.4940469264984131,
      "sampling/sampling_logp_difference/mean": 0.021541573107242584,
      "step": 65
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 385.0,
      "completions/max_terminated_length": 385.0,
      "completions/mean_length": 114.609375,
      "completions/mean_terminated_length": 114.609375,
      "completions/min_length": 31.0,
      "completions/min_terminated_length": 31.0,
      "entropy": 0.5083310008049011,
      "epoch": 0.1168141592920354,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.6382262277093083,
      "kl": 0.0006657812045887113,
      "learning_rate": 2.8761061946902654e-07,
      "loss": 0.0159,
      "num_tokens": 1044191.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.4574837684631348,
      "sampling/importance_sampling_ratio/mean": 1.0002875328063965,
      "sampling/importance_sampling_ratio/min": 0.6315723657608032,
      "sampling/sampling_logp_difference/max": 0.45954275131225586,
      "sampling/sampling_logp_difference/mean": 0.017609603703022003,
      "step": 66
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 458.0,
      "completions/max_terminated_length": 458.0,
      "completions/mean_length": 68.65625,
      "completions/mean_terminated_length": 68.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4105982780456543,
      "epoch": 0.11858407079646018,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 12.053850779982422,
      "kl": 0.0023665111511945724,
      "learning_rate": 2.920353982300885e-07,
      "loss": 0.3613,
      "num_tokens": 1058313.0,
      "reward": 0.84375,
      "reward_std": 0.3723389506340027,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.3561493158340454,
      "sampling/importance_sampling_ratio/mean": 0.9979859590530396,
      "sampling/importance_sampling_ratio/min": 0.4442462921142578,
      "sampling/sampling_logp_difference/max": 0.8113762140274048,
      "sampling/sampling_logp_difference/mean": 0.022402148693799973,
      "step": 67
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 230.0,
      "completions/max_terminated_length": 230.0,
      "completions/mean_length": 66.46875,
      "completions/mean_terminated_length": 66.46875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1998130977153778,
      "epoch": 0.12035398230088495,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.022656157015304077,
      "kl": 0.0007582076359540224,
      "learning_rate": 2.9646017699115047e-07,
      "loss": 0.0,
      "num_tokens": 1072263.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.412645936012268,
      "sampling/importance_sampling_ratio/mean": 0.9985370635986328,
      "sampling/importance_sampling_ratio/min": 0.593745231628418,
      "sampling/sampling_logp_difference/max": 0.5213049650192261,
      "sampling/sampling_logp_difference/mean": 0.018213676288723946,
      "step": 68
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 137.0,
      "completions/max_terminated_length": 137.0,
      "completions/mean_length": 54.5625,
      "completions/mean_terminated_length": 54.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.271050363779068,
      "epoch": 0.12212389380530973,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.013820374585321411,
      "kl": 0.00034594995668157935,
      "learning_rate": 3.008849557522124e-07,
      "loss": 0.0,
      "num_tokens": 1086347.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2971035242080688,
      "sampling/importance_sampling_ratio/mean": 0.999480664730072,
      "sampling/importance_sampling_ratio/min": 0.37043890357017517,
      "sampling/sampling_logp_difference/max": 0.9930667877197266,
      "sampling/sampling_logp_difference/mean": 0.011262361891567707,
      "step": 69
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 207.0,
      "completions/max_terminated_length": 207.0,
      "completions/mean_length": 57.328125,
      "completions/mean_terminated_length": 57.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30324071645736694,
      "epoch": 0.12389380530973451,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.125386142425997,
      "kl": 0.0008926556329242885,
      "learning_rate": 3.053097345132743e-07,
      "loss": -0.0308,
      "num_tokens": 1101872.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.3413037061691284,
      "sampling/importance_sampling_ratio/mean": 0.9995997548103333,
      "sampling/importance_sampling_ratio/min": 0.6632469892501831,
      "sampling/sampling_logp_difference/max": 0.41060781478881836,
      "sampling/sampling_logp_difference/mean": 0.015544610098004341,
      "step": 70
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 262.0,
      "completions/max_terminated_length": 262.0,
      "completions/mean_length": 97.671875,
      "completions/mean_terminated_length": 97.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.6352293491363525,
      "epoch": 0.1256637168141593,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.979445476665689,
      "kl": 0.001399570144712925,
      "learning_rate": 3.0973451327433626e-07,
      "loss": 0.1946,
      "num_tokens": 1118347.0,
      "reward": 0.375,
      "reward_std": 0.36435678601264954,
      "rewards/decision_reward_func/mean": 0.375,
      "rewards/decision_reward_func/std": 0.934353232383728,
      "sampling/importance_sampling_ratio/max": 1.7627366781234741,
      "sampling/importance_sampling_ratio/mean": 1.0006338357925415,
      "sampling/importance_sampling_ratio/min": 0.5739402174949646,
      "sampling/sampling_logp_difference/max": 0.5668675899505615,
      "sampling/sampling_logp_difference/mean": 0.02080698311328888,
      "step": 71
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 674.0,
      "completions/max_terminated_length": 674.0,
      "completions/mean_length": 80.890625,
      "completions/mean_terminated_length": 80.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3430714011192322,
      "epoch": 0.12743362831858407,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 12.02956264887036,
      "kl": 0.006966053508222103,
      "learning_rate": 3.141592920353982e-07,
      "loss": 0.1236,
      "num_tokens": 1133908.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9982129335403442,
      "sampling/importance_sampling_ratio/min": 0.31648826599121094,
      "sampling/sampling_logp_difference/max": 1.150469183921814,
      "sampling/sampling_logp_difference/mean": 0.02067970484495163,
      "step": 72
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 240.0,
      "completions/max_terminated_length": 240.0,
      "completions/mean_length": 92.75,
      "completions/mean_terminated_length": 92.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5193896293640137,
      "epoch": 0.12920353982300886,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.848363550830051,
      "kl": 0.010346418246626854,
      "learning_rate": 3.1858407079646014e-07,
      "loss": -0.0719,
      "num_tokens": 1149572.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.555552363395691,
      "sampling/importance_sampling_ratio/mean": 0.9990873336791992,
      "sampling/importance_sampling_ratio/min": 0.48628026247024536,
      "sampling/sampling_logp_difference/max": 0.7209701538085938,
      "sampling/sampling_logp_difference/mean": 0.02255423739552498,
      "step": 73
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 638.0,
      "completions/max_terminated_length": 638.0,
      "completions/mean_length": 64.125,
      "completions/mean_terminated_length": 64.125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30180686712265015,
      "epoch": 0.13097345132743363,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 10.872794222298856,
      "kl": 0.001606640056706965,
      "learning_rate": 3.230088495575221e-07,
      "loss": -0.3004,
      "num_tokens": 1163660.0,
      "reward": 0.875,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 1.7250492572784424,
      "sampling/importance_sampling_ratio/mean": 1.0003955364227295,
      "sampling/importance_sampling_ratio/min": 0.5589487552642822,
      "sampling/sampling_logp_difference/max": 0.5816974639892578,
      "sampling/sampling_logp_difference/mean": 0.02236917056143284,
      "step": 74
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 208.0,
      "completions/max_terminated_length": 208.0,
      "completions/mean_length": 44.59375,
      "completions/mean_terminated_length": 44.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2116633653640747,
      "epoch": 0.13274336283185842,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.8699103275799596,
      "kl": 0.006977086421102285,
      "learning_rate": 3.2743362831858407e-07,
      "loss": 0.0026,
      "num_tokens": 1176994.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.4212322235107422,
      "sampling/importance_sampling_ratio/mean": 0.9926056265830994,
      "sampling/importance_sampling_ratio/min": 0.6300075054168701,
      "sampling/sampling_logp_difference/max": 0.4620234966278076,
      "sampling/sampling_logp_difference/mean": 0.02601846680045128,
      "step": 75
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 355.0,
      "completions/max_terminated_length": 355.0,
      "completions/mean_length": 99.203125,
      "completions/mean_terminated_length": 99.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.41840457916259766,
      "epoch": 0.13451327433628318,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.016256057216556645,
      "kl": 0.0008459609234705567,
      "learning_rate": 3.3185840707964603e-07,
      "loss": 0.0,
      "num_tokens": 1194431.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000793218612671,
      "sampling/importance_sampling_ratio/min": 0.5363112092018127,
      "sampling/sampling_logp_difference/max": 0.8022520542144775,
      "sampling/sampling_logp_difference/mean": 0.02195250429213047,
      "step": 76
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 363.0,
      "completions/max_terminated_length": 363.0,
      "completions/mean_length": 101.984375,
      "completions/mean_terminated_length": 101.984375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.37862005829811096,
      "epoch": 0.13628318584070798,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.132955638345939,
      "kl": 0.0018726624548435211,
      "learning_rate": 3.36283185840708e-07,
      "loss": -0.3206,
      "num_tokens": 1210270.0,
      "reward": 0.78125,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.78125,
      "rewards/decision_reward_func/std": 0.6291528940200806,
      "sampling/importance_sampling_ratio/max": 1.5600541830062866,
      "sampling/importance_sampling_ratio/mean": 1.0004552602767944,
      "sampling/importance_sampling_ratio/min": 0.6160954833030701,
      "sampling/sampling_logp_difference/max": 0.48435330390930176,
      "sampling/sampling_logp_difference/mean": 0.014363247901201248,
      "step": 77
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 168.0,
      "completions/max_terminated_length": 168.0,
      "completions/mean_length": 64.0,
      "completions/mean_terminated_length": 64.0,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.3538378179073334,
      "epoch": 0.13805309734513274,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.268417053893387,
      "kl": 0.005304547026753426,
      "learning_rate": 3.4070796460176986e-07,
      "loss": -0.1266,
      "num_tokens": 1228782.0,
      "reward": 0.59375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.59375,
      "rewards/decision_reward_func/std": 0.8110105991363525,
      "sampling/importance_sampling_ratio/max": 1.5338397026062012,
      "sampling/importance_sampling_ratio/mean": 1.000978708267212,
      "sampling/importance_sampling_ratio/min": 0.598846971988678,
      "sampling/sampling_logp_difference/max": 0.512749195098877,
      "sampling/sampling_logp_difference/mean": 0.01858421415090561,
      "step": 78
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 227.0,
      "completions/max_terminated_length": 227.0,
      "completions/mean_length": 96.515625,
      "completions/mean_terminated_length": 96.515625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4295077621936798,
      "epoch": 0.13982300884955753,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.009828059132190585,
      "kl": 0.0005789292044937611,
      "learning_rate": 3.451327433628318e-07,
      "loss": 0.0,
      "num_tokens": 1245087.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.9877007007598877,
      "sampling/importance_sampling_ratio/mean": 1.0007420778274536,
      "sampling/importance_sampling_ratio/min": 0.6938974857330322,
      "sampling/sampling_logp_difference/max": 0.6869785785675049,
      "sampling/sampling_logp_difference/mean": 0.01571083441376686,
      "step": 79
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 345.0,
      "completions/max_terminated_length": 345.0,
      "completions/mean_length": 77.5,
      "completions/mean_terminated_length": 77.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2790239453315735,
      "epoch": 0.1415929203539823,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.747476493391317,
      "kl": 0.003009934676811099,
      "learning_rate": 3.495575221238938e-07,
      "loss": 0.268,
      "num_tokens": 1262559.0,
      "reward": 0.625,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9996301531791687,
      "sampling/importance_sampling_ratio/min": 0.4342047870159149,
      "sampling/sampling_logp_difference/max": 0.8342390060424805,
      "sampling/sampling_logp_difference/mean": 0.019856680184602737,
      "step": 80
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 297.0,
      "completions/max_terminated_length": 297.0,
      "completions/mean_length": 58.0,
      "completions/mean_terminated_length": 58.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.20376178622245789,
      "epoch": 0.1433628318584071,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 12.49039325869809,
      "kl": 0.0034272735938429832,
      "learning_rate": 3.5398230088495575e-07,
      "loss": -0.4271,
      "num_tokens": 1277551.0,
      "reward": 0.71875,
      "reward_std": 0.2561737596988678,
      "rewards/decision_reward_func/mean": 0.71875,
      "rewards/decision_reward_func/std": 0.7007648944854736,
      "sampling/importance_sampling_ratio/max": 1.3819938898086548,
      "sampling/importance_sampling_ratio/mean": 0.9996453523635864,
      "sampling/importance_sampling_ratio/min": 0.5875664353370667,
      "sampling/sampling_logp_difference/max": 0.5317659378051758,
      "sampling/sampling_logp_difference/mean": 0.0125980693846941,
      "step": 81
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 194.0,
      "completions/max_terminated_length": 194.0,
      "completions/mean_length": 63.875,
      "completions/mean_terminated_length": 63.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.23782622814178467,
      "epoch": 0.14513274336283186,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 13.890965293596501,
      "kl": 0.0006324282148852944,
      "learning_rate": 3.5840707964601767e-07,
      "loss": 0.1274,
      "num_tokens": 1294071.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.7391748428344727,
      "sampling/importance_sampling_ratio/mean": 1.0006330013275146,
      "sampling/importance_sampling_ratio/min": 0.5648089051246643,
      "sampling/sampling_logp_difference/max": 0.57126784324646,
      "sampling/sampling_logp_difference/mean": 0.015169214457273483,
      "step": 82
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 180.0,
      "completions/max_terminated_length": 180.0,
      "completions/mean_length": 48.328125,
      "completions/mean_terminated_length": 48.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.295986533164978,
      "epoch": 0.14690265486725665,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02545414431293979,
      "kl": 0.00065580167574808,
      "learning_rate": 3.6283185840707963e-07,
      "loss": 0.0,
      "num_tokens": 1309340.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3446919918060303,
      "sampling/importance_sampling_ratio/mean": 0.9991348385810852,
      "sampling/importance_sampling_ratio/min": 0.6952046751976013,
      "sampling/sampling_logp_difference/max": 0.36354899406433105,
      "sampling/sampling_logp_difference/mean": 0.016460547223687172,
      "step": 83
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 186.0,
      "completions/max_terminated_length": 186.0,
      "completions/mean_length": 60.15625,
      "completions/mean_terminated_length": 60.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2950069308280945,
      "epoch": 0.1486725663716814,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03425744965710435,
      "kl": 0.0017368867993354797,
      "learning_rate": 3.672566371681416e-07,
      "loss": 0.0,
      "num_tokens": 1324326.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3615399599075317,
      "sampling/importance_sampling_ratio/mean": 0.9983700513839722,
      "sampling/importance_sampling_ratio/min": 0.556420087814331,
      "sampling/sampling_logp_difference/max": 0.5862317085266113,
      "sampling/sampling_logp_difference/mean": 0.018328040838241577,
      "step": 84
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 300.0,
      "completions/max_terminated_length": 300.0,
      "completions/mean_length": 106.921875,
      "completions/mean_terminated_length": 106.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5123524069786072,
      "epoch": 0.1504424778761062,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 17.031908186631007,
      "kl": 0.002637058962136507,
      "learning_rate": 3.7168141592920356e-07,
      "loss": 0.3122,
      "num_tokens": 1341793.0,
      "reward": 0.21875,
      "reward_std": 0.4101392924785614,
      "rewards/decision_reward_func/mean": 0.21875,
      "rewards/decision_reward_func/std": 0.983494758605957,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0009326934814453,
      "sampling/importance_sampling_ratio/min": 0.5631783604621887,
      "sampling/sampling_logp_difference/max": 1.1850533485412598,
      "sampling/sampling_logp_difference/mean": 0.020340263843536377,
      "step": 85
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 218.0,
      "completions/max_terminated_length": 218.0,
      "completions/mean_length": 91.390625,
      "completions/mean_terminated_length": 91.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4329564571380615,
      "epoch": 0.15221238938053097,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.023204412462173665,
      "kl": 0.001921823131851852,
      "learning_rate": 3.761061946902654e-07,
      "loss": 0.0,
      "num_tokens": 1357770.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6110613346099854,
      "sampling/importance_sampling_ratio/mean": 1.0003230571746826,
      "sampling/importance_sampling_ratio/min": 0.46672725677490234,
      "sampling/sampling_logp_difference/max": 0.7620102167129517,
      "sampling/sampling_logp_difference/mean": 0.018431926146149635,
      "step": 86
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 232.0,
      "completions/max_terminated_length": 232.0,
      "completions/mean_length": 63.953125,
      "completions/mean_terminated_length": 63.953125,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.42276352643966675,
      "epoch": 0.15398230088495576,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 16.80517251352282,
      "kl": 0.00828557275235653,
      "learning_rate": 3.805309734513274e-07,
      "loss": 0.3641,
      "num_tokens": 1371687.0,
      "reward": 0.78125,
      "reward_std": 0.48935678601264954,
      "rewards/decision_reward_func/mean": 0.78125,
      "rewards/decision_reward_func/std": 0.6291528940200806,
      "sampling/importance_sampling_ratio/max": 1.7898368835449219,
      "sampling/importance_sampling_ratio/mean": 1.0000966787338257,
      "sampling/importance_sampling_ratio/min": 0.5008212327957153,
      "sampling/sampling_logp_difference/max": 0.6915061473846436,
      "sampling/sampling_logp_difference/mean": 0.019807137548923492,
      "step": 87
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 347.0,
      "completions/max_terminated_length": 347.0,
      "completions/mean_length": 96.0625,
      "completions/mean_terminated_length": 96.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3446410298347473,
      "epoch": 0.15575221238938053,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.858522314114232,
      "kl": 0.0018287475686520338,
      "learning_rate": 3.8495575221238935e-07,
      "loss": 0.3426,
      "num_tokens": 1388731.0,
      "reward": 0.21875,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.21875,
      "rewards/decision_reward_func/std": 0.983494758605957,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0008213520050049,
      "sampling/importance_sampling_ratio/min": 0.33204153180122375,
      "sampling/sampling_logp_difference/max": 1.1024951934814453,
      "sampling/sampling_logp_difference/mean": 0.01567666232585907,
      "step": 88
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 166.0,
      "completions/max_terminated_length": 166.0,
      "completions/mean_length": 85.3125,
      "completions/mean_terminated_length": 85.3125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.3201582133769989,
      "epoch": 0.15752212389380532,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014084128619798917,
      "kl": 0.000516217143740505,
      "learning_rate": 3.893805309734513e-07,
      "loss": 0.0,
      "num_tokens": 1404319.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4318376779556274,
      "sampling/importance_sampling_ratio/mean": 1.0020582675933838,
      "sampling/importance_sampling_ratio/min": 0.6632694602012634,
      "sampling/sampling_logp_difference/max": 0.41057395935058594,
      "sampling/sampling_logp_difference/mean": 0.015921611338853836,
      "step": 89
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 257.0,
      "completions/max_terminated_length": 257.0,
      "completions/mean_length": 63.234375,
      "completions/mean_terminated_length": 63.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27583983540534973,
      "epoch": 0.1592920353982301,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 15.744491824905102,
      "kl": 0.0023850714787840843,
      "learning_rate": 3.938053097345133e-07,
      "loss": 0.0948,
      "num_tokens": 1422062.0,
      "reward": 0.0625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.0625,
      "rewards/decision_reward_func/std": 1.0059348344802856,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9983652830123901,
      "sampling/importance_sampling_ratio/min": 0.6548058390617371,
      "sampling/sampling_logp_difference/max": 0.9749901294708252,
      "sampling/sampling_logp_difference/mean": 0.014708740636706352,
      "step": 90
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 17.34375,
      "completions/mean_terminated_length": 17.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06893637031316757,
      "epoch": 0.16106194690265488,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.19984842013006643,
      "kl": 0.002403866034001112,
      "learning_rate": 3.982300884955752e-07,
      "loss": 0.0,
      "num_tokens": 1439604.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.133671522140503,
      "sampling/importance_sampling_ratio/mean": 0.9994039535522461,
      "sampling/importance_sampling_ratio/min": 0.6547919511795044,
      "sampling/sampling_logp_difference/max": 0.4234377145767212,
      "sampling/sampling_logp_difference/mean": 0.007659217808395624,
      "step": 91
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 214.0,
      "completions/max_terminated_length": 214.0,
      "completions/mean_length": 36.953125,
      "completions/mean_terminated_length": 36.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.26978591084480286,
      "epoch": 0.16283185840707964,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 16.587903339877094,
      "kl": 0.003500830614939332,
      "learning_rate": 4.0265486725663716e-07,
      "loss": 0.1325,
      "num_tokens": 1455329.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.6246740818023682,
      "sampling/importance_sampling_ratio/mean": 1.0004960298538208,
      "sampling/importance_sampling_ratio/min": 0.5870731472969055,
      "sampling/sampling_logp_difference/max": 0.532605767250061,
      "sampling/sampling_logp_difference/mean": 0.014703018590807915,
      "step": 92
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 218.0,
      "completions/max_terminated_length": 218.0,
      "completions/mean_length": 84.109375,
      "completions/mean_terminated_length": 84.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.47278204560279846,
      "epoch": 0.16460176991150444,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 11.06688751319423,
      "kl": 0.003939971327781677,
      "learning_rate": 4.0707964601769913e-07,
      "loss": 0.1314,
      "num_tokens": 1471768.0,
      "reward": 0.03125,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0012634992599487,
      "sampling/importance_sampling_ratio/min": 0.46622326970100403,
      "sampling/sampling_logp_difference/max": 0.8460280895233154,
      "sampling/sampling_logp_difference/mean": 0.022663792595267296,
      "step": 93
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 399.0,
      "completions/max_terminated_length": 399.0,
      "completions/mean_length": 59.6875,
      "completions/mean_terminated_length": 59.6875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2976003885269165,
      "epoch": 0.1663716814159292,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.901967299367,
      "kl": 0.002563281450420618,
      "learning_rate": 4.1150442477876104e-07,
      "loss": -0.3815,
      "num_tokens": 1485444.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.3954286575317383,
      "sampling/importance_sampling_ratio/mean": 1.0000526905059814,
      "sampling/importance_sampling_ratio/min": 0.6176431775093079,
      "sampling/sampling_logp_difference/max": 0.481844425201416,
      "sampling/sampling_logp_difference/mean": 0.015158753842115402,
      "step": 94
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 233.0,
      "completions/max_terminated_length": 233.0,
      "completions/mean_length": 51.46875,
      "completions/mean_terminated_length": 51.46875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2984507083892822,
      "epoch": 0.168141592920354,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 13.83890318477302,
      "kl": 0.05415243282914162,
      "learning_rate": 4.1592920353982295e-07,
      "loss": 0.2832,
      "num_tokens": 1500914.0,
      "reward": 0.5,
      "reward_std": 0.34156501293182373,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.7520718574523926,
      "sampling/importance_sampling_ratio/mean": 1.0006399154663086,
      "sampling/importance_sampling_ratio/min": 0.4704074263572693,
      "sampling/sampling_logp_difference/max": 0.7541561126708984,
      "sampling/sampling_logp_difference/mean": 0.017951540648937225,
      "step": 95
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 129.0,
      "completions/max_terminated_length": 129.0,
      "completions/mean_length": 32.59375,
      "completions/mean_terminated_length": 32.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.13821932673454285,
      "epoch": 0.16991150442477876,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.19751930399724366,
      "kl": 0.022612882778048515,
      "learning_rate": 4.203539823008849e-07,
      "loss": 0.0001,
      "num_tokens": 1512440.0,
      "reward": -1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.7523033618927002,
      "sampling/importance_sampling_ratio/mean": 0.9995698928833008,
      "sampling/importance_sampling_ratio/min": 0.4824843406677246,
      "sampling/sampling_logp_difference/max": 0.7288068532943726,
      "sampling/sampling_logp_difference/mean": 0.02123548462986946,
      "step": 96
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 135.0,
      "completions/max_terminated_length": 135.0,
      "completions/mean_length": 21.15625,
      "completions/mean_terminated_length": 21.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2681628465652466,
      "epoch": 0.17168141592920355,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 84.58976760954279,
      "kl": 1.1065090894699097,
      "learning_rate": 4.247787610619469e-07,
      "loss": -0.0586,
      "num_tokens": 1522786.0,
      "reward": -0.03125,
      "reward_std": 0.519389271736145,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.9697527885437012,
      "sampling/importance_sampling_ratio/mean": 0.9997271299362183,
      "sampling/importance_sampling_ratio/min": 0.0831858441233635,
      "sampling/sampling_logp_difference/max": 2.486678123474121,
      "sampling/sampling_logp_difference/mean": 0.025643471628427505,
      "step": 97
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 22.0,
      "completions/max_terminated_length": 22.0,
      "completions/mean_length": 16.875,
      "completions/mean_terminated_length": 16.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06743040680885315,
      "epoch": 0.17345132743362832,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.39896373700566834,
      "kl": 0.010008303448557854,
      "learning_rate": 4.2920353982300885e-07,
      "loss": 0.0001,
      "num_tokens": 1536762.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5342676639556885,
      "sampling/importance_sampling_ratio/mean": 0.9996311068534851,
      "sampling/importance_sampling_ratio/min": 0.6176677346229553,
      "sampling/sampling_logp_difference/max": 0.48180460929870605,
      "sampling/sampling_logp_difference/mean": 0.012232774868607521,
      "step": 98
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 211.0,
      "completions/max_terminated_length": 211.0,
      "completions/mean_length": 39.625,
      "completions/mean_terminated_length": 39.625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2937965393066406,
      "epoch": 0.1752212389380531,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 17.05345220718161,
      "kl": 0.047442853450775146,
      "learning_rate": 4.3362831858407076e-07,
      "loss": 0.1838,
      "num_tokens": 1551058.0,
      "reward": 0.90625,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.90625,
      "rewards/decision_reward_func/std": 0.42608407139778137,
      "sampling/importance_sampling_ratio/max": 1.50980806350708,
      "sampling/importance_sampling_ratio/mean": 0.9996635913848877,
      "sampling/importance_sampling_ratio/min": 0.5629895925521851,
      "sampling/sampling_logp_difference/max": 0.5744941234588623,
      "sampling/sampling_logp_difference/mean": 0.012546870857477188,
      "step": 99
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 138.0,
      "completions/max_terminated_length": 138.0,
      "completions/mean_length": 43.921875,
      "completions/mean_terminated_length": 43.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25557002425193787,
      "epoch": 0.17699115044247787,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 23.867050724829383,
      "kl": 0.012761028483510017,
      "learning_rate": 4.380530973451327e-07,
      "loss": -0.2337,
      "num_tokens": 1569261.0,
      "reward": 0.875,
      "reward_std": 0.3265564441680908,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0012588500976562,
      "sampling/importance_sampling_ratio/min": 0.5261755585670471,
      "sampling/sampling_logp_difference/max": 0.8514184951782227,
      "sampling/sampling_logp_difference/mean": 0.017819223925471306,
      "step": 100
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 21.0,
      "completions/max_terminated_length": 21.0,
      "completions/mean_length": 16.0,
      "completions/mean_terminated_length": 16.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06638449430465698,
      "epoch": 0.17876106194690267,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.44630179415515814,
      "kl": 0.024098360911011696,
      "learning_rate": 4.424778761061947e-07,
      "loss": 0.0002,
      "num_tokens": 1581645.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2675453424453735,
      "sampling/importance_sampling_ratio/mean": 0.999424397945404,
      "sampling/importance_sampling_ratio/min": 0.6548995971679688,
      "sampling/sampling_logp_difference/max": 0.42327332496643066,
      "sampling/sampling_logp_difference/mean": 0.011951332911849022,
      "step": 101
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 150.0,
      "completions/max_terminated_length": 150.0,
      "completions/mean_length": 36.296875,
      "completions/mean_terminated_length": 36.296875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2074262946844101,
      "epoch": 0.18053097345132743,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 17.8257040036157,
      "kl": 0.054125286638736725,
      "learning_rate": 4.469026548672566e-07,
      "loss": -0.144,
      "num_tokens": 1593456.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.6750662326812744,
      "sampling/importance_sampling_ratio/mean": 0.9999299049377441,
      "sampling/importance_sampling_ratio/min": 0.6054568886756897,
      "sampling/sampling_logp_difference/max": 0.515852689743042,
      "sampling/sampling_logp_difference/mean": 0.012604659423232079,
      "step": 102
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 158.0,
      "completions/max_terminated_length": 158.0,
      "completions/mean_length": 44.046875,
      "completions/mean_terminated_length": 44.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.32559850811958313,
      "epoch": 0.18230088495575222,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 32.22001959212673,
      "kl": 0.13167370855808258,
      "learning_rate": 4.5132743362831857e-07,
      "loss": -0.3729,
      "num_tokens": 1606947.0,
      "reward": 0.25,
      "reward_std": 0.42078250646591187,
      "rewards/decision_reward_func/mean": 0.25,
      "rewards/decision_reward_func/std": 0.9759001135826111,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9995090961456299,
      "sampling/importance_sampling_ratio/min": 0.4825321435928345,
      "sampling/sampling_logp_difference/max": 0.830096960067749,
      "sampling/sampling_logp_difference/mean": 0.02871047519147396,
      "step": 103
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 246.0,
      "completions/max_terminated_length": 246.0,
      "completions/mean_length": 59.21875,
      "completions/mean_terminated_length": 59.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4233275055885315,
      "epoch": 0.184070796460177,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 32.05855820384328,
      "kl": 0.05426112934947014,
      "learning_rate": 4.557522123893805e-07,
      "loss": -0.4524,
      "num_tokens": 1620529.0,
      "reward": 0.46875,
      "reward_std": 0.5959457159042358,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9988217353820801,
      "sampling/importance_sampling_ratio/min": 0.47307318449020386,
      "sampling/sampling_logp_difference/max": 1.6695234775543213,
      "sampling/sampling_logp_difference/mean": 0.026059508323669434,
      "step": 104
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 81.0,
      "completions/max_terminated_length": 81.0,
      "completions/mean_length": 16.578125,
      "completions/mean_terminated_length": 16.578125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.14209511876106262,
      "epoch": 0.18584070796460178,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0777847234694372,
      "kl": 0.08551844954490662,
      "learning_rate": 4.6017699115044245e-07,
      "loss": 0.0008,
      "num_tokens": 1636598.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2430133819580078,
      "sampling/importance_sampling_ratio/mean": 1.001842737197876,
      "sampling/importance_sampling_ratio/min": 0.42892172932624817,
      "sampling/sampling_logp_difference/max": 0.8464808464050293,
      "sampling/sampling_logp_difference/mean": 0.009883169084787369,
      "step": 105
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 149.0,
      "completions/max_terminated_length": 149.0,
      "completions/mean_length": 35.625,
      "completions/mean_terminated_length": 35.625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1706276834011078,
      "epoch": 0.18761061946902655,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 16.242571617604863,
      "kl": 0.008568068966269493,
      "learning_rate": 4.646017699115044e-07,
      "loss": -0.3094,
      "num_tokens": 1652014.0,
      "reward": 0.875,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 1.7361836433410645,
      "sampling/importance_sampling_ratio/mean": 0.9986557960510254,
      "sampling/importance_sampling_ratio/min": 0.49548566341400146,
      "sampling/sampling_logp_difference/max": 0.7022168040275574,
      "sampling/sampling_logp_difference/mean": 0.017092259600758553,
      "step": 106
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 216.0,
      "completions/max_terminated_length": 216.0,
      "completions/mean_length": 53.09375,
      "completions/mean_terminated_length": 53.09375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2413337230682373,
      "epoch": 0.18938053097345134,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 22.783578598982626,
      "kl": 0.0069497618824243546,
      "learning_rate": 4.690265486725664e-07,
      "loss": -0.2593,
      "num_tokens": 1667156.0,
      "reward": 0.875,
      "reward_std": 0.3265564441680908,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 1.5598926544189453,
      "sampling/importance_sampling_ratio/mean": 0.9992170333862305,
      "sampling/importance_sampling_ratio/min": 0.733043909072876,
      "sampling/sampling_logp_difference/max": 0.44461703300476074,
      "sampling/sampling_logp_difference/mean": 0.01676351949572563,
      "step": 107
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 337.0,
      "completions/max_terminated_length": 337.0,
      "completions/mean_length": 30.765625,
      "completions/mean_terminated_length": 30.765625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17704860866069794,
      "epoch": 0.1911504424778761,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 22.062156584055394,
      "kl": 0.015616398304700851,
      "learning_rate": 4.734513274336283e-07,
      "loss": 0.584,
      "num_tokens": 1680021.0,
      "reward": 0.28125,
      "reward_std": 0.2561737596988678,
      "rewards/decision_reward_func/mean": 0.28125,
      "rewards/decision_reward_func/std": 0.9672207236289978,
      "sampling/importance_sampling_ratio/max": 1.6436699628829956,
      "sampling/importance_sampling_ratio/mean": 1.0013000965118408,
      "sampling/importance_sampling_ratio/min": 0.6871146559715271,
      "sampling/sampling_logp_difference/max": 0.4969315528869629,
      "sampling/sampling_logp_difference/mean": 0.013623833656311035,
      "step": 108
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 374.0,
      "completions/max_terminated_length": 374.0,
      "completions/mean_length": 83.0625,
      "completions/mean_terminated_length": 83.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2795030176639557,
      "epoch": 0.1929203539823009,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.5345931936458213,
      "kl": 0.0018449004273861647,
      "learning_rate": 4.778761061946903e-07,
      "loss": 0.0868,
      "num_tokens": 1695865.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.4435425996780396,
      "sampling/importance_sampling_ratio/mean": 0.9981499314308167,
      "sampling/importance_sampling_ratio/min": 0.6196099519729614,
      "sampling/sampling_logp_difference/max": 0.4786651134490967,
      "sampling/sampling_logp_difference/mean": 0.01591462641954422,
      "step": 109
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 181.0,
      "completions/max_terminated_length": 181.0,
      "completions/mean_length": 58.171875,
      "completions/mean_terminated_length": 58.171875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2562200129032135,
      "epoch": 0.19469026548672566,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02541889548481687,
      "kl": 0.0014256872236728668,
      "learning_rate": 4.823008849557521e-07,
      "loss": 0.0,
      "num_tokens": 1709716.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6915197372436523,
      "sampling/importance_sampling_ratio/mean": 0.9971394538879395,
      "sampling/importance_sampling_ratio/min": 0.535825788974762,
      "sampling/sampling_logp_difference/max": 0.6239461898803711,
      "sampling/sampling_logp_difference/mean": 0.017007606104016304,
      "step": 110
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 23.390625,
      "completions/mean_terminated_length": 23.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21091923117637634,
      "epoch": 0.19646017699115045,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 29.116072245725366,
      "kl": 0.023805273696780205,
      "learning_rate": 4.867256637168141e-07,
      "loss": 0.4792,
      "num_tokens": 1725309.0,
      "reward": 0.84375,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.825963020324707,
      "sampling/importance_sampling_ratio/mean": 0.9973196387290955,
      "sampling/importance_sampling_ratio/min": 0.6877333521842957,
      "sampling/sampling_logp_difference/max": 0.6021075248718262,
      "sampling/sampling_logp_difference/mean": 0.02469644322991371,
      "step": 111
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 144.0,
      "completions/max_terminated_length": 144.0,
      "completions/mean_length": 41.171875,
      "completions/mean_terminated_length": 41.171875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.337478369474411,
      "epoch": 0.19823008849557522,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 28.478295365787474,
      "kl": 0.019370798021554947,
      "learning_rate": 4.91150442477876e-07,
      "loss": -0.373,
      "num_tokens": 1746520.0,
      "reward": 0.65625,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.4072273969650269,
      "sampling/importance_sampling_ratio/mean": 1.0000637769699097,
      "sampling/importance_sampling_ratio/min": 0.5572885274887085,
      "sampling/sampling_logp_difference/max": 0.5846721529960632,
      "sampling/sampling_logp_difference/mean": 0.015861041843891144,
      "step": 112
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 281.0,
      "completions/max_terminated_length": 281.0,
      "completions/mean_length": 67.5,
      "completions/mean_terminated_length": 67.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.43532299995422363,
      "epoch": 0.2,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 11.864682675131935,
      "kl": 0.018026825040578842,
      "learning_rate": 4.95575221238938e-07,
      "loss": -0.2313,
      "num_tokens": 1765192.0,
      "reward": -0.1875,
      "reward_std": 0.6991121172904968,
      "rewards/decision_reward_func/mean": -0.1875,
      "rewards/decision_reward_func/std": 0.9900296926498413,
      "sampling/importance_sampling_ratio/max": 1.565968632698059,
      "sampling/importance_sampling_ratio/mean": 0.9986177682876587,
      "sampling/importance_sampling_ratio/min": 0.6226667761802673,
      "sampling/sampling_logp_difference/max": 0.4737436771392822,
      "sampling/sampling_logp_difference/mean": 0.01891300082206726,
      "step": 113
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 222.0,
      "completions/max_terminated_length": 222.0,
      "completions/mean_length": 72.5625,
      "completions/mean_terminated_length": 72.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3272761106491089,
      "epoch": 0.20176991150442478,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.146958292258678,
      "kl": 0.05377377197146416,
      "learning_rate": 5e-07,
      "loss": -0.005,
      "num_tokens": 1782668.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0012102127075195,
      "sampling/importance_sampling_ratio/min": 0.3370976150035858,
      "sampling/sampling_logp_difference/max": 1.0873827934265137,
      "sampling/sampling_logp_difference/mean": 0.024121500551700592,
      "step": 114
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 247.0,
      "completions/max_terminated_length": 247.0,
      "completions/mean_length": 82.59375,
      "completions/mean_terminated_length": 82.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.46006712317466736,
      "epoch": 0.20353982300884957,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 6.751666668869363,
      "kl": 0.07440268248319626,
      "learning_rate": 5.044247787610619e-07,
      "loss": 0.0712,
      "num_tokens": 1799058.0,
      "reward": 0.90625,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.90625,
      "rewards/decision_reward_func/std": 0.42608407139778137,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9985596537590027,
      "sampling/importance_sampling_ratio/min": 0.4345816969871521,
      "sampling/sampling_logp_difference/max": 0.972254753112793,
      "sampling/sampling_logp_difference/mean": 0.019559338688850403,
      "step": 115
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 184.0,
      "completions/max_terminated_length": 184.0,
      "completions/mean_length": 37.625,
      "completions/mean_terminated_length": 37.625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2306581437587738,
      "epoch": 0.20530973451327433,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09028985253936475,
      "kl": 0.003620689269155264,
      "learning_rate": 5.088495575221239e-07,
      "loss": 0.0,
      "num_tokens": 1815882.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6114362478256226,
      "sampling/importance_sampling_ratio/mean": 0.9963452816009521,
      "sampling/importance_sampling_ratio/min": 0.41382789611816406,
      "sampling/sampling_logp_difference/max": 0.8823051452636719,
      "sampling/sampling_logp_difference/mean": 0.019629651680588722,
      "step": 116
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 296.0,
      "completions/max_terminated_length": 296.0,
      "completions/mean_length": 45.75,
      "completions/mean_terminated_length": 45.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2207641899585724,
      "epoch": 0.20707964601769913,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.030791093669244915,
      "kl": 0.001684409799054265,
      "learning_rate": 5.132743362831859e-07,
      "loss": 0.0,
      "num_tokens": 1830506.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4853850603103638,
      "sampling/importance_sampling_ratio/mean": 0.9980629086494446,
      "sampling/importance_sampling_ratio/min": 0.444266676902771,
      "sampling/sampling_logp_difference/max": 0.8113303184509277,
      "sampling/sampling_logp_difference/mean": 0.028768619522452354,
      "step": 117
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 317.0,
      "completions/max_terminated_length": 317.0,
      "completions/mean_length": 64.484375,
      "completions/mean_terminated_length": 64.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3205970227718353,
      "epoch": 0.2088495575221239,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03203958114580459,
      "kl": 0.0018948880024254322,
      "learning_rate": 5.176991150442478e-07,
      "loss": 0.0,
      "num_tokens": 1845561.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5698038339614868,
      "sampling/importance_sampling_ratio/mean": 1.0007352828979492,
      "sampling/importance_sampling_ratio/min": 0.6012983322143555,
      "sampling/sampling_logp_difference/max": 0.5086641311645508,
      "sampling/sampling_logp_difference/mean": 0.021763615310192108,
      "step": 118
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 993.0,
      "completions/max_terminated_length": 993.0,
      "completions/mean_length": 140.8125,
      "completions/mean_terminated_length": 140.8125,
      "completions/min_length": 52.0,
      "completions/min_terminated_length": 52.0,
      "entropy": 0.5416373014450073,
      "epoch": 0.21061946902654868,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.007404796935460493,
      "kl": 0.0005785170360468328,
      "learning_rate": 5.221238938053097e-07,
      "loss": 0.0,
      "num_tokens": 1863789.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4254950284957886,
      "sampling/importance_sampling_ratio/mean": 1.000152349472046,
      "sampling/importance_sampling_ratio/min": 0.6323530077934265,
      "sampling/sampling_logp_difference/max": 0.45830750465393066,
      "sampling/sampling_logp_difference/mean": 0.01834021881222725,
      "step": 119
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 289.0,
      "completions/max_terminated_length": 289.0,
      "completions/mean_length": 49.9375,
      "completions/mean_terminated_length": 49.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1841767281293869,
      "epoch": 0.21238938053097345,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 14.962615729352883,
      "kl": 0.005641619674861431,
      "learning_rate": 5.265486725663717e-07,
      "loss": 0.2669,
      "num_tokens": 1878841.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.765735149383545,
      "sampling/importance_sampling_ratio/mean": 0.9993394613265991,
      "sampling/importance_sampling_ratio/min": 0.7040172815322876,
      "sampling/sampling_logp_difference/max": 0.568567156791687,
      "sampling/sampling_logp_difference/mean": 0.009488564915955067,
      "step": 120
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 144.0,
      "completions/max_terminated_length": 144.0,
      "completions/mean_length": 44.046875,
      "completions/mean_terminated_length": 44.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2508974075317383,
      "epoch": 0.21415929203539824,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 15.138540143090314,
      "kl": 0.003447249997407198,
      "learning_rate": 5.309734513274336e-07,
      "loss": -0.2376,
      "num_tokens": 1895772.0,
      "reward": 0.75,
      "reward_std": 0.25819888710975647,
      "rewards/decision_reward_func/mean": 0.75,
      "rewards/decision_reward_func/std": 0.6666666865348816,
      "sampling/importance_sampling_ratio/max": 1.6151796579360962,
      "sampling/importance_sampling_ratio/mean": 0.9982805252075195,
      "sampling/importance_sampling_ratio/min": 0.6492539644241333,
      "sampling/sampling_logp_difference/max": 0.4794461727142334,
      "sampling/sampling_logp_difference/mean": 0.017452435567975044,
      "step": 121
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 167.0,
      "completions/max_terminated_length": 167.0,
      "completions/mean_length": 38.140625,
      "completions/mean_terminated_length": 38.140625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3044247627258301,
      "epoch": 0.215929203539823,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 19.479083260753757,
      "kl": 0.01046132855117321,
      "learning_rate": 5.353982300884956e-07,
      "loss": 0.0352,
      "num_tokens": 1911141.0,
      "reward": 0.34375,
      "reward_std": 0.5827301740646362,
      "rewards/decision_reward_func/mean": 0.34375,
      "rewards/decision_reward_func/std": 0.9464847445487976,
      "sampling/importance_sampling_ratio/max": 1.5219072103500366,
      "sampling/importance_sampling_ratio/mean": 0.9994726181030273,
      "sampling/importance_sampling_ratio/min": 0.5666710734367371,
      "sampling/sampling_logp_difference/max": 0.5679762363433838,
      "sampling/sampling_logp_difference/mean": 0.016208771616220474,
      "step": 122
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 251.0,
      "completions/max_terminated_length": 251.0,
      "completions/mean_length": 105.75,
      "completions/mean_terminated_length": 105.75,
      "completions/min_length": 41.0,
      "completions/min_terminated_length": 41.0,
      "entropy": 0.519478440284729,
      "epoch": 0.2176991150442478,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.019650278961170106,
      "kl": 0.0011656515998765826,
      "learning_rate": 5.398230088495575e-07,
      "loss": 0.0,
      "num_tokens": 1927541.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.8163784742355347,
      "sampling/importance_sampling_ratio/mean": 1.000349998474121,
      "sampling/importance_sampling_ratio/min": 0.6328713893890381,
      "sampling/sampling_logp_difference/max": 0.5968446731567383,
      "sampling/sampling_logp_difference/mean": 0.019682027399539948,
      "step": 123
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 249.0,
      "completions/max_terminated_length": 249.0,
      "completions/mean_length": 106.96875,
      "completions/mean_terminated_length": 106.96875,
      "completions/min_length": 41.0,
      "completions/min_terminated_length": 41.0,
      "entropy": 0.45931580662727356,
      "epoch": 0.21946902654867256,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.008681335336834708,
      "kl": 0.0005357326008379459,
      "learning_rate": 5.442477876106194e-07,
      "loss": 0.0,
      "num_tokens": 1944499.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.929969310760498,
      "sampling/importance_sampling_ratio/mean": 1.0011227130889893,
      "sampling/importance_sampling_ratio/min": 0.6561875939369202,
      "sampling/sampling_logp_difference/max": 0.6575040817260742,
      "sampling/sampling_logp_difference/mean": 0.01847759075462818,
      "step": 124
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 254.0,
      "completions/max_terminated_length": 254.0,
      "completions/mean_length": 110.75,
      "completions/mean_terminated_length": 110.75,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.37561601400375366,
      "epoch": 0.22123893805309736,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.9753687076399795,
      "kl": 0.0010785853955894709,
      "learning_rate": 5.486725663716814e-07,
      "loss": -0.0333,
      "num_tokens": 1961459.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9995502829551697,
      "sampling/importance_sampling_ratio/min": 0.6624337434768677,
      "sampling/sampling_logp_difference/max": 1.584089994430542,
      "sampling/sampling_logp_difference/mean": 0.016993992030620575,
      "step": 125
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 279.0,
      "completions/max_terminated_length": 279.0,
      "completions/mean_length": 70.53125,
      "completions/mean_terminated_length": 70.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.31924882531166077,
      "epoch": 0.22300884955752212,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.031328937808989196,
      "kl": 0.0014360514469444752,
      "learning_rate": 5.530973451327434e-07,
      "loss": 0.0,
      "num_tokens": 1975653.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.8715254068374634,
      "sampling/importance_sampling_ratio/mean": 1.0022432804107666,
      "sampling/importance_sampling_ratio/min": 0.6149583458900452,
      "sampling/sampling_logp_difference/max": 0.6267538070678711,
      "sampling/sampling_logp_difference/mean": 0.025006473064422607,
      "step": 126
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 198.0,
      "completions/max_terminated_length": 198.0,
      "completions/mean_length": 82.21875,
      "completions/mean_terminated_length": 82.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3541611433029175,
      "epoch": 0.2247787610619469,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.013457624899861007,
      "kl": 0.0013129812432453036,
      "learning_rate": 5.575221238938052e-07,
      "loss": 0.0,
      "num_tokens": 1995827.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.574439525604248,
      "sampling/importance_sampling_ratio/mean": 0.9993070960044861,
      "sampling/importance_sampling_ratio/min": 0.5381397008895874,
      "sampling/sampling_logp_difference/max": 0.6196370124816895,
      "sampling/sampling_logp_difference/mean": 0.014719847589731216,
      "step": 127
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 240.0,
      "completions/max_terminated_length": 240.0,
      "completions/mean_length": 67.015625,
      "completions/mean_terminated_length": 67.015625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.44300490617752075,
      "epoch": 0.22654867256637168,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.134743959337715,
      "kl": 0.0014869027072563767,
      "learning_rate": 5.619469026548672e-07,
      "loss": 0.2426,
      "num_tokens": 2011460.0,
      "reward": 0.875,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 1.8721041679382324,
      "sampling/importance_sampling_ratio/mean": 0.9977869987487793,
      "sampling/importance_sampling_ratio/min": 0.4702486991882324,
      "sampling/sampling_logp_difference/max": 0.7544935941696167,
      "sampling/sampling_logp_difference/mean": 0.023253358900547028,
      "step": 128
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 382.0,
      "completions/max_terminated_length": 382.0,
      "completions/mean_length": 81.8125,
      "completions/mean_terminated_length": 81.8125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2518032491207123,
      "epoch": 0.22831858407079647,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.862384196324069,
      "kl": 0.0007762154564261436,
      "learning_rate": 5.663716814159291e-07,
      "loss": 0.1619,
      "num_tokens": 2027848.0,
      "reward": 0.6875,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.6875,
      "rewards/decision_reward_func/std": 0.7319250702857971,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0013949871063232,
      "sampling/importance_sampling_ratio/min": 0.5217262506484985,
      "sampling/sampling_logp_difference/max": 1.1203207969665527,
      "sampling/sampling_logp_difference/mean": 0.01854543387889862,
      "step": 129
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 257.0,
      "completions/max_terminated_length": 257.0,
      "completions/mean_length": 71.625,
      "completions/mean_terminated_length": 71.625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.299152135848999,
      "epoch": 0.23008849557522124,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014935968388077706,
      "kl": 0.0009952923282980919,
      "learning_rate": 5.707964601769911e-07,
      "loss": 0.0,
      "num_tokens": 2042800.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6621967554092407,
      "sampling/importance_sampling_ratio/mean": 1.0008690357208252,
      "sampling/importance_sampling_ratio/min": 0.4874541759490967,
      "sampling/sampling_logp_difference/max": 0.7185590267181396,
      "sampling/sampling_logp_difference/mean": 0.018116779625415802,
      "step": 130
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 324.0,
      "completions/max_terminated_length": 324.0,
      "completions/mean_length": 129.546875,
      "completions/mean_terminated_length": 129.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4606068730354309,
      "epoch": 0.23185840707964603,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.560787920244211,
      "kl": 0.001249688328243792,
      "learning_rate": 5.752212389380531e-07,
      "loss": 0.1933,
      "num_tokens": 2060803.0,
      "reward": 0.03125,
      "reward_std": 0.29578250646591187,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.5971571207046509,
      "sampling/importance_sampling_ratio/mean": 1.0004818439483643,
      "sampling/importance_sampling_ratio/min": 0.5036928653717041,
      "sampling/sampling_logp_difference/max": 0.685788631439209,
      "sampling/sampling_logp_difference/mean": 0.01681518740952015,
      "step": 131
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 326.0,
      "completions/max_terminated_length": 326.0,
      "completions/mean_length": 84.25,
      "completions/mean_terminated_length": 84.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3058205246925354,
      "epoch": 0.2336283185840708,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01254687731931507,
      "kl": 0.0007779281586408615,
      "learning_rate": 5.79646017699115e-07,
      "loss": 0.0,
      "num_tokens": 2078243.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0002914667129517,
      "sampling/importance_sampling_ratio/min": 0.4855709969997406,
      "sampling/sampling_logp_difference/max": 0.7224297523498535,
      "sampling/sampling_logp_difference/mean": 0.02297021448612213,
      "step": 132
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 245.0,
      "completions/max_terminated_length": 245.0,
      "completions/mean_length": 69.484375,
      "completions/mean_terminated_length": 69.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.26227444410324097,
      "epoch": 0.23539823008849559,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.016136415805378446,
      "kl": 0.0010202470002695918,
      "learning_rate": 5.84070796460177e-07,
      "loss": 0.0,
      "num_tokens": 2096418.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.781939148902893,
      "sampling/importance_sampling_ratio/mean": 0.9994803667068481,
      "sampling/importance_sampling_ratio/min": 0.5584296584129333,
      "sampling/sampling_logp_difference/max": 0.5826265811920166,
      "sampling/sampling_logp_difference/mean": 0.014151913113892078,
      "step": 133
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 215.0,
      "completions/max_terminated_length": 215.0,
      "completions/mean_length": 72.875,
      "completions/mean_terminated_length": 72.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24580150842666626,
      "epoch": 0.23716814159292035,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02503235510969517,
      "kl": 0.001629705191589892,
      "learning_rate": 5.88495575221239e-07,
      "loss": 0.0,
      "num_tokens": 2112890.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5154296159744263,
      "sampling/importance_sampling_ratio/mean": 1.0017865896224976,
      "sampling/importance_sampling_ratio/min": 0.6210728287696838,
      "sampling/sampling_logp_difference/max": 0.4763069152832031,
      "sampling/sampling_logp_difference/mean": 0.01634804531931877,
      "step": 134
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 207.0,
      "completions/max_terminated_length": 207.0,
      "completions/mean_length": 75.90625,
      "completions/mean_terminated_length": 75.90625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5235762000083923,
      "epoch": 0.23893805309734514,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01575589802905874,
      "kl": 0.001007390907034278,
      "learning_rate": 5.929203539823009e-07,
      "loss": 0.0,
      "num_tokens": 2129028.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.001007080078125,
      "sampling/importance_sampling_ratio/min": 0.6624895334243774,
      "sampling/sampling_logp_difference/max": 0.7274326086044312,
      "sampling/sampling_logp_difference/mean": 0.02209286577999592,
      "step": 135
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 239.0,
      "completions/max_terminated_length": 239.0,
      "completions/mean_length": 88.375,
      "completions/mean_terminated_length": 88.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5046366453170776,
      "epoch": 0.2407079646017699,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.578048939539401,
      "kl": 0.0036787153221666813,
      "learning_rate": 5.973451327433628e-07,
      "loss": 0.1217,
      "num_tokens": 2144508.0,
      "reward": 0.90625,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.90625,
      "rewards/decision_reward_func/std": 0.42608407139778137,
      "sampling/importance_sampling_ratio/max": 1.4554598331451416,
      "sampling/importance_sampling_ratio/mean": 0.9992860555648804,
      "sampling/importance_sampling_ratio/min": 0.25643423199653625,
      "sampling/sampling_logp_difference/max": 1.3608829975128174,
      "sampling/sampling_logp_difference/mean": 0.022801248356699944,
      "step": 136
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 123.0,
      "completions/max_terminated_length": 123.0,
      "completions/mean_length": 33.5625,
      "completions/mean_terminated_length": 33.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18427319824695587,
      "epoch": 0.2424778761061947,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03157405100169581,
      "kl": 0.0014091174816712737,
      "learning_rate": 6.017699115044248e-07,
      "loss": 0.0,
      "num_tokens": 2156688.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0007842779159546,
      "sampling/importance_sampling_ratio/min": 0.627287745475769,
      "sampling/sampling_logp_difference/max": 0.8145256042480469,
      "sampling/sampling_logp_difference/mean": 0.017598610371351242,
      "step": 137
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 159.0,
      "completions/max_terminated_length": 159.0,
      "completions/mean_length": 71.640625,
      "completions/mean_terminated_length": 71.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3980875611305237,
      "epoch": 0.24424778761061947,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.016564111690985255,
      "kl": 0.0008818708593025804,
      "learning_rate": 6.061946902654867e-07,
      "loss": 0.0,
      "num_tokens": 2173097.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4036502838134766,
      "sampling/importance_sampling_ratio/mean": 1.0016671419143677,
      "sampling/importance_sampling_ratio/min": 0.6285651922225952,
      "sampling/sampling_logp_difference/max": 0.4643155336380005,
      "sampling/sampling_logp_difference/mean": 0.01826133392751217,
      "step": 138
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 243.0,
      "completions/max_terminated_length": 243.0,
      "completions/mean_length": 81.828125,
      "completions/mean_terminated_length": 81.828125,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.4850948452949524,
      "epoch": 0.24601769911504426,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01713387326277667,
      "kl": 0.001043393975123763,
      "learning_rate": 6.106194690265486e-07,
      "loss": 0.0,
      "num_tokens": 2188382.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.910315752029419,
      "sampling/importance_sampling_ratio/mean": 0.9982123374938965,
      "sampling/importance_sampling_ratio/min": 0.5476705431938171,
      "sampling/sampling_logp_difference/max": 0.647268533706665,
      "sampling/sampling_logp_difference/mean": 0.020284462720155716,
      "step": 139
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 219.0,
      "completions/max_terminated_length": 219.0,
      "completions/mean_length": 99.71875,
      "completions/mean_terminated_length": 99.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3546068072319031,
      "epoch": 0.24778761061946902,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01725404162883253,
      "kl": 0.0017146074678748846,
      "learning_rate": 6.150442477876105e-07,
      "loss": 0.0,
      "num_tokens": 2204524.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3572512865066528,
      "sampling/importance_sampling_ratio/mean": 1.0001609325408936,
      "sampling/importance_sampling_ratio/min": 0.4417930245399475,
      "sampling/sampling_logp_difference/max": 0.8169138431549072,
      "sampling/sampling_logp_difference/mean": 0.01794680394232273,
      "step": 140
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 172.0,
      "completions/max_terminated_length": 172.0,
      "completions/mean_length": 35.140625,
      "completions/mean_terminated_length": 35.140625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18727999925613403,
      "epoch": 0.24955752212389382,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09529720865059939,
      "kl": 0.0010563858086243272,
      "learning_rate": 6.194690265486725e-07,
      "loss": 0.0,
      "num_tokens": 2216885.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9989814758300781,
      "sampling/importance_sampling_ratio/min": 0.7282373905181885,
      "sampling/sampling_logp_difference/max": 0.814521074295044,
      "sampling/sampling_logp_difference/mean": 0.018082180991768837,
      "step": 141
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 16.328125,
      "completions/mean_terminated_length": 16.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06842117011547089,
      "epoch": 0.2513274336283186,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.075752721630387,
      "kl": 0.004978060256689787,
      "learning_rate": 6.238938053097345e-07,
      "loss": -0.0013,
      "num_tokens": 2230346.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.266401767730713,
      "sampling/importance_sampling_ratio/mean": 1.0020692348480225,
      "sampling/importance_sampling_ratio/min": 0.6714641451835632,
      "sampling/sampling_logp_difference/max": 0.39829468727111816,
      "sampling/sampling_logp_difference/mean": 0.010533824563026428,
      "step": 142
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 164.0,
      "completions/max_terminated_length": 164.0,
      "completions/mean_length": 74.671875,
      "completions/mean_terminated_length": 74.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3647806644439697,
      "epoch": 0.25309734513274335,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.015980370154043362,
      "kl": 0.0007940245559439063,
      "learning_rate": 6.283185840707964e-07,
      "loss": 0.0,
      "num_tokens": 2245157.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.424361228942871,
      "sampling/importance_sampling_ratio/mean": 1.000733494758606,
      "sampling/importance_sampling_ratio/min": 0.7051947116851807,
      "sampling/sampling_logp_difference/max": 0.353723406791687,
      "sampling/sampling_logp_difference/mean": 0.015422477386891842,
      "step": 143
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 840.0,
      "completions/max_terminated_length": 840.0,
      "completions/mean_length": 141.390625,
      "completions/mean_terminated_length": 141.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5192233324050903,
      "epoch": 0.25486725663716814,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.03788217298924,
      "kl": 0.0028998591005802155,
      "learning_rate": 6.327433628318584e-07,
      "loss": -0.0586,
      "num_tokens": 2262686.0,
      "reward": -0.15625,
      "reward_std": 0.47978055477142334,
      "rewards/decision_reward_func/mean": -0.15625,
      "rewards/decision_reward_func/std": 0.9955257177352905,
      "sampling/importance_sampling_ratio/max": 1.6302465200424194,
      "sampling/importance_sampling_ratio/mean": 0.9998656511306763,
      "sampling/importance_sampling_ratio/min": 0.5490341782569885,
      "sampling/sampling_logp_difference/max": 0.5995945930480957,
      "sampling/sampling_logp_difference/mean": 0.019007448107004166,
      "step": 144
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 382.0,
      "completions/max_terminated_length": 382.0,
      "completions/mean_length": 101.109375,
      "completions/mean_terminated_length": 101.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3655405640602112,
      "epoch": 0.25663716814159293,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.607652442786013,
      "kl": 0.002844990696758032,
      "learning_rate": 6.371681415929203e-07,
      "loss": 0.0627,
      "num_tokens": 2279573.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9995540380477905,
      "sampling/importance_sampling_ratio/min": 0.6338145136833191,
      "sampling/sampling_logp_difference/max": 1.1002683639526367,
      "sampling/sampling_logp_difference/mean": 0.021086499094963074,
      "step": 145
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 141.0,
      "completions/max_terminated_length": 141.0,
      "completions/mean_length": 48.328125,
      "completions/mean_terminated_length": 48.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2803422212600708,
      "epoch": 0.2584070796460177,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02802108714282455,
      "kl": 0.0013906147796660662,
      "learning_rate": 6.415929203539822e-07,
      "loss": 0.0,
      "num_tokens": 2295178.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2905248403549194,
      "sampling/importance_sampling_ratio/mean": 0.9989687204360962,
      "sampling/importance_sampling_ratio/min": 0.4375634789466858,
      "sampling/sampling_logp_difference/max": 0.8265335559844971,
      "sampling/sampling_logp_difference/mean": 0.015749461948871613,
      "step": 146
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 187.0,
      "completions/max_terminated_length": 187.0,
      "completions/mean_length": 62.203125,
      "completions/mean_terminated_length": 62.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2617567479610443,
      "epoch": 0.26017699115044246,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.024533085491038015,
      "kl": 0.001395686762407422,
      "learning_rate": 6.460176991150442e-07,
      "loss": 0.0,
      "num_tokens": 2316247.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5753140449523926,
      "sampling/importance_sampling_ratio/mean": 0.9995524287223816,
      "sampling/importance_sampling_ratio/min": 0.7271898984909058,
      "sampling/sampling_logp_difference/max": 0.4544546604156494,
      "sampling/sampling_logp_difference/mean": 0.011754988692700863,
      "step": 147
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.84375,
      "completions/mean_terminated_length": 15.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07275533676147461,
      "epoch": 0.26194690265486725,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09102567414694034,
      "kl": 0.0017056685173884034,
      "learning_rate": 6.504424778761062e-07,
      "loss": 0.0,
      "num_tokens": 2330141.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.197696566581726,
      "sampling/importance_sampling_ratio/mean": 1.000204086303711,
      "sampling/importance_sampling_ratio/min": 0.8154329061508179,
      "sampling/sampling_logp_difference/max": 0.20403611660003662,
      "sampling/sampling_logp_difference/mean": 0.0072796279564499855,
      "step": 148
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 190.0,
      "completions/max_terminated_length": 190.0,
      "completions/mean_length": 38.5625,
      "completions/mean_terminated_length": 38.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.14959517121315002,
      "epoch": 0.26371681415929205,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 12.158302193342701,
      "kl": 0.009750444442033768,
      "learning_rate": 6.548672566371681e-07,
      "loss": -0.1449,
      "num_tokens": 2343185.0,
      "reward": -0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.7331476211547852,
      "sampling/importance_sampling_ratio/mean": 0.9996535778045654,
      "sampling/importance_sampling_ratio/min": 0.255138635635376,
      "sampling/sampling_logp_difference/max": 1.36594820022583,
      "sampling/sampling_logp_difference/mean": 0.016084929928183556,
      "step": 149
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 138.0,
      "completions/max_terminated_length": 138.0,
      "completions/mean_length": 35.59375,
      "completions/mean_terminated_length": 35.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1963880956172943,
      "epoch": 0.26548672566371684,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.1218924747110615,
      "kl": 0.005532016046345234,
      "learning_rate": 6.592920353982301e-07,
      "loss": -0.1457,
      "num_tokens": 2358039.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0004479885101318,
      "sampling/importance_sampling_ratio/min": 0.5923722982406616,
      "sampling/sampling_logp_difference/max": 1.9722204208374023,
      "sampling/sampling_logp_difference/mean": 0.013359785079956055,
      "step": 150
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 235.0,
      "completions/max_terminated_length": 235.0,
      "completions/mean_length": 78.109375,
      "completions/mean_terminated_length": 78.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2529459297657013,
      "epoch": 0.2672566371681416,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.044873586553780806,
      "kl": 0.005619180388748646,
      "learning_rate": 6.637168141592921e-07,
      "loss": 0.0,
      "num_tokens": 2373422.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.9478777647018433,
      "sampling/importance_sampling_ratio/mean": 1.0008846521377563,
      "sampling/importance_sampling_ratio/min": 0.45570361614227295,
      "sampling/sampling_logp_difference/max": 0.7859126925468445,
      "sampling/sampling_logp_difference/mean": 0.015898194164037704,
      "step": 151
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 305.0,
      "completions/max_terminated_length": 305.0,
      "completions/mean_length": 65.640625,
      "completions/mean_terminated_length": 65.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30609267950057983,
      "epoch": 0.26902654867256637,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.061460167266631,
      "kl": 0.010558098554611206,
      "learning_rate": 6.68141592920354e-07,
      "loss": -0.0973,
      "num_tokens": 2389335.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.5556261539459229,
      "sampling/importance_sampling_ratio/mean": 1.0001894235610962,
      "sampling/importance_sampling_ratio/min": 0.572333037853241,
      "sampling/sampling_logp_difference/max": 0.5580341815948486,
      "sampling/sampling_logp_difference/mean": 0.014833297580480576,
      "step": 152
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 162.0,
      "completions/max_terminated_length": 162.0,
      "completions/mean_length": 36.953125,
      "completions/mean_terminated_length": 36.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.216139018535614,
      "epoch": 0.27079646017699116,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.14097604253434792,
      "kl": 0.017395276576280594,
      "learning_rate": 6.72566371681416e-07,
      "loss": 0.0001,
      "num_tokens": 2404404.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.879258394241333,
      "sampling/importance_sampling_ratio/mean": 0.9984070062637329,
      "sampling/importance_sampling_ratio/min": 0.6819742321968079,
      "sampling/sampling_logp_difference/max": 0.6308772563934326,
      "sampling/sampling_logp_difference/mean": 0.012710087932646275,
      "step": 153
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 250.0,
      "completions/max_terminated_length": 250.0,
      "completions/mean_length": 48.265625,
      "completions/mean_terminated_length": 48.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1130940243601799,
      "epoch": 0.27256637168141595,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.12575044725568155,
      "kl": 0.007728739641606808,
      "learning_rate": 6.769911504424779e-07,
      "loss": 0.0,
      "num_tokens": 2420661.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4918367862701416,
      "sampling/importance_sampling_ratio/mean": 0.9995026588439941,
      "sampling/importance_sampling_ratio/min": 0.6138349175453186,
      "sampling/sampling_logp_difference/max": 0.48802924156188965,
      "sampling/sampling_logp_difference/mean": 0.020763784646987915,
      "step": 154
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 270.0,
      "completions/max_terminated_length": 270.0,
      "completions/mean_length": 59.953125,
      "completions/mean_terminated_length": 59.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21759197115898132,
      "epoch": 0.2743362831858407,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 18.00910866136154,
      "kl": 0.010943894274532795,
      "learning_rate": 6.814159292035397e-07,
      "loss": -0.221,
      "num_tokens": 2433986.0,
      "reward": 0.84375,
      "reward_std": 0.34860679507255554,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.969313383102417,
      "sampling/importance_sampling_ratio/mean": 1.002643346786499,
      "sampling/importance_sampling_ratio/min": 0.538426399230957,
      "sampling/sampling_logp_difference/max": 0.677685022354126,
      "sampling/sampling_logp_difference/mean": 0.024278851225972176,
      "step": 155
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 252.0,
      "completions/max_terminated_length": 252.0,
      "completions/mean_length": 41.84375,
      "completions/mean_terminated_length": 41.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24851931631565094,
      "epoch": 0.2761061946902655,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 19.56733698517216,
      "kl": 0.0038888424169272184,
      "learning_rate": 6.858407079646017e-07,
      "loss": -0.2139,
      "num_tokens": 2447512.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.663854956626892,
      "sampling/importance_sampling_ratio/mean": 1.0001685619354248,
      "sampling/importance_sampling_ratio/min": 0.46025779843330383,
      "sampling/sampling_logp_difference/max": 0.7759685516357422,
      "sampling/sampling_logp_difference/mean": 0.012126958929002285,
      "step": 156
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 254.0,
      "completions/max_terminated_length": 254.0,
      "completions/mean_length": 82.734375,
      "completions/mean_terminated_length": 82.734375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.24554727971553802,
      "epoch": 0.2778761061946903,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02984062761107293,
      "kl": 0.001689649187028408,
      "learning_rate": 6.902654867256636e-07,
      "loss": 0.0,
      "num_tokens": 2464983.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4411574602127075,
      "sampling/importance_sampling_ratio/mean": 1.000152349472046,
      "sampling/importance_sampling_ratio/min": 0.6802230477333069,
      "sampling/sampling_logp_difference/max": 0.38533449172973633,
      "sampling/sampling_logp_difference/mean": 0.013160878792405128,
      "step": 157
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 398.0,
      "completions/max_terminated_length": 398.0,
      "completions/mean_length": 88.640625,
      "completions/mean_terminated_length": 88.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2844574451446533,
      "epoch": 0.27964601769911507,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.589166739359197,
      "kl": 0.0038004093803465366,
      "learning_rate": 6.946902654867256e-07,
      "loss": 0.0108,
      "num_tokens": 2481360.0,
      "reward": 0.84375,
      "reward_std": 0.3723389506340027,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.5664699077606201,
      "sampling/importance_sampling_ratio/mean": 0.9985457062721252,
      "sampling/importance_sampling_ratio/min": 0.5478703379631042,
      "sampling/sampling_logp_difference/max": 0.6017166376113892,
      "sampling/sampling_logp_difference/mean": 0.017418205738067627,
      "step": 158
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 197.0,
      "completions/max_terminated_length": 197.0,
      "completions/mean_length": 29.828125,
      "completions/mean_terminated_length": 29.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.13992829620838165,
      "epoch": 0.2814159292035398,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 22.822138577571543,
      "kl": 0.007147709373384714,
      "learning_rate": 6.991150442477876e-07,
      "loss": -0.2478,
      "num_tokens": 2494325.0,
      "reward": 0.90625,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.90625,
      "rewards/decision_reward_func/std": 0.42608407139778137,
      "sampling/importance_sampling_ratio/max": 1.9513019323349,
      "sampling/importance_sampling_ratio/mean": 0.9980819225311279,
      "sampling/importance_sampling_ratio/min": 0.49336642026901245,
      "sampling/sampling_logp_difference/max": 0.70650315284729,
      "sampling/sampling_logp_difference/mean": 0.01531197689473629,
      "step": 159
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 151.0,
      "completions/max_terminated_length": 151.0,
      "completions/mean_length": 36.28125,
      "completions/mean_terminated_length": 36.28125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1655530482530594,
      "epoch": 0.2831858407079646,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.220842673661625,
      "kl": 0.005232499912381172,
      "learning_rate": 7.035398230088495e-07,
      "loss": 0.0035,
      "num_tokens": 2508055.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.6507823467254639,
      "sampling/importance_sampling_ratio/mean": 1.0039724111557007,
      "sampling/importance_sampling_ratio/min": 0.5558677315711975,
      "sampling/sampling_logp_difference/max": 0.5872249603271484,
      "sampling/sampling_logp_difference/mean": 0.01936100609600544,
      "step": 160
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 148.0,
      "completions/max_terminated_length": 148.0,
      "completions/mean_length": 37.234375,
      "completions/mean_terminated_length": 37.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.14813996851444244,
      "epoch": 0.2849557522123894,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06153866209942643,
      "kl": 0.0026678910944610834,
      "learning_rate": 7.079646017699115e-07,
      "loss": 0.0,
      "num_tokens": 2521126.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.862612009048462,
      "sampling/importance_sampling_ratio/mean": 0.9947205781936646,
      "sampling/importance_sampling_ratio/min": 0.5880116820335388,
      "sampling/sampling_logp_difference/max": 0.621979832649231,
      "sampling/sampling_logp_difference/mean": 0.024796247482299805,
      "step": 161
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 324.0,
      "completions/max_terminated_length": 324.0,
      "completions/mean_length": 97.875,
      "completions/mean_terminated_length": 97.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4262969493865967,
      "epoch": 0.2867256637168142,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.013023578679145853,
      "kl": 0.0014449837617576122,
      "learning_rate": 7.123893805309734e-07,
      "loss": 0.0,
      "num_tokens": 2538334.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.434495449066162,
      "sampling/importance_sampling_ratio/mean": 0.9998641014099121,
      "sampling/importance_sampling_ratio/min": 0.613407552242279,
      "sampling/sampling_logp_difference/max": 0.4887256622314453,
      "sampling/sampling_logp_difference/mean": 0.017236698418855667,
      "step": 162
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 328.0,
      "completions/max_terminated_length": 328.0,
      "completions/mean_length": 48.375,
      "completions/mean_terminated_length": 48.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21288928389549255,
      "epoch": 0.2884955752212389,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 15.800928761698737,
      "kl": 0.004314340651035309,
      "learning_rate": 7.168141592920353e-07,
      "loss": 0.37,
      "num_tokens": 2554374.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.3523565530776978,
      "sampling/importance_sampling_ratio/mean": 1.0010709762573242,
      "sampling/importance_sampling_ratio/min": 0.6108425259590149,
      "sampling/sampling_logp_difference/max": 0.4929161071777344,
      "sampling/sampling_logp_difference/mean": 0.012879086658358574,
      "step": 163
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 358.0,
      "completions/max_terminated_length": 358.0,
      "completions/mean_length": 97.3125,
      "completions/mean_terminated_length": 97.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21863287687301636,
      "epoch": 0.2902654867256637,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01218772305508651,
      "kl": 0.0015893366653472185,
      "learning_rate": 7.212389380530973e-07,
      "loss": 0.0,
      "num_tokens": 2569834.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000678539276123,
      "sampling/importance_sampling_ratio/min": 0.5868110060691833,
      "sampling/sampling_logp_difference/max": 0.9002819061279297,
      "sampling/sampling_logp_difference/mean": 0.012089526280760765,
      "step": 164
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 397.0,
      "completions/max_terminated_length": 397.0,
      "completions/mean_length": 85.046875,
      "completions/mean_terminated_length": 85.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.303574800491333,
      "epoch": 0.2920353982300885,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.2945564536762735,
      "kl": 0.003153447061777115,
      "learning_rate": 7.256637168141593e-07,
      "loss": 0.1266,
      "num_tokens": 2587805.0,
      "reward": 0.0,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.7874300479888916,
      "sampling/importance_sampling_ratio/mean": 0.9995529651641846,
      "sampling/importance_sampling_ratio/min": 0.6549264788627625,
      "sampling/sampling_logp_difference/max": 0.5807788372039795,
      "sampling/sampling_logp_difference/mean": 0.020131688565015793,
      "step": 165
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 216.0,
      "completions/max_terminated_length": 216.0,
      "completions/mean_length": 62.4375,
      "completions/mean_terminated_length": 62.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3352992534637451,
      "epoch": 0.2938053097345133,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05527729473556852,
      "kl": 0.006250219885259867,
      "learning_rate": 7.300884955752212e-07,
      "loss": 0.0,
      "num_tokens": 2603433.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.315501093864441,
      "sampling/importance_sampling_ratio/mean": 0.9993723630905151,
      "sampling/importance_sampling_ratio/min": 0.65906822681427,
      "sampling/sampling_logp_difference/max": 0.4169282913208008,
      "sampling/sampling_logp_difference/mean": 0.01699444092810154,
      "step": 166
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 645.0,
      "completions/max_terminated_length": 645.0,
      "completions/mean_length": 122.328125,
      "completions/mean_terminated_length": 122.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.34051042795181274,
      "epoch": 0.29557522123893804,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.2065456771565835,
      "kl": 0.004882376175373793,
      "learning_rate": 7.345132743362832e-07,
      "loss": 0.1907,
      "num_tokens": 2620878.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.4985473155975342,
      "sampling/importance_sampling_ratio/mean": 0.9992001056671143,
      "sampling/importance_sampling_ratio/min": 0.5491763353347778,
      "sampling/sampling_logp_difference/max": 0.5993356704711914,
      "sampling/sampling_logp_difference/mean": 0.015716129913926125,
      "step": 167
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 260.0,
      "completions/max_terminated_length": 260.0,
      "completions/mean_length": 83.671875,
      "completions/mean_terminated_length": 83.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4155896306037903,
      "epoch": 0.2973451327433628,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02412913910928674,
      "kl": 0.002413155511021614,
      "learning_rate": 7.389380530973452e-07,
      "loss": 0.0,
      "num_tokens": 2635865.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5505778789520264,
      "sampling/importance_sampling_ratio/mean": 0.9991369247436523,
      "sampling/importance_sampling_ratio/min": 0.5488505959510803,
      "sampling/sampling_logp_difference/max": 0.5999289751052856,
      "sampling/sampling_logp_difference/mean": 0.018513940274715424,
      "step": 168
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 254.0,
      "completions/max_terminated_length": 254.0,
      "completions/mean_length": 95.734375,
      "completions/mean_terminated_length": 95.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.31833508610725403,
      "epoch": 0.2991150442477876,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.018637248678998186,
      "kl": 0.0012480122968554497,
      "learning_rate": 7.433628318584071e-07,
      "loss": 0.0,
      "num_tokens": 2652520.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5856602191925049,
      "sampling/importance_sampling_ratio/mean": 1.0013822317123413,
      "sampling/importance_sampling_ratio/min": 0.573755145072937,
      "sampling/sampling_logp_difference/max": 0.55555260181427,
      "sampling/sampling_logp_difference/mean": 0.015990041196346283,
      "step": 169
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 402.0,
      "completions/max_terminated_length": 402.0,
      "completions/mean_length": 66.3125,
      "completions/mean_terminated_length": 66.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.36399298906326294,
      "epoch": 0.3008849557522124,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 11.85054564249182,
      "kl": 0.0069623468443751335,
      "learning_rate": 7.477876106194691e-07,
      "loss": 0.3686,
      "num_tokens": 2670876.0,
      "reward": 0.625,
      "reward_std": 0.36435678601264954,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.423884391784668,
      "sampling/importance_sampling_ratio/mean": 0.999387264251709,
      "sampling/importance_sampling_ratio/min": 0.5508221983909607,
      "sampling/sampling_logp_difference/max": 0.5963431596755981,
      "sampling/sampling_logp_difference/mean": 0.01819230616092682,
      "step": 170
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 210.0,
      "completions/max_terminated_length": 210.0,
      "completions/mean_length": 65.265625,
      "completions/mean_terminated_length": 65.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3241598606109619,
      "epoch": 0.30265486725663715,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03064381044639263,
      "kl": 0.0015449854545295238,
      "learning_rate": 7.522123893805308e-07,
      "loss": 0.0,
      "num_tokens": 2690733.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9951161742210388,
      "sampling/importance_sampling_ratio/min": 0.5531994104385376,
      "sampling/sampling_logp_difference/max": 0.7696962356567383,
      "sampling/sampling_logp_difference/mean": 0.02130197361111641,
      "step": 171
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 278.0,
      "completions/max_terminated_length": 278.0,
      "completions/mean_length": 60.546875,
      "completions/mean_terminated_length": 60.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.34832802414894104,
      "epoch": 0.30442477876106194,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01942212919039001,
      "kl": 0.0010704933665692806,
      "learning_rate": 7.566371681415928e-07,
      "loss": 0.0,
      "num_tokens": 2710400.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3826576471328735,
      "sampling/importance_sampling_ratio/mean": 0.9970792531967163,
      "sampling/importance_sampling_ratio/min": 0.6549003720283508,
      "sampling/sampling_logp_difference/max": 0.42327213287353516,
      "sampling/sampling_logp_difference/mean": 0.02165486104786396,
      "step": 172
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 292.0,
      "completions/max_terminated_length": 292.0,
      "completions/mean_length": 99.46875,
      "completions/mean_terminated_length": 99.46875,
      "completions/min_length": 33.0,
      "completions/min_terminated_length": 33.0,
      "entropy": 0.5507562160491943,
      "epoch": 0.30619469026548674,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.2067772660198575,
      "kl": 0.0015644296072423458,
      "learning_rate": 7.610619469026548e-07,
      "loss": 0.1168,
      "num_tokens": 2725870.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.6677830219268799,
      "sampling/importance_sampling_ratio/mean": 0.9991893768310547,
      "sampling/importance_sampling_ratio/min": 0.3073960840702057,
      "sampling/sampling_logp_difference/max": 1.1796181201934814,
      "sampling/sampling_logp_difference/mean": 0.0195973701775074,
      "step": 173
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 197.0,
      "completions/max_terminated_length": 197.0,
      "completions/mean_length": 65.59375,
      "completions/mean_terminated_length": 65.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2421538382768631,
      "epoch": 0.30796460176991153,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.029068109786864894,
      "kl": 0.0013104917015880346,
      "learning_rate": 7.654867256637167e-07,
      "loss": 0.0,
      "num_tokens": 2740260.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9995702505111694,
      "sampling/importance_sampling_ratio/min": 0.6254555583000183,
      "sampling/sampling_logp_difference/max": 0.7899044752120972,
      "sampling/sampling_logp_difference/mean": 0.022774621844291687,
      "step": 174
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 190.0,
      "completions/max_terminated_length": 190.0,
      "completions/mean_length": 75.921875,
      "completions/mean_terminated_length": 75.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3197605013847351,
      "epoch": 0.30973451327433627,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.6400337164230887,
      "kl": 0.0015748351579532027,
      "learning_rate": 7.699115044247787e-07,
      "loss": -0.0426,
      "num_tokens": 2753055.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.5528416633605957,
      "sampling/importance_sampling_ratio/mean": 0.9991353750228882,
      "sampling/importance_sampling_ratio/min": 0.6129763126373291,
      "sampling/sampling_logp_difference/max": 0.4894289970397949,
      "sampling/sampling_logp_difference/mean": 0.013405116274952888,
      "step": 175
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 113.0,
      "completions/max_terminated_length": 113.0,
      "completions/mean_length": 34.75,
      "completions/mean_terminated_length": 34.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1282852590084076,
      "epoch": 0.31150442477876106,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.965701958805946,
      "kl": 0.0016409424133598804,
      "learning_rate": 7.743362831858407e-07,
      "loss": 0.0365,
      "num_tokens": 2766895.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.306551456451416,
      "sampling/importance_sampling_ratio/mean": 1.00221848487854,
      "sampling/importance_sampling_ratio/min": 0.6547200679779053,
      "sampling/sampling_logp_difference/max": 0.42354750633239746,
      "sampling/sampling_logp_difference/mean": 0.012759683653712273,
      "step": 176
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 238.0,
      "completions/max_terminated_length": 238.0,
      "completions/mean_length": 86.859375,
      "completions/mean_terminated_length": 86.859375,
      "completions/min_length": 16.0,
      "completions/min_terminated_length": 16.0,
      "entropy": 0.3257979154586792,
      "epoch": 0.31327433628318585,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.1971454945435758,
      "kl": 0.0011299046454951167,
      "learning_rate": 7.787610619469026e-07,
      "loss": 0.0277,
      "num_tokens": 2784630.0,
      "reward": -0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.28737211227417,
      "sampling/importance_sampling_ratio/mean": 1.0015079975128174,
      "sampling/importance_sampling_ratio/min": 0.6338574886322021,
      "sampling/sampling_logp_difference/max": 0.4559311866760254,
      "sampling/sampling_logp_difference/mean": 0.014637185260653496,
      "step": 177
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 197.0,
      "completions/max_terminated_length": 197.0,
      "completions/mean_length": 35.921875,
      "completions/mean_terminated_length": 35.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24986636638641357,
      "epoch": 0.31504424778761064,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.890195329991731,
      "kl": 0.006909585557878017,
      "learning_rate": 7.831858407079646e-07,
      "loss": 0.2545,
      "num_tokens": 2797281.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.8822228908538818,
      "sampling/importance_sampling_ratio/mean": 0.999498724937439,
      "sampling/importance_sampling_ratio/min": 0.277087539434433,
      "sampling/sampling_logp_difference/max": 1.2834217548370361,
      "sampling/sampling_logp_difference/mean": 0.01401694305241108,
      "step": 178
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 171.0,
      "completions/max_terminated_length": 171.0,
      "completions/mean_length": 71.921875,
      "completions/mean_terminated_length": 71.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3881117105484009,
      "epoch": 0.3168141592920354,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0174497927871592,
      "kl": 0.001206007320433855,
      "learning_rate": 7.876106194690266e-07,
      "loss": 0.0,
      "num_tokens": 2812412.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.54652738571167,
      "sampling/importance_sampling_ratio/mean": 1.0000944137573242,
      "sampling/importance_sampling_ratio/min": 0.4926236569881439,
      "sampling/sampling_logp_difference/max": 0.7080097198486328,
      "sampling/sampling_logp_difference/mean": 0.01670251041650772,
      "step": 179
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 502.0,
      "completions/max_terminated_length": 502.0,
      "completions/mean_length": 81.234375,
      "completions/mean_terminated_length": 81.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.37518447637557983,
      "epoch": 0.3185840707964602,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.298833240783317,
      "kl": 0.0012631581630557775,
      "learning_rate": 7.920353982300884e-07,
      "loss": 0.1495,
      "num_tokens": 2829723.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9986304640769958,
      "sampling/importance_sampling_ratio/min": 0.6893054246902466,
      "sampling/sampling_logp_difference/max": 0.7487752437591553,
      "sampling/sampling_logp_difference/mean": 0.01565636694431305,
      "step": 180
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 209.0,
      "completions/max_terminated_length": 209.0,
      "completions/mean_length": 68.59375,
      "completions/mean_terminated_length": 68.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19516851007938385,
      "epoch": 0.32035398230088497,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.022769574972825203,
      "kl": 0.0016036881133913994,
      "learning_rate": 7.964601769911504e-07,
      "loss": 0.0,
      "num_tokens": 2843313.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.887073040008545,
      "sampling/importance_sampling_ratio/mean": 0.9991527795791626,
      "sampling/importance_sampling_ratio/min": 0.5863591432571411,
      "sampling/sampling_logp_difference/max": 0.6350269317626953,
      "sampling/sampling_logp_difference/mean": 0.019246196374297142,
      "step": 181
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 181.0,
      "completions/max_terminated_length": 181.0,
      "completions/mean_length": 39.78125,
      "completions/mean_terminated_length": 39.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2743300795555115,
      "epoch": 0.32212389380530976,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03835264627824913,
      "kl": 0.0019648310262709856,
      "learning_rate": 8.008849557522124e-07,
      "loss": 0.0,
      "num_tokens": 2858579.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.516083836555481,
      "sampling/importance_sampling_ratio/mean": 1.0001496076583862,
      "sampling/importance_sampling_ratio/min": 0.688202440738678,
      "sampling/sampling_logp_difference/max": 0.4161306619644165,
      "sampling/sampling_logp_difference/mean": 0.014625566080212593,
      "step": 182
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 334.0,
      "completions/max_terminated_length": 334.0,
      "completions/mean_length": 90.515625,
      "completions/mean_terminated_length": 90.515625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.40749549865722656,
      "epoch": 0.3238938053097345,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.021686218872358942,
      "kl": 0.0016722225118428469,
      "learning_rate": 8.053097345132743e-07,
      "loss": 0.0,
      "num_tokens": 2875172.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6169039011001587,
      "sampling/importance_sampling_ratio/mean": 1.0007729530334473,
      "sampling/importance_sampling_ratio/min": 0.6497799158096313,
      "sampling/sampling_logp_difference/max": 0.4805130958557129,
      "sampling/sampling_logp_difference/mean": 0.014663636684417725,
      "step": 183
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 292.0,
      "completions/max_terminated_length": 292.0,
      "completions/mean_length": 96.640625,
      "completions/mean_terminated_length": 96.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4288424253463745,
      "epoch": 0.3256637168141593,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.032833059064123725,
      "kl": 0.0040725404396653175,
      "learning_rate": 8.097345132743363e-07,
      "loss": 0.0,
      "num_tokens": 2891581.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0008609294891357,
      "sampling/importance_sampling_ratio/min": 0.6394206285476685,
      "sampling/sampling_logp_difference/max": 0.7237358093261719,
      "sampling/sampling_logp_difference/mean": 0.02272997982800007,
      "step": 184
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 82.953125,
      "completions/mean_terminated_length": 82.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4191206097602844,
      "epoch": 0.3274336283185841,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.017823889993753107,
      "kl": 0.0016019068425521255,
      "learning_rate": 8.141592920353983e-07,
      "loss": 0.0,
      "num_tokens": 2906906.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3651630878448486,
      "sampling/importance_sampling_ratio/mean": 0.998401403427124,
      "sampling/importance_sampling_ratio/min": 0.6447701454162598,
      "sampling/sampling_logp_difference/max": 0.4388613700866699,
      "sampling/sampling_logp_difference/mean": 0.020062603056430817,
      "step": 185
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 182.0,
      "completions/max_terminated_length": 182.0,
      "completions/mean_length": 67.65625,
      "completions/mean_terminated_length": 67.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3816743493080139,
      "epoch": 0.3292035398230089,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.018024805681682764,
      "kl": 0.0013134465552866459,
      "learning_rate": 8.185840707964602e-07,
      "loss": 0.0,
      "num_tokens": 2922388.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.706222653388977,
      "sampling/importance_sampling_ratio/mean": 0.9995259046554565,
      "sampling/importance_sampling_ratio/min": 0.5038576722145081,
      "sampling/sampling_logp_difference/max": 0.6854615211486816,
      "sampling/sampling_logp_difference/mean": 0.016284022480249405,
      "step": 186
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 88.015625,
      "completions/mean_terminated_length": 88.015625,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.4469081163406372,
      "epoch": 0.3309734513274336,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.371433628774608,
      "kl": 0.00285714166238904,
      "learning_rate": 8.230088495575221e-07,
      "loss": -0.0155,
      "num_tokens": 2938437.0,
      "reward": 0.9375,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.4655020236968994,
      "sampling/importance_sampling_ratio/mean": 0.9996187686920166,
      "sampling/importance_sampling_ratio/min": 0.6309859752655029,
      "sampling/sampling_logp_difference/max": 0.46047163009643555,
      "sampling/sampling_logp_difference/mean": 0.01801963523030281,
      "step": 187
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 373.0,
      "completions/max_terminated_length": 373.0,
      "completions/mean_length": 96.90625,
      "completions/mean_terminated_length": 96.90625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21593278646469116,
      "epoch": 0.3327433628318584,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.2014978144866415,
      "kl": 0.0016711852513253689,
      "learning_rate": 8.274336283185839e-07,
      "loss": 0.0088,
      "num_tokens": 2955631.0,
      "reward": 0.375,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.375,
      "rewards/decision_reward_func/std": 0.934353232383728,
      "sampling/importance_sampling_ratio/max": 1.4248566627502441,
      "sampling/importance_sampling_ratio/mean": 1.0010021924972534,
      "sampling/importance_sampling_ratio/min": 0.3327132761478424,
      "sampling/sampling_logp_difference/max": 1.1004741191864014,
      "sampling/sampling_logp_difference/mean": 0.015623951330780983,
      "step": 188
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 280.0,
      "completions/max_terminated_length": 280.0,
      "completions/mean_length": 96.140625,
      "completions/mean_terminated_length": 96.140625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.45695868134498596,
      "epoch": 0.3345132743362832,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.4257601631699997,
      "kl": 0.0024920753203332424,
      "learning_rate": 8.318584070796459e-07,
      "loss": 0.0294,
      "num_tokens": 2972056.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.6508358716964722,
      "sampling/importance_sampling_ratio/mean": 1.0013859272003174,
      "sampling/importance_sampling_ratio/min": 0.5403315424919128,
      "sampling/sampling_logp_difference/max": 0.615572452545166,
      "sampling/sampling_logp_difference/mean": 0.019048312678933144,
      "step": 189
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 226.0,
      "completions/max_terminated_length": 226.0,
      "completions/mean_length": 37.03125,
      "completions/mean_terminated_length": 37.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.13859203457832336,
      "epoch": 0.336283185840708,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.097821127529105,
      "kl": 0.005911725573241711,
      "learning_rate": 8.362831858407079e-07,
      "loss": 0.2139,
      "num_tokens": 2985642.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.4755970239639282,
      "sampling/importance_sampling_ratio/mean": 1.0006300210952759,
      "sampling/importance_sampling_ratio/min": 0.6624336838722229,
      "sampling/sampling_logp_difference/max": 0.41183483600616455,
      "sampling/sampling_logp_difference/mean": 0.01723603904247284,
      "step": 190
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 149.0,
      "completions/max_terminated_length": 149.0,
      "completions/mean_length": 53.203125,
      "completions/mean_terminated_length": 53.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3490672707557678,
      "epoch": 0.3380530973451327,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07868897743366499,
      "kl": 0.005042393691837788,
      "learning_rate": 8.407079646017698e-07,
      "loss": 0.0,
      "num_tokens": 2999239.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.3826329708099365,
      "sampling/importance_sampling_ratio/mean": 0.9992291927337646,
      "sampling/importance_sampling_ratio/min": 0.5984382033348083,
      "sampling/sampling_logp_difference/max": 0.5134320259094238,
      "sampling/sampling_logp_difference/mean": 0.021959085017442703,
      "step": 191
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 252.0,
      "completions/max_terminated_length": 252.0,
      "completions/mean_length": 71.734375,
      "completions/mean_terminated_length": 71.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.307967871427536,
      "epoch": 0.3398230088495575,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.025977733696619907,
      "kl": 0.0022195237688720226,
      "learning_rate": 8.451327433628318e-07,
      "loss": 0.0,
      "num_tokens": 3016038.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5421792268753052,
      "sampling/importance_sampling_ratio/mean": 1.0004302263259888,
      "sampling/importance_sampling_ratio/min": 0.5187493562698364,
      "sampling/sampling_logp_difference/max": 0.656334400177002,
      "sampling/sampling_logp_difference/mean": 0.012343363836407661,
      "step": 192
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 266.0,
      "completions/max_terminated_length": 266.0,
      "completions/mean_length": 65.75,
      "completions/mean_terminated_length": 65.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3567613661289215,
      "epoch": 0.3415929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0460271782944802,
      "kl": 0.016741083934903145,
      "learning_rate": 8.495575221238938e-07,
      "loss": 0.0001,
      "num_tokens": 3031846.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0010594129562378,
      "sampling/importance_sampling_ratio/min": 0.5105498433113098,
      "sampling/sampling_logp_difference/max": 0.7371149063110352,
      "sampling/sampling_logp_difference/mean": 0.019204776734113693,
      "step": 193
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 253.0,
      "completions/max_terminated_length": 253.0,
      "completions/mean_length": 66.328125,
      "completions/mean_terminated_length": 66.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3064947724342346,
      "epoch": 0.3433628318584071,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.828728528219838,
      "kl": 0.003697988111525774,
      "learning_rate": 8.539823008849557e-07,
      "loss": -0.1107,
      "num_tokens": 3046859.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0001063346862793,
      "sampling/importance_sampling_ratio/min": 0.6153205633163452,
      "sampling/sampling_logp_difference/max": 0.8466815948486328,
      "sampling/sampling_logp_difference/mean": 0.01753907836973667,
      "step": 194
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 359.0,
      "completions/max_terminated_length": 359.0,
      "completions/mean_length": 95.890625,
      "completions/mean_terminated_length": 95.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.45068109035491943,
      "epoch": 0.34513274336283184,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.027973118552110467,
      "kl": 0.002571109915152192,
      "learning_rate": 8.584070796460177e-07,
      "loss": 0.0,
      "num_tokens": 3062788.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000136137008667,
      "sampling/importance_sampling_ratio/min": 0.6952780485153198,
      "sampling/sampling_logp_difference/max": 0.8257927894592285,
      "sampling/sampling_logp_difference/mean": 0.015602972358465195,
      "step": 195
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 149.0,
      "completions/max_terminated_length": 149.0,
      "completions/mean_length": 47.3125,
      "completions/mean_terminated_length": 47.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.29853731393814087,
      "epoch": 0.34690265486725663,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07086573487077374,
      "kl": 0.035713452845811844,
      "learning_rate": 8.628318584070797e-07,
      "loss": 0.0001,
      "num_tokens": 3076456.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4578688144683838,
      "sampling/importance_sampling_ratio/mean": 0.9997751712799072,
      "sampling/importance_sampling_ratio/min": 0.3986743092536926,
      "sampling/sampling_logp_difference/max": 0.9196105003356934,
      "sampling/sampling_logp_difference/mean": 0.020105183124542236,
      "step": 196
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 162.0,
      "completions/max_terminated_length": 162.0,
      "completions/mean_length": 50.34375,
      "completions/mean_terminated_length": 50.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3566172122955322,
      "epoch": 0.3486725663716814,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.050854542449691606,
      "kl": 0.02495628222823143,
      "learning_rate": 8.672566371681415e-07,
      "loss": 0.0001,
      "num_tokens": 3090718.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6565725803375244,
      "sampling/importance_sampling_ratio/mean": 1.0001130104064941,
      "sampling/importance_sampling_ratio/min": 0.6099073886871338,
      "sampling/sampling_logp_difference/max": 0.5047507286071777,
      "sampling/sampling_logp_difference/mean": 0.017076190561056137,
      "step": 197
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 182.0,
      "completions/max_terminated_length": 182.0,
      "completions/mean_length": 88.65625,
      "completions/mean_terminated_length": 88.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.6213746070861816,
      "epoch": 0.3504424778761062,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 12.629178819001702,
      "kl": 0.003796565579250455,
      "learning_rate": 8.716814159292035e-07,
      "loss": 0.0921,
      "num_tokens": 3106456.0,
      "reward": 0.59375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.59375,
      "rewards/decision_reward_func/std": 0.8110105991363525,
      "sampling/importance_sampling_ratio/max": 1.5601270198822021,
      "sampling/importance_sampling_ratio/mean": 0.9996728897094727,
      "sampling/importance_sampling_ratio/min": 0.6842789053916931,
      "sampling/sampling_logp_difference/max": 0.4447672367095947,
      "sampling/sampling_logp_difference/mean": 0.021474361419677734,
      "step": 198
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 437.0,
      "completions/max_terminated_length": 437.0,
      "completions/mean_length": 84.3125,
      "completions/mean_terminated_length": 84.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3975464701652527,
      "epoch": 0.35221238938053095,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.019198297739022,
      "kl": 0.007026151288300753,
      "learning_rate": 8.761061946902655e-07,
      "loss": 0.2102,
      "num_tokens": 3122780.0,
      "reward": 0.65625,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.674494743347168,
      "sampling/importance_sampling_ratio/mean": 1.0001417398452759,
      "sampling/importance_sampling_ratio/min": 0.3294057250022888,
      "sampling/sampling_logp_difference/max": 1.1104650497436523,
      "sampling/sampling_logp_difference/mean": 0.020129112526774406,
      "step": 199
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 179.0,
      "completions/max_terminated_length": 179.0,
      "completions/mean_length": 40.71875,
      "completions/mean_terminated_length": 40.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19297020137310028,
      "epoch": 0.35398230088495575,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03618304098894548,
      "kl": 0.001933026360347867,
      "learning_rate": 8.805309734513274e-07,
      "loss": 0.0,
      "num_tokens": 3143066.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5666741132736206,
      "sampling/importance_sampling_ratio/mean": 0.9981307983398438,
      "sampling/importance_sampling_ratio/min": 0.40572628378868103,
      "sampling/sampling_logp_difference/max": 0.9020766019821167,
      "sampling/sampling_logp_difference/mean": 0.021881964057683945,
      "step": 200
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 13.6875,
      "completions/mean_terminated_length": 13.6875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06337769329547882,
      "epoch": 0.35575221238938054,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.3730626436437873,
      "kl": 0.005040082149207592,
      "learning_rate": 8.849557522123894e-07,
      "loss": 0.0001,
      "num_tokens": 3153878.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5257240533828735,
      "sampling/importance_sampling_ratio/mean": 0.9977389574050903,
      "sampling/importance_sampling_ratio/min": 0.4783879518508911,
      "sampling/sampling_logp_difference/max": 0.7373332977294922,
      "sampling/sampling_logp_difference/mean": 0.012821320444345474,
      "step": 201
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 205.0,
      "completions/max_terminated_length": 205.0,
      "completions/mean_length": 20.921875,
      "completions/mean_terminated_length": 20.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.16319364309310913,
      "epoch": 0.35752212389380533,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 29.22328448711984,
      "kl": 0.04330842196941376,
      "learning_rate": 8.893805309734513e-07,
      "loss": -0.4998,
      "num_tokens": 3168065.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.6684596538543701,
      "sampling/importance_sampling_ratio/mean": 0.9991862177848816,
      "sampling/importance_sampling_ratio/min": 0.6727373003959656,
      "sampling/sampling_logp_difference/max": 0.5119008421897888,
      "sampling/sampling_logp_difference/mean": 0.022439446300268173,
      "step": 202
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 189.0,
      "completions/max_terminated_length": 189.0,
      "completions/mean_length": 38.34375,
      "completions/mean_terminated_length": 38.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19670294225215912,
      "epoch": 0.35929203539823007,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1867018254244633,
      "kl": 0.03805701807141304,
      "learning_rate": 8.938053097345132e-07,
      "loss": 0.0002,
      "num_tokens": 3180439.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3781383037567139,
      "sampling/importance_sampling_ratio/mean": 1.00034761428833,
      "sampling/importance_sampling_ratio/min": 0.6061520576477051,
      "sampling/sampling_logp_difference/max": 0.500624418258667,
      "sampling/sampling_logp_difference/mean": 0.016402604058384895,
      "step": 203
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 124.0,
      "completions/max_terminated_length": 124.0,
      "completions/mean_length": 16.984375,
      "completions/mean_terminated_length": 16.984375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.08758561313152313,
      "epoch": 0.36106194690265486,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 67.96663604069774,
      "kl": 0.03069724701344967,
      "learning_rate": 8.982300884955752e-07,
      "loss": -0.3958,
      "num_tokens": 3194870.0,
      "reward": -0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": -0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.944921612739563,
      "sampling/importance_sampling_ratio/mean": 1.0006206035614014,
      "sampling/importance_sampling_ratio/min": 0.6622530221939087,
      "sampling/sampling_logp_difference/max": 0.6652216911315918,
      "sampling/sampling_logp_difference/mean": 0.010742882266640663,
      "step": 204
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.328125,
      "completions/mean_terminated_length": 15.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06518861651420593,
      "epoch": 0.36283185840707965,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09771435391875398,
      "kl": 0.02997596189379692,
      "learning_rate": 9.026548672566371e-07,
      "loss": 0.0002,
      "num_tokens": 3208795.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.0926145315170288,
      "sampling/importance_sampling_ratio/mean": 0.999653160572052,
      "sampling/importance_sampling_ratio/min": 0.6137433052062988,
      "sampling/sampling_logp_difference/max": 0.4881784915924072,
      "sampling/sampling_logp_difference/mean": 0.005968471057713032,
      "step": 205
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 195.0,
      "completions/max_terminated_length": 195.0,
      "completions/mean_length": 58.0625,
      "completions/mean_terminated_length": 58.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2882225811481476,
      "epoch": 0.36460176991150445,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.055160220710038725,
      "kl": 0.003100259928032756,
      "learning_rate": 9.07079646017699e-07,
      "loss": 0.0,
      "num_tokens": 3222911.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3295564651489258,
      "sampling/importance_sampling_ratio/mean": 0.9999579191207886,
      "sampling/importance_sampling_ratio/min": 0.6964197754859924,
      "sampling/sampling_logp_difference/max": 0.36180269718170166,
      "sampling/sampling_logp_difference/mean": 0.010741502977907658,
      "step": 206
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 450.0,
      "completions/max_terminated_length": 450.0,
      "completions/mean_length": 68.4375,
      "completions/mean_terminated_length": 68.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3214592933654785,
      "epoch": 0.3663716814159292,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.336265981679781,
      "kl": 0.005980383604764938,
      "learning_rate": 9.11504424778761e-07,
      "loss": -0.0132,
      "num_tokens": 3237387.0,
      "reward": 0.09375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.3671600818634033,
      "sampling/importance_sampling_ratio/mean": 1.0011777877807617,
      "sampling/importance_sampling_ratio/min": 0.6046066284179688,
      "sampling/sampling_logp_difference/max": 0.5031771659851074,
      "sampling/sampling_logp_difference/mean": 0.015451844781637192,
      "step": 207
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 531.0,
      "completions/max_terminated_length": 531.0,
      "completions/mean_length": 108.15625,
      "completions/mean_terminated_length": 108.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.35494041442871094,
      "epoch": 0.368141592920354,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.6888265767189843,
      "kl": 0.002561464672908187,
      "learning_rate": 9.159292035398229e-07,
      "loss": 0.0946,
      "num_tokens": 3254645.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9973541498184204,
      "sampling/importance_sampling_ratio/min": 0.6656482815742493,
      "sampling/sampling_logp_difference/max": 0.7017228603363037,
      "sampling/sampling_logp_difference/mean": 0.015536784194409847,
      "step": 208
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 207.0,
      "completions/max_terminated_length": 207.0,
      "completions/mean_length": 106.453125,
      "completions/mean_terminated_length": 106.453125,
      "completions/min_length": 63.0,
      "completions/min_terminated_length": 63.0,
      "entropy": 0.5013563632965088,
      "epoch": 0.36991150442477877,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05157514700743391,
      "kl": 0.0033573838882148266,
      "learning_rate": 9.203539823008849e-07,
      "loss": 0.0,
      "num_tokens": 3270530.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6783130168914795,
      "sampling/importance_sampling_ratio/mean": 0.9998264312744141,
      "sampling/importance_sampling_ratio/min": 0.5489054918289185,
      "sampling/sampling_logp_difference/max": 0.5998289585113525,
      "sampling/sampling_logp_difference/mean": 0.018359629437327385,
      "step": 209
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 189.0,
      "completions/max_terminated_length": 189.0,
      "completions/mean_length": 44.453125,
      "completions/mean_terminated_length": 44.453125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1675224006175995,
      "epoch": 0.37168141592920356,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03374225208507583,
      "kl": 0.001434197649359703,
      "learning_rate": 9.247787610619469e-07,
      "loss": 0.0,
      "num_tokens": 3285727.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.7428385019302368,
      "sampling/importance_sampling_ratio/mean": 0.9988299608230591,
      "sampling/importance_sampling_ratio/min": 0.5252789855003357,
      "sampling/sampling_logp_difference/max": 0.6438257694244385,
      "sampling/sampling_logp_difference/mean": 0.01490543782711029,
      "step": 210
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 165.0,
      "completions/max_terminated_length": 165.0,
      "completions/mean_length": 40.21875,
      "completions/mean_terminated_length": 40.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18517419695854187,
      "epoch": 0.3734513274336283,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.050757562647429574,
      "kl": 0.002568627241998911,
      "learning_rate": 9.292035398230088e-07,
      "loss": 0.0,
      "num_tokens": 3300285.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.381851315498352,
      "sampling/importance_sampling_ratio/mean": 0.9986093044281006,
      "sampling/importance_sampling_ratio/min": 0.609358549118042,
      "sampling/sampling_logp_difference/max": 0.4953484535217285,
      "sampling/sampling_logp_difference/mean": 0.012276686728000641,
      "step": 211
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 265.0,
      "completions/max_terminated_length": 265.0,
      "completions/mean_length": 71.78125,
      "completions/mean_terminated_length": 71.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2278621345758438,
      "epoch": 0.3752212389380531,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03204199727428532,
      "kl": 0.003042126540094614,
      "learning_rate": 9.336283185840708e-07,
      "loss": 0.0,
      "num_tokens": 3316879.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.281581997871399,
      "sampling/importance_sampling_ratio/mean": 0.9995130896568298,
      "sampling/importance_sampling_ratio/min": 0.6369761228561401,
      "sampling/sampling_logp_difference/max": 0.4510231018066406,
      "sampling/sampling_logp_difference/mean": 0.014066744595766068,
      "step": 212
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 175.0,
      "completions/max_terminated_length": 175.0,
      "completions/mean_length": 38.34375,
      "completions/mean_terminated_length": 38.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17596986889839172,
      "epoch": 0.3769911504424779,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09589961445803016,
      "kl": 0.004617048427462578,
      "learning_rate": 9.380530973451328e-07,
      "loss": 0.0,
      "num_tokens": 3331989.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.341760516166687,
      "sampling/importance_sampling_ratio/mean": 0.9994263648986816,
      "sampling/importance_sampling_ratio/min": 0.7102124094963074,
      "sampling/sampling_logp_difference/max": 0.342191219329834,
      "sampling/sampling_logp_difference/mean": 0.010547621175646782,
      "step": 213
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 182.0,
      "completions/max_terminated_length": 182.0,
      "completions/mean_length": 102.0,
      "completions/mean_terminated_length": 102.0,
      "completions/min_length": 32.0,
      "completions/min_terminated_length": 32.0,
      "entropy": 0.46252167224884033,
      "epoch": 0.3787610619469027,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03145555998263438,
      "kl": 0.003119957633316517,
      "learning_rate": 9.424778761061947e-07,
      "loss": 0.0,
      "num_tokens": 3348373.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6221009492874146,
      "sampling/importance_sampling_ratio/mean": 0.9998818039894104,
      "sampling/importance_sampling_ratio/min": 0.20803289115428925,
      "sampling/sampling_logp_difference/max": 1.570059061050415,
      "sampling/sampling_logp_difference/mean": 0.01729355938732624,
      "step": 214
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 248.0,
      "completions/max_terminated_length": 248.0,
      "completions/mean_length": 62.59375,
      "completions/mean_terminated_length": 62.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2621881365776062,
      "epoch": 0.3805309734513274,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.056861081675358534,
      "kl": 0.005555910989642143,
      "learning_rate": 9.469026548672566e-07,
      "loss": 0.0,
      "num_tokens": 3362795.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.7976553440093994,
      "sampling/importance_sampling_ratio/mean": 1.002416729927063,
      "sampling/importance_sampling_ratio/min": 0.5128350853919983,
      "sampling/sampling_logp_difference/max": 0.667801022529602,
      "sampling/sampling_logp_difference/mean": 0.01444577518850565,
      "step": 215
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 204.0,
      "completions/max_terminated_length": 204.0,
      "completions/mean_length": 59.53125,
      "completions/mean_terminated_length": 59.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3704109191894531,
      "epoch": 0.3823008849557522,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1242170837294012,
      "kl": 0.005496586672961712,
      "learning_rate": 9.513274336283185e-07,
      "loss": 0.0001,
      "num_tokens": 3378029.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5919092893600464,
      "sampling/importance_sampling_ratio/mean": 1.0035631656646729,
      "sampling/importance_sampling_ratio/min": 0.7009298801422119,
      "sampling/sampling_logp_difference/max": 0.4649341106414795,
      "sampling/sampling_logp_difference/mean": 0.020102720707654953,
      "step": 216
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 225.0,
      "completions/max_terminated_length": 225.0,
      "completions/mean_length": 98.078125,
      "completions/mean_terminated_length": 98.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3987380266189575,
      "epoch": 0.384070796460177,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.045585197120377674,
      "kl": 0.003970324993133545,
      "learning_rate": 9.557522123893805e-07,
      "loss": 0.0,
      "num_tokens": 3394690.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9998895525932312,
      "sampling/importance_sampling_ratio/min": 0.5717587471008301,
      "sampling/sampling_logp_difference/max": 0.921055793762207,
      "sampling/sampling_logp_difference/mean": 0.01475785207003355,
      "step": 217
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 199.0,
      "completions/max_terminated_length": 199.0,
      "completions/mean_length": 34.296875,
      "completions/mean_terminated_length": 34.296875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19475924968719482,
      "epoch": 0.3858407079646018,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.11738437086607027,
      "kl": 0.010515974834561348,
      "learning_rate": 9.601769911504426e-07,
      "loss": 0.0001,
      "num_tokens": 3407157.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5994064807891846,
      "sampling/importance_sampling_ratio/mean": 0.9995347261428833,
      "sampling/importance_sampling_ratio/min": 0.6554685831069946,
      "sampling/sampling_logp_difference/max": 0.469632625579834,
      "sampling/sampling_logp_difference/mean": 0.011333119124174118,
      "step": 218
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 180.0,
      "completions/max_terminated_length": 180.0,
      "completions/mean_length": 54.765625,
      "completions/mean_terminated_length": 54.765625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.33620890974998474,
      "epoch": 0.38761061946902653,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10464694120487733,
      "kl": 0.004381977953016758,
      "learning_rate": 9.646017699115042e-07,
      "loss": 0.0001,
      "num_tokens": 3421606.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5475236177444458,
      "sampling/importance_sampling_ratio/mean": 1.0017409324645996,
      "sampling/importance_sampling_ratio/min": 0.6921407580375671,
      "sampling/sampling_logp_difference/max": 0.43665599822998047,
      "sampling/sampling_logp_difference/mean": 0.022906150668859482,
      "step": 219
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 309.0,
      "completions/max_terminated_length": 309.0,
      "completions/mean_length": 52.09375,
      "completions/mean_terminated_length": 52.09375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17895236611366272,
      "epoch": 0.3893805309734513,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.487661476567311,
      "kl": 0.002079170662909746,
      "learning_rate": 9.690265486725663e-07,
      "loss": 0.184,
      "num_tokens": 3435900.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.4371217489242554,
      "sampling/importance_sampling_ratio/mean": 0.9985246658325195,
      "sampling/importance_sampling_ratio/min": 0.5759603381156921,
      "sampling/sampling_logp_difference/max": 0.5517165660858154,
      "sampling/sampling_logp_difference/mean": 0.017181089147925377,
      "step": 220
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 161.0,
      "completions/max_terminated_length": 161.0,
      "completions/mean_length": 41.65625,
      "completions/mean_terminated_length": 41.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19378438591957092,
      "epoch": 0.3911504424778761,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.057238480728486814,
      "kl": 0.0032039829529821873,
      "learning_rate": 9.734513274336282e-07,
      "loss": 0.0,
      "num_tokens": 3450294.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4914631843566895,
      "sampling/importance_sampling_ratio/mean": 0.9990211129188538,
      "sampling/importance_sampling_ratio/min": 0.6217343211174011,
      "sampling/sampling_logp_difference/max": 0.47524237632751465,
      "sampling/sampling_logp_difference/mean": 0.01241573877632618,
      "step": 221
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 206.0,
      "completions/max_terminated_length": 206.0,
      "completions/mean_length": 62.703125,
      "completions/mean_terminated_length": 62.703125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3654828667640686,
      "epoch": 0.3929203539823009,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.035383096163377675,
      "kl": 0.0059638191014528275,
      "learning_rate": 9.778761061946902e-07,
      "loss": 0.0,
      "num_tokens": 3465411.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5720010995864868,
      "sampling/importance_sampling_ratio/mean": 0.9991636276245117,
      "sampling/importance_sampling_ratio/min": 0.6438614726066589,
      "sampling/sampling_logp_difference/max": 0.4523494243621826,
      "sampling/sampling_logp_difference/mean": 0.01645643450319767,
      "step": 222
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 108.0,
      "completions/max_terminated_length": 108.0,
      "completions/mean_length": 32.265625,
      "completions/mean_terminated_length": 32.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.12503893673419952,
      "epoch": 0.39469026548672564,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.029294659331895984,
      "kl": 0.0009783116402104497,
      "learning_rate": 9.82300884955752e-07,
      "loss": 0.0,
      "num_tokens": 3479988.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2597873210906982,
      "sampling/importance_sampling_ratio/mean": 1.0013203620910645,
      "sampling/importance_sampling_ratio/min": 0.7212660312652588,
      "sampling/sampling_logp_difference/max": 0.32674723863601685,
      "sampling/sampling_logp_difference/mean": 0.009833626449108124,
      "step": 223
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 198.0,
      "completions/max_terminated_length": 198.0,
      "completions/mean_length": 57.65625,
      "completions/mean_terminated_length": 57.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.242695152759552,
      "epoch": 0.39646017699115044,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02587311644919685,
      "kl": 0.0017175667453557253,
      "learning_rate": 9.867256637168142e-07,
      "loss": 0.0,
      "num_tokens": 3495246.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.328568935394287,
      "sampling/importance_sampling_ratio/mean": 1.0001909732818604,
      "sampling/importance_sampling_ratio/min": 0.7182667851448059,
      "sampling/sampling_logp_difference/max": 0.3309142589569092,
      "sampling/sampling_logp_difference/mean": 0.008995315060019493,
      "step": 224
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 382.0,
      "completions/max_terminated_length": 382.0,
      "completions/mean_length": 96.734375,
      "completions/mean_terminated_length": 96.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.39507365226745605,
      "epoch": 0.39823008849557523,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.7512447226554966,
      "kl": 0.004321282729506493,
      "learning_rate": 9.91150442477876e-07,
      "loss": -0.0746,
      "num_tokens": 3511581.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.593572974205017,
      "sampling/importance_sampling_ratio/mean": 1.0000005960464478,
      "sampling/importance_sampling_ratio/min": 0.6780757904052734,
      "sampling/sampling_logp_difference/max": 0.46597862243652344,
      "sampling/sampling_logp_difference/mean": 0.01681409776210785,
      "step": 225
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 143.0,
      "completions/max_terminated_length": 143.0,
      "completions/mean_length": 35.171875,
      "completions/mean_terminated_length": 35.171875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22788852453231812,
      "epoch": 0.4,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05385673391397239,
      "kl": 0.004239211790263653,
      "learning_rate": 9.95575221238938e-07,
      "loss": 0.0,
      "num_tokens": 3523800.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0043925046920776,
      "sampling/importance_sampling_ratio/min": 0.7580122351646423,
      "sampling/sampling_logp_difference/max": 0.705294132232666,
      "sampling/sampling_logp_difference/mean": 0.016978923231363297,
      "step": 226
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 232.0,
      "completions/max_terminated_length": 232.0,
      "completions/mean_length": 43.015625,
      "completions/mean_terminated_length": 43.015625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1527884155511856,
      "epoch": 0.40176991150442476,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 21.641425147437015,
      "kl": 0.004469120409339666,
      "learning_rate": 1e-06,
      "loss": -0.1624,
      "num_tokens": 3537321.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0010275840759277,
      "sampling/importance_sampling_ratio/min": 0.4817606806755066,
      "sampling/sampling_logp_difference/max": 0.7303078174591064,
      "sampling/sampling_logp_difference/mean": 0.017776301130652428,
      "step": 227
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 212.0,
      "completions/max_terminated_length": 212.0,
      "completions/mean_length": 30.71875,
      "completions/mean_terminated_length": 30.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19440095126628876,
      "epoch": 0.40353982300884955,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 36.71682744138212,
      "kl": 0.010755006223917007,
      "learning_rate": 9.999994035998135e-07,
      "loss": 0.3461,
      "num_tokens": 3550775.0,
      "reward": 0.65625,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.5073049068450928,
      "sampling/importance_sampling_ratio/mean": 0.9993104934692383,
      "sampling/importance_sampling_ratio/min": 0.6966415047645569,
      "sampling/sampling_logp_difference/max": 0.41032326221466064,
      "sampling/sampling_logp_difference/mean": 0.01622116006910801,
      "step": 228
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 202.0,
      "completions/max_terminated_length": 202.0,
      "completions/mean_length": 107.890625,
      "completions/mean_terminated_length": 107.890625,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.4642508029937744,
      "epoch": 0.40530973451327434,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.054119781405429675,
      "kl": 0.0034284957218915224,
      "learning_rate": 9.99997614400677e-07,
      "loss": 0.0,
      "num_tokens": 3567168.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0001689195632935,
      "sampling/importance_sampling_ratio/min": 0.6158477663993835,
      "sampling/sampling_logp_difference/max": 0.7310385704040527,
      "sampling/sampling_logp_difference/mean": 0.01859310269355774,
      "step": 229
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 170.0,
      "completions/max_terminated_length": 170.0,
      "completions/mean_length": 52.40625,
      "completions/mean_terminated_length": 52.40625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2456488460302353,
      "epoch": 0.40707964601769914,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.062557665823258,
      "kl": 0.002922304905951023,
      "learning_rate": 9.999946324068587e-07,
      "loss": 0.0,
      "num_tokens": 3580618.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.969736933708191,
      "sampling/importance_sampling_ratio/mean": 0.9975132942199707,
      "sampling/importance_sampling_ratio/min": 0.6378993988037109,
      "sampling/sampling_logp_difference/max": 0.677899956703186,
      "sampling/sampling_logp_difference/mean": 0.019860180094838142,
      "step": 230
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 224.0,
      "completions/max_terminated_length": 224.0,
      "completions/mean_length": 39.734375,
      "completions/mean_terminated_length": 39.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18301242589950562,
      "epoch": 0.4088495575221239,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04358872525227839,
      "kl": 0.0017693135887384415,
      "learning_rate": 9.999904576254724e-07,
      "loss": 0.0,
      "num_tokens": 3595081.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3643784523010254,
      "sampling/importance_sampling_ratio/mean": 1.0005431175231934,
      "sampling/importance_sampling_ratio/min": 0.6537138223648071,
      "sampling/sampling_logp_difference/max": 0.4250856637954712,
      "sampling/sampling_logp_difference/mean": 0.013198630884289742,
      "step": 231
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 207.0,
      "completions/max_terminated_length": 207.0,
      "completions/mean_length": 73.890625,
      "completions/mean_terminated_length": 73.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2257222831249237,
      "epoch": 0.41061946902654867,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08980511226821004,
      "kl": 0.001283166348002851,
      "learning_rate": 9.999850900664773e-07,
      "loss": 0.0,
      "num_tokens": 3611618.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2656625509262085,
      "sampling/importance_sampling_ratio/mean": 0.9989427924156189,
      "sampling/importance_sampling_ratio/min": 0.512908935546875,
      "sampling/sampling_logp_difference/max": 0.6676568984985352,
      "sampling/sampling_logp_difference/mean": 0.012095805257558823,
      "step": 232
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 104.0,
      "completions/max_terminated_length": 104.0,
      "completions/mean_length": 32.453125,
      "completions/mean_terminated_length": 32.453125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.294692724943161,
      "epoch": 0.41238938053097346,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 21.452015476936687,
      "kl": 0.023687120527029037,
      "learning_rate": 9.999785297426788e-07,
      "loss": -0.1992,
      "num_tokens": 3623407.0,
      "reward": 0.09375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.6505494117736816,
      "sampling/importance_sampling_ratio/mean": 0.9979889988899231,
      "sampling/importance_sampling_ratio/min": 0.6730009913444519,
      "sampling/sampling_logp_difference/max": 0.5011081695556641,
      "sampling/sampling_logp_difference/mean": 0.01914994977414608,
      "step": 233
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 37.53125,
      "completions/mean_terminated_length": 37.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.23396769165992737,
      "epoch": 0.41415929203539825,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04800585746953848,
      "kl": 0.002836022526025772,
      "learning_rate": 9.999707766697265e-07,
      "loss": 0.0,
      "num_tokens": 3637537.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4313642978668213,
      "sampling/importance_sampling_ratio/mean": 1.0034034252166748,
      "sampling/importance_sampling_ratio/min": 0.6590465307235718,
      "sampling/sampling_logp_difference/max": 0.4169611930847168,
      "sampling/sampling_logp_difference/mean": 0.019834768027067184,
      "step": 234
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 184.0,
      "completions/max_terminated_length": 184.0,
      "completions/mean_length": 42.234375,
      "completions/mean_terminated_length": 42.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.15960168838500977,
      "epoch": 0.415929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.026987595816708707,
      "kl": 0.0025480028707534075,
      "learning_rate": 9.999618308661168e-07,
      "loss": 0.0,
      "num_tokens": 3650912.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5073378086090088,
      "sampling/importance_sampling_ratio/mean": 0.9996541738510132,
      "sampling/importance_sampling_ratio/min": 0.6348318457603455,
      "sampling/sampling_logp_difference/max": 0.454395055770874,
      "sampling/sampling_logp_difference/mean": 0.01629040017724037,
      "step": 235
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 246.0,
      "completions/max_terminated_length": 246.0,
      "completions/mean_length": 90.703125,
      "completions/mean_terminated_length": 90.703125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.38684922456741333,
      "epoch": 0.4176991150442478,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.010045925952474,
      "kl": 0.0022131020668894053,
      "learning_rate": 9.999516923531906e-07,
      "loss": 0.0773,
      "num_tokens": 3668093.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.644107699394226,
      "sampling/importance_sampling_ratio/mean": 1.0001626014709473,
      "sampling/importance_sampling_ratio/min": 0.6234187483787537,
      "sampling/sampling_logp_difference/max": 0.4971977472305298,
      "sampling/sampling_logp_difference/mean": 0.017248239368200302,
      "step": 236
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 247.0,
      "completions/max_terminated_length": 247.0,
      "completions/mean_length": 86.953125,
      "completions/mean_terminated_length": 86.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.476330429315567,
      "epoch": 0.4194690265486726,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.72996716584878,
      "kl": 0.0019192376639693975,
      "learning_rate": 9.99940361155134e-07,
      "loss": 0.0898,
      "num_tokens": 3682634.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.4441698789596558,
      "sampling/importance_sampling_ratio/mean": 1.0006109476089478,
      "sampling/importance_sampling_ratio/min": 0.6419512033462524,
      "sampling/sampling_logp_difference/max": 0.44324302673339844,
      "sampling/sampling_logp_difference/mean": 0.016820615157485008,
      "step": 237
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 214.0,
      "completions/max_terminated_length": 214.0,
      "completions/mean_length": 79.15625,
      "completions/mean_terminated_length": 79.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27041104435920715,
      "epoch": 0.42123893805309737,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.019962387727511497,
      "kl": 0.0013378332369029522,
      "learning_rate": 9.99927837298979e-07,
      "loss": 0.0,
      "num_tokens": 3698772.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.7013784646987915,
      "sampling/importance_sampling_ratio/mean": 0.9985455274581909,
      "sampling/importance_sampling_ratio/min": 0.6555176377296448,
      "sampling/sampling_logp_difference/max": 0.5314388275146484,
      "sampling/sampling_logp_difference/mean": 0.012149603106081486,
      "step": 238
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 202.0,
      "completions/max_terminated_length": 202.0,
      "completions/mean_length": 54.84375,
      "completions/mean_terminated_length": 54.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.20714148879051208,
      "epoch": 0.4230088495575221,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04988395372967355,
      "kl": 0.003713001497089863,
      "learning_rate": 9.999141208146027e-07,
      "loss": 0.0,
      "num_tokens": 3712506.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3076030015945435,
      "sampling/importance_sampling_ratio/mean": 0.9982582330703735,
      "sampling/importance_sampling_ratio/min": 0.5380278825759888,
      "sampling/sampling_logp_difference/max": 0.619844913482666,
      "sampling/sampling_logp_difference/mean": 0.014127662405371666,
      "step": 239
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 154.0,
      "completions/max_terminated_length": 154.0,
      "completions/mean_length": 37.328125,
      "completions/mean_terminated_length": 37.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1810479760169983,
      "epoch": 0.4247787610619469,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.14025310662941104,
      "kl": 0.006155351176857948,
      "learning_rate": 9.99899211734727e-07,
      "loss": 0.0,
      "num_tokens": 3727135.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4445059299468994,
      "sampling/importance_sampling_ratio/mean": 1.0028483867645264,
      "sampling/importance_sampling_ratio/min": 0.5484683513641357,
      "sampling/sampling_logp_difference/max": 0.60062575340271,
      "sampling/sampling_logp_difference/mean": 0.020090894773602486,
      "step": 240
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 13.234375,
      "completions/mean_terminated_length": 13.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06778696179389954,
      "epoch": 0.4265486725663717,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.32877278340164795,
      "kl": 0.003949641715735197,
      "learning_rate": 9.998831100949186e-07,
      "loss": 0.0,
      "num_tokens": 3738222.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4646263122558594,
      "sampling/importance_sampling_ratio/mean": 0.9988141059875488,
      "sampling/importance_sampling_ratio/min": 0.781985878944397,
      "sampling/sampling_logp_difference/max": 0.38160014152526855,
      "sampling/sampling_logp_difference/mean": 0.009022011421620846,
      "step": 241
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 148.0,
      "completions/max_terminated_length": 148.0,
      "completions/mean_length": 36.21875,
      "completions/mean_terminated_length": 36.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.14866621792316437,
      "epoch": 0.4283185840707965,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.030079433302675318,
      "kl": 0.0009830165654420853,
      "learning_rate": 9.998658159335901e-07,
      "loss": 0.0,
      "num_tokens": 3754716.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.250900387763977,
      "sampling/importance_sampling_ratio/mean": 0.9993738532066345,
      "sampling/importance_sampling_ratio/min": 0.7676602005958557,
      "sampling/sampling_logp_difference/max": 0.2644081115722656,
      "sampling/sampling_logp_difference/mean": 0.009230071678757668,
      "step": 242
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 264.0,
      "completions/max_terminated_length": 264.0,
      "completions/mean_length": 85.328125,
      "completions/mean_terminated_length": 85.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.38343095779418945,
      "epoch": 0.4300884955752212,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.0171205491946504,
      "kl": 0.0024617710150778294,
      "learning_rate": 9.998473292919985e-07,
      "loss": -0.0413,
      "num_tokens": 3771553.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.6167631149291992,
      "sampling/importance_sampling_ratio/mean": 1.0004315376281738,
      "sampling/importance_sampling_ratio/min": 0.3154713213443756,
      "sampling/sampling_logp_difference/max": 1.1536874771118164,
      "sampling/sampling_logp_difference/mean": 0.01735665649175644,
      "step": 243
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 152.0,
      "completions/max_terminated_length": 152.0,
      "completions/mean_length": 37.078125,
      "completions/mean_terminated_length": 37.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18088294565677643,
      "epoch": 0.431858407079646,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.12771523381725922,
      "kl": 0.006010980345308781,
      "learning_rate": 9.998276502142454e-07,
      "loss": 0.0,
      "num_tokens": 3788790.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2638511657714844,
      "sampling/importance_sampling_ratio/mean": 0.999843180179596,
      "sampling/importance_sampling_ratio/min": 0.6756933331489563,
      "sampling/sampling_logp_difference/max": 0.3920159339904785,
      "sampling/sampling_logp_difference/mean": 0.0102929025888443,
      "step": 244
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 176.0,
      "completions/max_terminated_length": 176.0,
      "completions/mean_length": 74.46875,
      "completions/mean_terminated_length": 74.46875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3529541492462158,
      "epoch": 0.4336283185840708,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03759443190383923,
      "kl": 0.003588553052395582,
      "learning_rate": 9.99806778747277e-07,
      "loss": 0.0,
      "num_tokens": 3802644.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.421198844909668,
      "sampling/importance_sampling_ratio/mean": 0.9967384934425354,
      "sampling/importance_sampling_ratio/min": 0.5428851842880249,
      "sampling/sampling_logp_difference/max": 0.6108574867248535,
      "sampling/sampling_logp_difference/mean": 0.024649962782859802,
      "step": 245
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 372.0,
      "completions/max_terminated_length": 372.0,
      "completions/mean_length": 67.40625,
      "completions/mean_terminated_length": 67.40625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3470374047756195,
      "epoch": 0.4353982300884956,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.051173374001041765,
      "kl": 0.004685951862484217,
      "learning_rate": 9.997847149408844e-07,
      "loss": 0.0,
      "num_tokens": 3817582.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.598049521446228,
      "sampling/importance_sampling_ratio/mean": 0.999319314956665,
      "sampling/importance_sampling_ratio/min": 0.5836055874824524,
      "sampling/sampling_logp_difference/max": 0.5385298728942871,
      "sampling/sampling_logp_difference/mean": 0.015922270715236664,
      "step": 246
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 291.0,
      "completions/max_terminated_length": 291.0,
      "completions/mean_length": 62.484375,
      "completions/mean_terminated_length": 62.484375,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.2849852740764618,
      "epoch": 0.43716814159292033,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.050583000758105276,
      "kl": 0.007301662117242813,
      "learning_rate": 9.997614588477033e-07,
      "loss": 0.0,
      "num_tokens": 3831645.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.3781495094299316,
      "sampling/importance_sampling_ratio/mean": 1.0004756450653076,
      "sampling/importance_sampling_ratio/min": 0.46459150314331055,
      "sampling/sampling_logp_difference/max": 0.766596794128418,
      "sampling/sampling_logp_difference/mean": 0.014835800975561142,
      "step": 247
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 295.0,
      "completions/max_terminated_length": 295.0,
      "completions/mean_length": 72.390625,
      "completions/mean_terminated_length": 72.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25849053263664246,
      "epoch": 0.4389380530973451,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.7871676009843105,
      "kl": 0.0029903594404459,
      "learning_rate": 9.997370105232132e-07,
      "loss": -0.1178,
      "num_tokens": 3847062.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.6574006080627441,
      "sampling/importance_sampling_ratio/mean": 1.0001531839370728,
      "sampling/importance_sampling_ratio/min": 0.5268588662147522,
      "sampling/sampling_logp_difference/max": 0.6408225297927856,
      "sampling/sampling_logp_difference/mean": 0.012619627639651299,
      "step": 248
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 196.0,
      "completions/max_terminated_length": 196.0,
      "completions/mean_length": 43.890625,
      "completions/mean_terminated_length": 43.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.16508735716342926,
      "epoch": 0.4407079646017699,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06435464819757303,
      "kl": 0.003507372457534075,
      "learning_rate": 9.99711370025738e-07,
      "loss": 0.0,
      "num_tokens": 3861119.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.8435330390930176,
      "sampling/importance_sampling_ratio/mean": 1.0004816055297852,
      "sampling/importance_sampling_ratio/min": 0.5295119285583496,
      "sampling/sampling_logp_difference/max": 0.6357995271682739,
      "sampling/sampling_logp_difference/mean": 0.0132959159091115,
      "step": 249
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 125.0,
      "completions/max_terminated_length": 125.0,
      "completions/mean_length": 36.046875,
      "completions/mean_terminated_length": 36.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.15764935314655304,
      "epoch": 0.4424778761061947,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05396769395261247,
      "kl": 0.00200331280939281,
      "learning_rate": 9.99684537416446e-07,
      "loss": 0.0,
      "num_tokens": 3879634.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4567588567733765,
      "sampling/importance_sampling_ratio/mean": 1.0007212162017822,
      "sampling/importance_sampling_ratio/min": 0.5999128818511963,
      "sampling/sampling_logp_difference/max": 0.5109708309173584,
      "sampling/sampling_logp_difference/mean": 0.008045634254813194,
      "step": 250
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 195.0,
      "completions/max_terminated_length": 195.0,
      "completions/mean_length": 95.9375,
      "completions/mean_terminated_length": 95.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3375559151172638,
      "epoch": 0.44424778761061945,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.022391294297959205,
      "kl": 0.0018936173291876912,
      "learning_rate": 9.996565127593489e-07,
      "loss": 0.0,
      "num_tokens": 3899038.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6154776811599731,
      "sampling/importance_sampling_ratio/mean": 0.9995601773262024,
      "sampling/importance_sampling_ratio/min": 0.6189666986465454,
      "sampling/sampling_logp_difference/max": 0.4797039031982422,
      "sampling/sampling_logp_difference/mean": 0.014130129478871822,
      "step": 251
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 170.0,
      "completions/max_terminated_length": 170.0,
      "completions/mean_length": 82.203125,
      "completions/mean_terminated_length": 82.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27879437804222107,
      "epoch": 0.44601769911504424,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05656063312383896,
      "kl": 0.0035494635812938213,
      "learning_rate": 9.996272961213022e-07,
      "loss": 0.0,
      "num_tokens": 3915467.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5963265895843506,
      "sampling/importance_sampling_ratio/mean": 0.9994061589241028,
      "sampling/importance_sampling_ratio/min": 0.4906884729862213,
      "sampling/sampling_logp_difference/max": 0.7119457721710205,
      "sampling/sampling_logp_difference/mean": 0.01221383735537529,
      "step": 252
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 331.0,
      "completions/max_terminated_length": 331.0,
      "completions/mean_length": 77.484375,
      "completions/mean_terminated_length": 77.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27256277203559875,
      "epoch": 0.44778761061946903,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.370429715106762,
      "kl": 0.004128012806177139,
      "learning_rate": 9.995968875720051e-07,
      "loss": 0.0854,
      "num_tokens": 3931418.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.5513712167739868,
      "sampling/importance_sampling_ratio/mean": 0.9996453523635864,
      "sampling/importance_sampling_ratio/min": 0.6445860862731934,
      "sampling/sampling_logp_difference/max": 0.4391469955444336,
      "sampling/sampling_logp_difference/mean": 0.013168378733098507,
      "step": 253
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 279.0,
      "completions/max_terminated_length": 279.0,
      "completions/mean_length": 104.703125,
      "completions/mean_terminated_length": 104.703125,
      "completions/min_length": 14.0,
      "completions/min_terminated_length": 14.0,
      "entropy": 0.39987921714782715,
      "epoch": 0.4495575221238938,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02226855702470426,
      "kl": 0.002329135313630104,
      "learning_rate": 9.995652871840006e-07,
      "loss": 0.0,
      "num_tokens": 3948695.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6002646684646606,
      "sampling/importance_sampling_ratio/mean": 1.0006269216537476,
      "sampling/importance_sampling_ratio/min": 0.6878425478935242,
      "sampling/sampling_logp_difference/max": 0.4701690673828125,
      "sampling/sampling_logp_difference/mean": 0.014145614579319954,
      "step": 254
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 173.0,
      "completions/max_terminated_length": 173.0,
      "completions/mean_length": 42.234375,
      "completions/mean_terminated_length": 42.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2794543504714966,
      "epoch": 0.45132743362831856,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 23.872798817189192,
      "kl": 0.0077172527089715,
      "learning_rate": 9.995324950326745e-07,
      "loss": 0.2481,
      "num_tokens": 3961158.0,
      "reward": 0.75,
      "reward_std": 0.25819888710975647,
      "rewards/decision_reward_func/mean": 0.75,
      "rewards/decision_reward_func/std": 0.6666666865348816,
      "sampling/importance_sampling_ratio/max": 1.440090298652649,
      "sampling/importance_sampling_ratio/mean": 0.9973623752593994,
      "sampling/importance_sampling_ratio/min": 0.5127883553504944,
      "sampling/sampling_logp_difference/max": 0.6678920984268188,
      "sampling/sampling_logp_difference/mean": 0.016313213855028152,
      "step": 255
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 210.0,
      "completions/max_terminated_length": 210.0,
      "completions/mean_length": 76.859375,
      "completions/mean_terminated_length": 76.859375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.366608589887619,
      "epoch": 0.45309734513274336,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.752881151929045,
      "kl": 0.0026165214367210865,
      "learning_rate": 9.994985111962555e-07,
      "loss": -0.0774,
      "num_tokens": 3976269.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.617125153541565,
      "sampling/importance_sampling_ratio/mean": 0.9991705417633057,
      "sampling/importance_sampling_ratio/min": 0.5433637499809265,
      "sampling/sampling_logp_difference/max": 0.6099762916564941,
      "sampling/sampling_logp_difference/mean": 0.018060646951198578,
      "step": 256
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 186.0,
      "completions/max_terminated_length": 186.0,
      "completions/mean_length": 39.828125,
      "completions/mean_terminated_length": 39.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.13335466384887695,
      "epoch": 0.45486725663716815,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10586924061032081,
      "kl": 0.03460051491856575,
      "learning_rate": 9.994633357558158e-07,
      "loss": 0.0001,
      "num_tokens": 3988818.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2890098094940186,
      "sampling/importance_sampling_ratio/mean": 1.0003187656402588,
      "sampling/importance_sampling_ratio/min": 0.530799150466919,
      "sampling/sampling_logp_difference/max": 0.6333715915679932,
      "sampling/sampling_logp_difference/mean": 0.012708578258752823,
      "step": 257
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 171.0,
      "completions/max_terminated_length": 171.0,
      "completions/mean_length": 82.9375,
      "completions/mean_terminated_length": 82.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4873947501182556,
      "epoch": 0.45663716814159294,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.046335198779911754,
      "kl": 0.004279019311070442,
      "learning_rate": 9.994269687952698e-07,
      "loss": 0.0,
      "num_tokens": 4005070.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.5861549377441406,
      "sampling/importance_sampling_ratio/mean": 0.9983864426612854,
      "sampling/importance_sampling_ratio/min": 0.583560585975647,
      "sampling/sampling_logp_difference/max": 0.5386070013046265,
      "sampling/sampling_logp_difference/mean": 0.019403012469410896,
      "step": 258
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 214.0,
      "completions/max_terminated_length": 214.0,
      "completions/mean_length": 55.25,
      "completions/mean_terminated_length": 55.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.28078144788742065,
      "epoch": 0.4584070796460177,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.051731198908428394,
      "kl": 0.00406224001199007,
      "learning_rate": 9.993894104013746e-07,
      "loss": 0.0,
      "num_tokens": 4031454.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4271976947784424,
      "sampling/importance_sampling_ratio/mean": 0.9989052414894104,
      "sampling/importance_sampling_ratio/min": 0.6807091236114502,
      "sampling/sampling_logp_difference/max": 0.38462018966674805,
      "sampling/sampling_logp_difference/mean": 0.012434137985110283,
      "step": 259
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 136.0,
      "completions/max_terminated_length": 136.0,
      "completions/mean_length": 38.859375,
      "completions/mean_terminated_length": 38.859375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17825648188591003,
      "epoch": 0.46017699115044247,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09457567520972249,
      "kl": 0.004964154213666916,
      "learning_rate": 9.993506606637296e-07,
      "loss": 0.0001,
      "num_tokens": 4045093.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.430456519126892,
      "sampling/importance_sampling_ratio/mean": 0.9988031387329102,
      "sampling/importance_sampling_ratio/min": 0.662390947341919,
      "sampling/sampling_logp_difference/max": 0.4118993282318115,
      "sampling/sampling_logp_difference/mean": 0.012801151722669601,
      "step": 260
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 174.0,
      "completions/max_terminated_length": 174.0,
      "completions/mean_length": 63.53125,
      "completions/mean_terminated_length": 63.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24471867084503174,
      "epoch": 0.46194690265486726,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07442819078368382,
      "kl": 0.014636065810918808,
      "learning_rate": 9.993107196747758e-07,
      "loss": 0.0001,
      "num_tokens": 4059527.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4217251539230347,
      "sampling/importance_sampling_ratio/mean": 0.9983421564102173,
      "sampling/importance_sampling_ratio/min": 0.6172206997871399,
      "sampling/sampling_logp_difference/max": 0.4825286865234375,
      "sampling/sampling_logp_difference/mean": 0.014853391796350479,
      "step": 261
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 284.0,
      "completions/max_terminated_length": 284.0,
      "completions/mean_length": 76.0625,
      "completions/mean_terminated_length": 76.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2663308382034302,
      "epoch": 0.46371681415929206,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03699252494847363,
      "kl": 0.002792743733152747,
      "learning_rate": 9.99269587529797e-07,
      "loss": 0.0,
      "num_tokens": 4076491.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4453238248825073,
      "sampling/importance_sampling_ratio/mean": 0.9987292289733887,
      "sampling/importance_sampling_ratio/min": 0.31267639994621277,
      "sampling/sampling_logp_difference/max": 1.1625864505767822,
      "sampling/sampling_logp_difference/mean": 0.010977178812026978,
      "step": 262
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 159.0,
      "completions/max_terminated_length": 159.0,
      "completions/mean_length": 25.109375,
      "completions/mean_terminated_length": 25.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22926849126815796,
      "epoch": 0.4654867256637168,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 31.72844486467987,
      "kl": 0.02525678463280201,
      "learning_rate": 9.99227264326918e-07,
      "loss": 0.4846,
      "num_tokens": 4090498.0,
      "reward": 0.84375,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.595948576927185,
      "sampling/importance_sampling_ratio/mean": 0.9998789429664612,
      "sampling/importance_sampling_ratio/min": 0.6622171401977539,
      "sampling/sampling_logp_difference/max": 0.46746826171875,
      "sampling/sampling_logp_difference/mean": 0.016610635444521904,
      "step": 263
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 21.0,
      "completions/max_terminated_length": 21.0,
      "completions/mean_length": 15.875,
      "completions/mean_terminated_length": 15.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.0870765745639801,
      "epoch": 0.4672566371681416,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.24809152797829495,
      "kl": 0.005051221698522568,
      "learning_rate": 9.991837501671048e-07,
      "loss": 0.0,
      "num_tokens": 4103130.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4061976671218872,
      "sampling/importance_sampling_ratio/mean": 1.0006778240203857,
      "sampling/importance_sampling_ratio/min": 0.730045735836029,
      "sampling/sampling_logp_difference/max": 0.3408893346786499,
      "sampling/sampling_logp_difference/mean": 0.02010222151875496,
      "step": 264
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 189.0,
      "completions/max_terminated_length": 189.0,
      "completions/mean_length": 81.34375,
      "completions/mean_terminated_length": 81.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3357410430908203,
      "epoch": 0.4690265486725664,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02930875122187649,
      "kl": 0.0031524254009127617,
      "learning_rate": 9.991390451541648e-07,
      "loss": 0.0,
      "num_tokens": 4118880.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4591114521026611,
      "sampling/importance_sampling_ratio/mean": 0.9978489279747009,
      "sampling/importance_sampling_ratio/min": 0.5481137037277222,
      "sampling/sampling_logp_difference/max": 0.6012725830078125,
      "sampling/sampling_logp_difference/mean": 0.01842043176293373,
      "step": 265
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 195.0,
      "completions/max_terminated_length": 195.0,
      "completions/mean_length": 76.6875,
      "completions/mean_terminated_length": 76.6875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.39841485023498535,
      "epoch": 0.47079646017699117,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09000805900037295,
      "kl": 0.09546725451946259,
      "learning_rate": 9.990931493947465e-07,
      "loss": 0.0002,
      "num_tokens": 4133548.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.588159203529358,
      "sampling/importance_sampling_ratio/mean": 0.9959626197814941,
      "sampling/importance_sampling_ratio/min": 0.4857686758041382,
      "sampling/sampling_logp_difference/max": 0.7220227718353271,
      "sampling/sampling_logp_difference/mean": 0.0178472138941288,
      "step": 266
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 245.0,
      "completions/max_terminated_length": 245.0,
      "completions/mean_length": 47.296875,
      "completions/mean_terminated_length": 47.296875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.15155848860740662,
      "epoch": 0.4725663716814159,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.025041931999307744,
      "kl": 0.006949643604457378,
      "learning_rate": 9.990460629983388e-07,
      "loss": 0.0,
      "num_tokens": 4147439.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.2860136032104492,
      "sampling/importance_sampling_ratio/mean": 0.9995124936103821,
      "sampling/importance_sampling_ratio/min": 0.6633288264274597,
      "sampling/sampling_logp_difference/max": 0.4104844331741333,
      "sampling/sampling_logp_difference/mean": 0.006444099824875593,
      "step": 267
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.25,
      "completions/mean_terminated_length": 15.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07502971589565277,
      "epoch": 0.4743362831858407,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06736313727533422,
      "kl": 0.0005349120474420488,
      "learning_rate": 9.98997786077271e-07,
      "loss": 0.0,
      "num_tokens": 4161759.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3041719198226929,
      "sampling/importance_sampling_ratio/mean": 1.00235915184021,
      "sampling/importance_sampling_ratio/min": 0.8297764658927917,
      "sampling/sampling_logp_difference/max": 0.26556825637817383,
      "sampling/sampling_logp_difference/mean": 0.0092760119587183,
      "step": 268
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 282.0,
      "completions/max_terminated_length": 282.0,
      "completions/mean_length": 61.265625,
      "completions/mean_terminated_length": 61.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.331500768661499,
      "epoch": 0.4761061946902655,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 25.38060991543653,
      "kl": 0.06742088496685028,
      "learning_rate": 9.989483187467125e-07,
      "loss": -0.0358,
      "num_tokens": 4178736.0,
      "reward": 0.5,
      "reward_std": 0.5501632690429688,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.854530930519104,
      "sampling/importance_sampling_ratio/mean": 0.9997930526733398,
      "sampling/importance_sampling_ratio/min": 0.4544433653354645,
      "sampling/sampling_logp_difference/max": 0.7886819839477539,
      "sampling/sampling_logp_difference/mean": 0.018123114481568336,
      "step": 269
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 297.0,
      "completions/max_terminated_length": 297.0,
      "completions/mean_length": 33.328125,
      "completions/mean_terminated_length": 33.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.13617177307605743,
      "epoch": 0.4778761061946903,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 29.547501296153417,
      "kl": 0.09512509405612946,
      "learning_rate": 9.988976611246728e-07,
      "loss": -0.7683,
      "num_tokens": 4190165.0,
      "reward": -0.3125,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": -0.3125,
      "rewards/decision_reward_func/std": 0.9574271440505981,
      "sampling/importance_sampling_ratio/max": 1.278420090675354,
      "sampling/importance_sampling_ratio/mean": 1.0012259483337402,
      "sampling/importance_sampling_ratio/min": 0.7977098822593689,
      "sampling/sampling_logp_difference/max": 0.24562501907348633,
      "sampling/sampling_logp_difference/mean": 0.00955257099121809,
      "step": 270
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 84.5,
      "completions/mean_terminated_length": 84.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3737594485282898,
      "epoch": 0.479646017699115,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.198167042441208,
      "kl": 0.002854880876839161,
      "learning_rate": 9.988458133320008e-07,
      "loss": -0.0762,
      "num_tokens": 4205285.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.346764087677002,
      "sampling/importance_sampling_ratio/mean": 0.9988805651664734,
      "sampling/importance_sampling_ratio/min": 0.5224089622497559,
      "sampling/sampling_logp_difference/max": 0.6493045091629028,
      "sampling/sampling_logp_difference/mean": 0.01678304933011532,
      "step": 271
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 292.0,
      "completions/max_terminated_length": 292.0,
      "completions/mean_length": 82.8125,
      "completions/mean_terminated_length": 82.8125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.43366530537605286,
      "epoch": 0.4814159292035398,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.031160634170031478,
      "kl": 0.004372789058834314,
      "learning_rate": 9.987927754923843e-07,
      "loss": 0.0,
      "num_tokens": 4224985.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6253509521484375,
      "sampling/importance_sampling_ratio/mean": 1.000022530555725,
      "sampling/importance_sampling_ratio/min": 0.417615681886673,
      "sampling/sampling_logp_difference/max": 0.8731937408447266,
      "sampling/sampling_logp_difference/mean": 0.017850469797849655,
      "step": 272
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 201.0,
      "completions/max_terminated_length": 201.0,
      "completions/mean_length": 93.890625,
      "completions/mean_terminated_length": 93.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30273085832595825,
      "epoch": 0.4831858407079646,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03724891110348457,
      "kl": 0.003369373269379139,
      "learning_rate": 9.987385477323506e-07,
      "loss": 0.0,
      "num_tokens": 4242722.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4414142370224,
      "sampling/importance_sampling_ratio/mean": 1.0018047094345093,
      "sampling/importance_sampling_ratio/min": 0.5525964498519897,
      "sampling/sampling_logp_difference/max": 0.5931272506713867,
      "sampling/sampling_logp_difference/mean": 0.020217468962073326,
      "step": 273
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 236.0,
      "completions/max_terminated_length": 236.0,
      "completions/mean_length": 103.703125,
      "completions/mean_terminated_length": 103.703125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3254879415035248,
      "epoch": 0.4849557522123894,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.028170574077244014,
      "kl": 0.004011090844869614,
      "learning_rate": 9.986831301812655e-07,
      "loss": 0.0,
      "num_tokens": 4260655.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3963828086853027,
      "sampling/importance_sampling_ratio/mean": 0.9998421669006348,
      "sampling/importance_sampling_ratio/min": 0.6422039270401001,
      "sampling/sampling_logp_difference/max": 0.44284939765930176,
      "sampling/sampling_logp_difference/mean": 0.012491639703512192,
      "step": 274
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 168.0,
      "completions/max_terminated_length": 168.0,
      "completions/mean_length": 103.25,
      "completions/mean_terminated_length": 103.25,
      "completions/min_length": 40.0,
      "completions/min_terminated_length": 40.0,
      "entropy": 0.4250674843788147,
      "epoch": 0.48672566371681414,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.024067047052095888,
      "kl": 0.002488392870873213,
      "learning_rate": 9.98626522971333e-07,
      "loss": 0.0,
      "num_tokens": 4277503.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6527782678604126,
      "sampling/importance_sampling_ratio/mean": 0.9999735355377197,
      "sampling/importance_sampling_ratio/min": 0.5634852647781372,
      "sampling/sampling_logp_difference/max": 0.5736141204833984,
      "sampling/sampling_logp_difference/mean": 0.016514642164111137,
      "step": 275
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 222.0,
      "completions/max_terminated_length": 222.0,
      "completions/mean_length": 57.203125,
      "completions/mean_terminated_length": 57.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30962735414505005,
      "epoch": 0.48849557522123893,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08756911954230388,
      "kl": 0.010065382346510887,
      "learning_rate": 9.985687262375956e-07,
      "loss": 0.0,
      "num_tokens": 4300284.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5002374649047852,
      "sampling/importance_sampling_ratio/mean": 1.0002117156982422,
      "sampling/importance_sampling_ratio/min": 0.7399517297744751,
      "sampling/sampling_logp_difference/max": 0.4056234359741211,
      "sampling/sampling_logp_difference/mean": 0.014174298383295536,
      "step": 276
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 242.0,
      "completions/max_terminated_length": 242.0,
      "completions/mean_length": 104.296875,
      "completions/mean_terminated_length": 104.296875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.3918093144893646,
      "epoch": 0.4902654867256637,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.026797512712986273,
      "kl": 0.0023234847467392683,
      "learning_rate": 9.985097401179333e-07,
      "loss": 0.0,
      "num_tokens": 4315727.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.5206025838851929,
      "sampling/importance_sampling_ratio/mean": 1.0002238750457764,
      "sampling/importance_sampling_ratio/min": 0.6069608926773071,
      "sampling/sampling_logp_difference/max": 0.49929094314575195,
      "sampling/sampling_logp_difference/mean": 0.014753839001059532,
      "step": 277
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 150.0,
      "completions/max_terminated_length": 150.0,
      "completions/mean_length": 56.5,
      "completions/mean_terminated_length": 56.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2540507912635803,
      "epoch": 0.4920353982300885,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09931648670257845,
      "kl": 0.012060517445206642,
      "learning_rate": 9.98449564753063e-07,
      "loss": 0.0001,
      "num_tokens": 4330927.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4970253705978394,
      "sampling/importance_sampling_ratio/mean": 0.999140202999115,
      "sampling/importance_sampling_ratio/min": 0.6079336404800415,
      "sampling/sampling_logp_difference/max": 0.49768948554992676,
      "sampling/sampling_logp_difference/mean": 0.012635363265872002,
      "step": 278
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 263.0,
      "completions/max_terminated_length": 263.0,
      "completions/mean_length": 95.515625,
      "completions/mean_terminated_length": 95.515625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.29506152868270874,
      "epoch": 0.49380530973451325,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02588733266619697,
      "kl": 0.005474093370139599,
      "learning_rate": 9.98388200286539e-07,
      "loss": 0.0,
      "num_tokens": 4346944.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.7718281745910645,
      "sampling/importance_sampling_ratio/mean": 0.9993443489074707,
      "sampling/importance_sampling_ratio/min": 0.6424316167831421,
      "sampling/sampling_logp_difference/max": 0.5720119476318359,
      "sampling/sampling_logp_difference/mean": 0.015047137625515461,
      "step": 279
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 275.0,
      "completions/max_terminated_length": 275.0,
      "completions/mean_length": 120.96875,
      "completions/mean_terminated_length": 120.96875,
      "completions/min_length": 39.0,
      "completions/min_terminated_length": 39.0,
      "entropy": 0.43145015835762024,
      "epoch": 0.49557522123893805,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.15511521376513152,
      "kl": 0.007133021019399166,
      "learning_rate": 9.98325646864753e-07,
      "loss": 0.0001,
      "num_tokens": 4363742.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.998978853225708,
      "sampling/importance_sampling_ratio/min": 0.41871336102485657,
      "sampling/sampling_logp_difference/max": 0.8705687522888184,
      "sampling/sampling_logp_difference/mean": 0.01744624227285385,
      "step": 280
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 327.0,
      "completions/max_terminated_length": 327.0,
      "completions/mean_length": 59.71875,
      "completions/mean_terminated_length": 59.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.33847692608833313,
      "epoch": 0.49734513274336284,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.5322890577873676,
      "kl": 0.009394319728016853,
      "learning_rate": 9.98261904636932e-07,
      "loss": -0.0016,
      "num_tokens": 4377820.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.586092233657837,
      "sampling/importance_sampling_ratio/mean": 1.0006672143936157,
      "sampling/importance_sampling_ratio/min": 0.6987010836601257,
      "sampling/sampling_logp_difference/max": 0.461273193359375,
      "sampling/sampling_logp_difference/mean": 0.015468766912817955,
      "step": 281
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 195.0,
      "completions/max_terminated_length": 195.0,
      "completions/mean_length": 30.8125,
      "completions/mean_terminated_length": 30.8125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22856585681438446,
      "epoch": 0.49911504424778763,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.20919807997035103,
      "kl": 0.013904878869652748,
      "learning_rate": 9.9819697375514e-07,
      "loss": 0.0001,
      "num_tokens": 4399776.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3514093160629272,
      "sampling/importance_sampling_ratio/mean": 1.0007905960083008,
      "sampling/importance_sampling_ratio/min": 0.7770977020263672,
      "sampling/sampling_logp_difference/max": 0.3011479377746582,
      "sampling/sampling_logp_difference/mean": 0.011299017816781998,
      "step": 282
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 200.0,
      "completions/max_terminated_length": 200.0,
      "completions/mean_length": 41.3125,
      "completions/mean_terminated_length": 41.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17480722069740295,
      "epoch": 0.5008849557522124,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.033366070697314384,
      "kl": 0.0021748305298388004,
      "learning_rate": 9.981308543742756e-07,
      "loss": 0.0,
      "num_tokens": 4416100.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2915865182876587,
      "sampling/importance_sampling_ratio/mean": 0.9992952346801758,
      "sampling/importance_sampling_ratio/min": 0.737482488155365,
      "sampling/sampling_logp_difference/max": 0.30451297760009766,
      "sampling/sampling_logp_difference/mean": 0.01027563028037548,
      "step": 283
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 205.0,
      "completions/max_terminated_length": 205.0,
      "completions/mean_length": 68.890625,
      "completions/mean_terminated_length": 68.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24031007289886475,
      "epoch": 0.5026548672566372,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.015676680037355565,
      "kl": 0.001302052871324122,
      "learning_rate": 9.980635466520736e-07,
      "loss": 0.0,
      "num_tokens": 4432957.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3076136112213135,
      "sampling/importance_sampling_ratio/mean": 1.0002350807189941,
      "sampling/importance_sampling_ratio/min": 0.7194289565086365,
      "sampling/sampling_logp_difference/max": 0.3292975425720215,
      "sampling/sampling_logp_difference/mean": 0.009223762899637222,
      "step": 284
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 308.0,
      "completions/max_terminated_length": 308.0,
      "completions/mean_length": 72.375,
      "completions/mean_terminated_length": 72.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3059983253479004,
      "epoch": 0.504424778761062,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03358499227826061,
      "kl": 0.004222136922180653,
      "learning_rate": 9.979950507491033e-07,
      "loss": 0.0,
      "num_tokens": 4453269.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.9277665615081787,
      "sampling/importance_sampling_ratio/mean": 0.999069333076477,
      "sampling/importance_sampling_ratio/min": 0.7011988759040833,
      "sampling/sampling_logp_difference/max": 0.6563620567321777,
      "sampling/sampling_logp_difference/mean": 0.014185376465320587,
      "step": 285
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 271.0,
      "completions/max_terminated_length": 271.0,
      "completions/mean_length": 100.25,
      "completions/mean_terminated_length": 100.25,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.46278679370880127,
      "epoch": 0.5061946902654867,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.349333059405631,
      "kl": 0.00824448186904192,
      "learning_rate": 9.979253668287685e-07,
      "loss": 0.0758,
      "num_tokens": 4470949.0,
      "reward": 0.0625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.0625,
      "rewards/decision_reward_func/std": 1.0059348344802856,
      "sampling/importance_sampling_ratio/max": 1.6193196773529053,
      "sampling/importance_sampling_ratio/mean": 1.000375747680664,
      "sampling/importance_sampling_ratio/min": 0.6411598324775696,
      "sampling/sampling_logp_difference/max": 0.4820060729980469,
      "sampling/sampling_logp_difference/mean": 0.017631113529205322,
      "step": 286
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 823.0,
      "completions/max_terminated_length": 823.0,
      "completions/mean_length": 140.109375,
      "completions/mean_terminated_length": 140.109375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.3196713924407959,
      "epoch": 0.5079646017699115,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.4901874515071363,
      "kl": 0.005851144902408123,
      "learning_rate": 9.978544950573073e-07,
      "loss": -0.0837,
      "num_tokens": 4489404.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9992465376853943,
      "sampling/importance_sampling_ratio/min": 0.6388933062553406,
      "sampling/sampling_logp_difference/max": 0.8145382404327393,
      "sampling/sampling_logp_difference/mean": 0.01222514733672142,
      "step": 287
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 193.0,
      "completions/max_terminated_length": 193.0,
      "completions/mean_length": 81.03125,
      "completions/mean_terminated_length": 81.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2995682954788208,
      "epoch": 0.5097345132743363,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.036957311092894976,
      "kl": 0.006040679756551981,
      "learning_rate": 9.977824356037915e-07,
      "loss": 0.0,
      "num_tokens": 4504238.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.493669033050537,
      "sampling/importance_sampling_ratio/mean": 1.0006897449493408,
      "sampling/importance_sampling_ratio/min": 0.6473813652992249,
      "sampling/sampling_logp_difference/max": 0.43481969833374023,
      "sampling/sampling_logp_difference/mean": 0.013412510976195335,
      "step": 288
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 236.0,
      "completions/max_terminated_length": 236.0,
      "completions/mean_length": 75.078125,
      "completions/mean_terminated_length": 75.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4171299338340759,
      "epoch": 0.511504424778761,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 15.761554871688615,
      "kl": 0.018951520323753357,
      "learning_rate": 9.97709188640126e-07,
      "loss": 0.1355,
      "num_tokens": 4521539.0,
      "reward": 0.625,
      "reward_std": 0.42078250646591187,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.690384030342102,
      "sampling/importance_sampling_ratio/mean": 0.9999475479125977,
      "sampling/importance_sampling_ratio/min": 0.5168097615242004,
      "sampling/sampling_logp_difference/max": 0.6600804328918457,
      "sampling/sampling_logp_difference/mean": 0.01776520349085331,
      "step": 289
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 150.0,
      "completions/max_terminated_length": 150.0,
      "completions/mean_length": 56.796875,
      "completions/mean_terminated_length": 56.796875,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.3325667381286621,
      "epoch": 0.5132743362831859,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 12.906319531110897,
      "kl": 0.01129073090851307,
      "learning_rate": 9.976347543410486e-07,
      "loss": -0.0576,
      "num_tokens": 4542166.0,
      "reward": 0.71875,
      "reward_std": 0.38319888710975647,
      "rewards/decision_reward_func/mean": 0.71875,
      "rewards/decision_reward_func/std": 0.7007648944854736,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0011260509490967,
      "sampling/importance_sampling_ratio/min": 0.6028571724891663,
      "sampling/sampling_logp_difference/max": 0.7271323204040527,
      "sampling/sampling_logp_difference/mean": 0.014359983615577221,
      "step": 290
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 352.0,
      "completions/max_terminated_length": 352.0,
      "completions/mean_length": 91.40625,
      "completions/mean_terminated_length": 91.40625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27161622047424316,
      "epoch": 0.5150442477876106,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.2989239760333975,
      "kl": 0.022020166739821434,
      "learning_rate": 9.975591328841304e-07,
      "loss": 0.1212,
      "num_tokens": 4559424.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0005266666412354,
      "sampling/importance_sampling_ratio/min": 0.4944953918457031,
      "sampling/sampling_logp_difference/max": 0.7871992588043213,
      "sampling/sampling_logp_difference/mean": 0.013765428215265274,
      "step": 291
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 300.0,
      "completions/max_terminated_length": 300.0,
      "completions/mean_length": 32.109375,
      "completions/mean_terminated_length": 32.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.20906458795070648,
      "epoch": 0.5168141592920354,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 30.28759677520763,
      "kl": 0.030800532549619675,
      "learning_rate": 9.974823244497737e-07,
      "loss": 0.1248,
      "num_tokens": 4573079.0,
      "reward": 0.46875,
      "reward_std": 0.3723389506340027,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9987730979919434,
      "sampling/importance_sampling_ratio/min": 0.42857232689857483,
      "sampling/sampling_logp_difference/max": 0.8472957611083984,
      "sampling/sampling_logp_difference/mean": 0.016848154366016388,
      "step": 292
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 235.0,
      "completions/max_terminated_length": 235.0,
      "completions/mean_length": 62.265625,
      "completions/mean_terminated_length": 62.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.275778591632843,
      "epoch": 0.5185840707964602,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 11.562227028990126,
      "kl": 0.030032262206077576,
      "learning_rate": 9.974043292212127e-07,
      "loss": 0.2261,
      "num_tokens": 4587704.0,
      "reward": 0.65625,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.631435751914978,
      "sampling/importance_sampling_ratio/mean": 0.9993401765823364,
      "sampling/importance_sampling_ratio/min": 0.1340518444776535,
      "sampling/sampling_logp_difference/max": 2.009528636932373,
      "sampling/sampling_logp_difference/mean": 0.016510870307683945,
      "step": 293
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 17.359375,
      "completions/mean_terminated_length": 17.359375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.049905285239219666,
      "epoch": 0.5203539823008849,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.29504253071905645,
      "kl": 0.008438438177108765,
      "learning_rate": 9.97325147384513e-07,
      "loss": 0.0001,
      "num_tokens": 4622127.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2581522464752197,
      "sampling/importance_sampling_ratio/mean": 0.9986610412597656,
      "sampling/importance_sampling_ratio/min": 0.8183937072753906,
      "sampling/sampling_logp_difference/max": 0.22964417934417725,
      "sampling/sampling_logp_difference/mean": 0.0056549059227108955,
      "step": 294
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 291.0,
      "completions/max_terminated_length": 291.0,
      "completions/mean_length": 111.421875,
      "completions/mean_terminated_length": 111.421875,
      "completions/min_length": 33.0,
      "completions/min_terminated_length": 33.0,
      "entropy": 0.5299802422523499,
      "epoch": 0.5221238938053098,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.4250467422790787,
      "kl": 0.0054356371983885765,
      "learning_rate": 9.97244779128571e-07,
      "loss": -0.0261,
      "num_tokens": 4638570.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.3723515272140503,
      "sampling/importance_sampling_ratio/mean": 0.9999358654022217,
      "sampling/importance_sampling_ratio/min": 0.6040018200874329,
      "sampling/sampling_logp_difference/max": 0.5041780471801758,
      "sampling/sampling_logp_difference/mean": 0.01785511150956154,
      "step": 295
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 182.0,
      "completions/max_terminated_length": 182.0,
      "completions/mean_length": 60.34375,
      "completions/mean_terminated_length": 60.34375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.18186187744140625,
      "epoch": 0.5238938053097345,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0889240132782423,
      "kl": 0.009175324812531471,
      "learning_rate": 9.971632246451127e-07,
      "loss": 0.0001,
      "num_tokens": 4652832.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.874014139175415,
      "sampling/importance_sampling_ratio/mean": 1.0008121728897095,
      "sampling/importance_sampling_ratio/min": 0.4280461370944977,
      "sampling/sampling_logp_difference/max": 0.8485243320465088,
      "sampling/sampling_logp_difference/mean": 0.008807593025267124,
      "step": 296
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 303.0,
      "completions/max_terminated_length": 303.0,
      "completions/mean_length": 69.3125,
      "completions/mean_terminated_length": 69.3125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.29337161779403687,
      "epoch": 0.5256637168141592,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2845039152124611,
      "kl": 0.01000886783003807,
      "learning_rate": 9.970804841286953e-07,
      "loss": 0.0001,
      "num_tokens": 4668708.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.484485387802124,
      "sampling/importance_sampling_ratio/mean": 0.9988928437232971,
      "sampling/importance_sampling_ratio/min": 0.5810959935188293,
      "sampling/sampling_logp_difference/max": 0.5428392887115479,
      "sampling/sampling_logp_difference/mean": 0.012892980128526688,
      "step": 297
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 248.0,
      "completions/max_terminated_length": 248.0,
      "completions/mean_length": 106.4375,
      "completions/mean_terminated_length": 106.4375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.44031625986099243,
      "epoch": 0.5274336283185841,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.971494631759584,
      "kl": 0.006387860979884863,
      "learning_rate": 9.96996557776704e-07,
      "loss": -0.0907,
      "num_tokens": 4684560.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.8611406087875366,
      "sampling/importance_sampling_ratio/mean": 0.9997141361236572,
      "sampling/importance_sampling_ratio/min": 0.6056256890296936,
      "sampling/sampling_logp_difference/max": 0.6211895942687988,
      "sampling/sampling_logp_difference/mean": 0.01681620627641678,
      "step": 298
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 196.0,
      "completions/max_terminated_length": 196.0,
      "completions/mean_length": 45.78125,
      "completions/mean_terminated_length": 45.78125,
      "completions/min_length": 13.0,
      "completions/min_terminated_length": 13.0,
      "entropy": 0.20236334204673767,
      "epoch": 0.5292035398230088,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0530022444996644,
      "kl": 0.008030351251363754,
      "learning_rate": 9.969114457893539e-07,
      "loss": 0.0001,
      "num_tokens": 4700482.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6515015363693237,
      "sampling/importance_sampling_ratio/mean": 0.9991205334663391,
      "sampling/importance_sampling_ratio/min": 0.5748081207275391,
      "sampling/sampling_logp_difference/max": 0.5537190437316895,
      "sampling/sampling_logp_difference/mean": 0.009639833122491837,
      "step": 299
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 266.0,
      "completions/max_terminated_length": 266.0,
      "completions/mean_length": 53.359375,
      "completions/mean_terminated_length": 53.359375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.14972491562366486,
      "epoch": 0.5309734513274337,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2657145861144557,
      "kl": 0.02420363575220108,
      "learning_rate": 9.96825148369688e-07,
      "loss": 0.0001,
      "num_tokens": 4718153.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6632829904556274,
      "sampling/importance_sampling_ratio/mean": 0.9984308481216431,
      "sampling/importance_sampling_ratio/min": 0.6867097616195679,
      "sampling/sampling_logp_difference/max": 0.5087933540344238,
      "sampling/sampling_logp_difference/mean": 0.011596485041081905,
      "step": 300
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 193.0,
      "completions/max_terminated_length": 193.0,
      "completions/mean_length": 86.640625,
      "completions/mean_terminated_length": 86.640625,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.4426685571670532,
      "epoch": 0.5327433628318584,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05573498068166531,
      "kl": 0.0059372428804636,
      "learning_rate": 9.967376657235778e-07,
      "loss": 0.0001,
      "num_tokens": 4736258.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.352237582206726,
      "sampling/importance_sampling_ratio/mean": 1.0006613731384277,
      "sampling/importance_sampling_ratio/min": 0.5942684412002563,
      "sampling/sampling_logp_difference/max": 0.5204241275787354,
      "sampling/sampling_logp_difference/mean": 0.017739780247211456,
      "step": 301
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 527.0,
      "completions/max_terminated_length": 527.0,
      "completions/mean_length": 75.59375,
      "completions/mean_terminated_length": 75.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3071618676185608,
      "epoch": 0.5345132743362832,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.4380444782477624,
      "kl": 0.05999153107404709,
      "learning_rate": 9.966489980597217e-07,
      "loss": 0.0002,
      "num_tokens": 4753320.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0008987188339233,
      "sampling/importance_sampling_ratio/min": 0.41841116547584534,
      "sampling/sampling_logp_difference/max": 0.8712906837463379,
      "sampling/sampling_logp_difference/mean": 0.01640094444155693,
      "step": 302
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 241.0,
      "completions/max_terminated_length": 241.0,
      "completions/mean_length": 68.9375,
      "completions/mean_terminated_length": 68.9375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.3281650245189667,
      "epoch": 0.536283185840708,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07769537274799407,
      "kl": 0.007740527857095003,
      "learning_rate": 9.965591455896455e-07,
      "loss": 0.0001,
      "num_tokens": 4770196.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6265430450439453,
      "sampling/importance_sampling_ratio/mean": 0.9986387491226196,
      "sampling/importance_sampling_ratio/min": 0.7393641471862793,
      "sampling/sampling_logp_difference/max": 0.48645687103271484,
      "sampling/sampling_logp_difference/mean": 0.017250174656510353,
      "step": 303
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 304.0,
      "completions/max_terminated_length": 304.0,
      "completions/mean_length": 104.25,
      "completions/mean_terminated_length": 104.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3975277245044708,
      "epoch": 0.5380530973451327,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2827678240213584,
      "kl": 0.022672219201922417,
      "learning_rate": 9.964681085277011e-07,
      "loss": 0.0001,
      "num_tokens": 4786692.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.001333475112915,
      "sampling/importance_sampling_ratio/min": 0.592957079410553,
      "sampling/sampling_logp_difference/max": 1.0330731868743896,
      "sampling/sampling_logp_difference/mean": 0.017484024167060852,
      "step": 304
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 414.0,
      "completions/max_terminated_length": 414.0,
      "completions/mean_length": 158.265625,
      "completions/mean_terminated_length": 158.265625,
      "completions/min_length": 40.0,
      "completions/min_terminated_length": 40.0,
      "entropy": 0.5590561628341675,
      "epoch": 0.5398230088495575,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.08652906915803,
      "kl": 0.005347556434571743,
      "learning_rate": 9.96375887091067e-07,
      "loss": 0.0594,
      "num_tokens": 4807045.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.7717801332473755,
      "sampling/importance_sampling_ratio/mean": 0.9994161128997803,
      "sampling/importance_sampling_ratio/min": 0.5527122616767883,
      "sampling/sampling_logp_difference/max": 0.5929176807403564,
      "sampling/sampling_logp_difference/mean": 0.01890292763710022,
      "step": 305
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 363.0,
      "completions/max_terminated_length": 363.0,
      "completions/mean_length": 142.390625,
      "completions/mean_terminated_length": 142.390625,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.36418986320495605,
      "epoch": 0.5415929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03143902386920403,
      "kl": 0.004250286612659693,
      "learning_rate": 9.962824814997464e-07,
      "loss": 0.0,
      "num_tokens": 4827262.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0011675357818604,
      "sampling/importance_sampling_ratio/min": 0.6306570172309875,
      "sampling/sampling_logp_difference/max": 0.7148250341415405,
      "sampling/sampling_logp_difference/mean": 0.01493034791201353,
      "step": 306
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 224.0,
      "completions/max_terminated_length": 224.0,
      "completions/mean_length": 60.234375,
      "completions/mean_terminated_length": 60.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.29062414169311523,
      "epoch": 0.5433628318584071,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07331000426106707,
      "kl": 0.011067867279052734,
      "learning_rate": 9.961878919765677e-07,
      "loss": 0.0001,
      "num_tokens": 4841629.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4982765913009644,
      "sampling/importance_sampling_ratio/mean": 0.9982985258102417,
      "sampling/importance_sampling_ratio/min": 0.5298459529876709,
      "sampling/sampling_logp_difference/max": 0.6351690292358398,
      "sampling/sampling_logp_difference/mean": 0.013999614864587784,
      "step": 307
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 312.0,
      "completions/max_terminated_length": 312.0,
      "completions/mean_length": 79.125,
      "completions/mean_terminated_length": 79.125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2869476079940796,
      "epoch": 0.5451327433628319,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0780303316547964,
      "kl": 0.014650448225438595,
      "learning_rate": 9.96092118747184e-07,
      "loss": 0.0001,
      "num_tokens": 4857221.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9994796514511108,
      "sampling/importance_sampling_ratio/min": 0.49689343571662903,
      "sampling/sampling_logp_difference/max": 0.7066882848739624,
      "sampling/sampling_logp_difference/mean": 0.018858671188354492,
      "step": 308
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 339.0,
      "completions/max_terminated_length": 339.0,
      "completions/mean_length": 72.328125,
      "completions/mean_terminated_length": 72.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25819018483161926,
      "epoch": 0.5469026548672566,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04189031557716253,
      "kl": 0.0059643518179655075,
      "learning_rate": 9.959951620400718e-07,
      "loss": 0.0001,
      "num_tokens": 4874138.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4584249258041382,
      "sampling/importance_sampling_ratio/mean": 0.9993717074394226,
      "sampling/importance_sampling_ratio/min": 0.6622897386550903,
      "sampling/sampling_logp_difference/max": 0.4120521545410156,
      "sampling/sampling_logp_difference/mean": 0.012498611584305763,
      "step": 309
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 209.0,
      "completions/max_terminated_length": 209.0,
      "completions/mean_length": 68.3125,
      "completions/mean_terminated_length": 68.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.401637464761734,
      "epoch": 0.5486725663716814,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.403602923707776,
      "kl": 0.013075219467282295,
      "learning_rate": 9.95897022086531e-07,
      "loss": 0.1173,
      "num_tokens": 4888862.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.3958466053009033,
      "sampling/importance_sampling_ratio/mean": 0.9982624650001526,
      "sampling/importance_sampling_ratio/min": 0.6836552619934082,
      "sampling/sampling_logp_difference/max": 0.38030147552490234,
      "sampling/sampling_logp_difference/mean": 0.016478929668664932,
      "step": 310
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 232.0,
      "completions/max_terminated_length": 232.0,
      "completions/mean_length": 49.859375,
      "completions/mean_terminated_length": 49.859375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3399659991264343,
      "epoch": 0.5504424778761062,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.15924851242976146,
      "kl": 0.015980035066604614,
      "learning_rate": 9.957976991206845e-07,
      "loss": 0.0001,
      "num_tokens": 4904533.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4301775693893433,
      "sampling/importance_sampling_ratio/mean": 1.001241683959961,
      "sampling/importance_sampling_ratio/min": 0.6126782298088074,
      "sampling/sampling_logp_difference/max": 0.4899153709411621,
      "sampling/sampling_logp_difference/mean": 0.01730850338935852,
      "step": 311
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 324.0,
      "completions/max_terminated_length": 324.0,
      "completions/mean_length": 83.75,
      "completions/mean_terminated_length": 83.75,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.41513192653656006,
      "epoch": 0.552212389380531,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04073152091838604,
      "kl": 0.006671360228210688,
      "learning_rate": 9.956971933794773e-07,
      "loss": 0.0001,
      "num_tokens": 4920229.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.8718671798706055,
      "sampling/importance_sampling_ratio/mean": 1.0000343322753906,
      "sampling/importance_sampling_ratio/min": 0.5747321844100952,
      "sampling/sampling_logp_difference/max": 0.6269364356994629,
      "sampling/sampling_logp_difference/mean": 0.01486043632030487,
      "step": 312
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 324.0,
      "completions/max_terminated_length": 324.0,
      "completions/mean_length": 135.65625,
      "completions/mean_terminated_length": 135.65625,
      "completions/min_length": 13.0,
      "completions/min_terminated_length": 13.0,
      "entropy": 0.4970802664756775,
      "epoch": 0.5539823008849557,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.021734362861208452,
      "kl": 0.0033400128595530987,
      "learning_rate": 9.955955051026758e-07,
      "loss": 0.0,
      "num_tokens": 4939343.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5506399869918823,
      "sampling/importance_sampling_ratio/mean": 1.0004029273986816,
      "sampling/importance_sampling_ratio/min": 0.710355281829834,
      "sampling/sampling_logp_difference/max": 0.43866777420043945,
      "sampling/sampling_logp_difference/mean": 0.01776507869362831,
      "step": 313
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 192.0,
      "completions/max_terminated_length": 192.0,
      "completions/mean_length": 52.359375,
      "completions/mean_terminated_length": 52.359375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2804589867591858,
      "epoch": 0.5557522123893806,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06747453302786525,
      "kl": 0.007154546212404966,
      "learning_rate": 9.954926345328678e-07,
      "loss": 0.0001,
      "num_tokens": 4954166.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5472042560577393,
      "sampling/importance_sampling_ratio/mean": 1.0000067949295044,
      "sampling/importance_sampling_ratio/min": 0.6417877674102783,
      "sampling/sampling_logp_difference/max": 0.4434976577758789,
      "sampling/sampling_logp_difference/mean": 0.014697583392262459,
      "step": 314
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 14.84375,
      "completions/mean_terminated_length": 14.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07681912183761597,
      "epoch": 0.5575221238938053,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.7951193652292655,
      "kl": 0.022772781550884247,
      "learning_rate": 9.953885819154614e-07,
      "loss": 0.0002,
      "num_tokens": 4975244.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5425488948822021,
      "sampling/importance_sampling_ratio/mean": 0.9988242983818054,
      "sampling/importance_sampling_ratio/min": 0.5694617629051208,
      "sampling/sampling_logp_difference/max": 0.5630636215209961,
      "sampling/sampling_logp_difference/mean": 0.018749669194221497,
      "step": 315
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 330.0,
      "completions/max_terminated_length": 330.0,
      "completions/mean_length": 106.96875,
      "completions/mean_terminated_length": 106.96875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4102153778076172,
      "epoch": 0.5592920353982301,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.188538697896643,
      "kl": 0.003499663434922695,
      "learning_rate": 9.952833474986846e-07,
      "loss": 0.04,
      "num_tokens": 4992490.0,
      "reward": -0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.5456247329711914,
      "sampling/importance_sampling_ratio/mean": 0.9988653063774109,
      "sampling/importance_sampling_ratio/min": 0.6146138310432434,
      "sampling/sampling_logp_difference/max": 0.48676109313964844,
      "sampling/sampling_logp_difference/mean": 0.020490184426307678,
      "step": 316
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 309.0,
      "completions/max_terminated_length": 309.0,
      "completions/mean_length": 119.140625,
      "completions/mean_terminated_length": 119.140625,
      "completions/min_length": 51.0,
      "completions/min_terminated_length": 51.0,
      "entropy": 0.6081708669662476,
      "epoch": 0.5610619469026549,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.7009228075577634,
      "kl": 0.0045472802594304085,
      "learning_rate": 9.951769315335843e-07,
      "loss": 0.0164,
      "num_tokens": 5009779.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.9710369110107422,
      "sampling/importance_sampling_ratio/mean": 1.0008913278579712,
      "sampling/importance_sampling_ratio/min": 0.5404453873634338,
      "sampling/sampling_logp_difference/max": 0.6785597801208496,
      "sampling/sampling_logp_difference/mean": 0.02135409787297249,
      "step": 317
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 726.0,
      "completions/max_terminated_length": 726.0,
      "completions/mean_length": 79.640625,
      "completions/mean_terminated_length": 79.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4556015729904175,
      "epoch": 0.5628318584070796,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.94893909065176,
      "kl": 0.004789283964782953,
      "learning_rate": 9.95069334274027e-07,
      "loss": 0.2709,
      "num_tokens": 5026012.0,
      "reward": 0.78125,
      "reward_std": 0.2561737596988678,
      "rewards/decision_reward_func/mean": 0.78125,
      "rewards/decision_reward_func/std": 0.6291528940200806,
      "sampling/importance_sampling_ratio/max": 1.5947656631469727,
      "sampling/importance_sampling_ratio/mean": 1.0000662803649902,
      "sampling/importance_sampling_ratio/min": 0.6957854628562927,
      "sampling/sampling_logp_difference/max": 0.46672677993774414,
      "sampling/sampling_logp_difference/mean": 0.02015022188425064,
      "step": 318
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 240.0,
      "completions/max_terminated_length": 240.0,
      "completions/mean_length": 41.609375,
      "completions/mean_terminated_length": 41.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30406153202056885,
      "epoch": 0.5646017699115045,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 12.714616664961063,
      "kl": 0.01399939227849245,
      "learning_rate": 9.949605559766967e-07,
      "loss": -0.1305,
      "num_tokens": 5041763.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.4089725017547607,
      "sampling/importance_sampling_ratio/mean": 0.9992440938949585,
      "sampling/importance_sampling_ratio/min": 0.6184074282646179,
      "sampling/sampling_logp_difference/max": 0.4806077480316162,
      "sampling/sampling_logp_difference/mean": 0.012585079297423363,
      "step": 319
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 193.0,
      "completions/max_terminated_length": 193.0,
      "completions/mean_length": 41.546875,
      "completions/mean_terminated_length": 41.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.17763113975524902,
      "epoch": 0.5663716814159292,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.14469667542750955,
      "kl": 0.017358124256134033,
      "learning_rate": 9.94850596901095e-07,
      "loss": 0.0001,
      "num_tokens": 5056518.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4221454858779907,
      "sampling/importance_sampling_ratio/mean": 1.0001862049102783,
      "sampling/importance_sampling_ratio/min": 0.586487889289856,
      "sampling/sampling_logp_difference/max": 0.5336031913757324,
      "sampling/sampling_logp_difference/mean": 0.01157684437930584,
      "step": 320
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 315.0,
      "completions/max_terminated_length": 315.0,
      "completions/mean_length": 76.484375,
      "completions/mean_terminated_length": 76.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3155892789363861,
      "epoch": 0.5681415929203539,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.021400392096172,
      "kl": 0.010425843298435211,
      "learning_rate": 9.947394573095402e-07,
      "loss": 0.2305,
      "num_tokens": 5075909.0,
      "reward": 0.625,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.4295427799224854,
      "sampling/importance_sampling_ratio/mean": 0.9993923902511597,
      "sampling/importance_sampling_ratio/min": 0.6148527264595032,
      "sampling/sampling_logp_difference/max": 0.4863724708557129,
      "sampling/sampling_logp_difference/mean": 0.01577897183597088,
      "step": 321
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 294.0,
      "completions/max_terminated_length": 294.0,
      "completions/mean_length": 86.734375,
      "completions/mean_terminated_length": 86.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3866017162799835,
      "epoch": 0.5699115044247788,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.040077809721334394,
      "kl": 0.007752751465886831,
      "learning_rate": 9.94627137467167e-07,
      "loss": 0.0001,
      "num_tokens": 5090708.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.6048550605773926,
      "sampling/importance_sampling_ratio/mean": 0.9996352195739746,
      "sampling/importance_sampling_ratio/min": 0.5155983567237854,
      "sampling/sampling_logp_difference/max": 0.6624271869659424,
      "sampling/sampling_logp_difference/mean": 0.014700477942824364,
      "step": 322
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 162.0,
      "completions/max_terminated_length": 162.0,
      "completions/mean_length": 35.953125,
      "completions/mean_terminated_length": 35.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1371205598115921,
      "epoch": 0.5716814159292035,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2096955708072268,
      "kl": 0.019187739118933678,
      "learning_rate": 9.945136376419258e-07,
      "loss": 0.0001,
      "num_tokens": 5105873.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3945482969284058,
      "sampling/importance_sampling_ratio/mean": 0.9994967579841614,
      "sampling/importance_sampling_ratio/min": 0.47429314255714417,
      "sampling/sampling_logp_difference/max": 0.7459297180175781,
      "sampling/sampling_logp_difference/mean": 0.007329988293349743,
      "step": 323
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 130.0,
      "completions/max_terminated_length": 130.0,
      "completions/mean_length": 33.78125,
      "completions/mean_terminated_length": 33.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.12136248499155045,
      "epoch": 0.5734513274336284,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.20048100584716425,
      "kl": 0.018594108521938324,
      "learning_rate": 9.943989581045819e-07,
      "loss": 0.0001,
      "num_tokens": 5122099.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.266669511795044,
      "sampling/importance_sampling_ratio/mean": 0.9998998641967773,
      "sampling/importance_sampling_ratio/min": 0.7039791941642761,
      "sampling/sampling_logp_difference/max": 0.35100650787353516,
      "sampling/sampling_logp_difference/mean": 0.005552534945309162,
      "step": 324
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 174.0,
      "completions/max_terminated_length": 174.0,
      "completions/mean_length": 55.84375,
      "completions/mean_terminated_length": 55.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3695143461227417,
      "epoch": 0.5752212389380531,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 12.740230354540556,
      "kl": 0.00976649858057499,
      "learning_rate": 9.942830991287149e-07,
      "loss": 0.0784,
      "num_tokens": 5135977.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.5773446559906006,
      "sampling/importance_sampling_ratio/mean": 0.9974552989006042,
      "sampling/importance_sampling_ratio/min": 0.6371076703071594,
      "sampling/sampling_logp_difference/max": 0.45574283599853516,
      "sampling/sampling_logp_difference/mean": 0.021933196112513542,
      "step": 325
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 229.0,
      "completions/max_terminated_length": 229.0,
      "completions/mean_length": 97.875,
      "completions/mean_terminated_length": 97.875,
      "completions/min_length": 35.0,
      "completions/min_terminated_length": 35.0,
      "entropy": 0.3761260509490967,
      "epoch": 0.5769911504424778,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0306591512793739,
      "kl": 0.003819515462964773,
      "learning_rate": 9.94166060990718e-07,
      "loss": 0.0,
      "num_tokens": 5151105.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6236847639083862,
      "sampling/importance_sampling_ratio/mean": 1.000258207321167,
      "sampling/importance_sampling_ratio/min": 0.5582404732704163,
      "sampling/sampling_logp_difference/max": 0.5829653739929199,
      "sampling/sampling_logp_difference/mean": 0.01616375334560871,
      "step": 326
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 171.0,
      "completions/max_terminated_length": 171.0,
      "completions/mean_length": 40.4375,
      "completions/mean_terminated_length": 40.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21063324809074402,
      "epoch": 0.5787610619469027,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.18340809261246058,
      "kl": 0.03675929456949234,
      "learning_rate": 9.940478439697972e-07,
      "loss": 0.0002,
      "num_tokens": 5165245.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6573799848556519,
      "sampling/importance_sampling_ratio/mean": 0.9981979727745056,
      "sampling/importance_sampling_ratio/min": 0.47290316224098206,
      "sampling/sampling_logp_difference/max": 0.7488646507263184,
      "sampling/sampling_logp_difference/mean": 0.012335125356912613,
      "step": 327
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 192.0,
      "completions/max_terminated_length": 192.0,
      "completions/mean_length": 54.9375,
      "completions/mean_terminated_length": 54.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.35798150300979614,
      "epoch": 0.5805309734513274,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.141605430777651,
      "kl": 0.05895555764436722,
      "learning_rate": 9.939284483479715e-07,
      "loss": 0.0999,
      "num_tokens": 5178681.0,
      "reward": 0.59375,
      "reward_std": 0.497555673122406,
      "rewards/decision_reward_func/mean": 0.59375,
      "rewards/decision_reward_func/std": 0.8110105991363525,
      "sampling/importance_sampling_ratio/max": 1.5644574165344238,
      "sampling/importance_sampling_ratio/mean": 1.0009150505065918,
      "sampling/importance_sampling_ratio/min": 0.5880892872810364,
      "sampling/sampling_logp_difference/max": 0.5308763980865479,
      "sampling/sampling_logp_difference/mean": 0.02186429128050804,
      "step": 328
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 18.0,
      "completions/max_terminated_length": 18.0,
      "completions/mean_length": 14.609375,
      "completions/mean_terminated_length": 14.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.05067729949951172,
      "epoch": 0.5823008849557522,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.5404423741608648,
      "kl": 0.02486887201666832,
      "learning_rate": 9.93807874410071e-07,
      "loss": 0.0002,
      "num_tokens": 5191696.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.265113115310669,
      "sampling/importance_sampling_ratio/mean": 0.9971482753753662,
      "sampling/importance_sampling_ratio/min": 0.7563673257827759,
      "sampling/sampling_logp_difference/max": 0.279228150844574,
      "sampling/sampling_logp_difference/mean": 0.009500461630523205,
      "step": 329
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 85.0,
      "completions/max_terminated_length": 85.0,
      "completions/mean_length": 15.828125,
      "completions/mean_terminated_length": 15.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1415385901927948,
      "epoch": 0.584070796460177,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.30440635525570386,
      "kl": 0.1242285966873169,
      "learning_rate": 9.936861224437372e-07,
      "loss": 0.0013,
      "num_tokens": 5205861.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5973737239837646,
      "sampling/importance_sampling_ratio/mean": 1.0021167993545532,
      "sampling/importance_sampling_ratio/min": 0.7834390997886658,
      "sampling/sampling_logp_difference/max": 0.46836090087890625,
      "sampling/sampling_logp_difference/mean": 0.008564083836972713,
      "step": 330
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 18.0,
      "completions/max_terminated_length": 18.0,
      "completions/mean_length": 13.65625,
      "completions/mean_terminated_length": 13.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.062073707580566406,
      "epoch": 0.5858407079646017,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08148900210167391,
      "kl": 0.1352459043264389,
      "learning_rate": 9.935631927394214e-07,
      "loss": 0.0014,
      "num_tokens": 5218223.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.6912966966629028,
      "sampling/importance_sampling_ratio/mean": 0.9999576807022095,
      "sampling/importance_sampling_ratio/min": 0.6084062457084656,
      "sampling/sampling_logp_difference/max": 0.5254955291748047,
      "sampling/sampling_logp_difference/mean": 0.011843099258840084,
      "step": 331
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 18.0,
      "completions/max_terminated_length": 18.0,
      "completions/mean_length": 14.96875,
      "completions/mean_terminated_length": 14.96875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07408523559570312,
      "epoch": 0.5876106194690266,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.37575585050410487,
      "kl": 0.12586016952991486,
      "learning_rate": 9.934390855903852e-07,
      "loss": 0.0011,
      "num_tokens": 5233069.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.1157681941986084,
      "sampling/importance_sampling_ratio/mean": 0.9991582632064819,
      "sampling/importance_sampling_ratio/min": 0.8067606091499329,
      "sampling/sampling_logp_difference/max": 0.21472835540771484,
      "sampling/sampling_logp_difference/mean": 0.005963312461972237,
      "step": 332
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 98.0,
      "completions/max_terminated_length": 98.0,
      "completions/mean_length": 14.796875,
      "completions/mean_terminated_length": 14.796875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.20496223866939545,
      "epoch": 0.5893805309734513,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2765369592767717,
      "kl": 0.2802959978580475,
      "learning_rate": 9.93313801292698e-07,
      "loss": 0.003,
      "num_tokens": 5243936.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.646456241607666,
      "sampling/importance_sampling_ratio/mean": 1.0015816688537598,
      "sampling/importance_sampling_ratio/min": 0.5725432634353638,
      "sampling/sampling_logp_difference/max": 0.5576670169830322,
      "sampling/sampling_logp_difference/mean": 0.008417494595050812,
      "step": 333
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 146.0,
      "completions/max_terminated_length": 146.0,
      "completions/mean_length": 19.640625,
      "completions/mean_terminated_length": 19.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1658213883638382,
      "epoch": 0.5911504424778761,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 38.715214411963814,
      "kl": 0.1244959682226181,
      "learning_rate": 9.93187340145239e-07,
      "loss": -0.506,
      "num_tokens": 5256345.0,
      "reward": 0.125,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.125,
      "rewards/decision_reward_func/std": 1.0,
      "sampling/importance_sampling_ratio/max": 1.4853169918060303,
      "sampling/importance_sampling_ratio/mean": 0.99802565574646,
      "sampling/importance_sampling_ratio/min": 0.6318823099136353,
      "sampling/sampling_logp_difference/max": 0.45905208587646484,
      "sampling/sampling_logp_difference/mean": 0.017435431480407715,
      "step": 334
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 83.0,
      "completions/max_terminated_length": 83.0,
      "completions/mean_length": 14.25,
      "completions/mean_terminated_length": 14.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.16205374896526337,
      "epoch": 0.5929203539823009,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 45.914292343197125,
      "kl": 0.5228198170661926,
      "learning_rate": 9.93059702449693e-07,
      "loss": -0.1383,
      "num_tokens": 5267081.0,
      "reward": -0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.230634093284607,
      "sampling/importance_sampling_ratio/mean": 1.0003182888031006,
      "sampling/importance_sampling_ratio/min": 0.8197073936462402,
      "sampling/sampling_logp_difference/max": 0.20752954483032227,
      "sampling/sampling_logp_difference/mean": 0.0072051300667226315,
      "step": 335
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 133.0,
      "completions/max_terminated_length": 133.0,
      "completions/mean_length": 15.390625,
      "completions/mean_terminated_length": 15.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1279073804616928,
      "epoch": 0.5946902654867257,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 31.76875433357429,
      "kl": 0.5445804595947266,
      "learning_rate": 9.929308885105534e-07,
      "loss": -0.2191,
      "num_tokens": 5279250.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.1882487535476685,
      "sampling/importance_sampling_ratio/mean": 1.0010457038879395,
      "sampling/importance_sampling_ratio/min": 0.7639153599739075,
      "sampling/sampling_logp_difference/max": 0.2692983150482178,
      "sampling/sampling_logp_difference/mean": 0.00909061636775732,
      "step": 336
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 98.0,
      "completions/max_terminated_length": 98.0,
      "completions/mean_length": 16.4375,
      "completions/mean_terminated_length": 16.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.12667866051197052,
      "epoch": 0.5964601769911504,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 50.528642905457026,
      "kl": 0.21811896562576294,
      "learning_rate": 9.928008986351186e-07,
      "loss": -0.4598,
      "num_tokens": 5290814.0,
      "reward": 0.625,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.3493925333023071,
      "sampling/importance_sampling_ratio/mean": 1.000293493270874,
      "sampling/importance_sampling_ratio/min": 0.7212910056114197,
      "sampling/sampling_logp_difference/max": 0.32671260833740234,
      "sampling/sampling_logp_difference/mean": 0.007266349159181118,
      "step": 337
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 283.0,
      "completions/max_terminated_length": 283.0,
      "completions/mean_length": 78.5625,
      "completions/mean_terminated_length": 78.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.40841299295425415,
      "epoch": 0.5982300884955752,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 22.944560152841685,
      "kl": 0.12588217854499817,
      "learning_rate": 9.926697331334924e-07,
      "loss": -0.3419,
      "num_tokens": 5306290.0,
      "reward": 0.15625,
      "reward_std": 0.7297805547714233,
      "rewards/decision_reward_func/mean": 0.15625,
      "rewards/decision_reward_func/std": 0.9955257177352905,
      "sampling/importance_sampling_ratio/max": 1.8606716394424438,
      "sampling/importance_sampling_ratio/mean": 1.0010006427764893,
      "sampling/importance_sampling_ratio/min": 0.4402218163013458,
      "sampling/sampling_logp_difference/max": 0.8204765319824219,
      "sampling/sampling_logp_difference/mean": 0.02122463285923004,
      "step": 338
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 168.0,
      "completions/max_terminated_length": 168.0,
      "completions/mean_length": 69.5,
      "completions/mean_terminated_length": 69.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3494517505168915,
      "epoch": 0.6,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03945891901988461,
      "kl": 0.004555393010377884,
      "learning_rate": 9.925373923185834e-07,
      "loss": 0.0001,
      "num_tokens": 5320178.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2857304811477661,
      "sampling/importance_sampling_ratio/mean": 0.9992601871490479,
      "sampling/importance_sampling_ratio/min": 0.6282122731208801,
      "sampling/sampling_logp_difference/max": 0.4648771286010742,
      "sampling/sampling_logp_difference/mean": 0.012271528132259846,
      "step": 339
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 183.0,
      "completions/max_terminated_length": 183.0,
      "completions/mean_length": 86.203125,
      "completions/mean_terminated_length": 86.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.544769287109375,
      "epoch": 0.6017699115044248,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 10.513692113628164,
      "kl": 0.012212522327899933,
      "learning_rate": 9.92403876506104e-07,
      "loss": -0.1197,
      "num_tokens": 5336207.0,
      "reward": 0.8125,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.8125,
      "rewards/decision_reward_func/std": 0.5875696539878845,
      "sampling/importance_sampling_ratio/max": 1.5996742248535156,
      "sampling/importance_sampling_ratio/mean": 1.00087571144104,
      "sampling/importance_sampling_ratio/min": 0.7218353748321533,
      "sampling/sampling_logp_difference/max": 0.4697999954223633,
      "sampling/sampling_logp_difference/mean": 0.018725665286183357,
      "step": 340
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 186.0,
      "completions/max_terminated_length": 186.0,
      "completions/mean_length": 50.984375,
      "completions/mean_terminated_length": 50.984375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.25952285528182983,
      "epoch": 0.6035398230088496,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.13402526271688212,
      "kl": 0.025582045316696167,
      "learning_rate": 9.922691860145696e-07,
      "loss": 0.0001,
      "num_tokens": 5350654.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.56446373462677,
      "sampling/importance_sampling_ratio/mean": 1.0010714530944824,
      "sampling/importance_sampling_ratio/min": 0.6776019930839539,
      "sampling/sampling_logp_difference/max": 0.4475431442260742,
      "sampling/sampling_logp_difference/mean": 0.015527051873505116,
      "step": 341
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 181.0,
      "completions/max_terminated_length": 181.0,
      "completions/mean_length": 51.3125,
      "completions/mean_terminated_length": 51.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.30644503235816956,
      "epoch": 0.6053097345132743,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.28312367214821127,
      "kl": 0.05292034149169922,
      "learning_rate": 9.921333211652977e-07,
      "loss": 0.0002,
      "num_tokens": 5365426.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.364556074142456,
      "sampling/importance_sampling_ratio/mean": 1.0007081031799316,
      "sampling/importance_sampling_ratio/min": 0.559749960899353,
      "sampling/sampling_logp_difference/max": 0.5802650451660156,
      "sampling/sampling_logp_difference/mean": 0.014037737622857094,
      "step": 342
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 252.0,
      "completions/max_terminated_length": 252.0,
      "completions/mean_length": 97.203125,
      "completions/mean_terminated_length": 97.203125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3141934275627136,
      "epoch": 0.6070796460176991,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.040306133956537274,
      "kl": 0.008490835316479206,
      "learning_rate": 9.919962822824083e-07,
      "loss": 0.0001,
      "num_tokens": 5381807.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.52422034740448,
      "sampling/importance_sampling_ratio/mean": 1.0011824369430542,
      "sampling/importance_sampling_ratio/min": 0.6413918733596802,
      "sampling/sampling_logp_difference/max": 0.44411468505859375,
      "sampling/sampling_logp_difference/mean": 0.011814025230705738,
      "step": 343
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 370.0,
      "completions/max_terminated_length": 370.0,
      "completions/mean_length": 112.453125,
      "completions/mean_terminated_length": 112.453125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3980545401573181,
      "epoch": 0.6088495575221239,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03816208034133788,
      "kl": 0.014165371656417847,
      "learning_rate": 9.918580696928205e-07,
      "loss": 0.0001,
      "num_tokens": 5400892.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.8213403224945068,
      "sampling/importance_sampling_ratio/mean": 0.999814510345459,
      "sampling/importance_sampling_ratio/min": 0.5459651947021484,
      "sampling/sampling_logp_difference/max": 0.6052000522613525,
      "sampling/sampling_logp_difference/mean": 0.022808555513620377,
      "step": 344
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 197.0,
      "completions/max_terminated_length": 197.0,
      "completions/mean_length": 65.78125,
      "completions/mean_terminated_length": 65.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3655658960342407,
      "epoch": 0.6106194690265486,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08708536054248218,
      "kl": 0.019657455384731293,
      "learning_rate": 9.91718683726255e-07,
      "loss": 0.0001,
      "num_tokens": 5417822.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4601846933364868,
      "sampling/importance_sampling_ratio/mean": 1.0012413263320923,
      "sampling/importance_sampling_ratio/min": 0.6977089643478394,
      "sampling/sampling_logp_difference/max": 0.37856292724609375,
      "sampling/sampling_logp_difference/mean": 0.016577202826738358,
      "step": 345
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 128.0,
      "completions/max_terminated_length": 128.0,
      "completions/mean_length": 35.328125,
      "completions/mean_terminated_length": 35.328125,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.22649946808815002,
      "epoch": 0.6123893805309735,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2680127798221453,
      "kl": 0.037691764533519745,
      "learning_rate": 9.915781247152308e-07,
      "loss": 0.0002,
      "num_tokens": 5431411.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6613584756851196,
      "sampling/importance_sampling_ratio/mean": 1.000566005706787,
      "sampling/importance_sampling_ratio/min": 0.6990428566932678,
      "sampling/sampling_logp_difference/max": 0.5076355934143066,
      "sampling/sampling_logp_difference/mean": 0.012610787525773048,
      "step": 346
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 213.0,
      "completions/max_terminated_length": 213.0,
      "completions/mean_length": 80.25,
      "completions/mean_terminated_length": 80.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.38664960861206055,
      "epoch": 0.6141592920353982,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07839437680152708,
      "kl": 0.013993092812597752,
      "learning_rate": 9.914363929950657e-07,
      "loss": 0.0001,
      "num_tokens": 5447107.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0008046627044678,
      "sampling/importance_sampling_ratio/min": 0.5363089442253113,
      "sampling/sampling_logp_difference/max": 1.106311321258545,
      "sampling/sampling_logp_difference/mean": 0.018526988103985786,
      "step": 347
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 241.0,
      "completions/max_terminated_length": 241.0,
      "completions/mean_length": 88.0,
      "completions/mean_terminated_length": 88.0,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.35303232073783875,
      "epoch": 0.6159292035398231,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1290413396036071,
      "kl": 0.03378347307443619,
      "learning_rate": 9.91293488903875e-07,
      "loss": 0.0002,
      "num_tokens": 5464083.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5305931568145752,
      "sampling/importance_sampling_ratio/mean": 1.0008561611175537,
      "sampling/importance_sampling_ratio/min": 0.6892523765563965,
      "sampling/sampling_logp_difference/max": 0.4256553649902344,
      "sampling/sampling_logp_difference/mean": 0.012805117294192314,
      "step": 348
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 403.0,
      "completions/max_terminated_length": 403.0,
      "completions/mean_length": 101.84375,
      "completions/mean_terminated_length": 101.84375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4014196991920471,
      "epoch": 0.6176991150442478,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05282522173640956,
      "kl": 0.012553641572594643,
      "learning_rate": 9.91149412782571e-07,
      "loss": 0.0001,
      "num_tokens": 5481881.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4656864404678345,
      "sampling/importance_sampling_ratio/mean": 0.9996223449707031,
      "sampling/importance_sampling_ratio/min": 0.6784743070602417,
      "sampling/sampling_logp_difference/max": 0.3879086971282959,
      "sampling/sampling_logp_difference/mean": 0.015167336910963058,
      "step": 349
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 433.0,
      "completions/max_terminated_length": 433.0,
      "completions/mean_length": 121.859375,
      "completions/mean_terminated_length": 121.859375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.42137038707733154,
      "epoch": 0.6194690265486725,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.17263349465215008,
      "kl": 0.021598845720291138,
      "learning_rate": 9.910041649748612e-07,
      "loss": 0.0002,
      "num_tokens": 5498240.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9998193979263306,
      "sampling/importance_sampling_ratio/min": 0.42447274923324585,
      "sampling/sampling_logp_difference/max": 0.8569074869155884,
      "sampling/sampling_logp_difference/mean": 0.015872254967689514,
      "step": 350
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 348.0,
      "completions/max_terminated_length": 348.0,
      "completions/mean_length": 120.828125,
      "completions/mean_terminated_length": 120.828125,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5624607801437378,
      "epoch": 0.6212389380530974,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07907780312988107,
      "kl": 0.04852885752916336,
      "learning_rate": 9.908577458272495e-07,
      "loss": 0.0005,
      "num_tokens": 5517605.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.870368242263794,
      "sampling/importance_sampling_ratio/mean": 1.0003533363342285,
      "sampling/importance_sampling_ratio/min": 0.5663659572601318,
      "sampling/sampling_logp_difference/max": 0.6261353492736816,
      "sampling/sampling_logp_difference/mean": 0.021383430808782578,
      "step": 351
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 382.0,
      "completions/max_terminated_length": 382.0,
      "completions/mean_length": 130.515625,
      "completions/mean_terminated_length": 130.515625,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.510712742805481,
      "epoch": 0.6230088495575221,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.14652883275587417,
      "kl": 0.037225618958473206,
      "learning_rate": 9.907101556890331e-07,
      "loss": 0.0004,
      "num_tokens": 5535334.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000260829925537,
      "sampling/importance_sampling_ratio/min": 0.6686792373657227,
      "sampling/sampling_logp_difference/max": 0.7186784744262695,
      "sampling/sampling_logp_difference/mean": 0.017669696360826492,
      "step": 352
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 313.0,
      "completions/max_terminated_length": 313.0,
      "completions/mean_length": 134.375,
      "completions/mean_terminated_length": 134.375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4618055820465088,
      "epoch": 0.6247787610619469,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0578213304417604,
      "kl": 0.034855760633945465,
      "learning_rate": 9.905613949123034e-07,
      "loss": 0.0004,
      "num_tokens": 5553918.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.6922149658203125,
      "sampling/importance_sampling_ratio/mean": 0.9999423027038574,
      "sampling/importance_sampling_ratio/min": 0.6325134038925171,
      "sampling/sampling_logp_difference/max": 0.5260382890701294,
      "sampling/sampling_logp_difference/mean": 0.01687721721827984,
      "step": 353
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 310.0,
      "completions/max_terminated_length": 310.0,
      "completions/mean_length": 122.09375,
      "completions/mean_terminated_length": 122.09375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.6637468338012695,
      "epoch": 0.6265486725663717,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05067320244235177,
      "kl": 0.02216402254998684,
      "learning_rate": 9.904114638519443e-07,
      "loss": 0.0003,
      "num_tokens": 5573492.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.7573411464691162,
      "sampling/importance_sampling_ratio/mean": 0.9993027448654175,
      "sampling/importance_sampling_ratio/min": 0.566085159778595,
      "sampling/sampling_logp_difference/max": 0.5690107345581055,
      "sampling/sampling_logp_difference/mean": 0.02189033478498459,
      "step": 354
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 303.0,
      "completions/max_terminated_length": 303.0,
      "completions/mean_length": 108.984375,
      "completions/mean_terminated_length": 108.984375,
      "completions/min_length": 51.0,
      "completions/min_terminated_length": 51.0,
      "entropy": 0.5064927339553833,
      "epoch": 0.6283185840707964,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.022769992272464255,
      "kl": 0.003910430707037449,
      "learning_rate": 9.902603628656311e-07,
      "loss": 0.0,
      "num_tokens": 5590211.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4734493494033813,
      "sampling/importance_sampling_ratio/mean": 1.0005618333816528,
      "sampling/importance_sampling_ratio/min": 0.6874695420265198,
      "sampling/sampling_logp_difference/max": 0.387606143951416,
      "sampling/sampling_logp_difference/mean": 0.017741823568940163,
      "step": 355
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 290.0,
      "completions/max_terminated_length": 290.0,
      "completions/mean_length": 146.703125,
      "completions/mean_terminated_length": 146.703125,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5128886103630066,
      "epoch": 0.6300884955752213,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05910555125404528,
      "kl": 0.06985120475292206,
      "learning_rate": 9.901080923138308e-07,
      "loss": 0.0006,
      "num_tokens": 5622528.0,
      "reward": -0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000289797782898,
      "sampling/importance_sampling_ratio/min": 0.6196466088294983,
      "sampling/sampling_logp_difference/max": 0.8414640426635742,
      "sampling/sampling_logp_difference/mean": 0.01770433411002159,
      "step": 356
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 868.0,
      "completions/max_terminated_length": 868.0,
      "completions/mean_length": 178.484375,
      "completions/mean_terminated_length": 178.484375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4746066629886627,
      "epoch": 0.631858407079646,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05989841681630618,
      "kl": 0.027999769896268845,
      "learning_rate": 9.899546525597997e-07,
      "loss": 0.0003,
      "num_tokens": 5644367.0,
      "reward": -0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.527292013168335,
      "sampling/importance_sampling_ratio/mean": 0.9995212554931641,
      "sampling/importance_sampling_ratio/min": 0.44542253017425537,
      "sampling/sampling_logp_difference/max": 0.8087320327758789,
      "sampling/sampling_logp_difference/mean": 0.017029207199811935,
      "step": 357
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 193.0,
      "completions/max_terminated_length": 193.0,
      "completions/mean_length": 112.890625,
      "completions/mean_terminated_length": 112.890625,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.620051383972168,
      "epoch": 0.6336283185840708,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.026391207446395157,
      "kl": 0.005611338187009096,
      "learning_rate": 9.898000439695843e-07,
      "loss": 0.0001,
      "num_tokens": 5661352.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.9880595207214355,
      "sampling/importance_sampling_ratio/mean": 1.0005333423614502,
      "sampling/importance_sampling_ratio/min": 0.7001907825469971,
      "sampling/sampling_logp_difference/max": 0.6871590614318848,
      "sampling/sampling_logp_difference/mean": 0.02070365846157074,
      "step": 358
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 345.0,
      "completions/max_terminated_length": 345.0,
      "completions/mean_length": 122.03125,
      "completions/mean_terminated_length": 122.03125,
      "completions/min_length": 29.0,
      "completions/min_terminated_length": 29.0,
      "entropy": 0.5711897611618042,
      "epoch": 0.6353982300884956,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.36483039142888785,
      "kl": 0.0318559929728508,
      "learning_rate": 9.896442669120187e-07,
      "loss": 0.0003,
      "num_tokens": 5679354.0,
      "reward": -0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.430940866470337,
      "sampling/importance_sampling_ratio/mean": 0.9996042847633362,
      "sampling/importance_sampling_ratio/min": 0.4420183300971985,
      "sampling/sampling_logp_difference/max": 0.816403865814209,
      "sampling/sampling_logp_difference/mean": 0.019348707050085068,
      "step": 359
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 175.0,
      "completions/max_terminated_length": 175.0,
      "completions/mean_length": 77.421875,
      "completions/mean_terminated_length": 77.421875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.31168049573898315,
      "epoch": 0.6371681415929203,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03541548343871878,
      "kl": 0.009070035070180893,
      "learning_rate": 9.894873217587245e-07,
      "loss": 0.0001,
      "num_tokens": 5697653.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5912870168685913,
      "sampling/importance_sampling_ratio/mean": 1.0003658533096313,
      "sampling/importance_sampling_ratio/min": 0.7069886326789856,
      "sampling/sampling_logp_difference/max": 0.4645431637763977,
      "sampling/sampling_logp_difference/mean": 0.01266229897737503,
      "step": 360
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 404.0,
      "completions/max_terminated_length": 404.0,
      "completions/mean_length": 158.796875,
      "completions/mean_terminated_length": 158.796875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.5861371755599976,
      "epoch": 0.6389380530973451,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.910836183478518,
      "kl": 0.04382926970720291,
      "learning_rate": 9.893292088841108e-07,
      "loss": 0.046,
      "num_tokens": 5719336.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.6420366764068604,
      "sampling/importance_sampling_ratio/mean": 1.0004842281341553,
      "sampling/importance_sampling_ratio/min": 0.48981279134750366,
      "sampling/sampling_logp_difference/max": 0.7137320041656494,
      "sampling/sampling_logp_difference/mean": 0.01918848231434822,
      "step": 361
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 300.0,
      "completions/max_terminated_length": 300.0,
      "completions/mean_length": 110.828125,
      "completions/mean_terminated_length": 110.828125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.431569904088974,
      "epoch": 0.6407079646017699,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.212237165655271,
      "kl": 0.0400247722864151,
      "learning_rate": 9.891699286653712e-07,
      "loss": -0.0838,
      "num_tokens": 5740701.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.780729055404663,
      "sampling/importance_sampling_ratio/mean": 1.0001611709594727,
      "sampling/importance_sampling_ratio/min": 0.5716253519058228,
      "sampling/sampling_logp_difference/max": 0.5770227909088135,
      "sampling/sampling_logp_difference/mean": 0.01780252903699875,
      "step": 362
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 620.0,
      "completions/max_terminated_length": 620.0,
      "completions/mean_length": 185.890625,
      "completions/mean_terminated_length": 185.890625,
      "completions/min_length": 18.0,
      "completions/min_terminated_length": 18.0,
      "entropy": 0.48092374205589294,
      "epoch": 0.6424778761061947,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.68364450863825,
      "kl": 0.029903601855039597,
      "learning_rate": 9.890094814824852e-07,
      "loss": -0.0082,
      "num_tokens": 5764518.0,
      "reward": 0.15625,
      "reward_std": 0.34860679507255554,
      "rewards/decision_reward_func/mean": 0.15625,
      "rewards/decision_reward_func/std": 0.9955257177352905,
      "sampling/importance_sampling_ratio/max": 1.4594154357910156,
      "sampling/importance_sampling_ratio/mean": 1.0001177787780762,
      "sampling/importance_sampling_ratio/min": 0.4550616145133972,
      "sampling/sampling_logp_difference/max": 0.7873225212097168,
      "sampling/sampling_logp_difference/mean": 0.017083294689655304,
      "step": 363
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 325.0,
      "completions/max_terminated_length": 325.0,
      "completions/mean_length": 98.375,
      "completions/mean_terminated_length": 98.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4051584005355835,
      "epoch": 0.6442477876106195,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.034731977046040044,
      "kl": 0.013031707145273685,
      "learning_rate": 9.888478677182154e-07,
      "loss": 0.0001,
      "num_tokens": 5782318.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3868625164031982,
      "sampling/importance_sampling_ratio/mean": 1.0004628896713257,
      "sampling/importance_sampling_ratio/min": 0.6924977898597717,
      "sampling/sampling_logp_difference/max": 0.3674502372741699,
      "sampling/sampling_logp_difference/mean": 0.013802760280668736,
      "step": 364
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 710.0,
      "completions/max_terminated_length": 710.0,
      "completions/mean_length": 104.421875,
      "completions/mean_terminated_length": 104.421875,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.4693533480167389,
      "epoch": 0.6460176991150443,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.0831511432804044,
      "kl": 0.024606188759207726,
      "learning_rate": 9.886850877581078e-07,
      "loss": -0.1668,
      "num_tokens": 5801369.0,
      "reward": -0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": -0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.9070464372634888,
      "sampling/importance_sampling_ratio/mean": 1.00021493434906,
      "sampling/importance_sampling_ratio/min": 0.4210282862186432,
      "sampling/sampling_logp_difference/max": 0.8650553226470947,
      "sampling/sampling_logp_difference/mean": 0.016472456976771355,
      "step": 365
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 86.09375,
      "completions/mean_terminated_length": 86.09375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.3831039369106293,
      "epoch": 0.647787610619469,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05046837503924455,
      "kl": 0.008188454434275627,
      "learning_rate": 9.885211419904903e-07,
      "loss": 0.0001,
      "num_tokens": 5816703.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4862701892852783,
      "sampling/importance_sampling_ratio/mean": 0.9999388456344604,
      "sampling/importance_sampling_ratio/min": 0.7058389782905579,
      "sampling/sampling_logp_difference/max": 0.3962697982788086,
      "sampling/sampling_logp_difference/mean": 0.018592428416013718,
      "step": 366
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 279.0,
      "completions/max_terminated_length": 279.0,
      "completions/mean_length": 82.09375,
      "completions/mean_terminated_length": 82.09375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.25472283363342285,
      "epoch": 0.6495575221238938,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04646732187995595,
      "kl": 0.018464451655745506,
      "learning_rate": 9.883560308064722e-07,
      "loss": 0.0001,
      "num_tokens": 5834549.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5658870935440063,
      "sampling/importance_sampling_ratio/mean": 0.9994906187057495,
      "sampling/importance_sampling_ratio/min": 0.7226534485816956,
      "sampling/sampling_logp_difference/max": 0.4484524726867676,
      "sampling/sampling_logp_difference/mean": 0.013782997615635395,
      "step": 367
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 247.0,
      "completions/max_terminated_length": 247.0,
      "completions/mean_length": 105.5625,
      "completions/mean_terminated_length": 105.5625,
      "completions/min_length": 13.0,
      "completions/min_terminated_length": 13.0,
      "entropy": 0.3836281895637512,
      "epoch": 0.6513274336283186,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05150506273323067,
      "kl": 0.016738034784793854,
      "learning_rate": 9.881897545999429e-07,
      "loss": 0.0002,
      "num_tokens": 5852649.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4687012434005737,
      "sampling/importance_sampling_ratio/mean": 1.0002315044403076,
      "sampling/importance_sampling_ratio/min": 0.6318727731704712,
      "sampling/sampling_logp_difference/max": 0.4590672254562378,
      "sampling/sampling_logp_difference/mean": 0.013846302404999733,
      "step": 368
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 399.0,
      "completions/max_terminated_length": 399.0,
      "completions/mean_length": 119.6875,
      "completions/mean_terminated_length": 119.6875,
      "completions/min_length": 14.0,
      "completions/min_terminated_length": 14.0,
      "entropy": 0.4400315582752228,
      "epoch": 0.6530973451327433,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04148350669275565,
      "kl": 0.017446907237172127,
      "learning_rate": 9.880223137675707e-07,
      "loss": 0.0002,
      "num_tokens": 5871701.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.461326003074646,
      "sampling/importance_sampling_ratio/mean": 1.0014960765838623,
      "sampling/importance_sampling_ratio/min": 0.6738423109054565,
      "sampling/sampling_logp_difference/max": 0.3947591781616211,
      "sampling/sampling_logp_difference/mean": 0.017619699239730835,
      "step": 369
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 417.0,
      "completions/max_terminated_length": 417.0,
      "completions/mean_length": 77.9375,
      "completions/mean_terminated_length": 77.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.16520312428474426,
      "epoch": 0.6548672566371682,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.952300200899005,
      "kl": 0.03090791031718254,
      "learning_rate": 9.87853708708803e-07,
      "loss": 0.1817,
      "num_tokens": 5892465.0,
      "reward": 0.65625,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.707594871520996,
      "sampling/importance_sampling_ratio/mean": 0.9993278980255127,
      "sampling/importance_sampling_ratio/min": 0.6782475709915161,
      "sampling/sampling_logp_difference/max": 0.535085916519165,
      "sampling/sampling_logp_difference/mean": 0.012654388323426247,
      "step": 370
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 355.0,
      "completions/max_terminated_length": 355.0,
      "completions/mean_length": 113.5625,
      "completions/mean_terminated_length": 113.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3643713891506195,
      "epoch": 0.6566371681415929,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.294525677618649,
      "kl": 0.0199178084731102,
      "learning_rate": 9.876839398258639e-07,
      "loss": 0.1817,
      "num_tokens": 5911253.0,
      "reward": 0.78125,
      "reward_std": 0.2561737596988678,
      "rewards/decision_reward_func/mean": 0.78125,
      "rewards/decision_reward_func/std": 0.6291528940200806,
      "sampling/importance_sampling_ratio/max": 1.5951619148254395,
      "sampling/importance_sampling_ratio/mean": 1.0007092952728271,
      "sampling/importance_sampling_ratio/min": 0.39645707607269287,
      "sampling/sampling_logp_difference/max": 0.9251875877380371,
      "sampling/sampling_logp_difference/mean": 0.014422750100493431,
      "step": 371
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 325.0,
      "completions/max_terminated_length": 325.0,
      "completions/mean_length": 106.671875,
      "completions/mean_terminated_length": 106.671875,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.39586547017097473,
      "epoch": 0.6584070796460177,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02170165706755283,
      "kl": 0.00442938506603241,
      "learning_rate": 9.875130075237543e-07,
      "loss": 0.0,
      "num_tokens": 5928768.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6555458307266235,
      "sampling/importance_sampling_ratio/mean": 1.0000089406967163,
      "sampling/importance_sampling_ratio/min": 0.7011512517929077,
      "sampling/sampling_logp_difference/max": 0.5041307210922241,
      "sampling/sampling_logp_difference/mean": 0.014346872456371784,
      "step": 372
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 506.0,
      "completions/max_terminated_length": 506.0,
      "completions/mean_length": 71.5,
      "completions/mean_terminated_length": 71.5,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.11664820462465286,
      "epoch": 0.6601769911504425,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.413113126385533,
      "kl": 0.0214911587536335,
      "learning_rate": 9.873409122102503e-07,
      "loss": 0.185,
      "num_tokens": 5946384.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.7430909872055054,
      "sampling/importance_sampling_ratio/mean": 0.9998141527175903,
      "sampling/importance_sampling_ratio/min": 0.7246072292327881,
      "sampling/sampling_logp_difference/max": 0.5556598901748657,
      "sampling/sampling_logp_difference/mean": 0.007141374982893467,
      "step": 373
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 350.0,
      "completions/max_terminated_length": 350.0,
      "completions/mean_length": 99.90625,
      "completions/mean_terminated_length": 99.90625,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.49403318762779236,
      "epoch": 0.6619469026548672,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.065293362113313,
      "kl": 0.009674089029431343,
      "learning_rate": 9.87167654295903e-07,
      "loss": 0.083,
      "num_tokens": 5965722.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.5598499774932861,
      "sampling/importance_sampling_ratio/mean": 0.9995803833007812,
      "sampling/importance_sampling_ratio/min": 0.5161864161491394,
      "sampling/sampling_logp_difference/max": 0.6612873077392578,
      "sampling/sampling_logp_difference/mean": 0.018438229337334633,
      "step": 374
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 191.0,
      "completions/max_terminated_length": 191.0,
      "completions/mean_length": 49.8125,
      "completions/mean_terminated_length": 49.8125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.1185794249176979,
      "epoch": 0.6637168141592921,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0955250497511546,
      "kl": 0.015874072909355164,
      "learning_rate": 9.869932341940358e-07,
      "loss": 0.0001,
      "num_tokens": 5981086.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0023404359817505,
      "sampling/importance_sampling_ratio/min": 0.5194261074066162,
      "sampling/sampling_logp_difference/max": 0.9786150455474854,
      "sampling/sampling_logp_difference/mean": 0.023584650829434395,
      "step": 375
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 191.0,
      "completions/max_terminated_length": 191.0,
      "completions/mean_length": 42.40625,
      "completions/mean_terminated_length": 42.40625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18201380968093872,
      "epoch": 0.6654867256637168,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.18587050428707652,
      "kl": 0.0745650976896286,
      "learning_rate": 9.868176523207463e-07,
      "loss": 0.0004,
      "num_tokens": 5994552.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3351178169250488,
      "sampling/importance_sampling_ratio/mean": 0.9991710186004639,
      "sampling/importance_sampling_ratio/min": 0.5561547875404358,
      "sampling/sampling_logp_difference/max": 0.586708664894104,
      "sampling/sampling_logp_difference/mean": 0.014691246673464775,
      "step": 376
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 218.0,
      "completions/max_terminated_length": 218.0,
      "completions/mean_length": 74.796875,
      "completions/mean_terminated_length": 74.796875,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.30755698680877686,
      "epoch": 0.6672566371681415,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10144206392078267,
      "kl": 0.0294176135212183,
      "learning_rate": 9.86640909094902e-07,
      "loss": 0.0001,
      "num_tokens": 6008331.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3973352909088135,
      "sampling/importance_sampling_ratio/mean": 0.9999659657478333,
      "sampling/importance_sampling_ratio/min": 0.40352514386177063,
      "sampling/sampling_logp_difference/max": 0.9075164794921875,
      "sampling/sampling_logp_difference/mean": 0.014053567312657833,
      "step": 377
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 232.0,
      "completions/max_terminated_length": 232.0,
      "completions/mean_length": 66.953125,
      "completions/mean_terminated_length": 66.953125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.25754937529563904,
      "epoch": 0.6690265486725664,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.230792858420169,
      "kl": 0.024930693209171295,
      "learning_rate": 9.864630049381424e-07,
      "loss": 0.0888,
      "num_tokens": 6022776.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.4337999820709229,
      "sampling/importance_sampling_ratio/mean": 0.9992684721946716,
      "sampling/importance_sampling_ratio/min": 0.6020495295524597,
      "sampling/sampling_logp_difference/max": 0.5074155330657959,
      "sampling/sampling_logp_difference/mean": 0.009889904409646988,
      "step": 378
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 181.0,
      "completions/max_terminated_length": 181.0,
      "completions/mean_length": 43.34375,
      "completions/mean_terminated_length": 43.34375,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.17238153517246246,
      "epoch": 0.6707964601769911,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1524987210184169,
      "kl": 0.1290230005979538,
      "learning_rate": 9.862839402748753e-07,
      "loss": 0.0006,
      "num_tokens": 6036398.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5416227579116821,
      "sampling/importance_sampling_ratio/mean": 1.0005288124084473,
      "sampling/importance_sampling_ratio/min": 0.6389191746711731,
      "sampling/sampling_logp_difference/max": 0.44797730445861816,
      "sampling/sampling_logp_difference/mean": 0.010033435188233852,
      "step": 379
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 144.0,
      "completions/max_terminated_length": 144.0,
      "completions/mean_length": 35.71875,
      "completions/mean_terminated_length": 35.71875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.1937960535287857,
      "epoch": 0.672566371681416,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.898301167597465,
      "kl": 0.202432781457901,
      "learning_rate": 9.861037155322776e-07,
      "loss": -0.178,
      "num_tokens": 6049132.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.343531608581543,
      "sampling/importance_sampling_ratio/mean": 1.001887321472168,
      "sampling/importance_sampling_ratio/min": 0.6436159610748291,
      "sampling/sampling_logp_difference/max": 0.44065308570861816,
      "sampling/sampling_logp_difference/mean": 0.013763444498181343,
      "step": 380
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 195.0,
      "completions/max_terminated_length": 195.0,
      "completions/mean_length": 38.84375,
      "completions/mean_terminated_length": 38.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.26760339736938477,
      "epoch": 0.6743362831858407,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 30.52928589488721,
      "kl": 0.09609369188547134,
      "learning_rate": 9.859223311402936e-07,
      "loss": -0.4911,
      "num_tokens": 6062402.0,
      "reward": -0.03125,
      "reward_std": 0.42516323924064636,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.7370104789733887,
      "sampling/importance_sampling_ratio/mean": 1.0004743337631226,
      "sampling/importance_sampling_ratio/min": 0.7212648987770081,
      "sampling/sampling_logp_difference/max": 0.5521655082702637,
      "sampling/sampling_logp_difference/mean": 0.014504313468933105,
      "step": 381
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 116.0,
      "completions/max_terminated_length": 116.0,
      "completions/mean_length": 22.125,
      "completions/mean_terminated_length": 22.125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.1711660772562027,
      "epoch": 0.6761061946902654,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 24.017188586007887,
      "kl": 0.3548486530780792,
      "learning_rate": 9.85739787531634e-07,
      "loss": -0.3051,
      "num_tokens": 6074586.0,
      "reward": -0.375,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": -0.375,
      "rewards/decision_reward_func/std": 0.934353232383728,
      "sampling/importance_sampling_ratio/max": 1.6289836168289185,
      "sampling/importance_sampling_ratio/mean": 1.0003910064697266,
      "sampling/importance_sampling_ratio/min": 0.5489668846130371,
      "sampling/sampling_logp_difference/max": 0.5997171401977539,
      "sampling/sampling_logp_difference/mean": 0.013700541108846664,
      "step": 382
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1171.0,
      "completions/max_terminated_length": 1171.0,
      "completions/mean_length": 59.65625,
      "completions/mean_terminated_length": 59.65625,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.17006395757198334,
      "epoch": 0.6778761061946903,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06476926527288644,
      "kl": 0.22285808622837067,
      "learning_rate": 9.85556085141775e-07,
      "loss": 0.0008,
      "num_tokens": 6088740.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.5189270973205566,
      "sampling/importance_sampling_ratio/mean": 0.9982098340988159,
      "sampling/importance_sampling_ratio/min": 0.7894731760025024,
      "sampling/sampling_logp_difference/max": 0.41800427436828613,
      "sampling/sampling_logp_difference/mean": 0.009094467386603355,
      "step": 383
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 112.0,
      "completions/max_terminated_length": 112.0,
      "completions/mean_length": 20.15625,
      "completions/mean_terminated_length": 20.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.15394248068332672,
      "epoch": 0.679646017699115,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 21.141641074846657,
      "kl": 0.3241913914680481,
      "learning_rate": 9.853712244089572e-07,
      "loss": 0.2209,
      "num_tokens": 6101086.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.425696611404419,
      "sampling/importance_sampling_ratio/mean": 1.0021278858184814,
      "sampling/importance_sampling_ratio/min": 0.6257392168045044,
      "sampling/sampling_logp_difference/max": 0.46882152557373047,
      "sampling/sampling_logp_difference/mean": 0.010466893203556538,
      "step": 384
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 17.1875,
      "completions/mean_terminated_length": 17.1875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06128706410527229,
      "epoch": 0.6814159292035398,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.4302172309995353,
      "kl": 0.2754594683647156,
      "learning_rate": 9.851852057741844e-07,
      "loss": 0.0027,
      "num_tokens": 6114986.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.1785469055175781,
      "sampling/importance_sampling_ratio/mean": 0.998342752456665,
      "sampling/importance_sampling_ratio/min": 0.8395922780036926,
      "sampling/sampling_logp_difference/max": 0.17483890056610107,
      "sampling/sampling_logp_difference/mean": 0.0034622247330844402,
      "step": 385
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 17.09375,
      "completions/mean_terminated_length": 17.09375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07266160845756531,
      "epoch": 0.6831858407079646,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.32445920449965693,
      "kl": 0.22553634643554688,
      "learning_rate": 9.849980296812231e-07,
      "loss": 0.0022,
      "num_tokens": 6126784.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3831980228424072,
      "sampling/importance_sampling_ratio/mean": 0.9980471134185791,
      "sampling/importance_sampling_ratio/min": 0.7396202683448792,
      "sampling/sampling_logp_difference/max": 0.3243982791900635,
      "sampling/sampling_logp_difference/mean": 0.01246151514351368,
      "step": 386
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1225.0,
      "completions/max_terminated_length": 1225.0,
      "completions/mean_length": 50.0625,
      "completions/mean_terminated_length": 50.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.36946654319763184,
      "epoch": 0.6849557522123894,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 10.48257294738574,
      "kl": 0.0802491158246994,
      "learning_rate": 9.848096965766002e-07,
      "loss": -0.1169,
      "num_tokens": 6141316.0,
      "reward": 0.625,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.3859869241714478,
      "sampling/importance_sampling_ratio/mean": 0.9985344409942627,
      "sampling/importance_sampling_ratio/min": 0.561350405216217,
      "sampling/sampling_logp_difference/max": 0.5774099826812744,
      "sampling/sampling_logp_difference/mean": 0.017032448202371597,
      "step": 387
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 197.0,
      "completions/max_terminated_length": 197.0,
      "completions/mean_length": 61.625,
      "completions/mean_terminated_length": 61.625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19366797804832458,
      "epoch": 0.6867256637168142,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.91991659893057,
      "kl": 0.08208681643009186,
      "learning_rate": 9.846202069096038e-07,
      "loss": -0.2283,
      "num_tokens": 6157004.0,
      "reward": 0.40625,
      "reward_std": 0.29578250646591187,
      "rewards/decision_reward_func/mean": 0.40625,
      "rewards/decision_reward_func/std": 0.9209855198860168,
      "sampling/importance_sampling_ratio/max": 1.6599446535110474,
      "sampling/importance_sampling_ratio/mean": 1.0001928806304932,
      "sampling/importance_sampling_ratio/min": 0.2644866704940796,
      "sampling/sampling_logp_difference/max": 1.3299643993377686,
      "sampling/sampling_logp_difference/mean": 0.010109896771609783,
      "step": 388
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1380.0,
      "completions/max_terminated_length": 1380.0,
      "completions/mean_length": 45.34375,
      "completions/mean_terminated_length": 45.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24957646429538727,
      "epoch": 0.6884955752212389,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.634748076840416,
      "kl": 0.20094981789588928,
      "learning_rate": 9.844295611322803e-07,
      "loss": -0.1671,
      "num_tokens": 6174466.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.3965059518814087,
      "sampling/importance_sampling_ratio/mean": 0.9998543858528137,
      "sampling/importance_sampling_ratio/min": 0.7887519001960754,
      "sampling/sampling_logp_difference/max": 0.33397340774536133,
      "sampling/sampling_logp_difference/mean": 0.0095017459243536,
      "step": 389
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1056.0,
      "completions/max_terminated_length": 1056.0,
      "completions/mean_length": 112.140625,
      "completions/mean_terminated_length": 112.140625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2890416085720062,
      "epoch": 0.6902654867256637,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 7.721867954809243,
      "kl": 0.016003964468836784,
      "learning_rate": 9.842377596994344e-07,
      "loss": 0.9181,
      "num_tokens": 6194203.0,
      "reward": 0.875,
      "reward_std": 0.42078250646591187,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 1.59376060962677,
      "sampling/importance_sampling_ratio/mean": 1.0002254247665405,
      "sampling/importance_sampling_ratio/min": 0.6421988010406494,
      "sampling/sampling_logp_difference/max": 0.4660964012145996,
      "sampling/sampling_logp_difference/mean": 0.010162144899368286,
      "step": 390
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 206.0,
      "completions/max_terminated_length": 206.0,
      "completions/mean_length": 61.90625,
      "completions/mean_terminated_length": 61.90625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.39551448822021484,
      "epoch": 0.6920353982300885,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.681739496403345,
      "kl": 0.016538335010409355,
      "learning_rate": 9.84044803068628e-07,
      "loss": -0.1438,
      "num_tokens": 6208901.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.596825122833252,
      "sampling/importance_sampling_ratio/mean": 1.0015642642974854,
      "sampling/importance_sampling_ratio/min": 0.6489729285240173,
      "sampling/sampling_logp_difference/max": 0.4680173397064209,
      "sampling/sampling_logp_difference/mean": 0.018785130232572556,
      "step": 391
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 838.0,
      "completions/max_terminated_length": 838.0,
      "completions/mean_length": 60.109375,
      "completions/mean_terminated_length": 60.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3214569389820099,
      "epoch": 0.6938053097345133,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.057294202411554675,
      "kl": 0.011079341173171997,
      "learning_rate": 9.838506917001784e-07,
      "loss": 0.0001,
      "num_tokens": 6223164.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.2777345180511475,
      "sampling/importance_sampling_ratio/mean": 1.0013971328735352,
      "sampling/importance_sampling_ratio/min": 0.6055299639701843,
      "sampling/sampling_logp_difference/max": 0.5016512870788574,
      "sampling/sampling_logp_difference/mean": 0.013057949021458626,
      "step": 392
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 712.0,
      "completions/max_terminated_length": 712.0,
      "completions/mean_length": 140.515625,
      "completions/mean_terminated_length": 140.515625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4342554211616516,
      "epoch": 0.695575221238938,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.046959634621339,
      "kl": 0.013974161818623543,
      "learning_rate": 9.836554260571577e-07,
      "loss": -0.0494,
      "num_tokens": 6242509.0,
      "reward": 0.625,
      "reward_std": 0.22360679507255554,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 1.4464900493621826,
      "sampling/importance_sampling_ratio/mean": 1.0002140998840332,
      "sampling/importance_sampling_ratio/min": 0.4558376967906952,
      "sampling/sampling_logp_difference/max": 0.7856185436248779,
      "sampling/sampling_logp_difference/mean": 0.016138717532157898,
      "step": 393
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1088.0,
      "completions/max_terminated_length": 1088.0,
      "completions/mean_length": 133.3125,
      "completions/mean_terminated_length": 133.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4057089686393738,
      "epoch": 0.6973451327433628,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.5408018723691908,
      "kl": 0.008893290534615517,
      "learning_rate": 9.834590066053917e-07,
      "loss": 0.4581,
      "num_tokens": 6262081.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.6015470027923584,
      "sampling/importance_sampling_ratio/mean": 0.9998269081115723,
      "sampling/importance_sampling_ratio/min": 0.540658175945282,
      "sampling/sampling_logp_difference/max": 0.6149680614471436,
      "sampling/sampling_logp_difference/mean": 0.013680309988558292,
      "step": 394
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 418.0,
      "completions/max_terminated_length": 418.0,
      "completions/mean_length": 88.953125,
      "completions/mean_terminated_length": 88.953125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.446719229221344,
      "epoch": 0.6991150442477876,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0329129135848926,
      "kl": 0.007592624984681606,
      "learning_rate": 9.832614338134595e-07,
      "loss": 0.0001,
      "num_tokens": 6276606.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4540531635284424,
      "sampling/importance_sampling_ratio/mean": 1.0001907348632812,
      "sampling/importance_sampling_ratio/min": 0.6069669127464294,
      "sampling/sampling_logp_difference/max": 0.4992809295654297,
      "sampling/sampling_logp_difference/mean": 0.01567765697836876,
      "step": 395
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 203.0,
      "completions/max_terminated_length": 203.0,
      "completions/mean_length": 108.09375,
      "completions/mean_terminated_length": 108.09375,
      "completions/min_length": 54.0,
      "completions/min_terminated_length": 54.0,
      "entropy": 0.41068732738494873,
      "epoch": 0.7008849557522124,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03007958844071075,
      "kl": 0.007306972984224558,
      "learning_rate": 9.8306270815269e-07,
      "loss": 0.0001,
      "num_tokens": 6293428.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.7739129066467285,
      "sampling/importance_sampling_ratio/mean": 1.0002840757369995,
      "sampling/importance_sampling_ratio/min": 0.5545526742935181,
      "sampling/sampling_logp_difference/max": 0.5895934104919434,
      "sampling/sampling_logp_difference/mean": 0.016644949093461037,
      "step": 396
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 278.0,
      "completions/max_terminated_length": 278.0,
      "completions/mean_length": 96.875,
      "completions/mean_terminated_length": 96.875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4401080906391144,
      "epoch": 0.7026548672566372,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.4586871026462416,
      "kl": 0.010626133531332016,
      "learning_rate": 9.828628300971638e-07,
      "loss": -0.0859,
      "num_tokens": 6310460.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.5749874114990234,
      "sampling/importance_sampling_ratio/mean": 1.0006617307662964,
      "sampling/importance_sampling_ratio/min": 0.5700552463531494,
      "sampling/sampling_logp_difference/max": 0.5620219707489014,
      "sampling/sampling_logp_difference/mean": 0.018485158681869507,
      "step": 397
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 179.0,
      "completions/max_terminated_length": 179.0,
      "completions/mean_length": 48.0625,
      "completions/mean_terminated_length": 48.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3010576069355011,
      "epoch": 0.7044247787610619,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.047274504860197054,
      "kl": 0.009327048435807228,
      "learning_rate": 9.826618001237099e-07,
      "loss": 0.0001,
      "num_tokens": 6324384.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6009161472320557,
      "sampling/importance_sampling_ratio/mean": 0.9997097253799438,
      "sampling/importance_sampling_ratio/min": 0.7120568752288818,
      "sampling/sampling_logp_difference/max": 0.47057604789733887,
      "sampling/sampling_logp_difference/mean": 0.011248650029301643,
      "step": 398
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 264.0,
      "completions/max_terminated_length": 264.0,
      "completions/mean_length": 65.53125,
      "completions/mean_terminated_length": 65.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3446110188961029,
      "epoch": 0.7061946902654868,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 13.730077407054516,
      "kl": 0.015162178315222263,
      "learning_rate": 9.82459618711906e-07,
      "loss": 0.275,
      "num_tokens": 6339490.0,
      "reward": 0.8125,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.8125,
      "rewards/decision_reward_func/std": 0.5875696539878845,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9998742341995239,
      "sampling/importance_sampling_ratio/min": 0.5365955829620361,
      "sampling/sampling_logp_difference/max": 0.877489447593689,
      "sampling/sampling_logp_difference/mean": 0.014317413792014122,
      "step": 399
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 61.59375,
      "completions/mean_terminated_length": 61.59375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.3405458927154541,
      "epoch": 0.7079646017699115,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.11617255689871578,
      "kl": 0.023211687803268433,
      "learning_rate": 9.822562863440755e-07,
      "loss": 0.0002,
      "num_tokens": 6357672.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3631829023361206,
      "sampling/importance_sampling_ratio/mean": 1.0004823207855225,
      "sampling/importance_sampling_ratio/min": 0.6919498443603516,
      "sampling/sampling_logp_difference/max": 0.3682417869567871,
      "sampling/sampling_logp_difference/mean": 0.01271877158433199,
      "step": 400
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 270.0,
      "completions/max_terminated_length": 270.0,
      "completions/mean_length": 91.984375,
      "completions/mean_terminated_length": 91.984375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3686937987804413,
      "epoch": 0.7097345132743362,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05523325223140905,
      "kl": 0.01296952087432146,
      "learning_rate": 9.820518035052889e-07,
      "loss": 0.0001,
      "num_tokens": 6374407.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4117615222930908,
      "sampling/importance_sampling_ratio/mean": 1.0009119510650635,
      "sampling/importance_sampling_ratio/min": 0.6341397166252136,
      "sampling/sampling_logp_difference/max": 0.4554860591888428,
      "sampling/sampling_logp_difference/mean": 0.013343364000320435,
      "step": 401
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 171.0,
      "completions/max_terminated_length": 171.0,
      "completions/mean_length": 69.921875,
      "completions/mean_terminated_length": 69.921875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.32802262902259827,
      "epoch": 0.7115044247787611,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08481747708423726,
      "kl": 0.017437882721424103,
      "learning_rate": 9.818461706833602e-07,
      "loss": 0.0001,
      "num_tokens": 6388386.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3902863264083862,
      "sampling/importance_sampling_ratio/mean": 0.9997358322143555,
      "sampling/importance_sampling_ratio/min": 0.6693275570869446,
      "sampling/sampling_logp_difference/max": 0.4014817476272583,
      "sampling/sampling_logp_difference/mean": 0.013436712324619293,
      "step": 402
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 590.0,
      "completions/max_terminated_length": 590.0,
      "completions/mean_length": 71.171875,
      "completions/mean_terminated_length": 71.171875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3199332654476166,
      "epoch": 0.7132743362831858,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.113427158524635,
      "kl": 0.01687893643975258,
      "learning_rate": 9.816393883688475e-07,
      "loss": -0.3956,
      "num_tokens": 6403565.0,
      "reward": 0.65625,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.65625,
      "rewards/decision_reward_func/std": 0.7605084180831909,
      "sampling/importance_sampling_ratio/max": 1.577026605606079,
      "sampling/importance_sampling_ratio/mean": 0.9993441700935364,
      "sampling/importance_sampling_ratio/min": 0.5030588507652283,
      "sampling/sampling_logp_difference/max": 0.6870481967926025,
      "sampling/sampling_logp_difference/mean": 0.013590632006525993,
      "step": 403
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 221.0,
      "completions/max_terminated_length": 221.0,
      "completions/mean_length": 82.15625,
      "completions/mean_terminated_length": 82.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.34127122163772583,
      "epoch": 0.7150442477876107,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.6826531843832764,
      "kl": 0.020201334729790688,
      "learning_rate": 9.814314570550505e-07,
      "loss": -0.0153,
      "num_tokens": 6424359.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.437758445739746,
      "sampling/importance_sampling_ratio/mean": 0.9995735883712769,
      "sampling/importance_sampling_ratio/min": 0.4791787564754486,
      "sampling/sampling_logp_difference/max": 0.7356815338134766,
      "sampling/sampling_logp_difference/mean": 0.013350830413401127,
      "step": 404
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 234.0,
      "completions/max_terminated_length": 234.0,
      "completions/mean_length": 59.859375,
      "completions/mean_terminated_length": 59.859375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.3270696699619293,
      "epoch": 0.7168141592920354,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06489359238192514,
      "kl": 0.022430576384067535,
      "learning_rate": 9.812223772380105e-07,
      "loss": 0.0002,
      "num_tokens": 6443438.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5900388956069946,
      "sampling/importance_sampling_ratio/mean": 0.9994983077049255,
      "sampling/importance_sampling_ratio/min": 0.645928144454956,
      "sampling/sampling_logp_difference/max": 0.4637584686279297,
      "sampling/sampling_logp_difference/mean": 0.014710272662341595,
      "step": 405
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 266.0,
      "completions/max_terminated_length": 266.0,
      "completions/mean_length": 82.078125,
      "completions/mean_terminated_length": 82.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.41037648916244507,
      "epoch": 0.7185840707964601,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04683548653083853,
      "kl": 0.02197330817580223,
      "learning_rate": 9.810121494165087e-07,
      "loss": 0.0001,
      "num_tokens": 6459123.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4226871728897095,
      "sampling/importance_sampling_ratio/mean": 0.9984689354896545,
      "sampling/importance_sampling_ratio/min": 0.6845190525054932,
      "sampling/sampling_logp_difference/max": 0.37903881072998047,
      "sampling/sampling_logp_difference/mean": 0.018058765679597855,
      "step": 406
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 155.0,
      "completions/max_terminated_length": 155.0,
      "completions/mean_length": 53.25,
      "completions/mean_terminated_length": 53.25,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.29711028933525085,
      "epoch": 0.720353982300885,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07698501154185808,
      "kl": 0.024243097752332687,
      "learning_rate": 9.808007740920645e-07,
      "loss": 0.0001,
      "num_tokens": 6472723.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.2674616575241089,
      "sampling/importance_sampling_ratio/mean": 1.0002200603485107,
      "sampling/importance_sampling_ratio/min": 0.5505961775779724,
      "sampling/sampling_logp_difference/max": 0.5967535972595215,
      "sampling/sampling_logp_difference/mean": 0.012171144597232342,
      "step": 407
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 346.0,
      "completions/max_terminated_length": 346.0,
      "completions/mean_length": 121.03125,
      "completions/mean_terminated_length": 121.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4948740005493164,
      "epoch": 0.7221238938053097,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.680780315466812,
      "kl": 0.015606466680765152,
      "learning_rate": 9.80588251768935e-07,
      "loss": 0.1649,
      "num_tokens": 6491157.0,
      "reward": 0.6875,
      "reward_std": 0.3811737596988678,
      "rewards/decision_reward_func/mean": 0.6875,
      "rewards/decision_reward_func/std": 0.7319250702857971,
      "sampling/importance_sampling_ratio/max": 1.3771916627883911,
      "sampling/importance_sampling_ratio/mean": 0.999821126461029,
      "sampling/importance_sampling_ratio/min": 0.7080368399620056,
      "sampling/sampling_logp_difference/max": 0.3452591896057129,
      "sampling/sampling_logp_difference/mean": 0.017087643966078758,
      "step": 408
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 229.0,
      "completions/max_terminated_length": 229.0,
      "completions/mean_length": 67.546875,
      "completions/mean_terminated_length": 67.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2259364277124405,
      "epoch": 0.7238938053097345,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.059875820545522006,
      "kl": 0.023707356303930283,
      "learning_rate": 9.803745829541137e-07,
      "loss": 0.0001,
      "num_tokens": 6507720.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2575604915618896,
      "sampling/importance_sampling_ratio/mean": 0.998282790184021,
      "sampling/importance_sampling_ratio/min": 0.3100183606147766,
      "sampling/sampling_logp_difference/max": 1.171123743057251,
      "sampling/sampling_logp_difference/mean": 0.011251486837863922,
      "step": 409
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 474.0,
      "completions/max_terminated_length": 474.0,
      "completions/mean_length": 111.15625,
      "completions/mean_terminated_length": 111.15625,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4915921688079834,
      "epoch": 0.7256637168141593,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.162438830181475,
      "kl": 0.011929306201636791,
      "learning_rate": 9.801597681573289e-07,
      "loss": -0.306,
      "num_tokens": 6524306.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0002862215042114,
      "sampling/importance_sampling_ratio/min": 0.619353711605072,
      "sampling/sampling_logp_difference/max": 0.8039919137954712,
      "sampling/sampling_logp_difference/mean": 0.019170641899108887,
      "step": 410
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 263.0,
      "completions/max_terminated_length": 263.0,
      "completions/mean_length": 94.578125,
      "completions/mean_terminated_length": 94.578125,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.373543918132782,
      "epoch": 0.727433628318584,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05289658409649595,
      "kl": 0.017881829291582108,
      "learning_rate": 9.799438078910432e-07,
      "loss": 0.0001,
      "num_tokens": 6539895.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3026437759399414,
      "sampling/importance_sampling_ratio/mean": 1.0004127025604248,
      "sampling/importance_sampling_ratio/min": 0.6380023956298828,
      "sampling/sampling_logp_difference/max": 0.4494132995605469,
      "sampling/sampling_logp_difference/mean": 0.014169419184327126,
      "step": 411
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 342.0,
      "completions/max_terminated_length": 342.0,
      "completions/mean_length": 94.734375,
      "completions/mean_terminated_length": 94.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25864890217781067,
      "epoch": 0.7292035398230089,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03554165373600353,
      "kl": 0.011999612674117088,
      "learning_rate": 9.797267026704514e-07,
      "loss": 0.0001,
      "num_tokens": 6558230.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3425296545028687,
      "sampling/importance_sampling_ratio/mean": 0.9993278980255127,
      "sampling/importance_sampling_ratio/min": 0.6613734364509583,
      "sampling/sampling_logp_difference/max": 0.4134366512298584,
      "sampling/sampling_logp_difference/mean": 0.011325197294354439,
      "step": 412
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 320.0,
      "completions/max_terminated_length": 320.0,
      "completions/mean_length": 96.0625,
      "completions/mean_terminated_length": 96.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.38927680253982544,
      "epoch": 0.7309734513274336,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03206544035807514,
      "kl": 0.009823622182011604,
      "learning_rate": 9.7950845301348e-07,
      "loss": 0.0001,
      "num_tokens": 6577162.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4312158823013306,
      "sampling/importance_sampling_ratio/mean": 1.0001397132873535,
      "sampling/importance_sampling_ratio/min": 0.41807880997657776,
      "sampling/sampling_logp_difference/max": 0.8720853328704834,
      "sampling/sampling_logp_difference/mean": 0.015506758354604244,
      "step": 413
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 483.0,
      "completions/max_terminated_length": 483.0,
      "completions/mean_length": 95.359375,
      "completions/mean_terminated_length": 95.359375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2924111783504486,
      "epoch": 0.7327433628318584,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.3881119270730435,
      "kl": 0.015888947993516922,
      "learning_rate": 9.792890594407855e-07,
      "loss": -0.0539,
      "num_tokens": 6601089.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.449302315711975,
      "sampling/importance_sampling_ratio/mean": 1.0000194311141968,
      "sampling/importance_sampling_ratio/min": 0.5437875986099243,
      "sampling/sampling_logp_difference/max": 0.6091965436935425,
      "sampling/sampling_logp_difference/mean": 0.011562496423721313,
      "step": 414
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 312.0,
      "completions/max_terminated_length": 312.0,
      "completions/mean_length": 96.40625,
      "completions/mean_terminated_length": 96.40625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.29087594151496887,
      "epoch": 0.7345132743362832,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.032274670059917146,
      "kl": 0.00947969127446413,
      "learning_rate": 9.790685224757532e-07,
      "loss": 0.0001,
      "num_tokens": 6623995.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4449760913848877,
      "sampling/importance_sampling_ratio/mean": 0.9994337558746338,
      "sampling/importance_sampling_ratio/min": 0.5143414735794067,
      "sampling/sampling_logp_difference/max": 0.6648678779602051,
      "sampling/sampling_logp_difference/mean": 0.016016721725463867,
      "step": 415
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 164.0,
      "completions/max_terminated_length": 164.0,
      "completions/mean_length": 65.4375,
      "completions/mean_terminated_length": 65.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2695866525173187,
      "epoch": 0.736283185840708,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10092692024078374,
      "kl": 0.025945590808987617,
      "learning_rate": 9.788468426444967e-07,
      "loss": 0.0001,
      "num_tokens": 6640055.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.255564570426941,
      "sampling/importance_sampling_ratio/mean": 1.000176191329956,
      "sampling/importance_sampling_ratio/min": 0.4881514608860016,
      "sampling/sampling_logp_difference/max": 0.7171295881271362,
      "sampling/sampling_logp_difference/mean": 0.01213115081191063,
      "step": 416
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 224.0,
      "completions/max_terminated_length": 224.0,
      "completions/mean_length": 59.21875,
      "completions/mean_terminated_length": 59.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27273327112197876,
      "epoch": 0.7380530973451327,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.046918830851836506,
      "kl": 0.011121374554932117,
      "learning_rate": 9.786240204758552e-07,
      "loss": 0.0001,
      "num_tokens": 6655957.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3171130418777466,
      "sampling/importance_sampling_ratio/mean": 0.9994080662727356,
      "sampling/importance_sampling_ratio/min": 0.5929533839225769,
      "sampling/sampling_logp_difference/max": 0.5226395130157471,
      "sampling/sampling_logp_difference/mean": 0.012166490778326988,
      "step": 417
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 722.0,
      "completions/max_terminated_length": 722.0,
      "completions/mean_length": 119.375,
      "completions/mean_terminated_length": 119.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4039311110973358,
      "epoch": 0.7398230088495575,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.238070768225619,
      "kl": 0.03022146411240101,
      "learning_rate": 9.784000565013933e-07,
      "loss": -0.0596,
      "num_tokens": 6675757.0,
      "reward": 0.15625,
      "reward_std": 0.3723389506340027,
      "rewards/decision_reward_func/mean": 0.15625,
      "rewards/decision_reward_func/std": 0.9955257177352905,
      "sampling/importance_sampling_ratio/max": 1.4661486148834229,
      "sampling/importance_sampling_ratio/mean": 1.0014400482177734,
      "sampling/importance_sampling_ratio/min": 0.40373674035072327,
      "sampling/sampling_logp_difference/max": 0.9069922566413879,
      "sampling/sampling_logp_difference/mean": 0.016016965731978416,
      "step": 418
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 317.0,
      "completions/max_terminated_length": 317.0,
      "completions/mean_length": 91.609375,
      "completions/mean_terminated_length": 91.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2584455907344818,
      "epoch": 0.7415929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.038183061521502,
      "kl": 0.012106215581297874,
      "learning_rate": 9.781749512553998e-07,
      "loss": 0.0001,
      "num_tokens": 6694612.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.4827467203140259,
      "sampling/importance_sampling_ratio/mean": 0.9982657432556152,
      "sampling/importance_sampling_ratio/min": 0.6861007213592529,
      "sampling/sampling_logp_difference/max": 0.393896222114563,
      "sampling/sampling_logp_difference/mean": 0.015866706147789955,
      "step": 419
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 309.0,
      "completions/max_terminated_length": 309.0,
      "completions/mean_length": 91.4375,
      "completions/mean_terminated_length": 91.4375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4179188907146454,
      "epoch": 0.7433628318584071,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02882059358884815,
      "kl": 0.006676926743239164,
      "learning_rate": 9.779487052748863e-07,
      "loss": 0.0001,
      "num_tokens": 6709984.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.3979723453521729,
      "sampling/importance_sampling_ratio/mean": 0.9994974732398987,
      "sampling/importance_sampling_ratio/min": 0.6180513501167297,
      "sampling/sampling_logp_difference/max": 0.4811837673187256,
      "sampling/sampling_logp_difference/mean": 0.01743045449256897,
      "step": 420
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 191.0,
      "completions/max_terminated_length": 191.0,
      "completions/mean_length": 92.171875,
      "completions/mean_terminated_length": 92.171875,
      "completions/min_length": 46.0,
      "completions/min_terminated_length": 46.0,
      "entropy": 0.46604567766189575,
      "epoch": 0.7451327433628319,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.029793712984509103,
      "kl": 0.008214936591684818,
      "learning_rate": 9.777213190995847e-07,
      "loss": 0.0001,
      "num_tokens": 6725323.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.729860782623291,
      "sampling/importance_sampling_ratio/mean": 1.0001142024993896,
      "sampling/importance_sampling_ratio/min": 0.6954634189605713,
      "sampling/sampling_logp_difference/max": 0.5480408668518066,
      "sampling/sampling_logp_difference/mean": 0.017748814076185226,
      "step": 421
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 300.0,
      "completions/max_terminated_length": 300.0,
      "completions/mean_length": 135.203125,
      "completions/mean_terminated_length": 135.203125,
      "completions/min_length": 49.0,
      "completions/min_terminated_length": 49.0,
      "entropy": 0.5472180843353271,
      "epoch": 0.7469026548672566,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 3.990843197760746,
      "kl": 0.013447487726807594,
      "learning_rate": 9.774927932719482e-07,
      "loss": 0.0167,
      "num_tokens": 6743560.0,
      "reward": -0.03125,
      "reward_std": 0.375,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0002206563949585,
      "sampling/importance_sampling_ratio/min": 0.6289763450622559,
      "sampling/sampling_logp_difference/max": 1.2453601360321045,
      "sampling/sampling_logp_difference/mean": 0.018746210262179375,
      "step": 422
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 624.0,
      "completions/max_terminated_length": 624.0,
      "completions/mean_length": 137.109375,
      "completions/mean_terminated_length": 137.109375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5125194191932678,
      "epoch": 0.7486725663716814,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.2868626079139616,
      "kl": 0.010398737154901028,
      "learning_rate": 9.77263128337148e-07,
      "loss": -0.098,
      "num_tokens": 6761343.0,
      "reward": -0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": -0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.3858596086502075,
      "sampling/importance_sampling_ratio/mean": 1.0002697706222534,
      "sampling/importance_sampling_ratio/min": 0.5133931636810303,
      "sampling/sampling_logp_difference/max": 0.6667132377624512,
      "sampling/sampling_logp_difference/mean": 0.017657287418842316,
      "step": 423
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 719.0,
      "completions/max_terminated_length": 719.0,
      "completions/mean_length": 116.3125,
      "completions/mean_terminated_length": 116.3125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2789403796195984,
      "epoch": 0.7504424778761062,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.833428871020576,
      "kl": 0.010706520639359951,
      "learning_rate": 9.770323248430727e-07,
      "loss": -0.2037,
      "num_tokens": 6778371.0,
      "reward": 0.09375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.3047248125076294,
      "sampling/importance_sampling_ratio/mean": 0.9989539980888367,
      "sampling/importance_sampling_ratio/min": 0.6344852447509766,
      "sampling/sampling_logp_difference/max": 0.4549412727355957,
      "sampling/sampling_logp_difference/mean": 0.011665822938084602,
      "step": 424
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 456.0,
      "completions/max_terminated_length": 456.0,
      "completions/mean_length": 112.546875,
      "completions/mean_terminated_length": 112.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.528927206993103,
      "epoch": 0.7522123893805309,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.167222849854241,
      "kl": 0.0181064922362566,
      "learning_rate": 9.768003833403276e-07,
      "loss": -0.0977,
      "num_tokens": 6796278.0,
      "reward": 0.75,
      "reward_std": 0.25819888710975647,
      "rewards/decision_reward_func/mean": 0.75,
      "rewards/decision_reward_func/std": 0.6666666865348816,
      "sampling/importance_sampling_ratio/max": 1.610059142112732,
      "sampling/importance_sampling_ratio/mean": 0.9996190667152405,
      "sampling/importance_sampling_ratio/min": 0.6272081136703491,
      "sampling/sampling_logp_difference/max": 0.4762709140777588,
      "sampling/sampling_logp_difference/mean": 0.01853884942829609,
      "step": 425
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 564.0,
      "completions/max_terminated_length": 564.0,
      "completions/mean_length": 47.734375,
      "completions/mean_terminated_length": 47.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3304079473018646,
      "epoch": 0.7539823008849558,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05113450168112818,
      "kl": 0.01000053621828556,
      "learning_rate": 9.765673043822324e-07,
      "loss": 0.0001,
      "num_tokens": 6813573.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9999748468399048,
      "sampling/importance_sampling_ratio/min": 0.7175012230873108,
      "sampling/sampling_logp_difference/max": 0.739088773727417,
      "sampling/sampling_logp_difference/mean": 0.014557043090462685,
      "step": 426
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 394.0,
      "completions/max_terminated_length": 394.0,
      "completions/mean_length": 137.03125,
      "completions/mean_terminated_length": 137.03125,
      "completions/min_length": 51.0,
      "completions/min_terminated_length": 51.0,
      "entropy": 0.4420037567615509,
      "epoch": 0.7557522123893805,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02800507514263248,
      "kl": 0.009223876520991325,
      "learning_rate": 9.763330885248204e-07,
      "loss": 0.0001,
      "num_tokens": 6831543.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4312554597854614,
      "sampling/importance_sampling_ratio/mean": 0.9998476505279541,
      "sampling/importance_sampling_ratio/min": 0.6370866894721985,
      "sampling/sampling_logp_difference/max": 0.4508495330810547,
      "sampling/sampling_logp_difference/mean": 0.015897046774625778,
      "step": 427
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 239.0,
      "completions/max_terminated_length": 239.0,
      "completions/mean_length": 50.671875,
      "completions/mean_terminated_length": 50.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1680561751127243,
      "epoch": 0.7575221238938054,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04842198314396375,
      "kl": 0.007218526676297188,
      "learning_rate": 9.760977363268373e-07,
      "loss": 0.0001,
      "num_tokens": 6846066.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2974542379379272,
      "sampling/importance_sampling_ratio/mean": 1.0001842975616455,
      "sampling/importance_sampling_ratio/min": 0.6183997988700867,
      "sampling/sampling_logp_difference/max": 0.4806201457977295,
      "sampling/sampling_logp_difference/mean": 0.011776283383369446,
      "step": 428
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 468.0,
      "completions/max_terminated_length": 468.0,
      "completions/mean_length": 98.734375,
      "completions/mean_terminated_length": 98.734375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21390385925769806,
      "epoch": 0.7592920353982301,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.798618793037985,
      "kl": 0.010727117769420147,
      "learning_rate": 9.758612483497394e-07,
      "loss": 0.0569,
      "num_tokens": 6863473.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.3427993059158325,
      "sampling/importance_sampling_ratio/mean": 0.999305248260498,
      "sampling/importance_sampling_ratio/min": 0.6970505714416504,
      "sampling/sampling_logp_difference/max": 0.3608973026275635,
      "sampling/sampling_logp_difference/mean": 0.010015236213803291,
      "step": 429
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 142.0,
      "completions/max_terminated_length": 142.0,
      "completions/mean_length": 34.109375,
      "completions/mean_terminated_length": 34.109375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2121407836675644,
      "epoch": 0.7610619469026548,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04642061517941747,
      "kl": 0.008114354684948921,
      "learning_rate": 9.756236251576924e-07,
      "loss": 0.0001,
      "num_tokens": 6878456.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.271593451499939,
      "sampling/importance_sampling_ratio/mean": 1.000221848487854,
      "sampling/importance_sampling_ratio/min": 0.6872886419296265,
      "sampling/sampling_logp_difference/max": 0.3750009536743164,
      "sampling/sampling_logp_difference/mean": 0.008882062509655952,
      "step": 430
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 647.0,
      "completions/max_terminated_length": 647.0,
      "completions/mean_length": 185.09375,
      "completions/mean_terminated_length": 185.09375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.44268298149108887,
      "epoch": 0.7628318584070797,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.191665702774492,
      "kl": 0.04039476066827774,
      "learning_rate": 9.753848673175707e-07,
      "loss": -0.1335,
      "num_tokens": 6899774.0,
      "reward": 0.1875,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.1875,
      "rewards/decision_reward_func/std": 0.9900296926498413,
      "sampling/importance_sampling_ratio/max": 1.6913667917251587,
      "sampling/importance_sampling_ratio/mean": 0.9997708797454834,
      "sampling/importance_sampling_ratio/min": 0.46763890981674194,
      "sampling/sampling_logp_difference/max": 0.7600588798522949,
      "sampling/sampling_logp_difference/mean": 0.01565675251185894,
      "step": 431
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 198.0,
      "completions/max_terminated_length": 198.0,
      "completions/mean_length": 43.234375,
      "completions/mean_terminated_length": 43.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1949003040790558,
      "epoch": 0.7646017699115044,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06551818397017477,
      "kl": 0.008975997567176819,
      "learning_rate": 9.751449753989546e-07,
      "loss": 0.0001,
      "num_tokens": 6915357.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4351768493652344,
      "sampling/importance_sampling_ratio/mean": 1.0001879930496216,
      "sampling/importance_sampling_ratio/min": 0.5978617668151855,
      "sampling/sampling_logp_difference/max": 0.5143957138061523,
      "sampling/sampling_logp_difference/mean": 0.011728991754353046,
      "step": 432
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 601.0,
      "completions/max_terminated_length": 601.0,
      "completions/mean_length": 109.34375,
      "completions/mean_terminated_length": 109.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3204788267612457,
      "epoch": 0.7663716814159292,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.4514060945140144,
      "kl": 0.016035258769989014,
      "learning_rate": 9.74903949974131e-07,
      "loss": 0.174,
      "num_tokens": 6933891.0,
      "reward": 0.71875,
      "reward_std": 0.2561737596988678,
      "rewards/decision_reward_func/mean": 0.71875,
      "rewards/decision_reward_func/std": 0.7007648944854736,
      "sampling/importance_sampling_ratio/max": 1.3704556226730347,
      "sampling/importance_sampling_ratio/mean": 1.000302791595459,
      "sampling/importance_sampling_ratio/min": 0.370818167924881,
      "sampling/sampling_logp_difference/max": 0.9920434951782227,
      "sampling/sampling_logp_difference/mean": 0.012358264066278934,
      "step": 433
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 122.0,
      "completions/max_terminated_length": 122.0,
      "completions/mean_length": 51.78125,
      "completions/mean_terminated_length": 51.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2988964915275574,
      "epoch": 0.768141592920354,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0646780144526037,
      "kl": 0.008315995335578918,
      "learning_rate": 9.746617916180905e-07,
      "loss": 0.0001,
      "num_tokens": 6948405.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2864738702774048,
      "sampling/importance_sampling_ratio/mean": 0.9995741844177246,
      "sampling/importance_sampling_ratio/min": 0.682759165763855,
      "sampling/sampling_logp_difference/max": 0.3816131353378296,
      "sampling/sampling_logp_difference/mean": 0.010760385543107986,
      "step": 434
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 483.0,
      "completions/max_terminated_length": 483.0,
      "completions/mean_length": 149.9375,
      "completions/mean_terminated_length": 149.9375,
      "completions/min_length": 22.0,
      "completions/min_terminated_length": 22.0,
      "entropy": 0.46958667039871216,
      "epoch": 0.7699115044247787,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.2892048962797666,
      "kl": 0.014349989593029022,
      "learning_rate": 9.744185009085256e-07,
      "loss": 0.058,
      "num_tokens": 6967809.0,
      "reward": 0.5,
      "reward_std": 0.34156501293182373,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.470845341682434,
      "sampling/importance_sampling_ratio/mean": 1.0003101825714111,
      "sampling/importance_sampling_ratio/min": 0.6346219778060913,
      "sampling/sampling_logp_difference/max": 0.4547257423400879,
      "sampling/sampling_logp_difference/mean": 0.01665390096604824,
      "step": 435
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 365.0,
      "completions/max_terminated_length": 365.0,
      "completions/mean_length": 66.078125,
      "completions/mean_terminated_length": 66.078125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1759306788444519,
      "epoch": 0.7716814159292036,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.144032453870604,
      "kl": 0.005398609209805727,
      "learning_rate": 9.741740784258311e-07,
      "loss": 0.189,
      "num_tokens": 6984070.0,
      "reward": 0.9375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.9375,
      "rewards/decision_reward_func/std": 0.35073620080947876,
      "sampling/importance_sampling_ratio/max": 1.762777328491211,
      "sampling/importance_sampling_ratio/mean": 0.998619794845581,
      "sampling/importance_sampling_ratio/min": 0.5538696050643921,
      "sampling/sampling_logp_difference/max": 0.5908260345458984,
      "sampling/sampling_logp_difference/mean": 0.01123537216335535,
      "step": 436
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 297.0,
      "completions/max_terminated_length": 297.0,
      "completions/mean_length": 123.640625,
      "completions/mean_terminated_length": 123.640625,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5412547588348389,
      "epoch": 0.7734513274336283,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.8359115425008254,
      "kl": 0.013453925028443336,
      "learning_rate": 9.739285247531017e-07,
      "loss": 0.0471,
      "num_tokens": 7002815.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000159502029419,
      "sampling/importance_sampling_ratio/min": 0.44841375946998596,
      "sampling/sampling_logp_difference/max": 1.0421335697174072,
      "sampling/sampling_logp_difference/mean": 0.019044142216444016,
      "step": 437
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 298.0,
      "completions/max_terminated_length": 298.0,
      "completions/mean_length": 108.671875,
      "completions/mean_terminated_length": 108.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4487525224685669,
      "epoch": 0.7752212389380531,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03290813086570887,
      "kl": 0.011676624417304993,
      "learning_rate": 9.736818404761302e-07,
      "loss": 0.0001,
      "num_tokens": 7020698.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5356125831604004,
      "sampling/importance_sampling_ratio/mean": 0.9999417662620544,
      "sampling/importance_sampling_ratio/min": 0.6842086911201477,
      "sampling/sampling_logp_difference/max": 0.42892932891845703,
      "sampling/sampling_logp_difference/mean": 0.018000802025198936,
      "step": 438
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 376.0,
      "completions/max_terminated_length": 376.0,
      "completions/mean_length": 142.734375,
      "completions/mean_terminated_length": 142.734375,
      "completions/min_length": 34.0,
      "completions/min_terminated_length": 34.0,
      "entropy": 0.558229386806488,
      "epoch": 0.7769911504424779,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.561573745299718,
      "kl": 0.01746143400669098,
      "learning_rate": 9.734340261834066e-07,
      "loss": -0.0123,
      "num_tokens": 7040329.0,
      "reward": -0.09375,
      "reward_std": 0.29578250646591187,
      "rewards/decision_reward_func/mean": -0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.3913923501968384,
      "sampling/importance_sampling_ratio/mean": 1.0002951622009277,
      "sampling/importance_sampling_ratio/min": 0.2626253366470337,
      "sampling/sampling_logp_difference/max": 1.337026834487915,
      "sampling/sampling_logp_difference/mean": 0.018860086798667908,
      "step": 439
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 347.0,
      "completions/max_terminated_length": 347.0,
      "completions/mean_length": 84.484375,
      "completions/mean_terminated_length": 84.484375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.43578213453292847,
      "epoch": 0.7787610619469026,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.487202414597359,
      "kl": 0.011191412806510925,
      "learning_rate": 9.73185082466117e-07,
      "loss": -0.1363,
      "num_tokens": 7054808.0,
      "reward": -0.1875,
      "reward_std": 0.3811737596988678,
      "rewards/decision_reward_func/mean": -0.1875,
      "rewards/decision_reward_func/std": 0.9900296926498413,
      "sampling/importance_sampling_ratio/max": 1.597430944442749,
      "sampling/importance_sampling_ratio/mean": 1.0004340410232544,
      "sampling/importance_sampling_ratio/min": 0.5151932239532471,
      "sampling/sampling_logp_difference/max": 0.6632132530212402,
      "sampling/sampling_logp_difference/mean": 0.015007372945547104,
      "step": 440
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 180.0,
      "completions/max_terminated_length": 180.0,
      "completions/mean_length": 68.921875,
      "completions/mean_terminated_length": 68.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22991178929805756,
      "epoch": 0.7805309734513274,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.0598608952177795,
      "kl": 0.021865490823984146,
      "learning_rate": 9.729350099181419e-07,
      "loss": -0.0892,
      "num_tokens": 7070659.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0008248090744019,
      "sampling/importance_sampling_ratio/min": 0.4950145483016968,
      "sampling/sampling_logp_difference/max": 0.9080419540405273,
      "sampling/sampling_logp_difference/mean": 0.013804643414914608,
      "step": 441
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 324.0,
      "completions/max_terminated_length": 324.0,
      "completions/mean_length": 105.8125,
      "completions/mean_terminated_length": 105.8125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.49717026948928833,
      "epoch": 0.7823008849557522,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04886234557938152,
      "kl": 0.01464010775089264,
      "learning_rate": 9.726838091360545e-07,
      "loss": 0.0001,
      "num_tokens": 7087271.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.9791940450668335,
      "sampling/importance_sampling_ratio/mean": 0.9998542666435242,
      "sampling/importance_sampling_ratio/min": 0.69510418176651,
      "sampling/sampling_logp_difference/max": 0.6826896667480469,
      "sampling/sampling_logp_difference/mean": 0.017835965380072594,
      "step": 442
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 474.0,
      "completions/max_terminated_length": 474.0,
      "completions/mean_length": 173.484375,
      "completions/mean_terminated_length": 173.484375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.5604503154754639,
      "epoch": 0.784070796460177,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.387832890411488,
      "kl": 0.013209588825702667,
      "learning_rate": 9.724314807191196e-07,
      "loss": 0.1046,
      "num_tokens": 7108870.0,
      "reward": -0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": -0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.5783559083938599,
      "sampling/importance_sampling_ratio/mean": 0.9999769926071167,
      "sampling/importance_sampling_ratio/min": 0.5818358659744263,
      "sampling/sampling_logp_difference/max": 0.5415668487548828,
      "sampling/sampling_logp_difference/mean": 0.018716178834438324,
      "step": 443
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 281.0,
      "completions/max_terminated_length": 281.0,
      "completions/mean_length": 101.421875,
      "completions/mean_terminated_length": 101.421875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.42693156003952026,
      "epoch": 0.7858407079646018,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.142223704463168,
      "kl": 0.01894482970237732,
      "learning_rate": 9.721780252692917e-07,
      "loss": -0.0856,
      "num_tokens": 7125889.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.4481490850448608,
      "sampling/importance_sampling_ratio/mean": 1.0003788471221924,
      "sampling/importance_sampling_ratio/min": 0.7009619474411011,
      "sampling/sampling_logp_difference/max": 0.370286226272583,
      "sampling/sampling_logp_difference/mean": 0.01637919247150421,
      "step": 444
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 309.0,
      "completions/max_terminated_length": 309.0,
      "completions/mean_length": 69.28125,
      "completions/mean_terminated_length": 69.28125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.376160204410553,
      "epoch": 0.7876106194690266,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.813332613168112,
      "kl": 0.01751999743282795,
      "learning_rate": 9.719234433912146e-07,
      "loss": 0.1039,
      "num_tokens": 7140835.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.3959933519363403,
      "sampling/importance_sampling_ratio/mean": 1.0019288063049316,
      "sampling/importance_sampling_ratio/min": 0.7030871510505676,
      "sampling/sampling_logp_difference/max": 0.35227441787719727,
      "sampling/sampling_logp_difference/mean": 0.01676108129322529,
      "step": 445
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 376.0,
      "completions/max_terminated_length": 376.0,
      "completions/mean_length": 104.9375,
      "completions/mean_terminated_length": 104.9375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3870048522949219,
      "epoch": 0.7893805309734513,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0487475996446023,
      "kl": 0.016693545505404472,
      "learning_rate": 9.716677356922192e-07,
      "loss": 0.0001,
      "num_tokens": 7156863.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3446028232574463,
      "sampling/importance_sampling_ratio/mean": 0.999738335609436,
      "sampling/importance_sampling_ratio/min": 0.6871829628944397,
      "sampling/sampling_logp_difference/max": 0.3751547336578369,
      "sampling/sampling_logp_difference/mean": 0.01365131326019764,
      "step": 446
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 276.0,
      "completions/max_terminated_length": 276.0,
      "completions/mean_length": 103.28125,
      "completions/mean_terminated_length": 103.28125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19822871685028076,
      "epoch": 0.7911504424778761,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04623919445889071,
      "kl": 0.016442574560642242,
      "learning_rate": 9.714109027823216e-07,
      "loss": 0.0001,
      "num_tokens": 7174289.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.3470251560211182,
      "sampling/importance_sampling_ratio/mean": 0.9989169836044312,
      "sampling/importance_sampling_ratio/min": 0.6370871663093567,
      "sampling/sampling_logp_difference/max": 0.4508488178253174,
      "sampling/sampling_logp_difference/mean": 0.007454845122992992,
      "step": 447
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 292.0,
      "completions/max_terminated_length": 292.0,
      "completions/mean_length": 80.140625,
      "completions/mean_terminated_length": 80.140625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2729114592075348,
      "epoch": 0.7929203539823009,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.5695436591573864,
      "kl": 0.021001402288675308,
      "learning_rate": 9.711529452742229e-07,
      "loss": 0.0173,
      "num_tokens": 7190314.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.5101569890975952,
      "sampling/importance_sampling_ratio/mean": 0.9994341135025024,
      "sampling/importance_sampling_ratio/min": 0.6054678559303284,
      "sampling/sampling_logp_difference/max": 0.5017538070678711,
      "sampling/sampling_logp_difference/mean": 0.010070646181702614,
      "step": 448
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 197.0,
      "completions/max_terminated_length": 197.0,
      "completions/mean_length": 91.5625,
      "completions/mean_terminated_length": 91.5625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.36996370553970337,
      "epoch": 0.7946902654867256,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.053565450688631694,
      "kl": 0.03582077473402023,
      "learning_rate": 9.708938637833064e-07,
      "loss": 0.0002,
      "num_tokens": 7208782.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4157929420471191,
      "sampling/importance_sampling_ratio/mean": 0.9996511936187744,
      "sampling/importance_sampling_ratio/min": 0.5518012642860413,
      "sampling/sampling_logp_difference/max": 0.5945672988891602,
      "sampling/sampling_logp_difference/mean": 0.015002947300672531,
      "step": 449
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 325.0,
      "completions/max_terminated_length": 325.0,
      "completions/mean_length": 101.875,
      "completions/mean_terminated_length": 101.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5070844888687134,
      "epoch": 0.7964601769911505,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.6667280847715173,
      "kl": 0.0295796487480402,
      "learning_rate": 9.706336589276374e-07,
      "loss": -0.0625,
      "num_tokens": 7226870.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.3591194152832031,
      "sampling/importance_sampling_ratio/mean": 1.0007846355438232,
      "sampling/importance_sampling_ratio/min": 0.6882472634315491,
      "sampling/sampling_logp_difference/max": 0.37360715866088867,
      "sampling/sampling_logp_difference/mean": 0.016192808747291565,
      "step": 450
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 354.0,
      "completions/max_terminated_length": 354.0,
      "completions/mean_length": 77.359375,
      "completions/mean_terminated_length": 77.359375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27710431814193726,
      "epoch": 0.7982300884955752,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.356774751619394,
      "kl": 0.03306623920798302,
      "learning_rate": 9.703723313279605e-07,
      "loss": 0.1667,
      "num_tokens": 7243117.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.5509188175201416,
      "sampling/importance_sampling_ratio/mean": 1.0007847547531128,
      "sampling/importance_sampling_ratio/min": 0.6969705820083618,
      "sampling/sampling_logp_difference/max": 0.43884754180908203,
      "sampling/sampling_logp_difference/mean": 0.012798595242202282,
      "step": 451
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 240.0,
      "completions/max_terminated_length": 240.0,
      "completions/mean_length": 60.84375,
      "completions/mean_terminated_length": 60.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3076930642127991,
      "epoch": 0.8,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09004043696286056,
      "kl": 0.025182174518704414,
      "learning_rate": 9.701098816076995e-07,
      "loss": 0.0002,
      "num_tokens": 7259971.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2837449312210083,
      "sampling/importance_sampling_ratio/mean": 1.0000991821289062,
      "sampling/importance_sampling_ratio/min": 0.4642953872680664,
      "sampling/sampling_logp_difference/max": 0.7672343254089355,
      "sampling/sampling_logp_difference/mean": 0.01526334322988987,
      "step": 452
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 302.0,
      "completions/max_terminated_length": 302.0,
      "completions/mean_length": 70.25,
      "completions/mean_terminated_length": 70.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3211529552936554,
      "epoch": 0.8017699115044248,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08582681747073204,
      "kl": 0.03589726239442825,
      "learning_rate": 9.698463103929541e-07,
      "loss": 0.0002,
      "num_tokens": 7275139.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.8302972316741943,
      "sampling/importance_sampling_ratio/mean": 1.0005204677581787,
      "sampling/importance_sampling_ratio/min": 0.6988446116447449,
      "sampling/sampling_logp_difference/max": 0.6044783592224121,
      "sampling/sampling_logp_difference/mean": 0.011284945532679558,
      "step": 453
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 213.0,
      "completions/max_terminated_length": 213.0,
      "completions/mean_length": 38.78125,
      "completions/mean_terminated_length": 38.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19094860553741455,
      "epoch": 0.8035398230088495,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.11639956999007596,
      "kl": 0.0388774499297142,
      "learning_rate": 9.695816183125003e-07,
      "loss": 0.0003,
      "num_tokens": 7288709.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.43803071975708,
      "sampling/importance_sampling_ratio/mean": 0.9972746968269348,
      "sampling/importance_sampling_ratio/min": 0.6347212195396423,
      "sampling/sampling_logp_difference/max": 0.45456933975219727,
      "sampling/sampling_logp_difference/mean": 0.011178547516465187,
      "step": 454
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 301.0,
      "completions/max_terminated_length": 301.0,
      "completions/mean_length": 104.15625,
      "completions/mean_terminated_length": 104.15625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.427107036113739,
      "epoch": 0.8053097345132744,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06527203937308146,
      "kl": 0.026374639943242073,
      "learning_rate": 9.693158059977877e-07,
      "loss": 0.0002,
      "num_tokens": 7306815.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5272523164749146,
      "sampling/importance_sampling_ratio/mean": 0.9991812705993652,
      "sampling/importance_sampling_ratio/min": 0.5485890507698059,
      "sampling/sampling_logp_difference/max": 0.6004055738449097,
      "sampling/sampling_logp_difference/mean": 0.016266994178295135,
      "step": 455
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 341.0,
      "completions/max_terminated_length": 341.0,
      "completions/mean_length": 98.703125,
      "completions/mean_terminated_length": 98.703125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.31570273637771606,
      "epoch": 0.8070796460176991,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07527148416420061,
      "kl": 0.029133901000022888,
      "learning_rate": 9.690488740829383e-07,
      "loss": 0.0002,
      "num_tokens": 7324908.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3732151985168457,
      "sampling/importance_sampling_ratio/mean": 1.0015473365783691,
      "sampling/importance_sampling_ratio/min": 0.6976590752601624,
      "sampling/sampling_logp_difference/max": 0.36002469062805176,
      "sampling/sampling_logp_difference/mean": 0.014895064756274223,
      "step": 456
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 328.0,
      "completions/max_terminated_length": 328.0,
      "completions/mean_length": 105.59375,
      "completions/mean_terminated_length": 105.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.38262939453125,
      "epoch": 0.8088495575221238,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06269113840649056,
      "kl": 0.021498549729585648,
      "learning_rate": 9.68780823204745e-07,
      "loss": 0.0002,
      "num_tokens": 7341666.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5470013618469238,
      "sampling/importance_sampling_ratio/mean": 0.9993369579315186,
      "sampling/importance_sampling_ratio/min": 0.685339093208313,
      "sampling/sampling_logp_difference/max": 0.43631839752197266,
      "sampling/sampling_logp_difference/mean": 0.014647518284618855,
      "step": 457
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 314.0,
      "completions/max_terminated_length": 314.0,
      "completions/mean_length": 101.40625,
      "completions/mean_terminated_length": 101.40625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.29837697744369507,
      "epoch": 0.8106194690265487,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06559806490152259,
      "kl": 0.018434328958392143,
      "learning_rate": 9.685116540026701e-07,
      "loss": 0.0002,
      "num_tokens": 7358172.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.436000108718872,
      "sampling/importance_sampling_ratio/mean": 1.0009186267852783,
      "sampling/importance_sampling_ratio/min": 0.6362788081169128,
      "sampling/sampling_logp_difference/max": 0.4521183967590332,
      "sampling/sampling_logp_difference/mean": 0.011897927150130272,
      "step": 458
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 792.0,
      "completions/max_terminated_length": 792.0,
      "completions/mean_length": 137.171875,
      "completions/mean_terminated_length": 137.171875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5502569675445557,
      "epoch": 0.8123893805309734,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.735182949968868,
      "kl": 0.01877054199576378,
      "learning_rate": 9.682413671188444e-07,
      "loss": 0.3218,
      "num_tokens": 7379463.0,
      "reward": -0.0625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": -0.0625,
      "rewards/decision_reward_func/std": 1.0059348344802856,
      "sampling/importance_sampling_ratio/max": 1.593226432800293,
      "sampling/importance_sampling_ratio/mean": 1.0002869367599487,
      "sampling/importance_sampling_ratio/min": 0.36082524061203003,
      "sampling/sampling_logp_difference/max": 1.0193616151809692,
      "sampling/sampling_logp_difference/mean": 0.017945023253560066,
      "step": 459
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 179.0,
      "completions/max_terminated_length": 179.0,
      "completions/mean_length": 36.671875,
      "completions/mean_terminated_length": 36.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.14537887275218964,
      "epoch": 0.8141592920353983,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.268007704313631,
      "kl": 0.01473191287368536,
      "learning_rate": 9.679699631980637e-07,
      "loss": -0.0339,
      "num_tokens": 7391026.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.2735956907272339,
      "sampling/importance_sampling_ratio/mean": 0.9991209506988525,
      "sampling/importance_sampling_ratio/min": 0.68517005443573,
      "sampling/sampling_logp_difference/max": 0.37808823585510254,
      "sampling/sampling_logp_difference/mean": 0.00664309598505497,
      "step": 460
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 199.0,
      "completions/max_terminated_length": 199.0,
      "completions/mean_length": 59.921875,
      "completions/mean_terminated_length": 59.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.257783442735672,
      "epoch": 0.815929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06853751772765107,
      "kl": 0.012552602216601372,
      "learning_rate": 9.6769744288779e-07,
      "loss": 0.0002,
      "num_tokens": 7404413.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5852875709533691,
      "sampling/importance_sampling_ratio/mean": 0.9996552467346191,
      "sampling/importance_sampling_ratio/min": 0.6521279215812683,
      "sampling/sampling_logp_difference/max": 0.4607658386230469,
      "sampling/sampling_logp_difference/mean": 0.01270623505115509,
      "step": 461
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 166.0,
      "completions/max_terminated_length": 166.0,
      "completions/mean_length": 16.140625,
      "completions/mean_terminated_length": 16.140625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18410514295101166,
      "epoch": 0.8176991150442477,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.17198806886123844,
      "kl": 0.02371949329972267,
      "learning_rate": 9.674238068381478e-07,
      "loss": 0.0003,
      "num_tokens": 7415350.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.637800693511963,
      "sampling/importance_sampling_ratio/mean": 0.9997628927230835,
      "sampling/importance_sampling_ratio/min": 0.42162972688674927,
      "sampling/sampling_logp_difference/max": 0.8636277914047241,
      "sampling/sampling_logp_difference/mean": 0.009763983078300953,
      "step": 462
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 327.0,
      "completions/max_terminated_length": 327.0,
      "completions/mean_length": 110.875,
      "completions/mean_terminated_length": 110.875,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.4821784496307373,
      "epoch": 0.8194690265486726,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07166451040935833,
      "kl": 0.017150908708572388,
      "learning_rate": 9.671490557019233e-07,
      "loss": 0.0002,
      "num_tokens": 7434286.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.764185905456543,
      "sampling/importance_sampling_ratio/mean": 1.0012612342834473,
      "sampling/importance_sampling_ratio/min": 0.6241204142570496,
      "sampling/sampling_logp_difference/max": 0.5676894187927246,
      "sampling/sampling_logp_difference/mean": 0.018654324114322662,
      "step": 463
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 573.0,
      "completions/max_terminated_length": 573.0,
      "completions/mean_length": 125.703125,
      "completions/mean_terminated_length": 125.703125,
      "completions/min_length": 48.0,
      "completions/min_terminated_length": 48.0,
      "entropy": 0.5399432182312012,
      "epoch": 0.8212389380530973,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07115499287856507,
      "kl": 0.020138971507549286,
      "learning_rate": 9.668731901345632e-07,
      "loss": 0.0002,
      "num_tokens": 7451675.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3973848819732666,
      "sampling/importance_sampling_ratio/mean": 1.000847578048706,
      "sampling/importance_sampling_ratio/min": 0.6998101472854614,
      "sampling/sampling_logp_difference/max": 0.3569462299346924,
      "sampling/sampling_logp_difference/mean": 0.01800524815917015,
      "step": 464
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 215.0,
      "completions/max_terminated_length": 215.0,
      "completions/mean_length": 61.75,
      "completions/mean_terminated_length": 61.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.46671062707901,
      "epoch": 0.8230088495575221,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08836532913771333,
      "kl": 0.02342853508889675,
      "learning_rate": 9.665962107941724e-07,
      "loss": 0.0003,
      "num_tokens": 7468491.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6156078577041626,
      "sampling/importance_sampling_ratio/mean": 1.0004723072052002,
      "sampling/importance_sampling_ratio/min": 0.6833137273788452,
      "sampling/sampling_logp_difference/max": 0.47971129417419434,
      "sampling/sampling_logp_difference/mean": 0.017525987699627876,
      "step": 465
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 327.0,
      "completions/max_terminated_length": 327.0,
      "completions/mean_length": 86.828125,
      "completions/mean_terminated_length": 86.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4605414867401123,
      "epoch": 0.8247787610619469,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06902940022709253,
      "kl": 0.015131618827581406,
      "learning_rate": 9.663181183415131e-07,
      "loss": 0.0002,
      "num_tokens": 7485584.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6584858894348145,
      "sampling/importance_sampling_ratio/mean": 0.9997375011444092,
      "sampling/importance_sampling_ratio/min": 0.5581821799278259,
      "sampling/sampling_logp_difference/max": 0.5830698013305664,
      "sampling/sampling_logp_difference/mean": 0.017058638855814934,
      "step": 466
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 386.0,
      "completions/max_terminated_length": 386.0,
      "completions/mean_length": 77.765625,
      "completions/mean_terminated_length": 77.765625,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.45054739713668823,
      "epoch": 0.8265486725663717,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.732709836704123,
      "kl": 0.028658758848905563,
      "learning_rate": 9.660389134400033e-07,
      "loss": 0.0982,
      "num_tokens": 7503057.0,
      "reward": -0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.000295639038086,
      "sampling/importance_sampling_ratio/min": 0.6674122214317322,
      "sampling/sampling_logp_difference/max": 1.2609937191009521,
      "sampling/sampling_logp_difference/mean": 0.021381115540862083,
      "step": 467
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 252.0,
      "completions/max_terminated_length": 252.0,
      "completions/mean_length": 109.78125,
      "completions/mean_terminated_length": 109.78125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3166119456291199,
      "epoch": 0.8283185840707965,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.17233483628358817,
      "kl": 0.018055278807878494,
      "learning_rate": 9.657585967557138e-07,
      "loss": 0.0002,
      "num_tokens": 7520851.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.999561071395874,
      "sampling/importance_sampling_ratio/min": 0.29186415672302246,
      "sampling/sampling_logp_difference/max": 1.2314667701721191,
      "sampling/sampling_logp_difference/mean": 0.012147579342126846,
      "step": 468
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 381.0,
      "completions/max_terminated_length": 381.0,
      "completions/mean_length": 108.1875,
      "completions/mean_terminated_length": 108.1875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.22439920902252197,
      "epoch": 0.8300884955752212,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.521774332190136,
      "kl": 0.0083629060536623,
      "learning_rate": 9.654771689573684e-07,
      "loss": -0.1336,
      "num_tokens": 7537055.0,
      "reward": 0.8125,
      "reward_std": 0.36435678601264954,
      "rewards/decision_reward_func/mean": 0.8125,
      "rewards/decision_reward_func/std": 0.5875696539878845,
      "sampling/importance_sampling_ratio/max": 1.7783924341201782,
      "sampling/importance_sampling_ratio/mean": 1.00020170211792,
      "sampling/importance_sampling_ratio/min": 0.6649647355079651,
      "sampling/sampling_logp_difference/max": 0.5757098197937012,
      "sampling/sampling_logp_difference/mean": 0.010277085937559605,
      "step": 469
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 178.0,
      "completions/max_terminated_length": 178.0,
      "completions/mean_length": 38.5,
      "completions/mean_terminated_length": 38.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1761656403541565,
      "epoch": 0.831858407079646,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.292711007458864,
      "kl": 0.0075834584422409534,
      "learning_rate": 9.651946307163416e-07,
      "loss": 0.0957,
      "num_tokens": 7550127.0,
      "reward": -0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.3057875633239746,
      "sampling/importance_sampling_ratio/mean": 1.001237154006958,
      "sampling/importance_sampling_ratio/min": 0.5409621000289917,
      "sampling/sampling_logp_difference/max": 0.6144061088562012,
      "sampling/sampling_logp_difference/mean": 0.013811590149998665,
      "step": 470
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 273.0,
      "completions/max_terminated_length": 273.0,
      "completions/mean_length": 96.671875,
      "completions/mean_terminated_length": 96.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5126727819442749,
      "epoch": 0.8336283185840708,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1124198074532935,
      "kl": 0.027021564543247223,
      "learning_rate": 9.64910982706657e-07,
      "loss": 0.0002,
      "num_tokens": 7573178.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5440689325332642,
      "sampling/importance_sampling_ratio/mean": 0.9985802173614502,
      "sampling/importance_sampling_ratio/min": 0.6255274415016174,
      "sampling/sampling_logp_difference/max": 0.4691600799560547,
      "sampling/sampling_logp_difference/mean": 0.019156504422426224,
      "step": 471
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 174.0,
      "completions/max_terminated_length": 174.0,
      "completions/mean_length": 79.421875,
      "completions/mean_terminated_length": 79.421875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3693407475948334,
      "epoch": 0.8353982300884956,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06026925116566931,
      "kl": 0.029721589758992195,
      "learning_rate": 9.646262256049852e-07,
      "loss": 0.0002,
      "num_tokens": 7586709.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.3058912754058838,
      "sampling/importance_sampling_ratio/mean": 0.9994974136352539,
      "sampling/importance_sampling_ratio/min": 0.5555564761161804,
      "sampling/sampling_logp_difference/max": 0.587785005569458,
      "sampling/sampling_logp_difference/mean": 0.01478197705000639,
      "step": 472
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 481.0,
      "completions/max_terminated_length": 481.0,
      "completions/mean_length": 59.390625,
      "completions/mean_terminated_length": 59.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21782368421554565,
      "epoch": 0.8371681415929203,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06599991621803777,
      "kl": 0.012956816703081131,
      "learning_rate": 9.643403600906432e-07,
      "loss": 0.0002,
      "num_tokens": 7600206.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.5796705484390259,
      "sampling/importance_sampling_ratio/mean": 0.9997380375862122,
      "sampling/importance_sampling_ratio/min": 0.6913318037986755,
      "sampling/sampling_logp_difference/max": 0.4572162628173828,
      "sampling/sampling_logp_difference/mean": 0.013016052544116974,
      "step": 473
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 311.0,
      "completions/max_terminated_length": 311.0,
      "completions/mean_length": 72.265625,
      "completions/mean_terminated_length": 72.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.27213335037231445,
      "epoch": 0.8389380530973451,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06018047672571673,
      "kl": 0.010599360801279545,
      "learning_rate": 9.640533868455918e-07,
      "loss": 0.0002,
      "num_tokens": 7618191.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4041589498519897,
      "sampling/importance_sampling_ratio/mean": 0.9997031092643738,
      "sampling/importance_sampling_ratio/min": 0.6552351117134094,
      "sampling/sampling_logp_difference/max": 0.4227612018585205,
      "sampling/sampling_logp_difference/mean": 0.010823149234056473,
      "step": 474
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 311.0,
      "completions/max_terminated_length": 311.0,
      "completions/mean_length": 68.75,
      "completions/mean_terminated_length": 68.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.37680357694625854,
      "epoch": 0.8407079646017699,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07623883161439017,
      "kl": 0.020373038947582245,
      "learning_rate": 9.637653065544349e-07,
      "loss": 0.0002,
      "num_tokens": 7634287.0,
      "reward": -0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6568858623504639,
      "sampling/importance_sampling_ratio/mean": 0.9996323585510254,
      "sampling/importance_sampling_ratio/min": 0.7048661112785339,
      "sampling/sampling_logp_difference/max": 0.5049397945404053,
      "sampling/sampling_logp_difference/mean": 0.015485817566514015,
      "step": 475
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 220.0,
      "completions/max_terminated_length": 220.0,
      "completions/mean_length": 93.6875,
      "completions/mean_terminated_length": 93.6875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4273343086242676,
      "epoch": 0.8424778761061947,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08576728790420027,
      "kl": 0.02765519730746746,
      "learning_rate": 9.634761199044165e-07,
      "loss": 0.0002,
      "num_tokens": 7650235.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5675156116485596,
      "sampling/importance_sampling_ratio/mean": 1.0012264251708984,
      "sampling/importance_sampling_ratio/min": 0.28429707884788513,
      "sampling/sampling_logp_difference/max": 1.2577356100082397,
      "sampling/sampling_logp_difference/mean": 0.01669345051050186,
      "step": 476
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 250.0,
      "completions/max_terminated_length": 250.0,
      "completions/mean_length": 57.390625,
      "completions/mean_terminated_length": 57.390625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1557103544473648,
      "epoch": 0.8442477876106195,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08774264129861918,
      "kl": 0.01975005678832531,
      "learning_rate": 9.63185827585421e-07,
      "loss": 0.0002,
      "num_tokens": 7666228.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.931877851486206,
      "sampling/importance_sampling_ratio/mean": 0.9998209476470947,
      "sampling/importance_sampling_ratio/min": 0.6257392168045044,
      "sampling/sampling_logp_difference/max": 0.6584925651550293,
      "sampling/sampling_logp_difference/mean": 0.010466136038303375,
      "step": 477
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 522.0,
      "completions/max_terminated_length": 522.0,
      "completions/mean_length": 75.609375,
      "completions/mean_terminated_length": 75.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3157695531845093,
      "epoch": 0.8460176991150442,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06225484644973683,
      "kl": 0.022523541003465652,
      "learning_rate": 9.628944302899695e-07,
      "loss": 0.0002,
      "num_tokens": 7682395.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5499114990234375,
      "sampling/importance_sampling_ratio/mean": 1.0018589496612549,
      "sampling/importance_sampling_ratio/min": 0.7029415965080261,
      "sampling/sampling_logp_difference/max": 0.4381978511810303,
      "sampling/sampling_logp_difference/mean": 0.015560553409159184,
      "step": 478
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 21.0,
      "completions/max_terminated_length": 21.0,
      "completions/mean_length": 17.0,
      "completions/mean_terminated_length": 17.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06959155946969986,
      "epoch": 0.8477876106194691,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.27239551020489455,
      "kl": 0.013529862277209759,
      "learning_rate": 9.6260192871322e-07,
      "loss": 0.0001,
      "num_tokens": 7695019.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.7910892963409424,
      "sampling/importance_sampling_ratio/mean": 1.000964879989624,
      "sampling/importance_sampling_ratio/min": 0.7308072447776794,
      "sampling/sampling_logp_difference/max": 0.5828239917755127,
      "sampling/sampling_logp_difference/mean": 0.0050699952989816666,
      "step": 479
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 192.0,
      "completions/max_terminated_length": 192.0,
      "completions/mean_length": 60.28125,
      "completions/mean_terminated_length": 60.28125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2767450213432312,
      "epoch": 0.8495575221238938,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05418953873731226,
      "kl": 0.009159858338534832,
      "learning_rate": 9.623083235529646e-07,
      "loss": 0.0001,
      "num_tokens": 7710733.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4479974508285522,
      "sampling/importance_sampling_ratio/mean": 0.9996775388717651,
      "sampling/importance_sampling_ratio/min": 0.6703086495399475,
      "sampling/sampling_logp_difference/max": 0.4000170826911926,
      "sampling/sampling_logp_difference/mean": 0.010580535978078842,
      "step": 480
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 219.0,
      "completions/max_terminated_length": 219.0,
      "completions/mean_length": 67.609375,
      "completions/mean_terminated_length": 67.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.26749616861343384,
      "epoch": 0.8513274336283185,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.041986870827581534,
      "kl": 0.007706194184720516,
      "learning_rate": 9.620136155096275e-07,
      "loss": 0.0001,
      "num_tokens": 7724564.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5395063161849976,
      "sampling/importance_sampling_ratio/mean": 0.9995875954627991,
      "sampling/importance_sampling_ratio/min": 0.5570306777954102,
      "sampling/sampling_logp_difference/max": 0.5851349830627441,
      "sampling/sampling_logp_difference/mean": 0.0122038209810853,
      "step": 481
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 192.0,
      "completions/max_terminated_length": 192.0,
      "completions/mean_length": 62.09375,
      "completions/mean_terminated_length": 62.09375,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.2679656147956848,
      "epoch": 0.8530973451327434,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06334004054578615,
      "kl": 0.013048840686678886,
      "learning_rate": 9.617178052862649e-07,
      "loss": 0.0001,
      "num_tokens": 7738714.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4340221881866455,
      "sampling/importance_sampling_ratio/mean": 1.0003671646118164,
      "sampling/importance_sampling_ratio/min": 0.6112040877342224,
      "sampling/sampling_logp_difference/max": 0.4923243522644043,
      "sampling/sampling_logp_difference/mean": 0.012869300320744514,
      "step": 482
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 263.0,
      "completions/max_terminated_length": 263.0,
      "completions/mean_length": 79.59375,
      "completions/mean_terminated_length": 79.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3289504051208496,
      "epoch": 0.8548672566371681,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04462738289464286,
      "kl": 0.014411314390599728,
      "learning_rate": 9.614208935885614e-07,
      "loss": 0.0001,
      "num_tokens": 7754512.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3118646144866943,
      "sampling/importance_sampling_ratio/mean": 1.0010559558868408,
      "sampling/importance_sampling_ratio/min": 0.6392340064048767,
      "sampling/sampling_logp_difference/max": 0.44748473167419434,
      "sampling/sampling_logp_difference/mean": 0.01603720150887966,
      "step": 483
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 251.0,
      "completions/max_terminated_length": 251.0,
      "completions/mean_length": 78.796875,
      "completions/mean_terminated_length": 78.796875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.28768643736839294,
      "epoch": 0.856637168141593,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.040511614927051826,
      "kl": 0.007257409393787384,
      "learning_rate": 9.6112288112483e-07,
      "loss": 0.0001,
      "num_tokens": 7771427.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4626340866088867,
      "sampling/importance_sampling_ratio/mean": 0.9994089603424072,
      "sampling/importance_sampling_ratio/min": 0.6803320646286011,
      "sampling/sampling_logp_difference/max": 0.3851742744445801,
      "sampling/sampling_logp_difference/mean": 0.010980907827615738,
      "step": 484
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 281.0,
      "completions/max_terminated_length": 281.0,
      "completions/mean_length": 110.84375,
      "completions/mean_terminated_length": 110.84375,
      "completions/min_length": 46.0,
      "completions/min_terminated_length": 46.0,
      "entropy": 0.4556252956390381,
      "epoch": 0.8584070796460177,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.054893842974183746,
      "kl": 0.013409667648375034,
      "learning_rate": 9.608237686060097e-07,
      "loss": 0.0001,
      "num_tokens": 7788601.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3639452457427979,
      "sampling/importance_sampling_ratio/mean": 1.0000271797180176,
      "sampling/importance_sampling_ratio/min": 0.561255931854248,
      "sampling/sampling_logp_difference/max": 0.5775783061981201,
      "sampling/sampling_logp_difference/mean": 0.0168587788939476,
      "step": 485
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 367.0,
      "completions/max_terminated_length": 367.0,
      "completions/mean_length": 62.03125,
      "completions/mean_terminated_length": 62.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3430402874946594,
      "epoch": 0.8601769911504424,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05995714263974936,
      "kl": 0.013722425326704979,
      "learning_rate": 9.605235567456635e-07,
      "loss": 0.0001,
      "num_tokens": 7803531.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6505627632141113,
      "sampling/importance_sampling_ratio/mean": 0.999412477016449,
      "sampling/importance_sampling_ratio/min": 0.6296095848083496,
      "sampling/sampling_logp_difference/max": 0.5011162757873535,
      "sampling/sampling_logp_difference/mean": 0.014488639310002327,
      "step": 486
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 561.0,
      "completions/max_terminated_length": 561.0,
      "completions/mean_length": 100.546875,
      "completions/mean_terminated_length": 100.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3184402883052826,
      "epoch": 0.8619469026548673,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.789551644989066,
      "kl": 0.014047389850020409,
      "learning_rate": 9.602222462599766e-07,
      "loss": 0.1634,
      "num_tokens": 7819838.0,
      "reward": -0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.4881186485290527,
      "sampling/importance_sampling_ratio/mean": 1.001170039176941,
      "sampling/importance_sampling_ratio/min": 0.6323857307434082,
      "sampling/sampling_logp_difference/max": 0.4582557678222656,
      "sampling/sampling_logp_difference/mean": 0.013673178851604462,
      "step": 487
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 482.0,
      "completions/max_terminated_length": 482.0,
      "completions/mean_length": 54.6875,
      "completions/mean_terminated_length": 54.6875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.36572539806365967,
      "epoch": 0.863716814159292,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 37.659219580786726,
      "kl": 0.07570485770702362,
      "learning_rate": 9.599198378677558e-07,
      "loss": 0.7396,
      "num_tokens": 7837898.0,
      "reward": 0.625,
      "reward_std": 0.42078250646591187,
      "rewards/decision_reward_func/mean": 0.625,
      "rewards/decision_reward_func/std": 0.7867957949638367,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0011978149414062,
      "sampling/importance_sampling_ratio/min": 0.7209191918373108,
      "sampling/sampling_logp_difference/max": 0.7518572807312012,
      "sampling/sampling_logp_difference/mean": 0.013867147266864777,
      "step": 488
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.046875,
      "completions/mean_terminated_length": 15.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06888234615325928,
      "epoch": 0.8654867256637168,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2142080578369344,
      "kl": 0.024057459086179733,
      "learning_rate": 9.596163322904269e-07,
      "loss": 0.0002,
      "num_tokens": 7851069.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3726508617401123,
      "sampling/importance_sampling_ratio/mean": 1.0013298988342285,
      "sampling/importance_sampling_ratio/min": 0.8065332174301147,
      "sampling/sampling_logp_difference/max": 0.3167438507080078,
      "sampling/sampling_logp_difference/mean": 0.006785606499761343,
      "step": 489
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 336.0,
      "completions/max_terminated_length": 336.0,
      "completions/mean_length": 88.53125,
      "completions/mean_terminated_length": 88.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.35719120502471924,
      "epoch": 0.8672566371681416,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0314355509412585,
      "kl": 0.011010359972715378,
      "learning_rate": 9.593117302520328e-07,
      "loss": 0.0001,
      "num_tokens": 7866143.0,
      "reward": -0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.511204719543457,
      "sampling/importance_sampling_ratio/mean": 0.9987002015113831,
      "sampling/importance_sampling_ratio/min": 0.6565293073654175,
      "sampling/sampling_logp_difference/max": 0.4207879304885864,
      "sampling/sampling_logp_difference/mean": 0.014091964811086655,
      "step": 490
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 404.0,
      "completions/max_terminated_length": 404.0,
      "completions/mean_length": 92.375,
      "completions/mean_terminated_length": 92.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4985180199146271,
      "epoch": 0.8690265486725663,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.644053057921895,
      "kl": 0.018009070307016373,
      "learning_rate": 9.590060324792325e-07,
      "loss": 0.1329,
      "num_tokens": 7881895.0,
      "reward": 0.03125,
      "reward_std": 0.29578250646591187,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.8456369638442993,
      "sampling/importance_sampling_ratio/mean": 1.0008618831634521,
      "sampling/importance_sampling_ratio/min": 0.6019213795661926,
      "sampling/sampling_logp_difference/max": 0.6128244400024414,
      "sampling/sampling_logp_difference/mean": 0.01921398565173149,
      "step": 491
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 211.0,
      "completions/max_terminated_length": 211.0,
      "completions/mean_length": 42.25,
      "completions/mean_terminated_length": 42.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2719489336013794,
      "epoch": 0.8707964601769912,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08532220988810806,
      "kl": 0.010976046323776245,
      "learning_rate": 9.58699239701299e-07,
      "loss": 0.0001,
      "num_tokens": 7898663.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.421367883682251,
      "sampling/importance_sampling_ratio/mean": 1.0002838373184204,
      "sampling/importance_sampling_ratio/min": 0.6368826031684875,
      "sampling/sampling_logp_difference/max": 0.4511699676513672,
      "sampling/sampling_logp_difference/mean": 0.012832563370466232,
      "step": 492
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 347.0,
      "completions/max_terminated_length": 347.0,
      "completions/mean_length": 58.21875,
      "completions/mean_terminated_length": 58.21875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1278567761182785,
      "epoch": 0.8725663716814159,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.595419109387096,
      "kl": 0.01883358508348465,
      "learning_rate": 9.58391352650117e-07,
      "loss": -0.5106,
      "num_tokens": 7911909.0,
      "reward": 0.84375,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.2526459693908691,
      "sampling/importance_sampling_ratio/mean": 1.0006308555603027,
      "sampling/importance_sampling_ratio/min": 0.4962599277496338,
      "sampling/sampling_logp_difference/max": 0.700655460357666,
      "sampling/sampling_logp_difference/mean": 0.010099800303578377,
      "step": 493
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 270.0,
      "completions/max_terminated_length": 270.0,
      "completions/mean_length": 40.125,
      "completions/mean_terminated_length": 40.125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.24204905331134796,
      "epoch": 0.8743362831858407,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06587748198923991,
      "kl": 0.009902874939143658,
      "learning_rate": 9.580823720601823e-07,
      "loss": 0.0001,
      "num_tokens": 7928429.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.445271372795105,
      "sampling/importance_sampling_ratio/mean": 1.0001397132873535,
      "sampling/importance_sampling_ratio/min": 0.7533186078071594,
      "sampling/sampling_logp_difference/max": 0.36829710006713867,
      "sampling/sampling_logp_difference/mean": 0.014522343873977661,
      "step": 494
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 206.0,
      "completions/max_terminated_length": 206.0,
      "completions/mean_length": 56.25,
      "completions/mean_terminated_length": 56.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2897334098815918,
      "epoch": 0.8761061946902655,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.1646682486535305,
      "kl": 0.1686052680015564,
      "learning_rate": 9.57772298668599e-07,
      "loss": -0.1076,
      "num_tokens": 7940925.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.336471676826477,
      "sampling/importance_sampling_ratio/mean": 1.0002723932266235,
      "sampling/importance_sampling_ratio/min": 0.7755419015884399,
      "sampling/sampling_logp_difference/max": 0.29003310203552246,
      "sampling/sampling_logp_difference/mean": 0.012305348180234432,
      "step": 495
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 178.0,
      "completions/max_terminated_length": 178.0,
      "completions/mean_length": 71.4375,
      "completions/mean_terminated_length": 71.4375,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.4769209027290344,
      "epoch": 0.8778761061946903,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 10.18835702156589,
      "kl": 0.060292646288871765,
      "learning_rate": 9.57461133215079e-07,
      "loss": -0.1092,
      "num_tokens": 7956665.0,
      "reward": -0.09375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": -0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.618024468421936,
      "sampling/importance_sampling_ratio/mean": 0.99906325340271,
      "sampling/importance_sampling_ratio/min": 0.6003147959709167,
      "sampling/sampling_logp_difference/max": 0.5103011131286621,
      "sampling/sampling_logp_difference/mean": 0.017318300902843475,
      "step": 496
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 188.0,
      "completions/max_terminated_length": 188.0,
      "completions/mean_length": 58.671875,
      "completions/mean_terminated_length": 58.671875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2595871090888977,
      "epoch": 0.879646017699115,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0416886480079275,
      "kl": 0.0072275553829967976,
      "learning_rate": 9.57148876441938e-07,
      "loss": 0.0001,
      "num_tokens": 7972820.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.45339035987854,
      "sampling/importance_sampling_ratio/mean": 1.000535011291504,
      "sampling/importance_sampling_ratio/min": 0.6839017868041992,
      "sampling/sampling_logp_difference/max": 0.3799409866333008,
      "sampling/sampling_logp_difference/mean": 0.010540075600147247,
      "step": 497
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 200.0,
      "completions/max_terminated_length": 200.0,
      "completions/mean_length": 39.84375,
      "completions/mean_terminated_length": 39.84375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2132524847984314,
      "epoch": 0.8814159292035398,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05927996815786505,
      "kl": 0.005652535706758499,
      "learning_rate": 9.568355290940966e-07,
      "loss": 0.0001,
      "num_tokens": 7987946.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.584210753440857,
      "sampling/importance_sampling_ratio/mean": 1.0005481243133545,
      "sampling/importance_sampling_ratio/min": 0.655351996421814,
      "sampling/sampling_logp_difference/max": 0.4600863456726074,
      "sampling/sampling_logp_difference/mean": 0.010919488966464996,
      "step": 498
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 437.0,
      "completions/max_terminated_length": 437.0,
      "completions/mean_length": 84.90625,
      "completions/mean_terminated_length": 84.90625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.296769917011261,
      "epoch": 0.8831858407079646,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.735124506983188,
      "kl": 0.016414642333984375,
      "learning_rate": 9.565210919190763e-07,
      "loss": 0.015,
      "num_tokens": 8003700.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.9363287687301636,
      "sampling/importance_sampling_ratio/mean": 0.9987481236457825,
      "sampling/importance_sampling_ratio/min": 0.7225105166435242,
      "sampling/sampling_logp_difference/max": 0.6607937812805176,
      "sampling/sampling_logp_difference/mean": 0.015974685549736023,
      "step": 499
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 178.0,
      "completions/max_terminated_length": 178.0,
      "completions/mean_length": 38.03125,
      "completions/mean_terminated_length": 38.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.19154030084609985,
      "epoch": 0.8849557522123894,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08183335721489599,
      "kl": 0.007817339152097702,
      "learning_rate": 9.562055656669987e-07,
      "loss": 0.0001,
      "num_tokens": 8020886.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.598867416381836,
      "sampling/importance_sampling_ratio/mean": 1.0001122951507568,
      "sampling/importance_sampling_ratio/min": 0.5727236270904541,
      "sampling/sampling_logp_difference/max": 0.5573520660400391,
      "sampling/sampling_logp_difference/mean": 0.013924205675721169,
      "step": 500
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 274.0,
      "completions/max_terminated_length": 274.0,
      "completions/mean_length": 97.796875,
      "completions/mean_terminated_length": 97.796875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3268697261810303,
      "epoch": 0.8867256637168142,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.029052867098632782,
      "kl": 0.007557909470051527,
      "learning_rate": 9.558889510905835e-07,
      "loss": 0.0001,
      "num_tokens": 8039145.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.647871494293213,
      "sampling/importance_sampling_ratio/mean": 1.000300407409668,
      "sampling/importance_sampling_ratio/min": 0.6561062932014465,
      "sampling/sampling_logp_difference/max": 0.49948447942733765,
      "sampling/sampling_logp_difference/mean": 0.013082124292850494,
      "step": 501
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 244.0,
      "completions/max_terminated_length": 244.0,
      "completions/mean_length": 94.796875,
      "completions/mean_terminated_length": 94.796875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.43946853280067444,
      "epoch": 0.8884955752212389,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04044782874748557,
      "kl": 0.058621086180210114,
      "learning_rate": 9.555712489451464e-07,
      "loss": 0.0002,
      "num_tokens": 8056556.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5867271423339844,
      "sampling/importance_sampling_ratio/mean": 0.9995195269584656,
      "sampling/importance_sampling_ratio/min": 0.6893123984336853,
      "sampling/sampling_logp_difference/max": 0.4616734981536865,
      "sampling/sampling_logp_difference/mean": 0.01584906503558159,
      "step": 502
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 374.0,
      "completions/max_terminated_length": 374.0,
      "completions/mean_length": 105.546875,
      "completions/mean_terminated_length": 105.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5053263306617737,
      "epoch": 0.8902654867256637,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.045677652748138,
      "kl": 0.2734725773334503,
      "learning_rate": 9.55252459988598e-07,
      "loss": 0.0963,
      "num_tokens": 8073807.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.57107675075531,
      "sampling/importance_sampling_ratio/mean": 0.9994295239448547,
      "sampling/importance_sampling_ratio/min": 0.5382452011108398,
      "sampling/sampling_logp_difference/max": 0.619441032409668,
      "sampling/sampling_logp_difference/mean": 0.018887978047132492,
      "step": 503
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 276.0,
      "completions/max_terminated_length": 276.0,
      "completions/mean_length": 61.453125,
      "completions/mean_terminated_length": 61.453125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.23520615696907043,
      "epoch": 0.8920353982300885,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.044142626143724856,
      "kl": 0.009435078129172325,
      "learning_rate": 9.549325849814418e-07,
      "loss": 0.0001,
      "num_tokens": 8089500.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4041417837142944,
      "sampling/importance_sampling_ratio/mean": 0.9998427629470825,
      "sampling/importance_sampling_ratio/min": 0.6066602468490601,
      "sampling/sampling_logp_difference/max": 0.499786376953125,
      "sampling/sampling_logp_difference/mean": 0.013650336302816868,
      "step": 504
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 895.0,
      "completions/max_terminated_length": 895.0,
      "completions/mean_length": 128.921875,
      "completions/mean_terminated_length": 128.921875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3584873080253601,
      "epoch": 0.8938053097345132,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.164030357296124,
      "kl": 0.006217114627361298,
      "learning_rate": 9.546116246867713e-07,
      "loss": 0.3262,
      "num_tokens": 8108295.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.430129885673523,
      "sampling/importance_sampling_ratio/mean": 1.0008875131607056,
      "sampling/importance_sampling_ratio/min": 0.6261174082756042,
      "sampling/sampling_logp_difference/max": 0.4682173728942871,
      "sampling/sampling_logp_difference/mean": 0.013849091716110706,
      "step": 505
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 209.0,
      "completions/max_terminated_length": 209.0,
      "completions/mean_length": 48.671875,
      "completions/mean_terminated_length": 48.671875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.19223593175411224,
      "epoch": 0.8955752212389381,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.298491366271188,
      "kl": 0.012916343286633492,
      "learning_rate": 9.542895798702701e-07,
      "loss": -0.1124,
      "num_tokens": 8121762.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.254530429840088,
      "sampling/importance_sampling_ratio/mean": 1.0021955966949463,
      "sampling/importance_sampling_ratio/min": 0.6728146076202393,
      "sampling/sampling_logp_difference/max": 0.3962855339050293,
      "sampling/sampling_logp_difference/mean": 0.012941524386405945,
      "step": 506
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 444.0,
      "completions/max_terminated_length": 444.0,
      "completions/mean_length": 109.03125,
      "completions/mean_terminated_length": 109.03125,
      "completions/min_length": 36.0,
      "completions/min_terminated_length": 36.0,
      "entropy": 0.4453309178352356,
      "epoch": 0.8973451327433628,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03895957066929234,
      "kl": 0.01071190182119608,
      "learning_rate": 9.539664513002084e-07,
      "loss": 0.0001,
      "num_tokens": 8138468.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.3871991634368896,
      "sampling/importance_sampling_ratio/mean": 1.0000834465026855,
      "sampling/importance_sampling_ratio/min": 0.5638471245765686,
      "sampling/sampling_logp_difference/max": 0.5729721784591675,
      "sampling/sampling_logp_difference/mean": 0.016829367727041245,
      "step": 507
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 251.0,
      "completions/max_terminated_length": 251.0,
      "completions/mean_length": 83.03125,
      "completions/mean_terminated_length": 83.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.34089598059654236,
      "epoch": 0.8991150442477877,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.044027304289342664,
      "kl": 0.009923753328621387,
      "learning_rate": 9.536422397474418e-07,
      "loss": 0.0001,
      "num_tokens": 8153046.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4149080514907837,
      "sampling/importance_sampling_ratio/mean": 1.0008606910705566,
      "sampling/importance_sampling_ratio/min": 0.6992542147636414,
      "sampling/sampling_logp_difference/max": 0.3577408790588379,
      "sampling/sampling_logp_difference/mean": 0.013866678811609745,
      "step": 508
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 223.0,
      "completions/max_terminated_length": 223.0,
      "completions/mean_length": 96.796875,
      "completions/mean_terminated_length": 96.796875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.3181552290916443,
      "epoch": 0.9008849557522124,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.035081746701379146,
      "kl": 0.009072013199329376,
      "learning_rate": 9.533169459854098e-07,
      "loss": 0.0001,
      "num_tokens": 8169865.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.546746850013733,
      "sampling/importance_sampling_ratio/mean": 0.9996622204780579,
      "sampling/importance_sampling_ratio/min": 0.7007312774658203,
      "sampling/sampling_logp_difference/max": 0.4361538887023926,
      "sampling/sampling_logp_difference/mean": 0.012515220791101456,
      "step": 509
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 134.0,
      "completions/max_terminated_length": 134.0,
      "completions/mean_length": 33.640625,
      "completions/mean_terminated_length": 33.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.16498476266860962,
      "epoch": 0.9026548672566371,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04012191510593879,
      "kl": 0.005203623324632645,
      "learning_rate": 9.529905707901333e-07,
      "loss": 0.0001,
      "num_tokens": 8182402.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2919236421585083,
      "sampling/importance_sampling_ratio/mean": 0.9999414086341858,
      "sampling/importance_sampling_ratio/min": 0.5896210074424744,
      "sampling/sampling_logp_difference/max": 0.5282753705978394,
      "sampling/sampling_logp_difference/mean": 0.008416219614446163,
      "step": 510
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 396.0,
      "completions/max_terminated_length": 396.0,
      "completions/mean_length": 113.21875,
      "completions/mean_terminated_length": 113.21875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.3625057339668274,
      "epoch": 0.904424778761062,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.568032198967511,
      "kl": 0.006654072552919388,
      "learning_rate": 9.526631149402134e-07,
      "loss": 0.0482,
      "num_tokens": 8201504.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.3120354413986206,
      "sampling/importance_sampling_ratio/mean": 1.0005406141281128,
      "sampling/importance_sampling_ratio/min": 0.5685505867004395,
      "sampling/sampling_logp_difference/max": 0.5646649599075317,
      "sampling/sampling_logp_difference/mean": 0.013463255017995834,
      "step": 511
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 318.0,
      "completions/max_terminated_length": 318.0,
      "completions/mean_length": 114.828125,
      "completions/mean_terminated_length": 114.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3808782994747162,
      "epoch": 0.9061946902654867,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.036424015039903675,
      "kl": 0.009641697630286217,
      "learning_rate": 9.523345792168288e-07,
      "loss": 0.0001,
      "num_tokens": 8218661.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.5945333242416382,
      "sampling/importance_sampling_ratio/mean": 0.9990466237068176,
      "sampling/importance_sampling_ratio/min": 0.498073011636734,
      "sampling/sampling_logp_difference/max": 0.6970086097717285,
      "sampling/sampling_logp_difference/mean": 0.015809332951903343,
      "step": 512
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 350.0,
      "completions/max_terminated_length": 350.0,
      "completions/mean_length": 89.890625,
      "completions/mean_terminated_length": 89.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.36214619874954224,
      "epoch": 0.9079646017699115,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02469736712140168,
      "kl": 0.009469065815210342,
      "learning_rate": 9.520049644037347e-07,
      "loss": 0.0001,
      "num_tokens": 8235598.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.2883288860321045,
      "sampling/importance_sampling_ratio/mean": 1.0004262924194336,
      "sampling/importance_sampling_ratio/min": 0.5304144620895386,
      "sampling/sampling_logp_difference/max": 0.634096622467041,
      "sampling/sampling_logp_difference/mean": 0.011682311072945595,
      "step": 513
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 239.0,
      "completions/max_terminated_length": 239.0,
      "completions/mean_length": 88.03125,
      "completions/mean_terminated_length": 88.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.44736549258232117,
      "epoch": 0.9097345132743363,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.036895631890060225,
      "kl": 0.016795285046100616,
      "learning_rate": 9.516742712872605e-07,
      "loss": 0.0002,
      "num_tokens": 8259312.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.6561634540557861,
      "sampling/importance_sampling_ratio/mean": 0.9994844198226929,
      "sampling/importance_sampling_ratio/min": 0.5980254411697388,
      "sampling/sampling_logp_difference/max": 0.5141220092773438,
      "sampling/sampling_logp_difference/mean": 0.017143195495009422,
      "step": 514
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 459.0,
      "completions/max_terminated_length": 459.0,
      "completions/mean_length": 140.765625,
      "completions/mean_terminated_length": 140.765625,
      "completions/min_length": 44.0,
      "completions/min_terminated_length": 44.0,
      "entropy": 0.46250495314598083,
      "epoch": 0.911504424778761,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.5997216064046285,
      "kl": 0.008927903138101101,
      "learning_rate": 9.513425006563078e-07,
      "loss": -0.1194,
      "num_tokens": 8278049.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.4426480531692505,
      "sampling/importance_sampling_ratio/mean": 1.0001449584960938,
      "sampling/importance_sampling_ratio/min": 0.6405063271522522,
      "sampling/sampling_logp_difference/max": 0.4454963207244873,
      "sampling/sampling_logp_difference/mean": 0.015760906040668488,
      "step": 515
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 335.0,
      "completions/max_terminated_length": 335.0,
      "completions/mean_length": 96.828125,
      "completions/mean_terminated_length": 96.828125,
      "completions/min_length": 8.0,
      "completions/min_terminated_length": 8.0,
      "entropy": 0.5690308809280396,
      "epoch": 0.9132743362831859,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0331973440363682,
      "kl": 0.009910114109516144,
      "learning_rate": 9.51009653302349e-07,
      "loss": 0.0001,
      "num_tokens": 8296374.0,
      "reward": -0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": -0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6122362613677979,
      "sampling/importance_sampling_ratio/mean": 0.9999167323112488,
      "sampling/importance_sampling_ratio/min": 0.5489570498466492,
      "sampling/sampling_logp_difference/max": 0.5997350215911865,
      "sampling/sampling_logp_difference/mean": 0.019404888153076172,
      "step": 516
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 591.0,
      "completions/max_terminated_length": 591.0,
      "completions/mean_length": 154.234375,
      "completions/mean_terminated_length": 154.234375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.36337944865226746,
      "epoch": 0.9150442477876106,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.664883462583671,
      "kl": 0.014331409707665443,
      "learning_rate": 9.506757300194248e-07,
      "loss": -0.0603,
      "num_tokens": 8318053.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.5481709241867065,
      "sampling/importance_sampling_ratio/mean": 0.9998996257781982,
      "sampling/importance_sampling_ratio/min": 0.6269489526748657,
      "sampling/sampling_logp_difference/max": 0.4668901562690735,
      "sampling/sampling_logp_difference/mean": 0.012643365189433098,
      "step": 517
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 584.0,
      "completions/max_terminated_length": 584.0,
      "completions/mean_length": 139.34375,
      "completions/mean_terminated_length": 139.34375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.43411684036254883,
      "epoch": 0.9168141592920354,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.140124683083626,
      "kl": 0.018226414918899536,
      "learning_rate": 9.50340731604143e-07,
      "loss": 0.2147,
      "num_tokens": 8336539.0,
      "reward": 0.09375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.09375,
      "rewards/decision_reward_func/std": 1.003466248512268,
      "sampling/importance_sampling_ratio/max": 1.62480890750885,
      "sampling/importance_sampling_ratio/mean": 0.9991416931152344,
      "sampling/importance_sampling_ratio/min": 0.236353799700737,
      "sampling/sampling_logp_difference/max": 1.4424254894256592,
      "sampling/sampling_logp_difference/mean": 0.015893306583166122,
      "step": 518
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 235.0,
      "completions/max_terminated_length": 235.0,
      "completions/mean_length": 95.546875,
      "completions/mean_terminated_length": 95.546875,
      "completions/min_length": 14.0,
      "completions/min_terminated_length": 14.0,
      "entropy": 0.5186383128166199,
      "epoch": 0.9185840707964602,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.039527739559405944,
      "kl": 0.011614596471190453,
      "learning_rate": 9.500046588556761e-07,
      "loss": 0.0001,
      "num_tokens": 8352046.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9996851682662964,
      "sampling/importance_sampling_ratio/min": 0.7078847289085388,
      "sampling/sampling_logp_difference/max": 0.9716105461120605,
      "sampling/sampling_logp_difference/mean": 0.017761271446943283,
      "step": 519
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 330.0,
      "completions/max_terminated_length": 330.0,
      "completions/mean_length": 107.421875,
      "completions/mean_terminated_length": 107.421875,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.3980678915977478,
      "epoch": 0.9203539823008849,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03050835034680889,
      "kl": 0.011641588993370533,
      "learning_rate": 9.496675125757594e-07,
      "loss": 0.0001,
      "num_tokens": 8368809.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.9738656282424927,
      "sampling/importance_sampling_ratio/mean": 1.0001617670059204,
      "sampling/importance_sampling_ratio/min": 0.6098880767822266,
      "sampling/sampling_logp_difference/max": 0.6799938678741455,
      "sampling/sampling_logp_difference/mean": 0.014947582967579365,
      "step": 520
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1060.0,
      "completions/max_terminated_length": 1060.0,
      "completions/mean_length": 140.09375,
      "completions/mean_terminated_length": 140.09375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.42774447798728943,
      "epoch": 0.9221238938053097,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.4898741180099533,
      "kl": 0.00820097140967846,
      "learning_rate": 9.493292935686894e-07,
      "loss": 0.261,
      "num_tokens": 8387039.0,
      "reward": 0.0,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.515367865562439,
      "sampling/importance_sampling_ratio/mean": 1.0005155801773071,
      "sampling/importance_sampling_ratio/min": 0.6541255116462708,
      "sampling/sampling_logp_difference/max": 0.4244561195373535,
      "sampling/sampling_logp_difference/mean": 0.01558619737625122,
      "step": 521
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 273.0,
      "completions/max_terminated_length": 273.0,
      "completions/mean_length": 118.859375,
      "completions/mean_terminated_length": 118.859375,
      "completions/min_length": 20.0,
      "completions/min_terminated_length": 20.0,
      "entropy": 0.499830037355423,
      "epoch": 0.9238938053097345,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0253552589765691,
      "kl": 0.009436726570129395,
      "learning_rate": 9.489900026413216e-07,
      "loss": 0.0001,
      "num_tokens": 8404118.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.5663814544677734,
      "sampling/importance_sampling_ratio/mean": 1.0002120733261108,
      "sampling/importance_sampling_ratio/min": 0.4994365870952606,
      "sampling/sampling_logp_difference/max": 0.6942746639251709,
      "sampling/sampling_logp_difference/mean": 0.018683871254324913,
      "step": 522
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 241.0,
      "completions/max_terminated_length": 241.0,
      "completions/mean_length": 91.046875,
      "completions/mean_terminated_length": 91.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4420931935310364,
      "epoch": 0.9256637168141593,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.032238059689454056,
      "kl": 0.009348719380795956,
      "learning_rate": 9.486496406030685e-07,
      "loss": 0.0001,
      "num_tokens": 8421017.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4921983480453491,
      "sampling/importance_sampling_ratio/mean": 0.9986906051635742,
      "sampling/importance_sampling_ratio/min": 0.7028223276138306,
      "sampling/sampling_logp_difference/max": 0.4002504348754883,
      "sampling/sampling_logp_difference/mean": 0.01746968738734722,
      "step": 523
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 205.0,
      "completions/max_terminated_length": 205.0,
      "completions/mean_length": 42.96875,
      "completions/mean_terminated_length": 42.96875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21450337767601013,
      "epoch": 0.9274336283185841,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06758275124603391,
      "kl": 0.006579137407243252,
      "learning_rate": 9.483082082658982e-07,
      "loss": 0.0001,
      "num_tokens": 8439863.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.434173345565796,
      "sampling/importance_sampling_ratio/mean": 0.9990978240966797,
      "sampling/importance_sampling_ratio/min": 0.5833903551101685,
      "sampling/sampling_logp_difference/max": 0.5388987064361572,
      "sampling/sampling_logp_difference/mean": 0.012227091938257217,
      "step": 524
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 411.0,
      "completions/max_terminated_length": 411.0,
      "completions/mean_length": 122.5,
      "completions/mean_terminated_length": 122.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.42718803882598877,
      "epoch": 0.9292035398230089,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.389959562645312,
      "kl": 0.010070500895380974,
      "learning_rate": 9.479657064443321e-07,
      "loss": -0.1146,
      "num_tokens": 8459079.0,
      "reward": 0.03125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.03125,
      "rewards/decision_reward_func/std": 1.0074130296707153,
      "sampling/importance_sampling_ratio/max": 1.6004486083984375,
      "sampling/importance_sampling_ratio/mean": 1.0002415180206299,
      "sampling/importance_sampling_ratio/min": 0.6741685271263123,
      "sampling/sampling_logp_difference/max": 0.47028398513793945,
      "sampling/sampling_logp_difference/mean": 0.016869788989424706,
      "step": 525
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 519.0,
      "completions/max_terminated_length": 519.0,
      "completions/mean_length": 95.09375,
      "completions/mean_terminated_length": 95.09375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5120389461517334,
      "epoch": 0.9309734513274336,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.8066848398685473,
      "kl": 0.02020437829196453,
      "learning_rate": 9.476221359554423e-07,
      "loss": 0.0886,
      "num_tokens": 8477517.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 1.464606523513794,
      "sampling/importance_sampling_ratio/mean": 0.9992693662643433,
      "sampling/importance_sampling_ratio/min": 0.7041698694229126,
      "sampling/sampling_logp_difference/max": 0.38158655166625977,
      "sampling/sampling_logp_difference/mean": 0.017616450786590576,
      "step": 526
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 453.0,
      "completions/max_terminated_length": 453.0,
      "completions/mean_length": 70.546875,
      "completions/mean_terminated_length": 70.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.260747492313385,
      "epoch": 0.9327433628318584,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.030913574652803144,
      "kl": 0.006850100588053465,
      "learning_rate": 9.472774976188513e-07,
      "loss": 0.0001,
      "num_tokens": 8493760.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.6097594499588013,
      "sampling/importance_sampling_ratio/mean": 1.001157522201538,
      "sampling/importance_sampling_ratio/min": 0.7061507105827332,
      "sampling/sampling_logp_difference/max": 0.47608476877212524,
      "sampling/sampling_logp_difference/mean": 0.009453845210373402,
      "step": 527
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 159.0,
      "completions/max_terminated_length": 159.0,
      "completions/mean_length": 37.4375,
      "completions/mean_terminated_length": 37.4375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.18946631252765656,
      "epoch": 0.9345132743362832,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06875360822757838,
      "kl": 0.009271876886487007,
      "learning_rate": 9.469317922567286e-07,
      "loss": 0.0002,
      "num_tokens": 8512924.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.5250033140182495,
      "sampling/importance_sampling_ratio/mean": 1.0006039142608643,
      "sampling/importance_sampling_ratio/min": 0.6538918614387512,
      "sampling/sampling_logp_difference/max": 0.42481327056884766,
      "sampling/sampling_logp_difference/mean": 0.010327961295843124,
      "step": 528
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 420.0,
      "completions/max_terminated_length": 420.0,
      "completions/mean_length": 70.53125,
      "completions/mean_terminated_length": 70.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.28202685713768005,
      "epoch": 0.9362831858407079,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.06847389682932,
      "kl": 0.01626504212617874,
      "learning_rate": 9.465850206937887e-07,
      "loss": 0.1768,
      "num_tokens": 8530238.0,
      "reward": 0.4375,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.4375,
      "rewards/decision_reward_func/std": 0.9063270092010498,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9993406534194946,
      "sampling/importance_sampling_ratio/min": 0.6998425126075745,
      "sampling/sampling_logp_difference/max": 0.7249191999435425,
      "sampling/sampling_logp_difference/mean": 0.012953529134392738,
      "step": 529
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 182.0,
      "completions/max_terminated_length": 182.0,
      "completions/mean_length": 62.25,
      "completions/mean_terminated_length": 62.25,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2565949559211731,
      "epoch": 0.9380530973451328,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03577417046315069,
      "kl": 0.0072803376242518425,
      "learning_rate": 9.462371837572906e-07,
      "loss": 0.0001,
      "num_tokens": 8545950.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.40841805934906,
      "sampling/importance_sampling_ratio/mean": 1.0001753568649292,
      "sampling/importance_sampling_ratio/min": 0.691235363483429,
      "sampling/sampling_logp_difference/max": 0.3692748546600342,
      "sampling/sampling_logp_difference/mean": 0.010468710213899612,
      "step": 530
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 193.0,
      "completions/max_terminated_length": 193.0,
      "completions/mean_length": 60.609375,
      "completions/mean_terminated_length": 60.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.26018762588500977,
      "epoch": 0.9398230088495575,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05116817286548551,
      "kl": 0.010109255090355873,
      "learning_rate": 9.45888282277034e-07,
      "loss": 0.0001,
      "num_tokens": 8566389.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2612920999526978,
      "sampling/importance_sampling_ratio/mean": 1.0020697116851807,
      "sampling/importance_sampling_ratio/min": 0.5850200653076172,
      "sampling/sampling_logp_difference/max": 0.5361090898513794,
      "sampling/sampling_logp_difference/mean": 0.013840936124324799,
      "step": 531
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 222.0,
      "completions/max_terminated_length": 222.0,
      "completions/mean_length": 66.421875,
      "completions/mean_terminated_length": 66.421875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2860739529132843,
      "epoch": 0.9415929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03885608048943523,
      "kl": 0.008966992609202862,
      "learning_rate": 9.455383170853585e-07,
      "loss": 0.0001,
      "num_tokens": 8582464.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.9738166332244873,
      "sampling/importance_sampling_ratio/mean": 0.9996252059936523,
      "sampling/importance_sampling_ratio/min": 0.5545295476913452,
      "sampling/sampling_logp_difference/max": 0.679969072341919,
      "sampling/sampling_logp_difference/mean": 0.01339352410286665,
      "step": 532
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 543.0,
      "completions/max_terminated_length": 543.0,
      "completions/mean_length": 108.578125,
      "completions/mean_terminated_length": 108.578125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.35880762338638306,
      "epoch": 0.9433628318584071,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.05997824834894199,
      "kl": 0.016652042046189308,
      "learning_rate": 9.451872890171419e-07,
      "loss": 0.0002,
      "num_tokens": 8609493.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.435707449913025,
      "sampling/importance_sampling_ratio/mean": 0.999294638633728,
      "sampling/importance_sampling_ratio/min": 0.47342342138290405,
      "sampling/sampling_logp_difference/max": 0.747765064239502,
      "sampling/sampling_logp_difference/mean": 0.013728786259889603,
      "step": 533
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 232.0,
      "completions/max_terminated_length": 232.0,
      "completions/mean_length": 88.546875,
      "completions/mean_terminated_length": 88.546875,
      "completions/min_length": 14.0,
      "completions/min_terminated_length": 14.0,
      "entropy": 0.45613330602645874,
      "epoch": 0.9451327433628318,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09313465584163444,
      "kl": 0.015598511323332787,
      "learning_rate": 9.448351989097962e-07,
      "loss": 0.0002,
      "num_tokens": 8626280.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3734320402145386,
      "sampling/importance_sampling_ratio/mean": 0.9996883869171143,
      "sampling/importance_sampling_ratio/min": 0.5861338376998901,
      "sampling/sampling_logp_difference/max": 0.5342071056365967,
      "sampling/sampling_logp_difference/mean": 0.014164279215037823,
      "step": 534
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 279.0,
      "completions/max_terminated_length": 279.0,
      "completions/mean_length": 70.375,
      "completions/mean_terminated_length": 70.375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.32646340131759644,
      "epoch": 0.9469026548672567,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03490572895714704,
      "kl": 0.00837424024939537,
      "learning_rate": 9.444820476032685e-07,
      "loss": 0.0001,
      "num_tokens": 8642800.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.33780038356781,
      "sampling/importance_sampling_ratio/mean": 0.9992144107818604,
      "sampling/importance_sampling_ratio/min": 0.6560214757919312,
      "sampling/sampling_logp_difference/max": 0.42156171798706055,
      "sampling/sampling_logp_difference/mean": 0.010708148591220379,
      "step": 535
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 269.0,
      "completions/max_terminated_length": 269.0,
      "completions/mean_length": 111.046875,
      "completions/mean_terminated_length": 111.046875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5900201201438904,
      "epoch": 0.9486725663716814,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0580239595566989,
      "kl": 0.018893791362643242,
      "learning_rate": 9.441278359400364e-07,
      "loss": 0.0002,
      "num_tokens": 8659587.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4816340208053589,
      "sampling/importance_sampling_ratio/mean": 1.0002975463867188,
      "sampling/importance_sampling_ratio/min": 0.6631470322608948,
      "sampling/sampling_logp_difference/max": 0.41075849533081055,
      "sampling/sampling_logp_difference/mean": 0.021431345492601395,
      "step": 536
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 220.0,
      "completions/max_terminated_length": 220.0,
      "completions/mean_length": 59.03125,
      "completions/mean_terminated_length": 59.03125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2766770124435425,
      "epoch": 0.9504424778761061,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08836968551118032,
      "kl": 0.00837495643645525,
      "learning_rate": 9.437725647651078e-07,
      "loss": 0.0001,
      "num_tokens": 8674293.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.0,
      "rewards/decision_reward_func/std": 1.0079052448272705,
      "sampling/importance_sampling_ratio/max": 1.759122371673584,
      "sampling/importance_sampling_ratio/mean": 0.9989175200462341,
      "sampling/importance_sampling_ratio/min": 0.5930877923965454,
      "sampling/sampling_logp_difference/max": 0.5648150444030762,
      "sampling/sampling_logp_difference/mean": 0.016760773956775665,
      "step": 537
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 190.0,
      "completions/max_terminated_length": 190.0,
      "completions/mean_length": 56.75,
      "completions/mean_terminated_length": 56.75,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3292761445045471,
      "epoch": 0.952212389380531,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.03191293281770827,
      "kl": 0.007732439786195755,
      "learning_rate": 9.434162349260178e-07,
      "loss": 0.0001,
      "num_tokens": 8688741.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.281243920326233,
      "sampling/importance_sampling_ratio/mean": 0.9997268915176392,
      "sampling/importance_sampling_ratio/min": 0.7160525321960449,
      "sampling/sampling_logp_difference/max": 0.3340017795562744,
      "sampling/sampling_logp_difference/mean": 0.0122067891061306,
      "step": 538
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 395.0,
      "completions/max_terminated_length": 395.0,
      "completions/mean_length": 76.296875,
      "completions/mean_terminated_length": 76.296875,
      "completions/min_length": 10.0,
      "completions/min_terminated_length": 10.0,
      "entropy": 0.35782527923583984,
      "epoch": 0.9539823008849557,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 10.676094231660771,
      "kl": 0.010862261056900024,
      "learning_rate": 9.430588472728269e-07,
      "loss": 0.0578,
      "num_tokens": 8704760.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0010135173797607,
      "sampling/importance_sampling_ratio/min": 0.6073746085166931,
      "sampling/sampling_logp_difference/max": 0.8597202301025391,
      "sampling/sampling_logp_difference/mean": 0.013356505893170834,
      "step": 539
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 243.0,
      "completions/max_terminated_length": 243.0,
      "completions/mean_length": 74.0,
      "completions/mean_terminated_length": 74.0,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3447580337524414,
      "epoch": 0.9557522123893806,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.105585897945515,
      "kl": 0.013816621154546738,
      "learning_rate": 9.427004026581196e-07,
      "loss": 0.1718,
      "num_tokens": 8719640.0,
      "reward": 0.84375,
      "reward_std": 0.23935678601264954,
      "rewards/decision_reward_func/mean": 0.84375,
      "rewards/decision_reward_func/std": 0.5409794449806213,
      "sampling/importance_sampling_ratio/max": 1.652379035949707,
      "sampling/importance_sampling_ratio/mean": 1.001023769378662,
      "sampling/importance_sampling_ratio/min": 0.7009364366531372,
      "sampling/sampling_logp_difference/max": 0.502216100692749,
      "sampling/sampling_logp_difference/mean": 0.013380978256464005,
      "step": 540
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 366.0,
      "completions/max_terminated_length": 366.0,
      "completions/mean_length": 123.703125,
      "completions/mean_terminated_length": 123.703125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3127603530883789,
      "epoch": 0.9575221238938053,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.913535613821327,
      "kl": 0.015127960592508316,
      "learning_rate": 9.423409019370014e-07,
      "loss": -0.202,
      "num_tokens": 8738533.0,
      "reward": 0.8125,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.8125,
      "rewards/decision_reward_func/std": 0.5875696539878845,
      "sampling/importance_sampling_ratio/max": 1.567948818206787,
      "sampling/importance_sampling_ratio/mean": 0.9999661445617676,
      "sampling/importance_sampling_ratio/min": 0.6265570521354675,
      "sampling/sampling_logp_difference/max": 0.4675154685974121,
      "sampling/sampling_logp_difference/mean": 0.012319693341851234,
      "step": 541
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 289.0,
      "completions/max_terminated_length": 289.0,
      "completions/mean_length": 87.265625,
      "completions/mean_terminated_length": 87.265625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4443648159503937,
      "epoch": 0.95929203539823,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08539988253972454,
      "kl": 0.018398810178041458,
      "learning_rate": 9.419803459670979e-07,
      "loss": 0.0002,
      "num_tokens": 8754758.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.574440360069275,
      "sampling/importance_sampling_ratio/mean": 0.9996380805969238,
      "sampling/importance_sampling_ratio/min": 0.5782991051673889,
      "sampling/sampling_logp_difference/max": 0.5476641654968262,
      "sampling/sampling_logp_difference/mean": 0.015303626656532288,
      "step": 542
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 219.0,
      "completions/max_terminated_length": 219.0,
      "completions/mean_length": 105.0625,
      "completions/mean_terminated_length": 105.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.5941020250320435,
      "epoch": 0.9610619469026549,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04467205979015622,
      "kl": 0.015284831635653973,
      "learning_rate": 9.416187356085512e-07,
      "loss": 0.0002,
      "num_tokens": 8770970.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.3732892274856567,
      "sampling/importance_sampling_ratio/mean": 1.0000840425491333,
      "sampling/importance_sampling_ratio/min": 0.27647536993026733,
      "sampling/sampling_logp_difference/max": 1.2856335639953613,
      "sampling/sampling_logp_difference/mean": 0.020597338676452637,
      "step": 543
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 312.0,
      "completions/max_terminated_length": 312.0,
      "completions/mean_length": 135.796875,
      "completions/mean_terminated_length": 135.796875,
      "completions/min_length": 50.0,
      "completions/min_terminated_length": 50.0,
      "entropy": 0.4649226665496826,
      "epoch": 0.9628318584070796,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0357517830995541,
      "kl": 0.014059186913073063,
      "learning_rate": 9.412560717240195e-07,
      "loss": 0.0001,
      "num_tokens": 8789613.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0002797842025757,
      "sampling/importance_sampling_ratio/min": 0.5656505823135376,
      "sampling/sampling_logp_difference/max": 0.6993000507354736,
      "sampling/sampling_logp_difference/mean": 0.017499998211860657,
      "step": 544
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 232.0,
      "completions/max_terminated_length": 232.0,
      "completions/mean_length": 33.875,
      "completions/mean_terminated_length": 33.875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1782006323337555,
      "epoch": 0.9646017699115044,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09206582105872552,
      "kl": 0.01672673039138317,
      "learning_rate": 9.408923551786742e-07,
      "loss": 0.0002,
      "num_tokens": 8804885.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4249764680862427,
      "sampling/importance_sampling_ratio/mean": 0.9994561672210693,
      "sampling/importance_sampling_ratio/min": 0.7034332752227783,
      "sampling/sampling_logp_difference/max": 0.3541553020477295,
      "sampling/sampling_logp_difference/mean": 0.011499391868710518,
      "step": 545
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 217.0,
      "completions/max_terminated_length": 217.0,
      "completions/mean_length": 41.890625,
      "completions/mean_terminated_length": 41.890625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.1523999273777008,
      "epoch": 0.9663716814159292,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.037107556403945,
      "kl": 0.0039587770588696,
      "learning_rate": 9.405275868401974e-07,
      "loss": 0.0001,
      "num_tokens": 8821678.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4257996082305908,
      "sampling/importance_sampling_ratio/mean": 0.999606728553772,
      "sampling/importance_sampling_ratio/min": 0.725745677947998,
      "sampling/sampling_logp_difference/max": 0.3547327518463135,
      "sampling/sampling_logp_difference/mean": 0.005943705327808857,
      "step": 546
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 392.0,
      "completions/max_terminated_length": 392.0,
      "completions/mean_length": 116.28125,
      "completions/mean_terminated_length": 116.28125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4968239665031433,
      "epoch": 0.968141592920354,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.457847220461041,
      "kl": 0.021092496812343597,
      "learning_rate": 9.40161767578781e-07,
      "loss": 0.1605,
      "num_tokens": 8838672.0,
      "reward": 0.8125,
      "reward_std": 0.25,
      "rewards/decision_reward_func/mean": 0.8125,
      "rewards/decision_reward_func/std": 0.5875696539878845,
      "sampling/importance_sampling_ratio/max": 1.600094199180603,
      "sampling/importance_sampling_ratio/mean": 1.0008270740509033,
      "sampling/importance_sampling_ratio/min": 0.26678600907325745,
      "sampling/sampling_logp_difference/max": 1.3213083744049072,
      "sampling/sampling_logp_difference/mean": 0.018169770017266273,
      "step": 547
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 206.0,
      "completions/max_terminated_length": 206.0,
      "completions/mean_length": 54.5,
      "completions/mean_terminated_length": 54.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2445429116487503,
      "epoch": 0.9699115044247788,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07124112019982765,
      "kl": 0.010958978906273842,
      "learning_rate": 9.397948982671236e-07,
      "loss": 0.0001,
      "num_tokens": 8852992.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.6126388311386108,
      "sampling/importance_sampling_ratio/mean": 1.0011223554611206,
      "sampling/importance_sampling_ratio/min": 0.7174001336097717,
      "sampling/sampling_logp_difference/max": 0.4778718948364258,
      "sampling/sampling_logp_difference/mean": 0.010054649785161018,
      "step": 548
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 229.0,
      "completions/max_terminated_length": 229.0,
      "completions/mean_length": 82.65625,
      "completions/mean_terminated_length": 82.65625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4400416612625122,
      "epoch": 0.9716814159292035,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.758155387149215,
      "kl": 0.01893545500934124,
      "learning_rate": 9.394269797804288e-07,
      "loss": 0.1139,
      "num_tokens": 8870970.0,
      "reward": 0.59375,
      "reward_std": 0.20155644416809082,
      "rewards/decision_reward_func/mean": 0.59375,
      "rewards/decision_reward_func/std": 0.8110105991363525,
      "sampling/importance_sampling_ratio/max": 1.4193644523620605,
      "sampling/importance_sampling_ratio/mean": 0.999426007270813,
      "sampling/importance_sampling_ratio/min": 0.4615151286125183,
      "sampling/sampling_logp_difference/max": 0.7732404470443726,
      "sampling/sampling_logp_difference/mean": 0.015387221239507198,
      "step": 549
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 429.0,
      "completions/max_terminated_length": 429.0,
      "completions/mean_length": 92.0625,
      "completions/mean_terminated_length": 92.0625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.4796140491962433,
      "epoch": 0.9734513274336283,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.630481719584537,
      "kl": 0.016388213261961937,
      "learning_rate": 9.390580129964035e-07,
      "loss": 0.2398,
      "num_tokens": 8886062.0,
      "reward": 0.875,
      "reward_std": 0.3265564441680908,
      "rewards/decision_reward_func/mean": 0.875,
      "rewards/decision_reward_func/std": 0.48795005679130554,
      "sampling/importance_sampling_ratio/max": 1.8056321144104004,
      "sampling/importance_sampling_ratio/mean": 0.9994021654129028,
      "sampling/importance_sampling_ratio/min": 0.6967775821685791,
      "sampling/sampling_logp_difference/max": 0.5909106731414795,
      "sampling/sampling_logp_difference/mean": 0.02025291696190834,
      "step": 550
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 135.0,
      "completions/max_terminated_length": 135.0,
      "completions/mean_length": 49.71875,
      "completions/mean_terminated_length": 49.71875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.21508081257343292,
      "epoch": 0.9752212389380531,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.066991866362209,
      "kl": 0.013176451437175274,
      "learning_rate": 9.386879987952549e-07,
      "loss": 0.0001,
      "num_tokens": 8899756.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4295161962509155,
      "sampling/importance_sampling_ratio/mean": 1.0003056526184082,
      "sampling/importance_sampling_ratio/min": 0.7088531851768494,
      "sampling/sampling_logp_difference/max": 0.35733604431152344,
      "sampling/sampling_logp_difference/mean": 0.009776454418897629,
      "step": 551
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 230.0,
      "completions/max_terminated_length": 230.0,
      "completions/mean_length": 61.96875,
      "completions/mean_terminated_length": 61.96875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.325038880109787,
      "epoch": 0.9769911504424779,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 13.860276512203823,
      "kl": 0.04639158397912979,
      "learning_rate": 9.383169380596892e-07,
      "loss": 0.1206,
      "num_tokens": 8916202.0,
      "reward": 0.5625,
      "reward_std": 0.17078250646591187,
      "rewards/decision_reward_func/mean": 0.5625,
      "rewards/decision_reward_func/std": 0.8333333730697632,
      "sampling/importance_sampling_ratio/max": 1.8170350790023804,
      "sampling/importance_sampling_ratio/mean": 0.9982390403747559,
      "sampling/importance_sampling_ratio/min": 0.6422437429428101,
      "sampling/sampling_logp_difference/max": 0.5972061157226562,
      "sampling/sampling_logp_difference/mean": 0.014898296445608139,
      "step": 552
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 240.0,
      "completions/max_terminated_length": 240.0,
      "completions/mean_length": 76.328125,
      "completions/mean_terminated_length": 76.328125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2448478639125824,
      "epoch": 0.9787610619469026,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10604575756165804,
      "kl": 0.0240221805870533,
      "learning_rate": 9.37944831674909e-07,
      "loss": 0.0002,
      "num_tokens": 8930895.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.4379768371582031,
      "sampling/importance_sampling_ratio/mean": 0.9995517134666443,
      "sampling/importance_sampling_ratio/min": 0.6523293852806091,
      "sampling/sampling_logp_difference/max": 0.4272056221961975,
      "sampling/sampling_logp_difference/mean": 0.010927535593509674,
      "step": 553
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 175.0,
      "completions/max_terminated_length": 175.0,
      "completions/mean_length": 66.859375,
      "completions/mean_terminated_length": 66.859375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.26345235109329224,
      "epoch": 0.9805309734513274,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.897098778736009,
      "kl": 0.02797047048807144,
      "learning_rate": 9.37571680528612e-07,
      "loss": -0.0969,
      "num_tokens": 8945126.0,
      "reward": 0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 1.0015347003936768,
      "sampling/importance_sampling_ratio/min": 0.5551202893257141,
      "sampling/sampling_logp_difference/max": 0.7081301212310791,
      "sampling/sampling_logp_difference/mean": 0.010650092735886574,
      "step": 554
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 205.0,
      "completions/max_terminated_length": 205.0,
      "completions/mean_length": 65.53125,
      "completions/mean_terminated_length": 65.53125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.2612670361995697,
      "epoch": 0.9823008849557522,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09061518316154932,
      "kl": 0.015878070145845413,
      "learning_rate": 9.371974855109874e-07,
      "loss": 0.0002,
      "num_tokens": 8962232.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.5268092155456543,
      "sampling/importance_sampling_ratio/mean": 1.0007729530334473,
      "sampling/importance_sampling_ratio/min": 0.6973515152931213,
      "sampling/sampling_logp_difference/max": 0.42318010330200195,
      "sampling/sampling_logp_difference/mean": 0.011717341840267181,
      "step": 555
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 199.0,
      "completions/max_terminated_length": 199.0,
      "completions/mean_length": 17.5,
      "completions/mean_terminated_length": 17.5,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.10745927691459656,
      "epoch": 0.984070796460177,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 23.680701551151667,
      "kl": 0.20216985046863556,
      "learning_rate": 9.368222475147153e-07,
      "loss": -0.623,
      "num_tokens": 8973160.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 1.8384283781051636,
      "sampling/importance_sampling_ratio/mean": 1.0007233619689941,
      "sampling/importance_sampling_ratio/min": 0.7110517024993896,
      "sampling/sampling_logp_difference/max": 0.6089110374450684,
      "sampling/sampling_logp_difference/mean": 0.01067114993929863,
      "step": 556
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 17.015625,
      "completions/mean_terminated_length": 17.015625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.06491994112730026,
      "epoch": 0.9858407079646018,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.2432899230676877,
      "kl": 0.1308480203151703,
      "learning_rate": 9.36445967434964e-07,
      "loss": 0.0012,
      "num_tokens": 8989801.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 0.5,
      "rewards/decision_reward_func/std": 0.8728715777397156,
      "sampling/importance_sampling_ratio/max": 1.1004294157028198,
      "sampling/importance_sampling_ratio/mean": 1.0001790523529053,
      "sampling/importance_sampling_ratio/min": 0.9071494936943054,
      "sampling/sampling_logp_difference/max": 0.09744799137115479,
      "sampling/sampling_logp_difference/mean": 0.0038687689229846,
      "step": 557
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 317.0,
      "completions/max_terminated_length": 317.0,
      "completions/mean_length": 91.8125,
      "completions/mean_terminated_length": 91.8125,
      "completions/min_length": 17.0,
      "completions/min_terminated_length": 17.0,
      "entropy": 0.34381309151649475,
      "epoch": 0.9876106194690265,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.125285207917033,
      "kl": 0.02384905517101288,
      "learning_rate": 9.360686461693872e-07,
      "loss": 0.1224,
      "num_tokens": 9005949.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.4320323467254639,
      "sampling/importance_sampling_ratio/mean": 1.0000149011611938,
      "sampling/importance_sampling_ratio/min": 0.7160810232162476,
      "sampling/sampling_logp_difference/max": 0.35909461975097656,
      "sampling/sampling_logp_difference/mean": 0.01379399560391903,
      "step": 558
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 111.0,
      "completions/max_terminated_length": 111.0,
      "completions/mean_length": 23.640625,
      "completions/mean_terminated_length": 23.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.25869446992874146,
      "epoch": 0.9893805309734514,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 14.470531190670112,
      "kl": 0.21281038224697113,
      "learning_rate": 9.356902846181228e-07,
      "loss": 0.1498,
      "num_tokens": 9019494.0,
      "reward": 0.375,
      "reward_std": 0.42078250646591187,
      "rewards/decision_reward_func/mean": 0.375,
      "rewards/decision_reward_func/std": 0.934353232383728,
      "sampling/importance_sampling_ratio/max": 1.56514573097229,
      "sampling/importance_sampling_ratio/mean": 1.0025410652160645,
      "sampling/importance_sampling_ratio/min": 0.7757983803749084,
      "sampling/sampling_logp_difference/max": 0.4479789733886719,
      "sampling/sampling_logp_difference/mean": 0.012579815462231636,
      "step": 559
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 194.0,
      "completions/max_terminated_length": 194.0,
      "completions/mean_length": 61.671875,
      "completions/mean_terminated_length": 61.671875,
      "completions/min_length": 16.0,
      "completions/min_terminated_length": 16.0,
      "entropy": 0.32114139199256897,
      "epoch": 0.9911504424778761,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.609035510367293,
      "kl": 0.018139678984880447,
      "learning_rate": 9.353108836837905e-07,
      "loss": 0.093,
      "num_tokens": 9036113.0,
      "reward": 0.53125,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.53125,
      "rewards/decision_reward_func/std": 0.8539125919342041,
      "sampling/importance_sampling_ratio/max": 1.6116080284118652,
      "sampling/importance_sampling_ratio/mean": 1.0010817050933838,
      "sampling/importance_sampling_ratio/min": 0.6868107914924622,
      "sampling/sampling_logp_difference/max": 0.4772324562072754,
      "sampling/sampling_logp_difference/mean": 0.012453297153115273,
      "step": 560
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 14.59375,
      "completions/mean_terminated_length": 14.59375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.07570166885852814,
      "epoch": 0.9929203539823008,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.20701388280592103,
      "kl": 0.19794254004955292,
      "learning_rate": 9.349304442714895e-07,
      "loss": 0.0019,
      "num_tokens": 9050007.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.2850689888000488,
      "sampling/importance_sampling_ratio/mean": 0.99912029504776,
      "sampling/importance_sampling_ratio/min": 0.8150334358215332,
      "sampling/sampling_logp_difference/max": 0.2508124113082886,
      "sampling/sampling_logp_difference/mean": 0.011712642386555672,
      "step": 561
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 152.0,
      "completions/max_terminated_length": 152.0,
      "completions/mean_length": 52.640625,
      "completions/mean_terminated_length": 52.640625,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.32283660769462585,
      "epoch": 0.9946902654867257,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.951665755188321,
      "kl": 0.12831877171993256,
      "learning_rate": 9.345489672887962e-07,
      "loss": -0.1108,
      "num_tokens": 9063488.0,
      "reward": 0.96875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": 0.96875,
      "rewards/decision_reward_func/std": 0.25,
      "sampling/importance_sampling_ratio/max": 1.4442304372787476,
      "sampling/importance_sampling_ratio/mean": 0.999579906463623,
      "sampling/importance_sampling_ratio/min": 0.6357463002204895,
      "sampling/sampling_logp_difference/max": 0.4529557228088379,
      "sampling/sampling_logp_difference/mean": 0.013744027353823185,
      "step": 562
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 317.0,
      "completions/max_terminated_length": 317.0,
      "completions/mean_length": 39.609375,
      "completions/mean_terminated_length": 39.609375,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.23001480102539062,
      "epoch": 0.9964601769911504,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 21.980215852115183,
      "kl": 0.34417223930358887,
      "learning_rate": 9.341664536457625e-07,
      "loss": -0.1346,
      "num_tokens": 9075783.0,
      "reward": -0.46875,
      "reward_std": 0.125,
      "rewards/decision_reward_func/mean": -0.46875,
      "rewards/decision_reward_func/std": 0.8903138637542725,
      "sampling/importance_sampling_ratio/max": 2.0,
      "sampling/importance_sampling_ratio/mean": 0.9994174838066101,
      "sampling/importance_sampling_ratio/min": 0.685285210609436,
      "sampling/sampling_logp_difference/max": 1.5750985145568848,
      "sampling/sampling_logp_difference/mean": 0.015202976763248444,
      "step": 563
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 20.0,
      "completions/max_terminated_length": 20.0,
      "completions/mean_length": 15.828125,
      "completions/mean_terminated_length": 15.828125,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.08043765276670456,
      "epoch": 0.9982300884955753,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.11158896786330862,
      "kl": 0.08205146342515945,
      "learning_rate": 9.337829042549133e-07,
      "loss": 0.0008,
      "num_tokens": 9087580.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/decision_reward_func/mean": 1.0,
      "rewards/decision_reward_func/std": 0.0,
      "sampling/importance_sampling_ratio/max": 1.4533406496047974,
      "sampling/importance_sampling_ratio/mean": 1.0018446445465088,
      "sampling/importance_sampling_ratio/min": 0.6155814528465271,
      "sampling/sampling_logp_difference/max": 0.48518800735473633,
      "sampling/sampling_logp_difference/mean": 0.01030774973332882,
      "step": 564
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 219.0,
      "completions/max_terminated_length": 219.0,
      "completions/mean_length": 53.546875,
      "completions/mean_terminated_length": 53.546875,
      "completions/min_length": 11.0,
      "completions/min_terminated_length": 11.0,
      "entropy": 0.3179180324077606,
      "epoch": 1.0,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.110451794643717,
      "kl": 0.07360747456550598,
      "learning_rate": 9.33398320031244e-07,
      "loss": -0.2468,
      "num_tokens": 9100799.0,
      "reward": 0.25,
      "reward_std": 0.25819888710975647,
      "rewards/decision_reward_func/mean": 0.25,
      "rewards/decision_reward_func/std": 0.9759001135826111,
      "sampling/importance_sampling_ratio/max": 1.5650666952133179,
      "sampling/importance_sampling_ratio/mean": 0.9979598522186279,
      "sampling/importance_sampling_ratio/min": 0.6005945205688477,
      "sampling/sampling_logp_difference/max": 0.5098352432250977,
      "sampling/sampling_logp_difference/mean": 0.017205916345119476,
      "step": 565
    }
  ],
  "logging_steps": 1,
  "max_steps": 2260,
  "num_input_tokens_seen": 9100799,
  "num_train_epochs": 4,
  "save_steps": 565,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}
