{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 292,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.03432003432003432,
      "grad_norm": 10.203938715992143,
      "learning_rate": 1.5e-07,
      "logits/chosen": -1.8046875,
      "logits/rejected": -1.734375,
      "logps/chosen": -0.28125,
      "logps/rejected": -0.283203125,
      "loss": 1.3126378059387207,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 2.9130210876464844,
      "rewards/margins": 0.8968315124511719,
      "rewards/rejected": 2.0161895751953125,
      "step": 10
    },
    {
      "epoch": 0.06864006864006864,
      "grad_norm": 34.812950350142344,
      "learning_rate": 3.166666666666666e-07,
      "logits/chosen": -1.734375,
      "logits/rejected": -1.71875,
      "logps/chosen": -0.28515625,
      "logps/rejected": -0.296875,
      "loss": 1.2969410419464111,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2256202697753906,
      "rewards/margins": 1.8563461303710938,
      "rewards/rejected": 1.3692741394042969,
      "step": 20
    },
    {
      "epoch": 0.10296010296010295,
      "grad_norm": 18.043733604549054,
      "learning_rate": 4.833333333333333e-07,
      "logits/chosen": -1.8828125,
      "logits/rejected": -1.828125,
      "logps/chosen": -0.2578125,
      "logps/rejected": -0.27734375,
      "loss": 1.2808767557144165,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 2.254518508911133,
      "rewards/margins": 0.9522342681884766,
      "rewards/rejected": 1.3022842407226562,
      "step": 30
    },
    {
      "epoch": 0.13728013728013727,
      "grad_norm": 14.522579405037842,
      "learning_rate": 4.985456442051682e-07,
      "logits/chosen": -1.7421875,
      "logits/rejected": -1.7578125,
      "logps/chosen": -0.28515625,
      "logps/rejected": -0.30078125,
      "loss": 1.2971984148025513,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 2.7966461181640625,
      "rewards/margins": 0.865142822265625,
      "rewards/rejected": 1.9315032958984375,
      "step": 40
    },
    {
      "epoch": 0.1716001716001716,
      "grad_norm": 9.44153398614957,
      "learning_rate": 4.935399618791793e-07,
      "logits/chosen": -1.6953125,
      "logits/rejected": -1.671875,
      "logps/chosen": -0.29296875,
      "logps/rejected": -0.314453125,
      "loss": 1.2938823699951172,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 3.333526611328125,
      "rewards/margins": 1.3265352249145508,
      "rewards/rejected": 2.006991386413574,
      "step": 50
    },
    {
      "epoch": 0.2059202059202059,
      "grad_norm": 7.8411658071056864,
      "learning_rate": 4.850368660819092e-07,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.8046875,
      "logps/chosen": -0.291015625,
      "logps/rejected": -0.310546875,
      "loss": 1.2829803228378296,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.5130538940429688,
      "rewards/margins": 0.6417007446289062,
      "rewards/rejected": 0.8713531494140625,
      "step": 60
    },
    {
      "epoch": 0.24024024024024024,
      "grad_norm": 28.26163110598253,
      "learning_rate": 4.731584675403184e-07,
      "logits/chosen": -1.65625,
      "logits/rejected": -1.640625,
      "logps/chosen": -0.30859375,
      "logps/rejected": -0.30078125,
      "loss": 1.3537184000015259,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.1701087951660156,
      "rewards/margins": 1.6411056518554688,
      "rewards/rejected": 1.5290031433105469,
      "step": 70
    },
    {
      "epoch": 0.27456027456027454,
      "grad_norm": 8.444828433926622,
      "learning_rate": 4.5807534881817183e-07,
      "logits/chosen": -1.7890625,
      "logits/rejected": -1.7109375,
      "logps/chosen": -0.294921875,
      "logps/rejected": -0.3125,
      "loss": 1.3036760091781616,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 2.192584991455078,
      "rewards/margins": 0.8953938484191895,
      "rewards/rejected": 1.2971911430358887,
      "step": 80
    },
    {
      "epoch": 0.3088803088803089,
      "grad_norm": 20.244100591742612,
      "learning_rate": 4.400041146246136e-07,
      "logits/chosen": -1.7578125,
      "logits/rejected": -1.6875,
      "logps/chosen": -0.3125,
      "logps/rejected": -0.3203125,
      "loss": 1.3008702993392944,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 3.269622802734375,
      "rewards/margins": 1.627532958984375,
      "rewards/rejected": 1.64208984375,
      "step": 90
    },
    {
      "epoch": 0.3432003432003432,
      "grad_norm": 7.960625256781812,
      "learning_rate": 4.1920428121079e-07,
      "logits/chosen": -1.71875,
      "logits/rejected": -1.6796875,
      "logps/chosen": -0.283203125,
      "logps/rejected": -0.330078125,
      "loss": 1.2717617750167847,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 2.0637588500976562,
      "rewards/margins": 1.0487861633300781,
      "rewards/rejected": 1.0149726867675781,
      "step": 100
    },
    {
      "epoch": 0.37752037752037754,
      "grad_norm": 36.274471498701175,
      "learning_rate": 3.959745495250818e-07,
      "logits/chosen": -1.9921875,
      "logits/rejected": -1.9140625,
      "logps/chosen": -0.275390625,
      "logps/rejected": -0.306640625,
      "loss": 1.2781312465667725,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 2.068091630935669,
      "rewards/margins": 1.2525169849395752,
      "rewards/rejected": 0.8155746459960938,
      "step": 110
    },
    {
      "epoch": 0.4118404118404118,
      "grad_norm": 32.62769321502908,
      "learning_rate": 3.7064851564718353e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.7109375,
      "logps/chosen": -0.287109375,
      "logps/rejected": -0.353515625,
      "loss": 1.2694454193115234,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 3.714447021484375,
      "rewards/margins": 2.37420654296875,
      "rewards/rejected": 1.340240478515625,
      "step": 120
    },
    {
      "epoch": 0.44616044616044614,
      "grad_norm": 8.666298132652223,
      "learning_rate": 3.4358988010236103e-07,
      "logits/chosen": -1.765625,
      "logits/rejected": -1.765625,
      "logps/chosen": -0.322265625,
      "logps/rejected": -0.3046875,
      "loss": 1.377197504043579,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 2.362274169921875,
      "rewards/margins": 1.509368896484375,
      "rewards/rejected": 0.8529052734375,
      "step": 130
    },
    {
      "epoch": 0.4804804804804805,
      "grad_norm": 17.229310324932,
      "learning_rate": 3.1518722485366754e-07,
      "logits/chosen": -1.90625,
      "logits/rejected": -1.875,
      "logps/chosen": -0.294921875,
      "logps/rejected": -0.3203125,
      "loss": 1.3077561855316162,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.7135581970214844,
      "rewards/margins": 1.8327293395996094,
      "rewards/rejected": 0.880828857421875,
      "step": 140
    },
    {
      "epoch": 0.5148005148005148,
      "grad_norm": 8.648271514841943,
      "learning_rate": 2.8584843297836277e-07,
      "logits/chosen": -1.8203125,
      "logits/rejected": -1.8046875,
      "logps/chosen": -0.28125,
      "logps/rejected": -0.33984375,
      "loss": 1.2540769577026367,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 2.5254592895507812,
      "rewards/margins": 1.1835556030273438,
      "rewards/rejected": 1.3419036865234375,
      "step": 150
    },
    {
      "epoch": 0.5491205491205491,
      "grad_norm": 49.83862434409446,
      "learning_rate": 2.5599483116609544e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.6953125,
      "logps/chosen": -0.298828125,
      "logps/rejected": -0.33203125,
      "loss": 1.273056983947754,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 2.8408203125,
      "rewards/margins": 1.439300537109375,
      "rewards/rejected": 1.401519775390625,
      "step": 160
    },
    {
      "epoch": 0.5834405834405835,
      "grad_norm": 25.723956384719525,
      "learning_rate": 2.2605513915689874e-07,
      "logits/chosen": -1.796875,
      "logits/rejected": -1.7578125,
      "logps/chosen": -0.28515625,
      "logps/rejected": -0.44140625,
      "loss": 1.2432403564453125,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 2.3864269256591797,
      "rewards/margins": 1.9962539672851562,
      "rewards/rejected": 0.39017295837402344,
      "step": 170
    },
    {
      "epoch": 0.6177606177606177,
      "grad_norm": 73.0500683832465,
      "learning_rate": 1.9645931300952795e-07,
      "logits/chosen": -1.9296875,
      "logits/rejected": -1.8828125,
      "logps/chosen": -0.30078125,
      "logps/rejected": -0.373046875,
      "loss": 1.3145594596862793,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.933912515640259,
      "rewards/margins": 1.966554880142212,
      "rewards/rejected": 0.9673576354980469,
      "step": 180
    },
    {
      "epoch": 0.6520806520806521,
      "grad_norm": 10.570188964120872,
      "learning_rate": 1.6763237061535008e-07,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.78125,
      "logps/chosen": -0.279296875,
      "logps/rejected": -0.33984375,
      "loss": 1.2554752826690674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.1903152465820312,
      "rewards/margins": 1.0904598236083984,
      "rewards/rejected": 1.0998554229736328,
      "step": 190
    },
    {
      "epoch": 0.6864006864006864,
      "grad_norm": 8.018004144482099,
      "learning_rate": 1.3998828812795932e-07,
      "logits/chosen": -1.7734375,
      "logits/rejected": -1.6953125,
      "logps/chosen": -0.3046875,
      "logps/rejected": -0.349609375,
      "loss": 1.2610969543457031,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4741287231445312,
      "rewards/margins": 1.5025177001953125,
      "rewards/rejected": 0.9716110229492188,
      "step": 200
    },
    {
      "epoch": 0.7207207207207207,
      "grad_norm": 13.917673494093787,
      "learning_rate": 1.1392405496029076e-07,
      "logits/chosen": -1.859375,
      "logits/rejected": -1.828125,
      "logps/chosen": -0.291015625,
      "logps/rejected": -0.37890625,
      "loss": 1.2743440866470337,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.3770370483398438,
      "rewards/margins": 1.9193801879882812,
      "rewards/rejected": 0.4576568603515625,
      "step": 210
    },
    {
      "epoch": 0.7550407550407551,
      "grad_norm": 7.293229087932327,
      "learning_rate": 8.981397272385738e-08,
      "logits/chosen": -1.90625,
      "logits/rejected": -1.8203125,
      "logps/chosen": -0.296875,
      "logps/rejected": -0.359375,
      "loss": 1.2591865062713623,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8160285949707031,
      "rewards/margins": 1.0967063903808594,
      "rewards/rejected": 0.7193222045898438,
      "step": 220
    },
    {
      "epoch": 0.7893607893607893,
      "grad_norm": 14.94242869882904,
      "learning_rate": 6.800427998154968e-08,
      "logits/chosen": -1.9609375,
      "logits/rejected": -1.875,
      "logps/chosen": -0.302734375,
      "logps/rejected": -0.439453125,
      "loss": 1.2168002128601074,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 3.1121368408203125,
      "rewards/margins": 1.8566741943359375,
      "rewards/rejected": 1.255462646484375,
      "step": 230
    },
    {
      "epoch": 0.8236808236808236,
      "grad_norm": 6.26084322691984,
      "learning_rate": 4.8808180006509356e-08,
      "logits/chosen": -1.8671875,
      "logits/rejected": -1.78125,
      "logps/chosen": -0.2890625,
      "logps/rejected": -0.375,
      "loss": 1.2628111839294434,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 2.628856658935547,
      "rewards/margins": 1.8339424133300781,
      "rewards/rejected": 0.7949142456054688,
      "step": 240
    },
    {
      "epoch": 0.858000858000858,
      "grad_norm": 7.036498888058669,
      "learning_rate": 3.250134295213053e-08,
      "logits/chosen": -1.8515625,
      "logits/rejected": -1.78125,
      "logps/chosen": -0.306640625,
      "logps/rejected": -0.328125,
      "loss": 1.3013354539871216,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.027191162109375,
      "rewards/margins": 1.3493232727050781,
      "rewards/rejected": 0.6778678894042969,
      "step": 250
    },
    {
      "epoch": 0.8923208923208923,
      "grad_norm": 26.20577632403144,
      "learning_rate": 1.9317947025340232e-08,
      "logits/chosen": -1.8125,
      "logits/rejected": -1.734375,
      "logps/chosen": -0.30859375,
      "logps/rejected": -0.390625,
      "loss": 1.249146819114685,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 3.2939453125,
      "rewards/margins": 2.5830841064453125,
      "rewards/rejected": 0.7108612060546875,
      "step": 260
    },
    {
      "epoch": 0.9266409266409267,
      "grad_norm": 9.303768228574064,
      "learning_rate": 9.447315514833353e-09,
      "logits/chosen": -1.8359375,
      "logits/rejected": -1.734375,
      "logps/chosen": -0.28515625,
      "logps/rejected": -0.31640625,
      "loss": 1.2721102237701416,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.105621337890625,
      "rewards/margins": 1.446533203125,
      "rewards/rejected": 0.659088134765625,
      "step": 270
    },
    {
      "epoch": 0.960960960960961,
      "grad_norm": 6.727511885545749,
      "learning_rate": 3.0311979690147426e-09,
      "logits/chosen": -1.78125,
      "logits/rejected": -1.6640625,
      "logps/chosen": -0.291015625,
      "logps/rejected": -0.33203125,
      "loss": 1.2691892385482788,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.3777236938476562,
      "rewards/margins": 1.4847488403320312,
      "rewards/rejected": 0.892974853515625,
      "step": 280
    },
    {
      "epoch": 0.9952809952809952,
      "grad_norm": 6.168691664518052,
      "learning_rate": 1.6173456793908135e-10,
      "logits/chosen": -1.9140625,
      "logits/rejected": -1.8828125,
      "logps/chosen": -0.287109375,
      "logps/rejected": -0.302734375,
      "loss": 1.300573706626892,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.7515792846679688,
      "rewards/margins": 1.7214431762695312,
      "rewards/rejected": 1.0301361083984375,
      "step": 290
    },
    {
      "epoch": 1.0,
      "step": 292,
      "total_flos": 0.0,
      "train_loss": 1.2802449281085027,
      "train_runtime": 10579.5096,
      "train_samples_per_second": 1.762,
      "train_steps_per_second": 0.028
    }
  ],
  "logging_steps": 10,
  "max_steps": 292,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}