{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.06652991073903643,
  "eval_steps": 500,
  "global_step": 75,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004435327382602428,
      "grad_norm": 14.0,
      "learning_rate": 2.5e-07,
      "logits/chosen": -2.144705295562744,
      "logits/rejected": -2.0726897716522217,
      "logps/chosen": -0.2952641248703003,
      "logps/rejected": -0.3042605519294739,
      "loss": 0.9013038873672485,
      "loss/core": 0.8998275995254517,
      "loss/preference_sft": 0.2952641248703003,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.2125720977783203,
      "rewards/margins": 1.2788416147232056,
      "rewards/rejected": 0.9337307810783386,
      "ronpo/logit": 0.1278841495513916,
      "ronpo/residual": -0.8721157908439636,
      "ronpo/residual_abs": 0.9368273615837097,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 5
    },
    {
      "epoch": 0.008870654765204856,
      "grad_norm": 210.0,
      "learning_rate": 4.997252228714278e-07,
      "logits/chosen": -2.165788412094116,
      "logits/rejected": -2.1329445838928223,
      "logps/chosen": -0.24870172142982483,
      "logps/rejected": -0.28110265731811523,
      "loss": 0.9554009437561035,
      "loss/core": 0.9541573524475098,
      "loss/preference_sft": 0.24870172142982483,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 3.0928869247436523,
      "rewards/margins": 1.3761483430862427,
      "rewards/rejected": 1.7167381048202515,
      "ronpo/logit": 0.13761483132839203,
      "ronpo/residual": -0.862385094165802,
      "ronpo/residual_abs": 0.9450987577438354,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 10
    },
    {
      "epoch": 0.013305982147807285,
      "grad_norm": 11.3125,
      "learning_rate": 4.9017129689421e-07,
      "logits/chosen": -2.123521089553833,
      "logits/rejected": -2.0528640747070312,
      "logps/chosen": -0.30137166380882263,
      "logps/rejected": -0.2893705666065216,
      "loss": 0.9385660290718079,
      "loss/core": 0.937059223651886,
      "loss/preference_sft": 0.30137166380882263,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.6111516952514648,
      "rewards/margins": 0.4742206931114197,
      "rewards/rejected": 1.13693106174469,
      "ronpo/logit": 0.04742206260561943,
      "ronpo/residual": -0.9525778889656067,
      "ronpo/residual_abs": 0.9525778889656067,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 15
    },
    {
      "epoch": 0.017741309530409713,
      "grad_norm": 12.5,
      "learning_rate": 4.6747659310219757e-07,
      "logits/chosen": -2.2110671997070312,
      "logits/rejected": -2.0166139602661133,
      "logps/chosen": -0.2711886465549469,
      "logps/rejected": -0.29270288348197937,
      "loss": 1.0191020965576172,
      "loss/core": 1.0177462100982666,
      "loss/preference_sft": 0.2711886465549469,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.056633949279785,
      "rewards/margins": 0.49368467926979065,
      "rewards/rejected": 1.5629494190216064,
      "ronpo/logit": 0.04936846345663071,
      "ronpo/residual": -0.9506314992904663,
      "ronpo/residual_abs": 0.9506314992904663,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 20
    },
    {
      "epoch": 0.02217663691301214,
      "grad_norm": 6.59375,
      "learning_rate": 4.3288283381591725e-07,
      "logits/chosen": -2.171949625015259,
      "logits/rejected": -2.0527703762054443,
      "logps/chosen": -0.28833886981010437,
      "logps/rejected": -0.29443103075027466,
      "loss": 0.9356898069381714,
      "loss/core": 0.934248149394989,
      "loss/preference_sft": 0.28833886981010437,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.5931298732757568,
      "rewards/margins": 0.5489386320114136,
      "rewards/rejected": 1.0441911220550537,
      "ronpo/logit": 0.0548938624560833,
      "ronpo/residual": -0.945106029510498,
      "ronpo/residual_abs": 0.945106029510498,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 25
    },
    {
      "epoch": 0.02661196429561457,
      "grad_norm": 49.75,
      "learning_rate": 3.882827885312998e-07,
      "logits/chosen": -2.277024269104004,
      "logits/rejected": -2.2178688049316406,
      "logps/chosen": -0.30705174803733826,
      "logps/rejected": -0.3033962845802307,
      "loss": 1.0691256523132324,
      "loss/core": 1.0675902366638184,
      "loss/preference_sft": 0.30705174803733826,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.114480495452881,
      "rewards/margins": 1.2642418146133423,
      "rewards/rejected": 0.8502384424209595,
      "ronpo/logit": 0.126424178481102,
      "ronpo/residual": -0.8735758662223816,
      "ronpo/residual_abs": 0.9844897389411926,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 30
    },
    {
      "epoch": 0.031047291678216998,
      "grad_norm": 19.75,
      "learning_rate": 3.361167125710832e-07,
      "logits/chosen": -2.1667983531951904,
      "logits/rejected": -2.209552764892578,
      "logps/chosen": -0.29950323700904846,
      "logps/rejected": -0.3074646592140198,
      "loss": 0.9153361320495605,
      "loss/core": 0.9138385057449341,
      "loss/preference_sft": 0.29950323700904846,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.2793786525726318,
      "rewards/margins": 0.6898711919784546,
      "rewards/rejected": 0.5895074009895325,
      "ronpo/logit": 0.06898711621761322,
      "ronpo/residual": -0.9310129284858704,
      "ronpo/residual_abs": 0.9501181840896606,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 35
    },
    {
      "epoch": 0.035482619060819426,
      "grad_norm": 14.5,
      "learning_rate": 2.7923883049302066e-07,
      "logits/chosen": -2.0911293029785156,
      "logits/rejected": -1.996106743812561,
      "logps/chosen": -0.2962394654750824,
      "logps/rejected": -0.2883327603340149,
      "loss": 1.2223423719406128,
      "loss/core": 1.2208611965179443,
      "loss/preference_sft": 0.2962394654750824,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.883702516555786,
      "rewards/margins": 1.178934097290039,
      "rewards/rejected": 1.7047685384750366,
      "ronpo/logit": 0.11789341270923615,
      "ronpo/residual": -0.882106602191925,
      "ronpo/residual_abs": 0.9954751133918762,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 40
    },
    {
      "epoch": 0.03991794644342186,
      "grad_norm": 17.5,
      "learning_rate": 2.2076116950697937e-07,
      "logits/chosen": -2.2753586769104004,
      "logits/rejected": -2.194331645965576,
      "logps/chosen": -0.2942467927932739,
      "logps/rejected": -0.2941977381706238,
      "loss": 0.960641086101532,
      "loss/core": 0.959169864654541,
      "loss/preference_sft": 0.2942467927932739,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.3440639972686768,
      "rewards/margins": 1.245705008506775,
      "rewards/rejected": 1.0983591079711914,
      "ronpo/logit": 0.12457048892974854,
      "ronpo/residual": -0.8754295110702515,
      "ronpo/residual_abs": 0.9467538595199585,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 45
    },
    {
      "epoch": 0.04435327382602428,
      "grad_norm": 6.03125,
      "learning_rate": 1.6388328742891678e-07,
      "logits/chosen": -2.3468501567840576,
      "logits/rejected": -2.2160422801971436,
      "logps/chosen": -0.2907429337501526,
      "logps/rejected": -0.299108624458313,
      "loss": 1.6810009479522705,
      "loss/core": 1.6795470714569092,
      "loss/preference_sft": 0.2907429337501526,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.491801381111145,
      "rewards/margins": -0.29926931858062744,
      "rewards/rejected": 1.7910706996917725,
      "ronpo/logit": -0.029926935210824013,
      "ronpo/residual": -1.0299268960952759,
      "ronpo/residual_abs": 1.0562489032745361,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 50
    },
    {
      "epoch": 0.048788601208626714,
      "grad_norm": 5.1875,
      "learning_rate": 1.1171721146870014e-07,
      "logits/chosen": -2.2220540046691895,
      "logits/rejected": -2.159651279449463,
      "logps/chosen": -0.29590311646461487,
      "logps/rejected": -0.3085595369338989,
      "loss": 1.2529202699661255,
      "loss/core": 1.2514407634735107,
      "loss/preference_sft": 0.29590311646461487,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7223021984100342,
      "rewards/margins": 0.2412434071302414,
      "rewards/rejected": 1.4810588359832764,
      "ronpo/logit": 0.02412433736026287,
      "ronpo/residual": -0.9758756756782532,
      "ronpo/residual_abs": 1.0081208944320679,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 55
    },
    {
      "epoch": 0.05322392859122914,
      "grad_norm": 121.0,
      "learning_rate": 6.711716618408281e-08,
      "logits/chosen": -1.9089536666870117,
      "logits/rejected": -1.977799654006958,
      "logps/chosen": -0.3068787455558777,
      "logps/rejected": -0.29860004782676697,
      "loss": 1.5921037197113037,
      "loss/core": 1.590569257736206,
      "loss/preference_sft": 0.3068787455558777,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 3.075084924697876,
      "rewards/margins": 1.1153666973114014,
      "rewards/rejected": 1.9597183465957642,
      "ronpo/logit": 0.1115366593003273,
      "ronpo/residual": -0.8884633183479309,
      "ronpo/residual_abs": 0.9980802536010742,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 60
    },
    {
      "epoch": 0.05765925597383157,
      "grad_norm": 25.25,
      "learning_rate": 3.2523406897802444e-08,
      "logits/chosen": -2.136443614959717,
      "logits/rejected": -2.083848476409912,
      "logps/chosen": -0.2734604477882385,
      "logps/rejected": -0.28085511922836304,
      "loss": 0.8904159665107727,
      "loss/core": 0.88904869556427,
      "loss/preference_sft": 0.2734604477882385,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9123761653900146,
      "rewards/margins": 0.9041536450386047,
      "rewards/rejected": 1.0082225799560547,
      "ronpo/logit": 0.09041537344455719,
      "ronpo/residual": -0.909584641456604,
      "ronpo/residual_abs": 0.93060702085495,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 65
    },
    {
      "epoch": 0.062094583356433995,
      "grad_norm": 8.1875,
      "learning_rate": 9.828703105789981e-09,
      "logits/chosen": -2.1384823322296143,
      "logits/rejected": -2.0494384765625,
      "logps/chosen": -0.29097050428390503,
      "logps/rejected": -0.3023843765258789,
      "loss": 0.8998498916625977,
      "loss/core": 0.8983950614929199,
      "loss/preference_sft": 0.29097050428390503,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2245758771896362,
      "rewards/margins": 0.5744353532791138,
      "rewards/rejected": 0.6501404643058777,
      "ronpo/logit": 0.057443536818027496,
      "ronpo/residual": -0.9425565004348755,
      "ronpo/residual_abs": 0.9425565004348755,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 70
    },
    {
      "epoch": 0.06652991073903643,
      "grad_norm": 59.25,
      "learning_rate": 2.7477712857215675e-10,
      "logits/chosen": -2.183204412460327,
      "logits/rejected": -2.0384721755981445,
      "logps/chosen": -0.27401086688041687,
      "logps/rejected": -0.27478593587875366,
      "loss": 1.1840168237686157,
      "loss/core": 1.182646632194519,
      "loss/preference_sft": 0.27401086688041687,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6213176250457764,
      "rewards/margins": 0.029666978865861893,
      "rewards/rejected": 1.5916506052017212,
      "ronpo/logit": 0.002966696862131357,
      "ronpo/residual": -0.9970332980155945,
      "ronpo/residual_abs": 0.9970332980155945,
      "ronpo/target": 1.0,
      "ronpo/target_abs": 1.0,
      "ronpo/weight": 1.0,
      "ronpo/weight_max": 1.0,
      "step": 75
    },
    {
      "epoch": 0.06652991073903643,
      "step": 75,
      "total_flos": 0.0,
      "train_loss": 1.094520975748698,
      "train_runtime": 689.8738,
      "train_samples_per_second": 1.739,
      "train_steps_per_second": 0.109
    }
  ],
  "logging_steps": 5,
  "max_steps": 75,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}