{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989528795811519,
  "eval_steps": 200,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0020942408376963353,
      "grad_norm": 19.463237762451172,
      "learning_rate": 0.0,
      "logits/chosen": 2.203179359436035,
      "logits/rejected": 2.035616397857666,
      "logps/chosen": -257.4821472167969,
      "logps/ref_chosen": -257.3987731933594,
      "logps/ref_rejected": -199.99972534179688,
      "logps/rejected": -199.93338012695312,
      "loss": 45.6501,
      "r_dpo/chosen_len": 260.25,
      "r_dpo/length_delta": 46.875,
      "r_dpo/regularization_term": 4.6875,
      "r_dpo/rejected_len": 213.375,
      "step": 1
    },
    {
      "epoch": 0.020942408376963352,
      "grad_norm": 18.84970474243164,
      "learning_rate": 9.375e-08,
      "logits/chosen": 1.9343596696853638,
      "logits/rejected": 1.8566161394119263,
      "logps/chosen": -290.1725158691406,
      "logps/ref_chosen": -290.0645751953125,
      "logps/ref_rejected": -255.95347595214844,
      "logps/rejected": -256.05364990234375,
      "loss": 43.0812,
      "r_dpo/chosen_len": 295.9757080078125,
      "r_dpo/length_delta": 49.59722137451172,
      "r_dpo/regularization_term": 4.959722518920898,
      "r_dpo/rejected_len": 246.37847900390625,
      "step": 10
    },
    {
      "epoch": 0.041884816753926704,
      "grad_norm": 19.003461837768555,
      "learning_rate": 1.9791666666666664e-07,
      "logits/chosen": 1.8805112838745117,
      "logits/rejected": 1.8091768026351929,
      "logps/chosen": -289.0137023925781,
      "logps/ref_chosen": -288.9098815917969,
      "logps/ref_rejected": -254.6271514892578,
      "logps/rejected": -254.85696411132812,
      "loss": 41.2955,
      "r_dpo/chosen_len": 295.1187438964844,
      "r_dpo/length_delta": 54.203125,
      "r_dpo/regularization_term": 5.42031192779541,
      "r_dpo/rejected_len": 240.91561889648438,
      "step": 20
    },
    {
      "epoch": 0.06282722513089005,
      "grad_norm": 21.310501098632812,
      "learning_rate": 3.020833333333333e-07,
      "logits/chosen": 1.8696081638336182,
      "logits/rejected": 1.818481683731079,
      "logps/chosen": -269.52618408203125,
      "logps/ref_chosen": -270.3978576660156,
      "logps/ref_rejected": -247.89431762695312,
      "logps/rejected": -247.4498291015625,
      "loss": 43.6841,
      "r_dpo/chosen_len": 272.8062438964844,
      "r_dpo/length_delta": 25.693750381469727,
      "r_dpo/regularization_term": 2.5693750381469727,
      "r_dpo/rejected_len": 247.1125030517578,
      "step": 30
    },
    {
      "epoch": 0.08376963350785341,
      "grad_norm": 19.77940559387207,
      "learning_rate": 4.0625e-07,
      "logits/chosen": 1.8189456462860107,
      "logits/rejected": 1.8397687673568726,
      "logps/chosen": -275.187744140625,
      "logps/ref_chosen": -274.24859619140625,
      "logps/ref_rejected": -261.91693115234375,
      "logps/rejected": -264.21368408203125,
      "loss": 41.7583,
      "r_dpo/chosen_len": 284.7875061035156,
      "r_dpo/length_delta": 33.97187423706055,
      "r_dpo/regularization_term": 3.3971877098083496,
      "r_dpo/rejected_len": 250.8156280517578,
      "step": 40
    },
    {
      "epoch": 0.10471204188481675,
      "grad_norm": 23.84255027770996,
      "learning_rate": 4.999932966293553e-07,
      "logits/chosen": 1.8647778034210205,
      "logits/rejected": 1.9068819284439087,
      "logps/chosen": -280.23028564453125,
      "logps/ref_chosen": -270.4697570800781,
      "logps/ref_rejected": -271.0193176269531,
      "logps/rejected": -282.69647216796875,
      "loss": 43.553,
      "r_dpo/chosen_len": 294.27813720703125,
      "r_dpo/length_delta": 37.4375,
      "r_dpo/regularization_term": 3.7437500953674316,
      "r_dpo/rejected_len": 256.84063720703125,
      "step": 50
    },
    {
      "epoch": 0.1256544502617801,
      "grad_norm": 34.24372482299805,
      "learning_rate": 4.991893270335525e-07,
      "logits/chosen": 1.659531831741333,
      "logits/rejected": 1.6509793996810913,
      "logps/chosen": -291.05303955078125,
      "logps/ref_chosen": -273.5165710449219,
      "logps/ref_rejected": -257.33099365234375,
      "logps/rejected": -275.82696533203125,
      "loss": 46.1736,
      "r_dpo/chosen_len": 277.1875,
      "r_dpo/length_delta": 29.518749237060547,
      "r_dpo/regularization_term": 2.9518749713897705,
      "r_dpo/rejected_len": 247.6687469482422,
      "step": 60
    },
    {
      "epoch": 0.14659685863874344,
      "grad_norm": 31.267230987548828,
      "learning_rate": 4.970496218214204e-07,
      "logits/chosen": 1.6715799570083618,
      "logits/rejected": 1.7152721881866455,
      "logps/chosen": -303.29656982421875,
      "logps/ref_chosen": -279.4599914550781,
      "logps/ref_rejected": -274.3787536621094,
      "logps/rejected": -300.0,
      "loss": 41.8889,
      "r_dpo/chosen_len": 269.234375,
      "r_dpo/length_delta": 14.634374618530273,
      "r_dpo/regularization_term": 1.4634376764297485,
      "r_dpo/rejected_len": 254.60000610351562,
      "step": 70
    },
    {
      "epoch": 0.16753926701570682,
      "grad_norm": 86.9355697631836,
      "learning_rate": 4.935856505068998e-07,
      "logits/chosen": 1.5558438301086426,
      "logits/rejected": 1.589551568031311,
      "logps/chosen": -319.4896545410156,
      "logps/ref_chosen": -272.08538818359375,
      "logps/ref_rejected": -255.73300170898438,
      "logps/rejected": -304.3583984375,
      "loss": 44.2864,
      "r_dpo/chosen_len": 270.97186279296875,
      "r_dpo/length_delta": 32.712501525878906,
      "r_dpo/regularization_term": 3.2712502479553223,
      "r_dpo/rejected_len": 238.25936889648438,
      "step": 80
    },
    {
      "epoch": 0.18848167539267016,
      "grad_norm": 53.2010498046875,
      "learning_rate": 4.8881598109976e-07,
      "logits/chosen": 1.4091914892196655,
      "logits/rejected": 1.4223301410675049,
      "logps/chosen": -342.06500244140625,
      "logps/ref_chosen": -269.1466369628906,
      "logps/ref_rejected": -247.70956420898438,
      "logps/rejected": -320.5975341796875,
      "loss": 43.205,
      "r_dpo/chosen_len": 278.11248779296875,
      "r_dpo/length_delta": 47.27812576293945,
      "r_dpo/regularization_term": 4.727812767028809,
      "r_dpo/rejected_len": 230.83438110351562,
      "step": 90
    },
    {
      "epoch": 0.2094240837696335,
      "grad_norm": 221.70379638671875,
      "learning_rate": 4.827661805750437e-07,
      "logits/chosen": 1.171535849571228,
      "logits/rejected": 1.1951229572296143,
      "logps/chosen": -339.74676513671875,
      "logps/ref_chosen": -265.9335632324219,
      "logps/ref_rejected": -270.045654296875,
      "logps/rejected": -358.4281921386719,
      "loss": 41.9093,
      "r_dpo/chosen_len": 278.3656311035156,
      "r_dpo/length_delta": 22.96875,
      "r_dpo/regularization_term": 2.296875,
      "r_dpo/rejected_len": 255.39688110351562,
      "step": 100
    },
    {
      "epoch": 0.23036649214659685,
      "grad_norm": 206.1243438720703,
      "learning_rate": 4.75468677825789e-07,
      "logits/chosen": 1.2150897979736328,
      "logits/rejected": 1.3317502737045288,
      "logps/chosen": -387.6827087402344,
      "logps/ref_chosen": -265.02532958984375,
      "logps/ref_rejected": -259.968017578125,
      "logps/rejected": -389.2899475097656,
      "loss": 36.8666,
      "r_dpo/chosen_len": 287.140625,
      "r_dpo/length_delta": 50.765625,
      "r_dpo/regularization_term": 5.076562881469727,
      "r_dpo/rejected_len": 236.375,
      "step": 110
    },
    {
      "epoch": 0.2513089005235602,
      "grad_norm": 155.615966796875,
      "learning_rate": 4.669625898336438e-07,
      "logits/chosen": 1.069676399230957,
      "logits/rejected": 1.0872914791107178,
      "logps/chosen": -421.203125,
      "logps/ref_chosen": -262.50323486328125,
      "logps/ref_rejected": -268.14996337890625,
      "logps/rejected": -438.32208251953125,
      "loss": 40.1069,
      "r_dpo/chosen_len": 266.70001220703125,
      "r_dpo/length_delta": 16.190624237060547,
      "r_dpo/regularization_term": 1.6190624237060547,
      "r_dpo/rejected_len": 250.50936889648438,
      "step": 120
    },
    {
      "epoch": 0.27225130890052357,
      "grad_norm": 440.5760498046875,
      "learning_rate": 4.5729351198915705e-07,
      "logits/chosen": 0.7952426671981812,
      "logits/rejected": 0.9414566159248352,
      "logps/chosen": -471.2674865722656,
      "logps/ref_chosen": -267.5928039550781,
      "logps/ref_rejected": -272.86810302734375,
      "logps/rejected": -491.12835693359375,
      "loss": 41.9211,
      "r_dpo/chosen_len": 268.75,
      "r_dpo/length_delta": 19.178125381469727,
      "r_dpo/regularization_term": 1.9178125858306885,
      "r_dpo/rejected_len": 249.57186889648438,
      "step": 130
    },
    {
      "epoch": 0.2931937172774869,
      "grad_norm": 194.5912322998047,
      "learning_rate": 4.4651327368569684e-07,
      "logits/chosen": 0.6943808794021606,
      "logits/rejected": 0.750653088092804,
      "logps/chosen": -524.4630126953125,
      "logps/ref_chosen": -268.4759521484375,
      "logps/ref_rejected": -254.6080780029297,
      "logps/rejected": -514.9945678710938,
      "loss": 40.9788,
      "r_dpo/chosen_len": 261.46875,
      "r_dpo/length_delta": 21.350000381469727,
      "r_dpo/regularization_term": 2.135000228881836,
      "r_dpo/rejected_len": 240.11874389648438,
      "step": 140
    },
    {
      "epoch": 0.31413612565445026,
      "grad_norm": 271.27703857421875,
      "learning_rate": 4.346796604970912e-07,
      "logits/chosen": 0.33445867896080017,
      "logits/rejected": 0.435607373714447,
      "logps/chosen": -838.4150390625,
      "logps/ref_chosen": -290.3619384765625,
      "logps/ref_rejected": -252.4545135498047,
      "logps/rejected": -750.5610961914062,
      "loss": 32.2283,
      "r_dpo/chosen_len": 286.29998779296875,
      "r_dpo/length_delta": 48.62187576293945,
      "r_dpo/regularization_term": 4.862187385559082,
      "r_dpo/rejected_len": 237.67813110351562,
      "step": 150
    },
    {
      "epoch": 0.33507853403141363,
      "grad_norm": 1304.436279296875,
      "learning_rate": 4.218561044282098e-07,
      "logits/chosen": -0.5341374278068542,
      "logits/rejected": -0.3209038972854614,
      "logps/chosen": -1746.661376953125,
      "logps/ref_chosen": -272.7589111328125,
      "logps/ref_rejected": -252.6876678466797,
      "logps/rejected": -1500.4178466796875,
      "loss": 23.2094,
      "r_dpo/chosen_len": 271.48748779296875,
      "r_dpo/length_delta": 41.578125,
      "r_dpo/regularization_term": 4.157812118530273,
      "r_dpo/rejected_len": 229.9093780517578,
      "step": 160
    },
    {
      "epoch": 0.35602094240837695,
      "grad_norm": 672.2740478515625,
      "learning_rate": 4.081113438988443e-07,
      "logits/chosen": -0.30694156885147095,
      "logits/rejected": -0.16359911859035492,
      "logps/chosen": -2336.08984375,
      "logps/ref_chosen": -281.5549621582031,
      "logps/ref_rejected": -238.20858764648438,
      "logps/rejected": -2039.7515869140625,
      "loss": 10.9533,
      "r_dpo/chosen_len": 288.2718811035156,
      "r_dpo/length_delta": 47.34375,
      "r_dpo/regularization_term": 4.734375476837158,
      "r_dpo/rejected_len": 240.92813110351562,
      "step": 170
    },
    {
      "epoch": 0.3769633507853403,
      "grad_norm": 437.60687255859375,
      "learning_rate": 3.935190552834828e-07,
      "logits/chosen": -0.3067488968372345,
      "logits/rejected": -0.20810024440288544,
      "logps/chosen": -2306.261474609375,
      "logps/ref_chosen": -278.9187927246094,
      "logps/ref_rejected": -248.9375457763672,
      "logps/rejected": -2002.85546875,
      "loss": 9.0173,
      "r_dpo/chosen_len": 269.1656188964844,
      "r_dpo/length_delta": 41.087501525878906,
      "r_dpo/regularization_term": 4.108750343322754,
      "r_dpo/rejected_len": 228.078125,
      "step": 180
    },
    {
      "epoch": 0.39790575916230364,
      "grad_norm": 321.61346435546875,
      "learning_rate": 3.781574579820464e-07,
      "logits/chosen": 0.40285807847976685,
      "logits/rejected": 0.36570462584495544,
      "logps/chosen": -2637.16845703125,
      "logps/ref_chosen": -272.6444091796875,
      "logps/ref_rejected": -251.23965454101562,
      "logps/rejected": -2281.21337890625,
      "loss": 7.2881,
      "r_dpo/chosen_len": 280.03436279296875,
      "r_dpo/length_delta": 43.209373474121094,
      "r_dpo/regularization_term": 4.320937156677246,
      "r_dpo/rejected_len": 236.8249969482422,
      "step": 190
    },
    {
      "epoch": 0.418848167539267,
      "grad_norm": 389.5361022949219,
      "learning_rate": 3.621088951385353e-07,
      "logits/chosen": 0.8133895993232727,
      "logits/rejected": 0.7692806720733643,
      "logps/chosen": -2436.361083984375,
      "logps/ref_chosen": -247.26144409179688,
      "logps/ref_rejected": -240.1345977783203,
      "logps/rejected": -2211.449951171875,
      "loss": 6.6608,
      "r_dpo/chosen_len": 249.71249389648438,
      "r_dpo/length_delta": 27.921875,
      "r_dpo/regularization_term": 2.7921876907348633,
      "r_dpo/rejected_len": 221.79061889648438,
      "step": 200
    },
    {
      "epoch": 0.418848167539267,
      "eval_logits/chosen": 0.6076958775520325,
      "eval_logits/rejected": 0.5855662822723389,
      "eval_logps/chosen": -2819.177978515625,
      "eval_logps/ref_chosen": -281.4849548339844,
      "eval_logps/ref_rejected": -261.800537109375,
      "eval_logps/rejected": -2438.321044921875,
      "eval_loss": 0.7287593483924866,
      "eval_r_dpo/chosen_len": 294.4800109863281,
      "eval_r_dpo/length_delta": 44.61000061035156,
      "eval_r_dpo/regularization_term": 4.461000442504883,
      "eval_r_dpo/rejected_len": 249.8699951171875,
      "eval_runtime": 97.4324,
      "eval_samples_per_second": 20.527,
      "eval_steps_per_second": 1.283,
      "step": 200
    },
    {
      "epoch": 0.4397905759162304,
      "grad_norm": 347.1392517089844,
      "learning_rate": 3.454593922550693e-07,
      "logits/chosen": 0.6368786692619324,
      "logits/rejected": 0.5997772216796875,
      "logps/chosen": -2782.99755859375,
      "logps/ref_chosen": -280.40606689453125,
      "logps/ref_rejected": -259.3473205566406,
      "logps/rejected": -2457.272705078125,
      "loss": 6.155,
      "r_dpo/chosen_len": 282.6156311035156,
      "r_dpo/length_delta": 36.171875,
      "r_dpo/regularization_term": 3.6171875,
      "r_dpo/rejected_len": 246.44375610351562,
      "step": 210
    },
    {
      "epoch": 0.4607329842931937,
      "grad_norm": 168.1587371826172,
      "learning_rate": 3.2829819606729477e-07,
      "logits/chosen": 0.771518886089325,
      "logits/rejected": 0.7589208483695984,
      "logps/chosen": -2625.2099609375,
      "logps/ref_chosen": -276.30694580078125,
      "logps/ref_rejected": -268.7990417480469,
      "logps/rejected": -2493.98193359375,
      "loss": 5.5739,
      "r_dpo/chosen_len": 265.546875,
      "r_dpo/length_delta": 18.515625,
      "r_dpo/regularization_term": 1.8515625,
      "r_dpo/rejected_len": 247.03125,
      "step": 220
    },
    {
      "epoch": 0.4816753926701571,
      "grad_norm": 290.4512939453125,
      "learning_rate": 3.1071729615293424e-07,
      "logits/chosen": 0.2807890474796295,
      "logits/rejected": 0.27411508560180664,
      "logps/chosen": -2629.66943359375,
      "logps/ref_chosen": -265.4647216796875,
      "logps/ref_rejected": -251.6260223388672,
      "logps/rejected": -2396.610107421875,
      "loss": 5.3014,
      "r_dpo/chosen_len": 268.1812438964844,
      "r_dpo/length_delta": 31.868749618530273,
      "r_dpo/regularization_term": 3.1868748664855957,
      "r_dpo/rejected_len": 236.3125,
      "step": 230
    },
    {
      "epoch": 0.5026178010471204,
      "grad_norm": 200.7110137939453,
      "learning_rate": 2.9281093183781403e-07,
      "logits/chosen": 0.6274879574775696,
      "logits/rejected": 0.589483916759491,
      "logps/chosen": -2683.999267578125,
      "logps/ref_chosen": -262.2581481933594,
      "logps/ref_rejected": -251.36373901367188,
      "logps/rejected": -2375.33642578125,
      "loss": 5.475,
      "r_dpo/chosen_len": 275.421875,
      "r_dpo/length_delta": 37.65937423706055,
      "r_dpo/regularization_term": 3.765937328338623,
      "r_dpo/rejected_len": 237.7624969482422,
      "step": 240
    },
    {
      "epoch": 0.5235602094240838,
      "grad_norm": 328.9794921875,
      "learning_rate": 2.7467508704251135e-07,
      "logits/chosen": 0.4434322416782379,
      "logits/rejected": 0.4112902581691742,
      "logps/chosen": -2788.759765625,
      "logps/ref_chosen": -282.21917724609375,
      "logps/ref_rejected": -262.7974548339844,
      "logps/rejected": -2449.12939453125,
      "loss": 4.8484,
      "r_dpo/chosen_len": 280.37811279296875,
      "r_dpo/length_delta": 41.196876525878906,
      "r_dpo/regularization_term": 4.119687557220459,
      "r_dpo/rejected_len": 239.18124389648438,
      "step": 250
    },
    {
      "epoch": 0.5445026178010471,
      "grad_norm": 174.2191925048828,
      "learning_rate": 2.5640697577740815e-07,
      "logits/chosen": 0.510331392288208,
      "logits/rejected": 0.46630868315696716,
      "logps/chosen": -2726.13232421875,
      "logps/ref_chosen": -280.39166259765625,
      "logps/ref_rejected": -263.56573486328125,
      "logps/rejected": -2545.450439453125,
      "loss": 5.0081,
      "r_dpo/chosen_len": 273.4468688964844,
      "r_dpo/length_delta": 25.353124618530273,
      "r_dpo/regularization_term": 2.5353128910064697,
      "r_dpo/rejected_len": 248.09375,
      "step": 260
    },
    {
      "epoch": 0.5654450261780105,
      "grad_norm": 183.47354125976562,
      "learning_rate": 2.381045210440644e-07,
      "logits/chosen": 0.6359630823135376,
      "logits/rejected": 0.6186918020248413,
      "logps/chosen": -2736.114501953125,
      "logps/ref_chosen": -271.36041259765625,
      "logps/ref_rejected": -262.174072265625,
      "logps/rejected": -2623.23974609375,
      "loss": 5.0971,
      "r_dpo/chosen_len": 274.98126220703125,
      "r_dpo/length_delta": 19.128124237060547,
      "r_dpo/regularization_term": 1.9128128290176392,
      "r_dpo/rejected_len": 255.85311889648438,
      "step": 270
    },
    {
      "epoch": 0.5863874345549738,
      "grad_norm": 193.64608764648438,
      "learning_rate": 2.1986582993616925e-07,
      "logits/chosen": 0.2083236277103424,
      "logits/rejected": 0.16970780491828918,
      "logps/chosen": -2850.24462890625,
      "logps/ref_chosen": -293.512939453125,
      "logps/ref_rejected": -253.37246704101562,
      "logps/rejected": -2385.664794921875,
      "loss": 5.0361,
      "r_dpo/chosen_len": 287.9437561035156,
      "r_dpo/length_delta": 52.75312423706055,
      "r_dpo/regularization_term": 5.275312423706055,
      "r_dpo/rejected_len": 235.1906280517578,
      "step": 280
    },
    {
      "epoch": 0.6073298429319371,
      "grad_norm": 193.0575408935547,
      "learning_rate": 2.0178866775369774e-07,
      "logits/chosen": 0.7391489148139954,
      "logits/rejected": 0.714061975479126,
      "logps/chosen": -2967.319091796875,
      "logps/ref_chosen": -294.9578857421875,
      "logps/ref_rejected": -288.91802978515625,
      "logps/rejected": -2823.838623046875,
      "loss": 4.9628,
      "r_dpo/chosen_len": 298.48748779296875,
      "r_dpo/length_delta": 22.706249237060547,
      "r_dpo/regularization_term": 2.270624876022339,
      "r_dpo/rejected_len": 275.78125,
      "step": 290
    },
    {
      "epoch": 0.6282722513089005,
      "grad_norm": 167.63522338867188,
      "learning_rate": 1.839699339491937e-07,
      "logits/chosen": 0.733379065990448,
      "logits/rejected": 0.6921027898788452,
      "logps/chosen": -2751.304931640625,
      "logps/ref_chosen": -267.9673767089844,
      "logps/ref_rejected": -262.1747131347656,
      "logps/rejected": -2606.73583984375,
      "loss": 4.8425,
      "r_dpo/chosen_len": 269.3999938964844,
      "r_dpo/length_delta": 20.831249237060547,
      "r_dpo/regularization_term": 2.083124876022339,
      "r_dpo/rejected_len": 248.56875610351562,
      "step": 300
    },
    {
      "epoch": 0.6492146596858639,
      "grad_norm": 373.60211181640625,
      "learning_rate": 1.6650514271527465e-07,
      "logits/chosen": 0.473613440990448,
      "logits/rejected": 0.41956576704978943,
      "logps/chosen": -2961.7822265625,
      "logps/ref_chosen": -286.36895751953125,
      "logps/ref_rejected": -274.4711608886719,
      "logps/rejected": -2710.29736328125,
      "loss": 4.8166,
      "r_dpo/chosen_len": 296.7749938964844,
      "r_dpo/length_delta": 32.959373474121094,
      "r_dpo/regularization_term": 3.2959372997283936,
      "r_dpo/rejected_len": 263.81561279296875,
      "step": 310
    },
    {
      "epoch": 0.6701570680628273,
      "grad_norm": 178.9756317138672,
      "learning_rate": 1.4948791099758052e-07,
      "logits/chosen": 0.6881324648857117,
      "logits/rejected": 0.6306595802307129,
      "logps/chosen": -2834.637451171875,
      "logps/ref_chosen": -276.7845764160156,
      "logps/ref_rejected": -258.2370300292969,
      "logps/rejected": -2472.187744140625,
      "loss": 4.5986,
      "r_dpo/chosen_len": 282.578125,
      "r_dpo/length_delta": 45.78437423706055,
      "r_dpo/regularization_term": 4.578437805175781,
      "r_dpo/rejected_len": 236.7937469482422,
      "step": 320
    },
    {
      "epoch": 0.6910994764397905,
      "grad_norm": 184.68521118164062,
      "learning_rate": 1.3300945667758012e-07,
      "logits/chosen": 0.6964876651763916,
      "logits/rejected": 0.6566891670227051,
      "logps/chosen": -2763.50390625,
      "logps/ref_chosen": -275.9217529296875,
      "logps/ref_rejected": -267.70782470703125,
      "logps/rejected": -2724.64453125,
      "loss": 4.854,
      "r_dpo/chosen_len": 270.0531311035156,
      "r_dpo/length_delta": 12.415624618530273,
      "r_dpo/regularization_term": 1.2415623664855957,
      "r_dpo/rejected_len": 257.63751220703125,
      "step": 330
    },
    {
      "epoch": 0.7120418848167539,
      "grad_norm": 233.2672882080078,
      "learning_rate": 1.1715810961514072e-07,
      "logits/chosen": 0.45854058861732483,
      "logits/rejected": 0.4010751247406006,
      "logps/chosen": -2628.388671875,
      "logps/ref_chosen": -251.37319946289062,
      "logps/ref_rejected": -240.65988159179688,
      "logps/rejected": -2376.85595703125,
      "loss": 4.7891,
      "r_dpo/chosen_len": 258.17498779296875,
      "r_dpo/length_delta": 32.990623474121094,
      "r_dpo/regularization_term": 3.2990622520446777,
      "r_dpo/rejected_len": 225.1843719482422,
      "step": 340
    },
    {
      "epoch": 0.7329842931937173,
      "grad_norm": 246.9112548828125,
      "learning_rate": 1.0201883817182949e-07,
      "logits/chosen": 0.6637452840805054,
      "logits/rejected": 0.5826420187950134,
      "logps/chosen": -2949.142333984375,
      "logps/ref_chosen": -290.9896240234375,
      "logps/ref_rejected": -269.58502197265625,
      "logps/rejected": -2537.853271484375,
      "loss": 4.6654,
      "r_dpo/chosen_len": 284.6187438964844,
      "r_dpo/length_delta": 46.57500076293945,
      "r_dpo/regularization_term": 4.657500267028809,
      "r_dpo/rejected_len": 238.0437469482422,
      "step": 350
    },
    {
      "epoch": 0.7539267015706806,
      "grad_norm": 221.2776336669922,
      "learning_rate": 8.76727937529367e-08,
      "logits/chosen": 0.47227534651756287,
      "logits/rejected": 0.4251709580421448,
      "logps/chosen": -2778.8525390625,
      "logps/ref_chosen": -272.4156799316406,
      "logps/ref_rejected": -261.71087646484375,
      "logps/rejected": -2545.5712890625,
      "loss": 4.7021,
      "r_dpo/chosen_len": 275.23748779296875,
      "r_dpo/length_delta": 30.956249237060547,
      "r_dpo/regularization_term": 3.095625162124634,
      "r_dpo/rejected_len": 244.28125,
      "step": 360
    },
    {
      "epoch": 0.774869109947644,
      "grad_norm": 179.33848571777344,
      "learning_rate": 7.419687580962222e-08,
      "logits/chosen": 0.4994059205055237,
      "logits/rejected": 0.43345919251441956,
      "logps/chosen": -2812.973876953125,
      "logps/ref_chosen": -295.822998046875,
      "logps/ref_rejected": -263.49639892578125,
      "logps/rejected": -2578.302734375,
      "loss": 4.31,
      "r_dpo/chosen_len": 277.0718688964844,
      "r_dpo/length_delta": 33.87187576293945,
      "r_dpo/regularization_term": 3.3871874809265137,
      "r_dpo/rejected_len": 243.1999969482422,
      "step": 370
    },
    {
      "epoch": 0.7958115183246073,
      "grad_norm": 127.59620666503906,
      "learning_rate": 6.166331963291519e-08,
      "logits/chosen": 0.36374586820602417,
      "logits/rejected": 0.3168891966342926,
      "logps/chosen": -2917.382568359375,
      "logps/ref_chosen": -292.1449890136719,
      "logps/ref_rejected": -262.7100524902344,
      "logps/rejected": -2657.446533203125,
      "loss": 4.4736,
      "r_dpo/chosen_len": 289.8687438964844,
      "r_dpo/length_delta": 33.578125,
      "r_dpo/regularization_term": 3.3578124046325684,
      "r_dpo/rejected_len": 256.2906188964844,
      "step": 380
    },
    {
      "epoch": 0.8167539267015707,
      "grad_norm": 180.08554077148438,
      "learning_rate": 5.013930914912476e-08,
      "logits/chosen": 0.6072987914085388,
      "logits/rejected": 0.5558676719665527,
      "logps/chosen": -2953.486083984375,
      "logps/ref_chosen": -286.49505615234375,
      "logps/ref_rejected": -274.98065185546875,
      "logps/rejected": -2727.49951171875,
      "loss": 4.5136,
      "r_dpo/chosen_len": 290.92498779296875,
      "r_dpo/length_delta": 31.337499618530273,
      "r_dpo/regularization_term": 3.1337499618530273,
      "r_dpo/rejected_len": 259.5874938964844,
      "step": 390
    },
    {
      "epoch": 0.837696335078534,
      "grad_norm": 181.38357543945312,
      "learning_rate": 3.968661679220467e-08,
      "logits/chosen": 0.6835483312606812,
      "logits/rejected": 0.6234606504440308,
      "logps/chosen": -2877.609130859375,
      "logps/ref_chosen": -289.52215576171875,
      "logps/ref_rejected": -254.75961303710938,
      "logps/rejected": -2525.186279296875,
      "loss": 4.5388,
      "r_dpo/chosen_len": 283.69061279296875,
      "r_dpo/length_delta": 40.421875,
      "r_dpo/regularization_term": 4.042187690734863,
      "r_dpo/rejected_len": 243.2687530517578,
      "step": 400
    },
    {
      "epoch": 0.837696335078534,
      "eval_logits/chosen": 0.6320263743400574,
      "eval_logits/rejected": 0.5333738327026367,
      "eval_logps/chosen": -2936.774658203125,
      "eval_logps/ref_chosen": -281.4849548339844,
      "eval_logps/ref_rejected": -261.800537109375,
      "eval_logps/rejected": -2579.16845703125,
      "eval_loss": 0.5469310879707336,
      "eval_r_dpo/chosen_len": 294.4800109863281,
      "eval_r_dpo/length_delta": 44.61000061035156,
      "eval_r_dpo/regularization_term": 4.461000442504883,
      "eval_r_dpo/rejected_len": 249.8699951171875,
      "eval_runtime": 96.769,
      "eval_samples_per_second": 20.668,
      "eval_steps_per_second": 1.292,
      "step": 400
    },
    {
      "epoch": 0.8586387434554974,
      "grad_norm": 192.98953247070312,
      "learning_rate": 3.036127238347164e-08,
      "logits/chosen": 0.8459997177124023,
      "logits/rejected": 0.8034231066703796,
      "logps/chosen": -2868.36279296875,
      "logps/ref_chosen": -282.5110778808594,
      "logps/ref_rejected": -269.2117004394531,
      "logps/rejected": -2660.260009765625,
      "loss": 4.6056,
      "r_dpo/chosen_len": 283.82501220703125,
      "r_dpo/length_delta": 27.762500762939453,
      "r_dpo/regularization_term": 2.7762503623962402,
      "r_dpo/rejected_len": 256.0625,
      "step": 410
    },
    {
      "epoch": 0.8795811518324608,
      "grad_norm": 174.4208526611328,
      "learning_rate": 2.2213262793589482e-08,
      "logits/chosen": 0.7089523077011108,
      "logits/rejected": 0.6533206105232239,
      "logps/chosen": -3067.21728515625,
      "logps/ref_chosen": -300.7202453613281,
      "logps/ref_rejected": -276.92340087890625,
      "logps/rejected": -2775.168701171875,
      "loss": 4.254,
      "r_dpo/chosen_len": 298.9937438964844,
      "r_dpo/length_delta": 37.178123474121094,
      "r_dpo/regularization_term": 3.7178127765655518,
      "r_dpo/rejected_len": 261.81561279296875,
      "step": 420
    },
    {
      "epoch": 0.900523560209424,
      "grad_norm": 184.81227111816406,
      "learning_rate": 1.5286263996730026e-08,
      "logits/chosen": 0.6772060990333557,
      "logits/rejected": 0.5796979665756226,
      "logps/chosen": -2999.145263671875,
      "logps/ref_chosen": -287.7877502441406,
      "logps/ref_rejected": -262.2414245605469,
      "logps/rejected": -2588.61669921875,
      "loss": 4.3751,
      "r_dpo/chosen_len": 292.76251220703125,
      "r_dpo/length_delta": 49.0,
      "r_dpo/regularization_term": 4.900000095367432,
      "r_dpo/rejected_len": 243.7624969482422,
      "step": 430
    },
    {
      "epoch": 0.9214659685863874,
      "grad_norm": 175.2359619140625,
      "learning_rate": 9.617406953185136e-09,
      "logits/chosen": 0.6800681352615356,
      "logits/rejected": 0.6796324849128723,
      "logps/chosen": -2927.792236328125,
      "logps/ref_chosen": -286.9573669433594,
      "logps/ref_rejected": -285.1015319824219,
      "logps/rejected": -2881.3701171875,
      "loss": 4.664,
      "r_dpo/chosen_len": 289.5218811035156,
      "r_dpo/length_delta": 13.934374809265137,
      "r_dpo/regularization_term": 1.3934376239776611,
      "r_dpo/rejected_len": 275.5874938964844,
      "step": 440
    },
    {
      "epoch": 0.9424083769633508,
      "grad_norm": 213.96315002441406,
      "learning_rate": 5.2370785753763356e-09,
      "logits/chosen": 0.702254056930542,
      "logits/rejected": 0.6395606994628906,
      "logps/chosen": -2904.09423828125,
      "logps/ref_chosen": -288.1899719238281,
      "logps/ref_rejected": -258.13909912109375,
      "logps/rejected": -2574.60986328125,
      "loss": 4.4079,
      "r_dpo/chosen_len": 285.5,
      "r_dpo/length_delta": 39.22187423706055,
      "r_dpo/regularization_term": 3.9221878051757812,
      "r_dpo/rejected_len": 246.2781219482422,
      "step": 450
    },
    {
      "epoch": 0.9633507853403142,
      "grad_norm": 308.9233703613281,
      "learning_rate": 2.168758844148272e-09,
      "logits/chosen": 0.7860502600669861,
      "logits/rejected": 0.6886175870895386,
      "logps/chosen": -2969.99609375,
      "logps/ref_chosen": -304.67333984375,
      "logps/ref_rejected": -275.60491943359375,
      "logps/rejected": -2588.6171875,
      "loss": 4.4801,
      "r_dpo/chosen_len": 291.12188720703125,
      "r_dpo/length_delta": 44.39374923706055,
      "r_dpo/regularization_term": 4.439374923706055,
      "r_dpo/rejected_len": 246.72811889648438,
      "step": 460
    },
    {
      "epoch": 0.9842931937172775,
      "grad_norm": 133.6565704345703,
      "learning_rate": 4.288949484559934e-10,
      "logits/chosen": 0.6798645853996277,
      "logits/rejected": 0.6197268962860107,
      "logps/chosen": -2756.23876953125,
      "logps/ref_chosen": -270.48248291015625,
      "logps/ref_rejected": -253.36062622070312,
      "logps/rejected": -2583.438232421875,
      "loss": 4.3438,
      "r_dpo/chosen_len": 271.95623779296875,
      "r_dpo/length_delta": 25.784374237060547,
      "r_dpo/regularization_term": 2.578437566757202,
      "r_dpo/rejected_len": 246.171875,
      "step": 470
    },
    {
      "epoch": 0.9989528795811519,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 17.04746322311945,
      "train_runtime": 7166.7787,
      "train_samples_per_second": 8.53,
      "train_steps_per_second": 0.067
    }
  ],
  "logging_steps": 10,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}