8390 lines
249 KiB
JSON
8390 lines
249 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.9913544668587897,
|
|
"eval_steps": 173,
|
|
"global_step": 519,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0038424591738712775,
|
|
"grad_norm": 201.0,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -3.849609375,
|
|
"logits/rejected": -3.916015625,
|
|
"logps/chosen": -331.75,
|
|
"logps/rejected": -160.3125,
|
|
"loss": 1.998,
|
|
"nll_loss": 1.03076171875,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.007684918347742555,
|
|
"grad_norm": 552.0,
|
|
"learning_rate": 1.923076923076923e-08,
|
|
"logits/chosen": -3.83984375,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -375.5,
|
|
"logps/rejected": -239.6875,
|
|
"loss": 2.002,
|
|
"nll_loss": 1.03271484375,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.011527377521613832,
|
|
"grad_norm": 984.0,
|
|
"learning_rate": 3.846153846153846e-08,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.87890625,
|
|
"logps/chosen": -531.125,
|
|
"logps/rejected": -266.25,
|
|
"loss": 1.918,
|
|
"nll_loss": 0.93359375,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": 0.05517578125,
|
|
"rewards/margins": -0.00506591796875,
|
|
"rewards/rejected": 0.060302734375,
|
|
"step": 3
|
|
},
|
|
{
|
|
"epoch": 0.01536983669548511,
|
|
"grad_norm": 6496.0,
|
|
"learning_rate": 5.7692307692307695e-08,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.974609375,
|
|
"logps/chosen": -466.0,
|
|
"logps/rejected": -251.625,
|
|
"loss": 1.958,
|
|
"nll_loss": 0.90087890625,
|
|
"rewards/accuracies": 0.1875,
|
|
"rewards/chosen": -0.31854248046875,
|
|
"rewards/margins": -0.2250823974609375,
|
|
"rewards/rejected": -0.0929412841796875,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.01921229586935639,
|
|
"grad_norm": 370.0,
|
|
"learning_rate": 7.692307692307692e-08,
|
|
"logits/chosen": -3.88671875,
|
|
"logits/rejected": -3.91796875,
|
|
"logps/chosen": -429.25,
|
|
"logps/rejected": -275.375,
|
|
"loss": 1.8857,
|
|
"nll_loss": 0.9248046875,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.1102294921875,
|
|
"rewards/margins": 0.070526123046875,
|
|
"rewards/rejected": 0.03985595703125,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.023054755043227664,
|
|
"grad_norm": 356.0,
|
|
"learning_rate": 9.615384615384616e-08,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.861328125,
|
|
"logps/chosen": -415.0,
|
|
"logps/rejected": -236.875,
|
|
"loss": 2.0488,
|
|
"nll_loss": 1.05712890625,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": -0.07568359375,
|
|
"rewards/margins": -0.044677734375,
|
|
"rewards/rejected": -0.03106689453125,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.026897214217098942,
|
|
"grad_norm": 464.0,
|
|
"learning_rate": 1.1538461538461539e-07,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -433.0,
|
|
"logps/rejected": -237.0,
|
|
"loss": 1.7891,
|
|
"nll_loss": 0.83935546875,
|
|
"rewards/accuracies": 0.46875,
|
|
"rewards/chosen": -0.079345703125,
|
|
"rewards/margins": 0.11199951171875,
|
|
"rewards/rejected": -0.1910400390625,
|
|
"step": 7
|
|
},
|
|
{
|
|
"epoch": 0.03073967339097022,
|
|
"grad_norm": 274.0,
|
|
"learning_rate": 1.346153846153846e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.908203125,
|
|
"logps/chosen": -424.125,
|
|
"logps/rejected": -225.6875,
|
|
"loss": 1.9102,
|
|
"nll_loss": 0.9326171875,
|
|
"rewards/accuracies": 0.25,
|
|
"rewards/chosen": 0.03765869140625,
|
|
"rewards/margins": -0.007080078125,
|
|
"rewards/rejected": 0.0447845458984375,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.0345821325648415,
|
|
"grad_norm": 262.0,
|
|
"learning_rate": 1.5384615384615385e-07,
|
|
"logits/chosen": -3.89453125,
|
|
"logits/rejected": -3.916015625,
|
|
"logps/chosen": -319.25,
|
|
"logps/rejected": -228.0,
|
|
"loss": 2.0381,
|
|
"nll_loss": 1.0556640625,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": -0.0657958984375,
|
|
"rewards/margins": -0.0295257568359375,
|
|
"rewards/rejected": -0.0361328125,
|
|
"step": 9
|
|
},
|
|
{
|
|
"epoch": 0.03842459173871278,
|
|
"grad_norm": 844.0,
|
|
"learning_rate": 1.7307692307692305e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.76953125,
|
|
"logps/chosen": -418.25,
|
|
"logps/rejected": -229.4375,
|
|
"loss": 1.7871,
|
|
"nll_loss": 0.8037109375,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.020751953125,
|
|
"rewards/margins": -0.005279541015625,
|
|
"rewards/rejected": 0.025634765625,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.04226705091258405,
|
|
"grad_norm": 660.0,
|
|
"learning_rate": 1.9230769230769231e-07,
|
|
"logits/chosen": -3.849609375,
|
|
"logits/rejected": -3.916015625,
|
|
"logps/chosen": -482.625,
|
|
"logps/rejected": -236.5625,
|
|
"loss": 1.9746,
|
|
"nll_loss": 1.0166015625,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": 0.08978271484375,
|
|
"rewards/margins": 0.062255859375,
|
|
"rewards/rejected": 0.02752685546875,
|
|
"step": 11
|
|
},
|
|
{
|
|
"epoch": 0.04610951008645533,
|
|
"grad_norm": 294.0,
|
|
"learning_rate": 2.1153846153846152e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.951171875,
|
|
"logps/chosen": -485.75,
|
|
"logps/rejected": -241.5,
|
|
"loss": 1.876,
|
|
"nll_loss": 0.908203125,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.075927734375,
|
|
"rewards/margins": 0.03619384765625,
|
|
"rewards/rejected": 0.039642333984375,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.049951969260326606,
|
|
"grad_norm": 406.0,
|
|
"learning_rate": 2.3076923076923078e-07,
|
|
"logits/chosen": -3.865234375,
|
|
"logits/rejected": -3.896484375,
|
|
"logps/chosen": -420.875,
|
|
"logps/rejected": -273.875,
|
|
"loss": 1.7861,
|
|
"nll_loss": 0.76904296875,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": -0.12744140625,
|
|
"rewards/margins": -0.10357666015625,
|
|
"rewards/rejected": -0.0240478515625,
|
|
"step": 13
|
|
},
|
|
{
|
|
"epoch": 0.053794428434197884,
|
|
"grad_norm": 344.0,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -3.86328125,
|
|
"logits/rejected": -3.919921875,
|
|
"logps/chosen": -423.0,
|
|
"logps/rejected": -220.0,
|
|
"loss": 1.9404,
|
|
"nll_loss": 0.93115234375,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": -0.072509765625,
|
|
"rewards/margins": -0.134765625,
|
|
"rewards/rejected": 0.062103271484375,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.05763688760806916,
|
|
"grad_norm": 450.0,
|
|
"learning_rate": 2.692307692307692e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -510.5,
|
|
"logps/rejected": -235.375,
|
|
"loss": 1.8848,
|
|
"nll_loss": 0.83935546875,
|
|
"rewards/accuracies": 0.21875,
|
|
"rewards/chosen": -0.23681640625,
|
|
"rewards/margins": -0.24786376953125,
|
|
"rewards/rejected": 0.010345458984375,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.06147934678194044,
|
|
"grad_norm": 350.0,
|
|
"learning_rate": 2.884615384615384e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.88671875,
|
|
"logps/chosen": -424.625,
|
|
"logps/rejected": -245.625,
|
|
"loss": 2.1211,
|
|
"nll_loss": 1.111572265625,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": -0.001739501953125,
|
|
"rewards/margins": -0.14471435546875,
|
|
"rewards/rejected": 0.1431427001953125,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.06532180595581172,
|
|
"grad_norm": 652.0,
|
|
"learning_rate": 3.076923076923077e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -433.25,
|
|
"logps/rejected": -224.25,
|
|
"loss": 1.9551,
|
|
"nll_loss": 0.95361328125,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": -0.075927734375,
|
|
"rewards/margins": -0.0660400390625,
|
|
"rewards/rejected": -0.0103759765625,
|
|
"step": 17
|
|
},
|
|
{
|
|
"epoch": 0.069164265129683,
|
|
"grad_norm": 712.0,
|
|
"learning_rate": 3.269230769230769e-07,
|
|
"logits/chosen": -3.8359375,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -456.625,
|
|
"logps/rejected": -248.0625,
|
|
"loss": 1.918,
|
|
"nll_loss": 0.9453125,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": 0.0758056640625,
|
|
"rewards/margins": 0.027679443359375,
|
|
"rewards/rejected": 0.048309326171875,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.07300672430355427,
|
|
"grad_norm": 496.0,
|
|
"learning_rate": 3.461538461538461e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -441.0,
|
|
"logps/rejected": -268.75,
|
|
"loss": 1.8633,
|
|
"nll_loss": 0.91162109375,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.02435302734375,
|
|
"rewards/margins": 0.09130859375,
|
|
"rewards/rejected": -0.0675048828125,
|
|
"step": 19
|
|
},
|
|
{
|
|
"epoch": 0.07684918347742556,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 3.6538461538461534e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -429.625,
|
|
"logps/rejected": -198.125,
|
|
"loss": 1.9121,
|
|
"nll_loss": 0.978515625,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.072509765625,
|
|
"rewards/margins": 0.1571044921875,
|
|
"rewards/rejected": -0.084503173828125,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.08069164265129683,
|
|
"grad_norm": 500.0,
|
|
"learning_rate": 3.8461538461538463e-07,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.94140625,
|
|
"logps/chosen": -469.875,
|
|
"logps/rejected": -213.25,
|
|
"loss": 1.8916,
|
|
"nll_loss": 0.9443359375,
|
|
"rewards/accuracies": 0.40625,
|
|
"rewards/chosen": -0.0640411376953125,
|
|
"rewards/margins": 0.11895751953125,
|
|
"rewards/rejected": -0.18243408203125,
|
|
"step": 21
|
|
},
|
|
{
|
|
"epoch": 0.0845341018251681,
|
|
"grad_norm": 402.0,
|
|
"learning_rate": 4.0384615384615386e-07,
|
|
"logits/chosen": -3.873046875,
|
|
"logits/rejected": -3.8984375,
|
|
"logps/chosen": -452.625,
|
|
"logps/rejected": -239.625,
|
|
"loss": 1.9355,
|
|
"nll_loss": 0.96923828125,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.04156494140625,
|
|
"rewards/margins": 0.04156494140625,
|
|
"rewards/rejected": -0.00048828125,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.08837656099903939,
|
|
"grad_norm": 284.0,
|
|
"learning_rate": 4.2307692307692304e-07,
|
|
"logits/chosen": -3.705078125,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -423.875,
|
|
"logps/rejected": -177.375,
|
|
"loss": 1.9336,
|
|
"nll_loss": 0.93896484375,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": 0.1396484375,
|
|
"rewards/margins": -0.009246826171875,
|
|
"rewards/rejected": 0.1484375,
|
|
"step": 23
|
|
},
|
|
{
|
|
"epoch": 0.09221902017291066,
|
|
"grad_norm": 255.0,
|
|
"learning_rate": 4.423076923076923e-07,
|
|
"logits/chosen": -3.865234375,
|
|
"logits/rejected": -3.919921875,
|
|
"logps/chosen": -422.25,
|
|
"logps/rejected": -182.9375,
|
|
"loss": 2.0938,
|
|
"nll_loss": 1.12353515625,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.127349853515625,
|
|
"rewards/margins": 0.0706329345703125,
|
|
"rewards/rejected": 0.056884765625,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.09606147934678194,
|
|
"grad_norm": 824.0,
|
|
"learning_rate": 4.6153846153846156e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.81640625,
|
|
"logps/chosen": -460.5,
|
|
"logps/rejected": -213.125,
|
|
"loss": 1.9287,
|
|
"nll_loss": 0.91748046875,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.000244140625,
|
|
"rewards/margins": -0.02911376953125,
|
|
"rewards/rejected": 0.029205322265625,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.09990393852065321,
|
|
"grad_norm": 318.0,
|
|
"learning_rate": 4.807692307692307e-07,
|
|
"logits/chosen": -3.890625,
|
|
"logits/rejected": -3.912109375,
|
|
"logps/chosen": -542.25,
|
|
"logps/rejected": -294.75,
|
|
"loss": 1.9258,
|
|
"nll_loss": 1.013671875,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.2484130859375,
|
|
"rewards/margins": 0.26861572265625,
|
|
"rewards/rejected": -0.0208740234375,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.1037463976945245,
|
|
"grad_norm": 430.0,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.939453125,
|
|
"logps/chosen": -460.25,
|
|
"logps/rejected": -222.375,
|
|
"loss": 1.9092,
|
|
"nll_loss": 0.94140625,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": 0.120635986328125,
|
|
"rewards/margins": 0.02752685546875,
|
|
"rewards/rejected": 0.0932769775390625,
|
|
"step": 27
|
|
},
|
|
{
|
|
"epoch": 0.10758885686839577,
|
|
"grad_norm": 216.0,
|
|
"learning_rate": 5.192307692307692e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -400.5,
|
|
"logps/rejected": -222.75,
|
|
"loss": 1.834,
|
|
"nll_loss": 0.8955078125,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.14508056640625,
|
|
"rewards/margins": 0.10870361328125,
|
|
"rewards/rejected": 0.03643798828125,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.11143131604226705,
|
|
"grad_norm": 362.0,
|
|
"learning_rate": 5.384615384615384e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.88671875,
|
|
"logps/chosen": -394.25,
|
|
"logps/rejected": -207.625,
|
|
"loss": 1.7959,
|
|
"nll_loss": 0.8515625,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.1758270263671875,
|
|
"rewards/margins": 0.11029052734375,
|
|
"rewards/rejected": 0.0654296875,
|
|
"step": 29
|
|
},
|
|
{
|
|
"epoch": 0.11527377521613832,
|
|
"grad_norm": 300.0,
|
|
"learning_rate": 5.576923076923077e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -437.25,
|
|
"logps/rejected": -228.875,
|
|
"loss": 1.7559,
|
|
"nll_loss": 0.794921875,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": 0.1015625,
|
|
"rewards/margins": 0.09283447265625,
|
|
"rewards/rejected": 0.0085296630859375,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.11911623439000961,
|
|
"grad_norm": 348.0,
|
|
"learning_rate": 5.769230769230768e-07,
|
|
"logits/chosen": -3.953125,
|
|
"logits/rejected": -3.982421875,
|
|
"logps/chosen": -377.875,
|
|
"logps/rejected": -225.5625,
|
|
"loss": 1.9229,
|
|
"nll_loss": 0.97314453125,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.2431640625,
|
|
"rewards/margins": 0.094879150390625,
|
|
"rewards/rejected": 0.148651123046875,
|
|
"step": 31
|
|
},
|
|
{
|
|
"epoch": 0.12295869356388088,
|
|
"grad_norm": 316.0,
|
|
"learning_rate": 5.961538461538461e-07,
|
|
"logits/chosen": -3.841796875,
|
|
"logits/rejected": -3.94140625,
|
|
"logps/chosen": -485.5,
|
|
"logps/rejected": -237.625,
|
|
"loss": 1.8027,
|
|
"nll_loss": 0.87060546875,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.20697021484375,
|
|
"rewards/margins": 0.1358642578125,
|
|
"rewards/rejected": 0.070709228515625,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.12680115273775217,
|
|
"grad_norm": 356.0,
|
|
"learning_rate": 6.153846153846154e-07,
|
|
"logits/chosen": -3.8984375,
|
|
"logits/rejected": -3.93359375,
|
|
"logps/chosen": -491.75,
|
|
"logps/rejected": -238.0,
|
|
"loss": 1.8301,
|
|
"nll_loss": 0.85791015625,
|
|
"rewards/accuracies": 0.28125,
|
|
"rewards/chosen": 0.098388671875,
|
|
"rewards/margins": 0.00518798828125,
|
|
"rewards/rejected": 0.093170166015625,
|
|
"step": 33
|
|
},
|
|
{
|
|
"epoch": 0.13064361191162344,
|
|
"grad_norm": 568.0,
|
|
"learning_rate": 6.346153846153845e-07,
|
|
"logits/chosen": -3.8671875,
|
|
"logits/rejected": -3.935546875,
|
|
"logps/chosen": -476.25,
|
|
"logps/rejected": -244.125,
|
|
"loss": 1.9102,
|
|
"nll_loss": 0.9716796875,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.383544921875,
|
|
"rewards/margins": 0.195648193359375,
|
|
"rewards/rejected": 0.187713623046875,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.1344860710854947,
|
|
"grad_norm": 792.0,
|
|
"learning_rate": 6.538461538461538e-07,
|
|
"logits/chosen": -3.798828125,
|
|
"logits/rejected": -3.7578125,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -229.875,
|
|
"loss": 1.7861,
|
|
"nll_loss": 0.79736328125,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": 0.12078857421875,
|
|
"rewards/margins": -0.0350341796875,
|
|
"rewards/rejected": 0.155364990234375,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.138328530259366,
|
|
"grad_norm": 268.0,
|
|
"learning_rate": 6.730769230769231e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.87109375,
|
|
"logps/chosen": -525.5,
|
|
"logps/rejected": -291.125,
|
|
"loss": 1.7363,
|
|
"nll_loss": 0.82958984375,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.4359130859375,
|
|
"rewards/margins": 0.311248779296875,
|
|
"rewards/rejected": 0.124420166015625,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.14217098943323728,
|
|
"grad_norm": 330.0,
|
|
"learning_rate": 6.923076923076922e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.94921875,
|
|
"logps/chosen": -417.25,
|
|
"logps/rejected": -210.0,
|
|
"loss": 1.8779,
|
|
"nll_loss": 0.94140625,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.2935791015625,
|
|
"rewards/margins": 0.176239013671875,
|
|
"rewards/rejected": 0.11712646484375,
|
|
"step": 37
|
|
},
|
|
{
|
|
"epoch": 0.14601344860710855,
|
|
"grad_norm": 424.0,
|
|
"learning_rate": 7.115384615384616e-07,
|
|
"logits/chosen": -3.8515625,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -450.375,
|
|
"logps/rejected": -243.5,
|
|
"loss": 1.7471,
|
|
"nll_loss": 0.834716796875,
|
|
"rewards/accuracies": 0.46875,
|
|
"rewards/chosen": 0.6263427734375,
|
|
"rewards/margins": 0.33740234375,
|
|
"rewards/rejected": 0.289764404296875,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.14985590778097982,
|
|
"grad_norm": 444.0,
|
|
"learning_rate": 7.307692307692307e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.916015625,
|
|
"logps/chosen": -401.0,
|
|
"logps/rejected": -203.125,
|
|
"loss": 1.8438,
|
|
"nll_loss": 0.907470703125,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.343109130859375,
|
|
"rewards/margins": 0.17431640625,
|
|
"rewards/rejected": 0.169036865234375,
|
|
"step": 39
|
|
},
|
|
{
|
|
"epoch": 0.15369836695485112,
|
|
"grad_norm": 6432.0,
|
|
"learning_rate": 7.5e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.927734375,
|
|
"logps/chosen": -446.625,
|
|
"logps/rejected": -183.875,
|
|
"loss": 1.8359,
|
|
"nll_loss": 0.865478515625,
|
|
"rewards/accuracies": 0.46875,
|
|
"rewards/chosen": 0.12066650390625,
|
|
"rewards/margins": 0.0828857421875,
|
|
"rewards/rejected": 0.03790283203125,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.1575408261287224,
|
|
"grad_norm": 256.0,
|
|
"learning_rate": 7.692307692307693e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.93359375,
|
|
"logps/chosen": -499.0,
|
|
"logps/rejected": -236.625,
|
|
"loss": 1.8291,
|
|
"nll_loss": 0.90771484375,
|
|
"rewards/accuracies": 0.46875,
|
|
"rewards/chosen": 0.394775390625,
|
|
"rewards/margins": 0.2154541015625,
|
|
"rewards/rejected": 0.1793212890625,
|
|
"step": 41
|
|
},
|
|
{
|
|
"epoch": 0.16138328530259366,
|
|
"grad_norm": 680.0,
|
|
"learning_rate": 7.884615384615384e-07,
|
|
"logits/chosen": -3.798828125,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -465.5,
|
|
"logps/rejected": -310.875,
|
|
"loss": 1.8613,
|
|
"nll_loss": 0.83837890625,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.4775390625,
|
|
"rewards/margins": -0.01446533203125,
|
|
"rewards/rejected": 0.49249267578125,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.16522574447646493,
|
|
"grad_norm": 211.0,
|
|
"learning_rate": 8.076923076923077e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -378.25,
|
|
"logps/rejected": -229.0,
|
|
"loss": 1.8418,
|
|
"nll_loss": 0.9267578125,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 0.384033203125,
|
|
"rewards/margins": 0.2515869140625,
|
|
"rewards/rejected": 0.132049560546875,
|
|
"step": 43
|
|
},
|
|
{
|
|
"epoch": 0.1690682036503362,
|
|
"grad_norm": 245.0,
|
|
"learning_rate": 8.269230769230768e-07,
|
|
"logits/chosen": -3.869140625,
|
|
"logits/rejected": -3.94140625,
|
|
"logps/chosen": -429.5,
|
|
"logps/rejected": -245.625,
|
|
"loss": 1.9141,
|
|
"nll_loss": 1.01123046875,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 0.4970703125,
|
|
"rewards/margins": 0.300994873046875,
|
|
"rewards/rejected": 0.196319580078125,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.1729106628242075,
|
|
"grad_norm": 332.0,
|
|
"learning_rate": 8.461538461538461e-07,
|
|
"logits/chosen": -3.720703125,
|
|
"logits/rejected": -3.751953125,
|
|
"logps/chosen": -445.0,
|
|
"logps/rejected": -230.5,
|
|
"loss": 1.6924,
|
|
"nll_loss": 0.80322265625,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": 0.4691162109375,
|
|
"rewards/margins": 0.35498046875,
|
|
"rewards/rejected": 0.11376953125,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.17675312199807877,
|
|
"grad_norm": 872.0,
|
|
"learning_rate": 8.653846153846154e-07,
|
|
"logits/chosen": -3.8359375,
|
|
"logits/rejected": -3.91796875,
|
|
"logps/chosen": -362.875,
|
|
"logps/rejected": -232.25,
|
|
"loss": 1.791,
|
|
"nll_loss": 0.8330078125,
|
|
"rewards/accuracies": 0.46875,
|
|
"rewards/chosen": 0.17926025390625,
|
|
"rewards/margins": 0.0855712890625,
|
|
"rewards/rejected": 0.094085693359375,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.18059558117195004,
|
|
"grad_norm": 214.0,
|
|
"learning_rate": 8.846153846153846e-07,
|
|
"logits/chosen": -3.845703125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -408.625,
|
|
"logps/rejected": -277.9375,
|
|
"loss": 1.7949,
|
|
"nll_loss": 0.8955078125,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.552734375,
|
|
"rewards/margins": 0.309478759765625,
|
|
"rewards/rejected": 0.2427978515625,
|
|
"step": 47
|
|
},
|
|
{
|
|
"epoch": 0.1844380403458213,
|
|
"grad_norm": 414.0,
|
|
"learning_rate": 9.038461538461538e-07,
|
|
"logits/chosen": -3.908203125,
|
|
"logits/rejected": -4.05859375,
|
|
"logps/chosen": -471.5,
|
|
"logps/rejected": -273.75,
|
|
"loss": 1.9482,
|
|
"nll_loss": 1.03076171875,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.517578125,
|
|
"rewards/margins": 0.333740234375,
|
|
"rewards/rejected": 0.183837890625,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.1882804995196926,
|
|
"grad_norm": 230.0,
|
|
"learning_rate": 9.230769230769231e-07,
|
|
"logits/chosen": -3.873046875,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -438.75,
|
|
"logps/rejected": -196.3125,
|
|
"loss": 1.7773,
|
|
"nll_loss": 0.91552734375,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 0.92529296875,
|
|
"rewards/margins": 0.56561279296875,
|
|
"rewards/rejected": 0.360107421875,
|
|
"step": 49
|
|
},
|
|
{
|
|
"epoch": 0.19212295869356388,
|
|
"grad_norm": 268.0,
|
|
"learning_rate": 9.423076923076923e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -409.75,
|
|
"logps/rejected": -201.375,
|
|
"loss": 1.9004,
|
|
"nll_loss": 1.02197265625,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.558837890625,
|
|
"rewards/margins": 0.357513427734375,
|
|
"rewards/rejected": 0.2017822265625,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.19596541786743515,
|
|
"grad_norm": 222.0,
|
|
"learning_rate": 9.615384615384615e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -400.75,
|
|
"logps/rejected": -203.5,
|
|
"loss": 1.8877,
|
|
"nll_loss": 0.97314453125,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 0.578857421875,
|
|
"rewards/margins": 0.289703369140625,
|
|
"rewards/rejected": 0.289459228515625,
|
|
"step": 51
|
|
},
|
|
{
|
|
"epoch": 0.19980787704130643,
|
|
"grad_norm": 328.0,
|
|
"learning_rate": 9.807692307692306e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.91015625,
|
|
"logps/chosen": -535.25,
|
|
"logps/rejected": -238.625,
|
|
"loss": 1.7109,
|
|
"nll_loss": 0.853515625,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": 0.81005859375,
|
|
"rewards/margins": 0.5323486328125,
|
|
"rewards/rejected": 0.277587890625,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.20365033621517772,
|
|
"grad_norm": 204.0,
|
|
"learning_rate": 1e-06,
|
|
"logits/chosen": -3.865234375,
|
|
"logits/rejected": -3.869140625,
|
|
"logps/chosen": -518.0,
|
|
"logps/rejected": -265.25,
|
|
"loss": 1.7949,
|
|
"nll_loss": 0.9453125,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.7431640625,
|
|
"rewards/margins": 0.526611328125,
|
|
"rewards/rejected": 0.2174072265625,
|
|
"step": 53
|
|
},
|
|
{
|
|
"epoch": 0.207492795389049,
|
|
"grad_norm": 157.0,
|
|
"learning_rate": 9.978723404255318e-07,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -365.25,
|
|
"logps/rejected": -191.625,
|
|
"loss": 1.6094,
|
|
"nll_loss": 0.763916015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.649169921875,
|
|
"rewards/margins": 0.5145263671875,
|
|
"rewards/rejected": 0.1346435546875,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.21133525456292027,
|
|
"grad_norm": 330.0,
|
|
"learning_rate": 9.957446808510637e-07,
|
|
"logits/chosen": -3.806640625,
|
|
"logits/rejected": -3.95703125,
|
|
"logps/chosen": -488.375,
|
|
"logps/rejected": -217.75,
|
|
"loss": 1.7256,
|
|
"nll_loss": 0.8583984375,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": 0.7841796875,
|
|
"rewards/margins": 0.453125,
|
|
"rewards/rejected": 0.3309326171875,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.21517771373679154,
|
|
"grad_norm": 212.0,
|
|
"learning_rate": 9.936170212765958e-07,
|
|
"logits/chosen": -3.958984375,
|
|
"logits/rejected": -3.94140625,
|
|
"logps/chosen": -594.0,
|
|
"logps/rejected": -280.75,
|
|
"loss": 1.7373,
|
|
"nll_loss": 0.97705078125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.64453125,
|
|
"rewards/margins": 1.099853515625,
|
|
"rewards/rejected": 0.545318603515625,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.21902017291066284,
|
|
"grad_norm": 306.0,
|
|
"learning_rate": 9.914893617021276e-07,
|
|
"logits/chosen": -3.7734375,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -504.25,
|
|
"logps/rejected": -262.4375,
|
|
"loss": 1.7422,
|
|
"nll_loss": 0.93359375,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": 1.24462890625,
|
|
"rewards/margins": 0.76220703125,
|
|
"rewards/rejected": 0.483642578125,
|
|
"step": 57
|
|
},
|
|
{
|
|
"epoch": 0.2228626320845341,
|
|
"grad_norm": 300.0,
|
|
"learning_rate": 9.893617021276595e-07,
|
|
"logits/chosen": -3.80859375,
|
|
"logits/rejected": -3.904296875,
|
|
"logps/chosen": -441.75,
|
|
"logps/rejected": -294.25,
|
|
"loss": 1.8174,
|
|
"nll_loss": 0.9765625,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": 1.0126953125,
|
|
"rewards/margins": 0.60302734375,
|
|
"rewards/rejected": 0.4107666015625,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.22670509125840538,
|
|
"grad_norm": 532.0,
|
|
"learning_rate": 9.872340425531914e-07,
|
|
"logits/chosen": -3.89453125,
|
|
"logits/rejected": -3.998046875,
|
|
"logps/chosen": -554.5,
|
|
"logps/rejected": -272.75,
|
|
"loss": 1.7188,
|
|
"nll_loss": 0.95068359375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.412109375,
|
|
"rewards/margins": 0.9609375,
|
|
"rewards/rejected": 0.449951171875,
|
|
"step": 59
|
|
},
|
|
{
|
|
"epoch": 0.23054755043227665,
|
|
"grad_norm": 246.0,
|
|
"learning_rate": 9.851063829787235e-07,
|
|
"logits/chosen": -3.787109375,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -421.125,
|
|
"logps/rejected": -206.125,
|
|
"loss": 1.7148,
|
|
"nll_loss": 0.9423828125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 1.0615234375,
|
|
"rewards/margins": 0.94189453125,
|
|
"rewards/rejected": 0.1209716796875,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.23439000960614795,
|
|
"grad_norm": 402.0,
|
|
"learning_rate": 9.829787234042553e-07,
|
|
"logits/chosen": -3.73046875,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -379.375,
|
|
"logps/rejected": -232.75,
|
|
"loss": 1.7725,
|
|
"nll_loss": 0.916015625,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.9814453125,
|
|
"rewards/margins": 0.55731201171875,
|
|
"rewards/rejected": 0.4244384765625,
|
|
"step": 61
|
|
},
|
|
{
|
|
"epoch": 0.23823246878001922,
|
|
"grad_norm": 225.0,
|
|
"learning_rate": 9.808510638297872e-07,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -482.875,
|
|
"logps/rejected": -251.5,
|
|
"loss": 1.8809,
|
|
"nll_loss": 1.0751953125,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.24072265625,
|
|
"rewards/margins": 0.7862548828125,
|
|
"rewards/rejected": 0.454345703125,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.2420749279538905,
|
|
"grad_norm": 194.0,
|
|
"learning_rate": 9.78723404255319e-07,
|
|
"logits/chosen": -3.83984375,
|
|
"logits/rejected": -3.86328125,
|
|
"logps/chosen": -449.0,
|
|
"logps/rejected": -204.6875,
|
|
"loss": 1.6621,
|
|
"nll_loss": 0.8916015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.21044921875,
|
|
"rewards/margins": 0.93170166015625,
|
|
"rewards/rejected": 0.2799072265625,
|
|
"step": 63
|
|
},
|
|
{
|
|
"epoch": 0.24591738712776176,
|
|
"grad_norm": 648.0,
|
|
"learning_rate": 9.765957446808511e-07,
|
|
"logits/chosen": -3.8828125,
|
|
"logits/rejected": -3.951171875,
|
|
"logps/chosen": -424.75,
|
|
"logps/rejected": -255.375,
|
|
"loss": 1.6562,
|
|
"nll_loss": 0.80712890625,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.93212890625,
|
|
"rewards/margins": 0.546630859375,
|
|
"rewards/rejected": 0.38623046875,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.24975984630163303,
|
|
"grad_norm": 338.0,
|
|
"learning_rate": 9.74468085106383e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.861328125,
|
|
"logps/chosen": -459.5,
|
|
"logps/rejected": -254.625,
|
|
"loss": 1.7275,
|
|
"nll_loss": 0.94287109375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 1.396484375,
|
|
"rewards/margins": 0.883819580078125,
|
|
"rewards/rejected": 0.514404296875,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.25360230547550433,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 9.723404255319149e-07,
|
|
"logits/chosen": -3.84765625,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -393.375,
|
|
"logps/rejected": -264.625,
|
|
"loss": 1.7061,
|
|
"nll_loss": 0.92138671875,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 1.35595703125,
|
|
"rewards/margins": 0.9586181640625,
|
|
"rewards/rejected": 0.3973388671875,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.2574447646493756,
|
|
"grad_norm": 290.0,
|
|
"learning_rate": 9.702127659574467e-07,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.904296875,
|
|
"logps/chosen": -546.25,
|
|
"logps/rejected": -251.375,
|
|
"loss": 1.5322,
|
|
"nll_loss": 0.8447265625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.0361328125,
|
|
"rewards/margins": 1.5693359375,
|
|
"rewards/rejected": 0.4716796875,
|
|
"step": 67
|
|
},
|
|
{
|
|
"epoch": 0.2612872238232469,
|
|
"grad_norm": 141.0,
|
|
"learning_rate": 9.680851063829786e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.748046875,
|
|
"logps/chosen": -440.5,
|
|
"logps/rejected": -211.875,
|
|
"loss": 1.6191,
|
|
"nll_loss": 0.84814453125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.22216796875,
|
|
"rewards/margins": 1.02178955078125,
|
|
"rewards/rejected": 0.2000732421875,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.26512968299711814,
|
|
"grad_norm": 334.0,
|
|
"learning_rate": 9.659574468085105e-07,
|
|
"logits/chosen": -3.90234375,
|
|
"logits/rejected": -3.9140625,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -236.125,
|
|
"loss": 1.7119,
|
|
"nll_loss": 0.91748046875,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": 1.617919921875,
|
|
"rewards/margins": 0.8857421875,
|
|
"rewards/rejected": 0.7354736328125,
|
|
"step": 69
|
|
},
|
|
{
|
|
"epoch": 0.2689721421709894,
|
|
"grad_norm": 446.0,
|
|
"learning_rate": 9.638297872340426e-07,
|
|
"logits/chosen": -3.794921875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -415.625,
|
|
"logps/rejected": -207.375,
|
|
"loss": 1.8086,
|
|
"nll_loss": 0.98046875,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.29638671875,
|
|
"rewards/margins": 0.8109130859375,
|
|
"rewards/rejected": 0.486328125,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.2728146013448607,
|
|
"grad_norm": 270.0,
|
|
"learning_rate": 9.617021276595744e-07,
|
|
"logits/chosen": -3.869140625,
|
|
"logits/rejected": -3.9453125,
|
|
"logps/chosen": -434.25,
|
|
"logps/rejected": -247.8125,
|
|
"loss": 1.7041,
|
|
"nll_loss": 0.8876953125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 1.5185546875,
|
|
"rewards/margins": 0.822265625,
|
|
"rewards/rejected": 0.69708251953125,
|
|
"step": 71
|
|
},
|
|
{
|
|
"epoch": 0.276657060518732,
|
|
"grad_norm": 520.0,
|
|
"learning_rate": 9.595744680851063e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -479.25,
|
|
"logps/rejected": -250.875,
|
|
"loss": 1.7227,
|
|
"nll_loss": 0.9169921875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.48583984375,
|
|
"rewards/margins": 1.148681640625,
|
|
"rewards/rejected": 0.33642578125,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.2804995196926033,
|
|
"grad_norm": 223.0,
|
|
"learning_rate": 9.574468085106384e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.896484375,
|
|
"logps/chosen": -463.75,
|
|
"logps/rejected": -237.9375,
|
|
"loss": 1.6895,
|
|
"nll_loss": 0.9013671875,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.5615234375,
|
|
"rewards/margins": 0.919921875,
|
|
"rewards/rejected": 0.64312744140625,
|
|
"step": 73
|
|
},
|
|
{
|
|
"epoch": 0.28434197886647455,
|
|
"grad_norm": 170.0,
|
|
"learning_rate": 9.553191489361702e-07,
|
|
"logits/chosen": -3.701171875,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -411.375,
|
|
"logps/rejected": -223.25,
|
|
"loss": 1.7793,
|
|
"nll_loss": 0.978515625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.32763671875,
|
|
"rewards/margins": 0.823486328125,
|
|
"rewards/rejected": 0.50537109375,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.2881844380403458,
|
|
"grad_norm": 624.0,
|
|
"learning_rate": 9.531914893617021e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.8984375,
|
|
"logps/chosen": -453.25,
|
|
"logps/rejected": -228.5625,
|
|
"loss": 1.6973,
|
|
"nll_loss": 0.91943359375,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.42236328125,
|
|
"rewards/margins": 0.9970703125,
|
|
"rewards/rejected": 0.42431640625,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.2920268972142171,
|
|
"grad_norm": 262.0,
|
|
"learning_rate": 9.51063829787234e-07,
|
|
"logits/chosen": -3.83984375,
|
|
"logits/rejected": -3.8515625,
|
|
"logps/chosen": -447.5,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.6963,
|
|
"nll_loss": 0.93115234375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 1.6591796875,
|
|
"rewards/margins": 1.05126953125,
|
|
"rewards/rejected": 0.609130859375,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.29586935638808837,
|
|
"grad_norm": 247.0,
|
|
"learning_rate": 9.489361702127659e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.888671875,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -272.0,
|
|
"loss": 1.8164,
|
|
"nll_loss": 1.01904296875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 1.693359375,
|
|
"rewards/margins": 0.8424072265625,
|
|
"rewards/rejected": 0.854736328125,
|
|
"step": 77
|
|
},
|
|
{
|
|
"epoch": 0.29971181556195964,
|
|
"grad_norm": 240.0,
|
|
"learning_rate": 9.468085106382978e-07,
|
|
"logits/chosen": -3.869140625,
|
|
"logits/rejected": -4.013671875,
|
|
"logps/chosen": -486.75,
|
|
"logps/rejected": -240.75,
|
|
"loss": 1.708,
|
|
"nll_loss": 1.0048828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4267578125,
|
|
"rewards/margins": 1.7763671875,
|
|
"rewards/rejected": 0.65087890625,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.3035542747358309,
|
|
"grad_norm": 131.0,
|
|
"learning_rate": 9.446808510638298e-07,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.88671875,
|
|
"logps/chosen": -398.75,
|
|
"logps/rejected": -195.625,
|
|
"loss": 1.6045,
|
|
"nll_loss": 0.86669921875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.63720703125,
|
|
"rewards/margins": 1.23681640625,
|
|
"rewards/rejected": 0.4012451171875,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.30739673390970224,
|
|
"grad_norm": 233.0,
|
|
"learning_rate": 9.425531914893617e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.919921875,
|
|
"logps/chosen": -408.25,
|
|
"logps/rejected": -239.0,
|
|
"loss": 1.6426,
|
|
"nll_loss": 0.87939453125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.8408203125,
|
|
"rewards/margins": 1.06689453125,
|
|
"rewards/rejected": 0.776123046875,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.3112391930835735,
|
|
"grad_norm": 176.0,
|
|
"learning_rate": 9.404255319148936e-07,
|
|
"logits/chosen": -3.880859375,
|
|
"logits/rejected": -3.9296875,
|
|
"logps/chosen": -485.25,
|
|
"logps/rejected": -259.125,
|
|
"loss": 1.5156,
|
|
"nll_loss": 0.84033203125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0283203125,
|
|
"rewards/margins": 1.4931640625,
|
|
"rewards/rejected": 0.53704833984375,
|
|
"step": 81
|
|
},
|
|
{
|
|
"epoch": 0.3150816522574448,
|
|
"grad_norm": 402.0,
|
|
"learning_rate": 9.382978723404255e-07,
|
|
"logits/chosen": -3.8359375,
|
|
"logits/rejected": -3.923828125,
|
|
"logps/chosen": -415.375,
|
|
"logps/rejected": -200.125,
|
|
"loss": 1.707,
|
|
"nll_loss": 0.97998046875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.74658203125,
|
|
"rewards/margins": 1.249267578125,
|
|
"rewards/rejected": 0.497161865234375,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.31892411143131605,
|
|
"grad_norm": 222.0,
|
|
"learning_rate": 9.361702127659575e-07,
|
|
"logits/chosen": -3.8671875,
|
|
"logits/rejected": -3.87109375,
|
|
"logps/chosen": -493.125,
|
|
"logps/rejected": -297.0,
|
|
"loss": 1.7607,
|
|
"nll_loss": 0.9794921875,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": 1.927734375,
|
|
"rewards/margins": 1.022216796875,
|
|
"rewards/rejected": 0.91009521484375,
|
|
"step": 83
|
|
},
|
|
{
|
|
"epoch": 0.3227665706051873,
|
|
"grad_norm": 2992.0,
|
|
"learning_rate": 9.340425531914892e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.916015625,
|
|
"logps/chosen": -495.5,
|
|
"logps/rejected": -232.625,
|
|
"loss": 1.5625,
|
|
"nll_loss": 0.8759765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7822265625,
|
|
"rewards/margins": 1.5244140625,
|
|
"rewards/rejected": 0.2606201171875,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.3266090297790586,
|
|
"grad_norm": 240.0,
|
|
"learning_rate": 9.319148936170212e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -569.75,
|
|
"logps/rejected": -259.75,
|
|
"loss": 1.5625,
|
|
"nll_loss": 0.8759765625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.078125,
|
|
"rewards/margins": 1.6181640625,
|
|
"rewards/rejected": 0.4600830078125,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.33045148895292986,
|
|
"grad_norm": 324.0,
|
|
"learning_rate": 9.297872340425531e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -245.625,
|
|
"loss": 1.8457,
|
|
"nll_loss": 1.02783203125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 1.209228515625,
|
|
"rewards/margins": 0.81768798828125,
|
|
"rewards/rejected": 0.39013671875,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.33429394812680113,
|
|
"grad_norm": 278.0,
|
|
"learning_rate": 9.27659574468085e-07,
|
|
"logits/chosen": -3.88671875,
|
|
"logits/rejected": -3.94140625,
|
|
"logps/chosen": -611.75,
|
|
"logps/rejected": -295.375,
|
|
"loss": 1.5273,
|
|
"nll_loss": 0.853515625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.8388671875,
|
|
"rewards/margins": 1.8564453125,
|
|
"rewards/rejected": 0.9833984375,
|
|
"step": 87
|
|
},
|
|
{
|
|
"epoch": 0.3381364073006724,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 9.255319148936169e-07,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -362.0,
|
|
"logps/rejected": -228.0,
|
|
"loss": 1.6787,
|
|
"nll_loss": 0.87060546875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.53759765625,
|
|
"rewards/margins": 0.8037109375,
|
|
"rewards/rejected": 0.7355804443359375,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.34197886647454373,
|
|
"grad_norm": 225.0,
|
|
"learning_rate": 9.234042553191489e-07,
|
|
"logits/chosen": -3.806640625,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -440.75,
|
|
"logps/rejected": -244.75,
|
|
"loss": 1.6602,
|
|
"nll_loss": 0.92333984375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.98095703125,
|
|
"rewards/margins": 1.285888671875,
|
|
"rewards/rejected": 0.6971435546875,
|
|
"step": 89
|
|
},
|
|
{
|
|
"epoch": 0.345821325648415,
|
|
"grad_norm": 178.0,
|
|
"learning_rate": 9.212765957446809e-07,
|
|
"logits/chosen": -3.841796875,
|
|
"logits/rejected": -3.869140625,
|
|
"logps/chosen": -425.25,
|
|
"logps/rejected": -206.0,
|
|
"loss": 1.7734,
|
|
"nll_loss": 1.0419921875,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.939453125,
|
|
"rewards/margins": 1.39013671875,
|
|
"rewards/rejected": 0.552001953125,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.34966378482228627,
|
|
"grad_norm": 832.0,
|
|
"learning_rate": 9.191489361702127e-07,
|
|
"logits/chosen": -3.806640625,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -242.75,
|
|
"loss": 1.7354,
|
|
"nll_loss": 1.03955078125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.5537109375,
|
|
"rewards/margins": 1.72607421875,
|
|
"rewards/rejected": 0.8270263671875,
|
|
"step": 91
|
|
},
|
|
{
|
|
"epoch": 0.35350624399615754,
|
|
"grad_norm": 195.0,
|
|
"learning_rate": 9.170212765957447e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.810546875,
|
|
"logps/chosen": -395.875,
|
|
"logps/rejected": -225.75,
|
|
"loss": 1.5938,
|
|
"nll_loss": 0.85400390625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.58544921875,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": 0.400390625,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.3573487031700288,
|
|
"grad_norm": 832.0,
|
|
"learning_rate": 9.148936170212766e-07,
|
|
"logits/chosen": -3.798828125,
|
|
"logits/rejected": -3.869140625,
|
|
"logps/chosen": -458.75,
|
|
"logps/rejected": -206.5625,
|
|
"loss": 1.6338,
|
|
"nll_loss": 0.88916015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.89453125,
|
|
"rewards/margins": 1.24462890625,
|
|
"rewards/rejected": 0.64971923828125,
|
|
"step": 93
|
|
},
|
|
{
|
|
"epoch": 0.3611911623439001,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 9.127659574468085e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -364.875,
|
|
"logps/rejected": -184.75,
|
|
"loss": 1.4531,
|
|
"nll_loss": 0.72216796875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 1.5322265625,
|
|
"rewards/margins": 1.21240234375,
|
|
"rewards/rejected": 0.31890869140625,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.36503362151777136,
|
|
"grad_norm": 177.0,
|
|
"learning_rate": 9.106382978723404e-07,
|
|
"logits/chosen": -3.8671875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -464.375,
|
|
"logps/rejected": -212.5,
|
|
"loss": 1.6123,
|
|
"nll_loss": 0.92822265625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.1669921875,
|
|
"rewards/margins": 1.567138671875,
|
|
"rewards/rejected": 0.601806640625,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.3688760806916426,
|
|
"grad_norm": 142.0,
|
|
"learning_rate": 9.085106382978724e-07,
|
|
"logits/chosen": -3.744140625,
|
|
"logits/rejected": -3.7578125,
|
|
"logps/chosen": -415.5,
|
|
"logps/rejected": -202.875,
|
|
"loss": 1.5967,
|
|
"nll_loss": 0.876953125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.84375,
|
|
"rewards/margins": 1.455078125,
|
|
"rewards/rejected": 0.3896484375,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.37271853986551395,
|
|
"grad_norm": 688.0,
|
|
"learning_rate": 9.063829787234041e-07,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.830078125,
|
|
"logps/chosen": -421.25,
|
|
"logps/rejected": -219.625,
|
|
"loss": 1.5859,
|
|
"nll_loss": 0.83154296875,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.98876953125,
|
|
"rewards/margins": 1.2889404296875,
|
|
"rewards/rejected": 0.704345703125,
|
|
"step": 97
|
|
},
|
|
{
|
|
"epoch": 0.3765609990393852,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 9.042553191489361e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -358.5,
|
|
"logps/rejected": -222.5,
|
|
"loss": 1.5762,
|
|
"nll_loss": 0.8349609375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.83935546875,
|
|
"rewards/margins": 1.26416015625,
|
|
"rewards/rejected": 0.576080322265625,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.3804034582132565,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 9.02127659574468e-07,
|
|
"logits/chosen": -3.73046875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -516.5,
|
|
"logps/rejected": -262.125,
|
|
"loss": 1.6494,
|
|
"nll_loss": 0.9033203125,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.11328125,
|
|
"rewards/margins": 1.32177734375,
|
|
"rewards/rejected": 0.79345703125,
|
|
"step": 99
|
|
},
|
|
{
|
|
"epoch": 0.38424591738712777,
|
|
"grad_norm": 206.0,
|
|
"learning_rate": 9e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -482.5,
|
|
"logps/rejected": -189.1875,
|
|
"loss": 1.6758,
|
|
"nll_loss": 1.04296875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.2978515625,
|
|
"rewards/margins": 1.8857421875,
|
|
"rewards/rejected": 0.412139892578125,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.38808837656099904,
|
|
"grad_norm": 185.0,
|
|
"learning_rate": 8.978723404255318e-07,
|
|
"logits/chosen": -3.7734375,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -412.75,
|
|
"logps/rejected": -194.8125,
|
|
"loss": 1.6689,
|
|
"nll_loss": 0.98193359375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.828125,
|
|
"rewards/margins": 1.462890625,
|
|
"rewards/rejected": 0.3673095703125,
|
|
"step": 101
|
|
},
|
|
{
|
|
"epoch": 0.3919308357348703,
|
|
"grad_norm": 280.0,
|
|
"learning_rate": 8.957446808510638e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.904296875,
|
|
"logps/chosen": -420.25,
|
|
"logps/rejected": -193.875,
|
|
"loss": 1.6377,
|
|
"nll_loss": 0.9443359375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.99169921875,
|
|
"rewards/margins": 1.43017578125,
|
|
"rewards/rejected": 0.5598297119140625,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.3957732949087416,
|
|
"grad_norm": 253.0,
|
|
"learning_rate": 8.936170212765957e-07,
|
|
"logits/chosen": -3.78515625,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -433.75,
|
|
"logps/rejected": -236.5,
|
|
"loss": 1.5654,
|
|
"nll_loss": 0.85791015625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.103515625,
|
|
"rewards/margins": 1.560546875,
|
|
"rewards/rejected": 0.54339599609375,
|
|
"step": 103
|
|
},
|
|
{
|
|
"epoch": 0.39961575408261285,
|
|
"grad_norm": 163.0,
|
|
"learning_rate": 8.914893617021276e-07,
|
|
"logits/chosen": -3.6953125,
|
|
"logits/rejected": -3.7734375,
|
|
"logps/chosen": -376.5,
|
|
"logps/rejected": -212.5,
|
|
"loss": 1.5381,
|
|
"nll_loss": 0.79052734375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 1.59912109375,
|
|
"rewards/margins": 1.1611328125,
|
|
"rewards/rejected": 0.439605712890625,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.4034582132564842,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 8.893617021276595e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.814453125,
|
|
"logps/chosen": -436.125,
|
|
"logps/rejected": -217.75,
|
|
"loss": 1.5127,
|
|
"nll_loss": 0.83203125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.3046875,
|
|
"rewards/margins": 1.68896484375,
|
|
"rewards/rejected": 0.618316650390625,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.40730067243035545,
|
|
"grad_norm": 179.0,
|
|
"learning_rate": 8.872340425531915e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.919921875,
|
|
"logps/chosen": -437.875,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.5703,
|
|
"nll_loss": 0.88525390625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.314453125,
|
|
"rewards/margins": 1.50537109375,
|
|
"rewards/rejected": 0.8076171875,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.4111431316042267,
|
|
"grad_norm": 568.0,
|
|
"learning_rate": 8.851063829787234e-07,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -331.75,
|
|
"logps/rejected": -245.0,
|
|
"loss": 1.8213,
|
|
"nll_loss": 0.9931640625,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": 1.51513671875,
|
|
"rewards/margins": 0.80279541015625,
|
|
"rewards/rejected": 0.71295166015625,
|
|
"step": 107
|
|
},
|
|
{
|
|
"epoch": 0.414985590778098,
|
|
"grad_norm": 262.0,
|
|
"learning_rate": 8.829787234042553e-07,
|
|
"logits/chosen": -3.845703125,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -526.5,
|
|
"logps/rejected": -339.125,
|
|
"loss": 1.6846,
|
|
"nll_loss": 0.95263671875,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 2.5693359375,
|
|
"rewards/margins": 1.49658203125,
|
|
"rewards/rejected": 1.076416015625,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.41882804995196926,
|
|
"grad_norm": 239.0,
|
|
"learning_rate": 8.808510638297872e-07,
|
|
"logits/chosen": -3.86328125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -380.0,
|
|
"logps/rejected": -252.0,
|
|
"loss": 1.583,
|
|
"nll_loss": 0.806640625,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.0732421875,
|
|
"rewards/margins": 1.227813720703125,
|
|
"rewards/rejected": 0.84814453125,
|
|
"step": 109
|
|
},
|
|
{
|
|
"epoch": 0.42267050912584053,
|
|
"grad_norm": 181.0,
|
|
"learning_rate": 8.787234042553191e-07,
|
|
"logits/chosen": -3.8203125,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -431.0,
|
|
"logps/rejected": -215.5,
|
|
"loss": 1.5312,
|
|
"nll_loss": 0.83447265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2900390625,
|
|
"rewards/margins": 1.55859375,
|
|
"rewards/rejected": 0.7327880859375,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.4265129682997118,
|
|
"grad_norm": 216.0,
|
|
"learning_rate": 8.76595744680851e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.787109375,
|
|
"logps/chosen": -496.5,
|
|
"logps/rejected": -243.0,
|
|
"loss": 1.4531,
|
|
"nll_loss": 0.82568359375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.6123046875,
|
|
"rewards/margins": 2.0751953125,
|
|
"rewards/rejected": 0.53875732421875,
|
|
"step": 111
|
|
},
|
|
{
|
|
"epoch": 0.4303554274735831,
|
|
"grad_norm": 334.0,
|
|
"learning_rate": 8.744680851063829e-07,
|
|
"logits/chosen": -3.8359375,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -272.125,
|
|
"loss": 1.6562,
|
|
"nll_loss": 0.93603515625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.3095703125,
|
|
"rewards/margins": 1.46484375,
|
|
"rewards/rejected": 0.842041015625,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.43419788664745435,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 8.723404255319149e-07,
|
|
"logits/chosen": -3.86328125,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -274.375,
|
|
"loss": 1.5752,
|
|
"nll_loss": 0.84716796875,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 2.09619140625,
|
|
"rewards/margins": 1.38818359375,
|
|
"rewards/rejected": 0.7090301513671875,
|
|
"step": 113
|
|
},
|
|
{
|
|
"epoch": 0.43804034582132567,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 8.702127659574467e-07,
|
|
"logits/chosen": -3.7734375,
|
|
"logits/rejected": -3.837890625,
|
|
"logps/chosen": -408.25,
|
|
"logps/rejected": -222.0,
|
|
"loss": 1.5449,
|
|
"nll_loss": 0.82666015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.9853515625,
|
|
"rewards/margins": 1.350830078125,
|
|
"rewards/rejected": 0.634185791015625,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.44188280499519694,
|
|
"grad_norm": 280.0,
|
|
"learning_rate": 8.680851063829787e-07,
|
|
"logits/chosen": -3.806640625,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -415.75,
|
|
"logps/rejected": -229.875,
|
|
"loss": 1.7617,
|
|
"nll_loss": 1.0244140625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.1650390625,
|
|
"rewards/margins": 1.3291015625,
|
|
"rewards/rejected": 0.83917236328125,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.4457252641690682,
|
|
"grad_norm": 203.0,
|
|
"learning_rate": 8.659574468085106e-07,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.912109375,
|
|
"logps/chosen": -489.75,
|
|
"logps/rejected": -260.25,
|
|
"loss": 1.5342,
|
|
"nll_loss": 0.83984375,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 2.8349609375,
|
|
"rewards/margins": 2.1044921875,
|
|
"rewards/rejected": 0.730224609375,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.4495677233429395,
|
|
"grad_norm": 221.0,
|
|
"learning_rate": 8.638297872340426e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -225.6875,
|
|
"loss": 1.6465,
|
|
"nll_loss": 1.00146484375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2255859375,
|
|
"rewards/margins": 1.85791015625,
|
|
"rewards/rejected": 0.367401123046875,
|
|
"step": 117
|
|
},
|
|
{
|
|
"epoch": 0.45341018251681076,
|
|
"grad_norm": 193.0,
|
|
"learning_rate": 8.617021276595744e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -476.5,
|
|
"logps/rejected": -241.375,
|
|
"loss": 1.5322,
|
|
"nll_loss": 0.88427734375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4482421875,
|
|
"rewards/margins": 1.818359375,
|
|
"rewards/rejected": 0.6304931640625,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.457252641690682,
|
|
"grad_norm": 244.0,
|
|
"learning_rate": 8.595744680851064e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.837890625,
|
|
"logps/chosen": -415.25,
|
|
"logps/rejected": -211.375,
|
|
"loss": 1.5801,
|
|
"nll_loss": 0.89697265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9833984375,
|
|
"rewards/margins": 1.55419921875,
|
|
"rewards/rejected": 0.43115234375,
|
|
"step": 119
|
|
},
|
|
{
|
|
"epoch": 0.4610951008645533,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 8.574468085106383e-07,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.9296875,
|
|
"logps/chosen": -435.5,
|
|
"logps/rejected": -182.9375,
|
|
"loss": 1.6582,
|
|
"nll_loss": 1.03955078125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.2822265625,
|
|
"rewards/margins": 1.94140625,
|
|
"rewards/rejected": 0.343353271484375,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.46493756003842457,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 8.553191489361702e-07,
|
|
"logits/chosen": -3.857421875,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -374.0,
|
|
"logps/rejected": -227.0625,
|
|
"loss": 1.5195,
|
|
"nll_loss": 0.79638671875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 1.83447265625,
|
|
"rewards/margins": 1.39111328125,
|
|
"rewards/rejected": 0.441497802734375,
|
|
"step": 121
|
|
},
|
|
{
|
|
"epoch": 0.4687800192122959,
|
|
"grad_norm": 196.0,
|
|
"learning_rate": 8.53191489361702e-07,
|
|
"logits/chosen": -3.853515625,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -524.5,
|
|
"logps/rejected": -263.875,
|
|
"loss": 1.5488,
|
|
"nll_loss": 0.900390625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.013671875,
|
|
"rewards/margins": 2.142333984375,
|
|
"rewards/rejected": 0.869903564453125,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.47262247838616717,
|
|
"grad_norm": 716.0,
|
|
"learning_rate": 8.51063829787234e-07,
|
|
"logits/chosen": -3.80859375,
|
|
"logits/rejected": -3.841796875,
|
|
"logps/chosen": -375.75,
|
|
"logps/rejected": -198.75,
|
|
"loss": 1.6484,
|
|
"nll_loss": 0.9248046875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9375,
|
|
"rewards/margins": 1.326171875,
|
|
"rewards/rejected": 0.6134033203125,
|
|
"step": 123
|
|
},
|
|
{
|
|
"epoch": 0.47646493756003844,
|
|
"grad_norm": 644.0,
|
|
"learning_rate": 8.489361702127658e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -335.0,
|
|
"logps/rejected": -193.625,
|
|
"loss": 1.6699,
|
|
"nll_loss": 0.92333984375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 1.8984375,
|
|
"rewards/margins": 1.21142578125,
|
|
"rewards/rejected": 0.6873779296875,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.4803073967339097,
|
|
"grad_norm": 1040.0,
|
|
"learning_rate": 8.468085106382978e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.935546875,
|
|
"logps/chosen": -372.5,
|
|
"logps/rejected": -206.625,
|
|
"loss": 1.582,
|
|
"nll_loss": 0.8134765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.759521484375,
|
|
"rewards/margins": 1.14892578125,
|
|
"rewards/rejected": 0.610595703125,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.484149855907781,
|
|
"grad_norm": 306.0,
|
|
"learning_rate": 8.446808510638298e-07,
|
|
"logits/chosen": -3.841796875,
|
|
"logits/rejected": -3.8671875,
|
|
"logps/chosen": -513.5,
|
|
"logps/rejected": -284.75,
|
|
"loss": 1.5312,
|
|
"nll_loss": 0.8662109375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.8369140625,
|
|
"rewards/margins": 1.92578125,
|
|
"rewards/rejected": 0.9114990234375,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.48799231508165225,
|
|
"grad_norm": 528.0,
|
|
"learning_rate": 8.425531914893617e-07,
|
|
"logits/chosen": -3.76171875,
|
|
"logits/rejected": -3.802734375,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -220.875,
|
|
"loss": 1.7197,
|
|
"nll_loss": 0.92041015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3447265625,
|
|
"rewards/margins": 1.479736328125,
|
|
"rewards/rejected": 0.8623046875,
|
|
"step": 127
|
|
},
|
|
{
|
|
"epoch": 0.4918347742555235,
|
|
"grad_norm": 222.0,
|
|
"learning_rate": 8.404255319148936e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -451.25,
|
|
"logps/rejected": -213.0,
|
|
"loss": 1.71,
|
|
"nll_loss": 1.0576171875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.408203125,
|
|
"rewards/margins": 1.86767578125,
|
|
"rewards/rejected": 0.5413818359375,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.4956772334293948,
|
|
"grad_norm": 166.0,
|
|
"learning_rate": 8.382978723404255e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -480.5,
|
|
"logps/rejected": -206.0625,
|
|
"loss": 1.5322,
|
|
"nll_loss": 0.8984375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.5556640625,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": 0.525115966796875,
|
|
"step": 129
|
|
},
|
|
{
|
|
"epoch": 0.49951969260326606,
|
|
"grad_norm": 242.0,
|
|
"learning_rate": 8.361702127659575e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.81640625,
|
|
"logps/chosen": -407.5,
|
|
"logps/rejected": -200.375,
|
|
"loss": 1.5566,
|
|
"nll_loss": 0.8857421875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 2.3193359375,
|
|
"rewards/margins": 1.712158203125,
|
|
"rewards/rejected": 0.608642578125,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.5033621517771374,
|
|
"grad_norm": 199.0,
|
|
"learning_rate": 8.340425531914893e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -436.75,
|
|
"logps/rejected": -217.75,
|
|
"loss": 1.6855,
|
|
"nll_loss": 1.021484375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.21875,
|
|
"rewards/margins": 1.65380859375,
|
|
"rewards/rejected": 0.5675048828125,
|
|
"step": 131
|
|
},
|
|
{
|
|
"epoch": 0.5072046109510087,
|
|
"grad_norm": 201.0,
|
|
"learning_rate": 8.319148936170213e-07,
|
|
"logits/chosen": -3.93359375,
|
|
"logits/rejected": -3.87109375,
|
|
"logps/chosen": -496.75,
|
|
"logps/rejected": -270.875,
|
|
"loss": 1.6523,
|
|
"nll_loss": 1.0185546875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.2138671875,
|
|
"rewards/margins": 2.18994140625,
|
|
"rewards/rejected": 1.02911376953125,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.5110470701248799,
|
|
"grad_norm": 356.0,
|
|
"learning_rate": 8.297872340425532e-07,
|
|
"logits/chosen": -3.87890625,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -429.0,
|
|
"logps/rejected": -217.9375,
|
|
"loss": 1.7441,
|
|
"nll_loss": 1.0126953125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2900390625,
|
|
"rewards/margins": 1.745849609375,
|
|
"rewards/rejected": 0.5435791015625,
|
|
"step": 133
|
|
},
|
|
{
|
|
"epoch": 0.5148895292987512,
|
|
"grad_norm": 916.0,
|
|
"learning_rate": 8.27659574468085e-07,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -443.75,
|
|
"logps/rejected": -265.375,
|
|
"loss": 1.5186,
|
|
"nll_loss": 0.869384765625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.380859375,
|
|
"rewards/margins": 1.884765625,
|
|
"rewards/rejected": 0.49267578125,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.5187319884726225,
|
|
"grad_norm": 245.0,
|
|
"learning_rate": 8.255319148936169e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.794921875,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -258.125,
|
|
"loss": 1.4658,
|
|
"nll_loss": 0.86181640625,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.0078125,
|
|
"rewards/margins": 2.462890625,
|
|
"rewards/rejected": 0.547607421875,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.5225744476464937,
|
|
"grad_norm": 135.0,
|
|
"learning_rate": 8.234042553191489e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.794921875,
|
|
"logps/chosen": -411.75,
|
|
"logps/rejected": -227.5,
|
|
"loss": 1.4697,
|
|
"nll_loss": 0.78857421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4384765625,
|
|
"rewards/margins": 1.73291015625,
|
|
"rewards/rejected": 0.7071533203125,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.526416906820365,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 8.212765957446808e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.908203125,
|
|
"logps/chosen": -428.5,
|
|
"logps/rejected": -201.5,
|
|
"loss": 1.4756,
|
|
"nll_loss": 0.8212890625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.50390625,
|
|
"rewards/margins": 1.955810546875,
|
|
"rewards/rejected": 0.5482177734375,
|
|
"step": 137
|
|
},
|
|
{
|
|
"epoch": 0.5302593659942363,
|
|
"grad_norm": 362.0,
|
|
"learning_rate": 8.191489361702127e-07,
|
|
"logits/chosen": -3.7265625,
|
|
"logits/rejected": -3.787109375,
|
|
"logps/chosen": -477.625,
|
|
"logps/rejected": -222.25,
|
|
"loss": 1.5947,
|
|
"nll_loss": 0.9228515625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.46875,
|
|
"rewards/margins": 1.81732177734375,
|
|
"rewards/rejected": 0.6524658203125,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.5341018251681076,
|
|
"grad_norm": 136.0,
|
|
"learning_rate": 8.170212765957446e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.806640625,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -194.0,
|
|
"loss": 1.6787,
|
|
"nll_loss": 0.97802734375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7744140625,
|
|
"rewards/margins": 1.4150390625,
|
|
"rewards/rejected": 0.36053466796875,
|
|
"step": 139
|
|
},
|
|
{
|
|
"epoch": 0.5379442843419788,
|
|
"grad_norm": 326.0,
|
|
"learning_rate": 8.148936170212766e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.83984375,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -260.0,
|
|
"loss": 1.4521,
|
|
"nll_loss": 0.86328125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.01171875,
|
|
"rewards/margins": 2.3251953125,
|
|
"rewards/rejected": 0.69134521484375,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.5417867435158501,
|
|
"grad_norm": 184.0,
|
|
"learning_rate": 8.127659574468084e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.771484375,
|
|
"logps/chosen": -489.75,
|
|
"logps/rejected": -273.625,
|
|
"loss": 1.4404,
|
|
"nll_loss": 0.81787109375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.955078125,
|
|
"rewards/margins": 2.04150390625,
|
|
"rewards/rejected": 0.913299560546875,
|
|
"step": 141
|
|
},
|
|
{
|
|
"epoch": 0.5456292026897214,
|
|
"grad_norm": 154.0,
|
|
"learning_rate": 8.106382978723404e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -298.75,
|
|
"logps/rejected": -211.125,
|
|
"loss": 1.7109,
|
|
"nll_loss": 0.96142578125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 1.9013671875,
|
|
"rewards/margins": 1.1875,
|
|
"rewards/rejected": 0.7139892578125,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.5494716618635928,
|
|
"grad_norm": 524.0,
|
|
"learning_rate": 8.085106382978723e-07,
|
|
"logits/chosen": -3.87109375,
|
|
"logits/rejected": -3.95703125,
|
|
"logps/chosen": -451.25,
|
|
"logps/rejected": -257.875,
|
|
"loss": 1.7227,
|
|
"nll_loss": 0.9521484375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.51171875,
|
|
"rewards/margins": 1.38330078125,
|
|
"rewards/rejected": 1.128662109375,
|
|
"step": 143
|
|
},
|
|
{
|
|
"epoch": 0.553314121037464,
|
|
"grad_norm": 167.0,
|
|
"learning_rate": 8.063829787234043e-07,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -449.375,
|
|
"logps/rejected": -251.375,
|
|
"loss": 1.541,
|
|
"nll_loss": 0.94580078125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 2.7060546875,
|
|
"rewards/margins": 2.216796875,
|
|
"rewards/rejected": 0.494140625,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.5571565802113353,
|
|
"grad_norm": 219.0,
|
|
"learning_rate": 8.042553191489362e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.841796875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -244.625,
|
|
"loss": 1.4922,
|
|
"nll_loss": 0.8291015625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.2265625,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": 0.6923828125,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.5609990393852066,
|
|
"grad_norm": 193.0,
|
|
"learning_rate": 8.02127659574468e-07,
|
|
"logits/chosen": -3.7734375,
|
|
"logits/rejected": -3.763671875,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -241.875,
|
|
"loss": 1.6504,
|
|
"nll_loss": 0.9228515625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7861328125,
|
|
"rewards/margins": 1.5780029296875,
|
|
"rewards/rejected": 1.208892822265625,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.5648414985590778,
|
|
"grad_norm": 192.0,
|
|
"learning_rate": 8e-07,
|
|
"logits/chosen": -3.783203125,
|
|
"logits/rejected": -3.9140625,
|
|
"logps/chosen": -393.5,
|
|
"logps/rejected": -205.875,
|
|
"loss": 1.5732,
|
|
"nll_loss": 0.92822265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.501953125,
|
|
"rewards/margins": 1.74072265625,
|
|
"rewards/rejected": 0.760040283203125,
|
|
"step": 147
|
|
},
|
|
{
|
|
"epoch": 0.5686839577329491,
|
|
"grad_norm": 149.0,
|
|
"learning_rate": 7.978723404255318e-07,
|
|
"logits/chosen": -3.7734375,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -394.75,
|
|
"logps/rejected": -230.25,
|
|
"loss": 1.5508,
|
|
"nll_loss": 0.86376953125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.474609375,
|
|
"rewards/margins": 1.82373046875,
|
|
"rewards/rejected": 0.6494140625,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.5725264169068204,
|
|
"grad_norm": 235.0,
|
|
"learning_rate": 7.957446808510638e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.74609375,
|
|
"logps/chosen": -389.875,
|
|
"logps/rejected": -216.875,
|
|
"loss": 1.583,
|
|
"nll_loss": 0.86181640625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1279296875,
|
|
"rewards/margins": 1.52880859375,
|
|
"rewards/rejected": 0.60052490234375,
|
|
"step": 149
|
|
},
|
|
{
|
|
"epoch": 0.5763688760806917,
|
|
"grad_norm": 255.0,
|
|
"learning_rate": 7.936170212765957e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.76953125,
|
|
"logps/chosen": -428.75,
|
|
"logps/rejected": -234.875,
|
|
"loss": 1.6641,
|
|
"nll_loss": 0.98828125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 2.5654296875,
|
|
"rewards/margins": 1.751953125,
|
|
"rewards/rejected": 0.81561279296875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.5802113352545629,
|
|
"grad_norm": 264.0,
|
|
"learning_rate": 7.914893617021276e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.97265625,
|
|
"logps/chosen": -431.75,
|
|
"logps/rejected": -231.0,
|
|
"loss": 1.6074,
|
|
"nll_loss": 0.9482421875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.5546875,
|
|
"rewards/margins": 1.7177734375,
|
|
"rewards/rejected": 0.83740234375,
|
|
"step": 151
|
|
},
|
|
{
|
|
"epoch": 0.5840537944284342,
|
|
"grad_norm": 207.0,
|
|
"learning_rate": 7.893617021276595e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -501.25,
|
|
"logps/rejected": -279.0,
|
|
"loss": 1.4629,
|
|
"nll_loss": 0.83984375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1220703125,
|
|
"rewards/margins": 2.240234375,
|
|
"rewards/rejected": 0.882568359375,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.5878962536023055,
|
|
"grad_norm": 704.0,
|
|
"learning_rate": 7.872340425531915e-07,
|
|
"logits/chosen": -3.833984375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -543.5,
|
|
"logps/rejected": -278.5,
|
|
"loss": 1.6807,
|
|
"nll_loss": 0.96923828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.64501953125,
|
|
"rewards/margins": 1.675048828125,
|
|
"rewards/rejected": 0.9708251953125,
|
|
"step": 153
|
|
},
|
|
{
|
|
"epoch": 0.5917387127761767,
|
|
"grad_norm": 157.0,
|
|
"learning_rate": 7.851063829787234e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -485.625,
|
|
"logps/rejected": -243.5,
|
|
"loss": 1.5781,
|
|
"nll_loss": 0.96435546875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.8017578125,
|
|
"rewards/margins": 2.04345703125,
|
|
"rewards/rejected": 0.7607421875,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.595581171950048,
|
|
"grad_norm": 756.0,
|
|
"learning_rate": 7.829787234042553e-07,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -449.75,
|
|
"logps/rejected": -244.0,
|
|
"loss": 1.6514,
|
|
"nll_loss": 0.81591796875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.001953125,
|
|
"rewards/margins": 1.68798828125,
|
|
"rewards/rejected": 1.31463623046875,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.5994236311239193,
|
|
"grad_norm": 214.0,
|
|
"learning_rate": 7.808510638297872e-07,
|
|
"logits/chosen": -3.732421875,
|
|
"logits/rejected": -3.837890625,
|
|
"logps/chosen": -518.625,
|
|
"logps/rejected": -285.25,
|
|
"loss": 1.5527,
|
|
"nll_loss": 0.91845703125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.2080078125,
|
|
"rewards/margins": 2.19140625,
|
|
"rewards/rejected": 1.0189208984375,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.6032660902977905,
|
|
"grad_norm": 162.0,
|
|
"learning_rate": 7.787234042553192e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -495.75,
|
|
"logps/rejected": -224.5,
|
|
"loss": 1.5039,
|
|
"nll_loss": 0.8828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.91796875,
|
|
"rewards/margins": 2.1611328125,
|
|
"rewards/rejected": 0.7611083984375,
|
|
"step": 157
|
|
},
|
|
{
|
|
"epoch": 0.6071085494716618,
|
|
"grad_norm": 280.0,
|
|
"learning_rate": 7.765957446808509e-07,
|
|
"logits/chosen": -3.73046875,
|
|
"logits/rejected": -3.73828125,
|
|
"logps/chosen": -422.625,
|
|
"logps/rejected": -256.5,
|
|
"loss": 1.5732,
|
|
"nll_loss": 0.873046875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.63671875,
|
|
"rewards/margins": 1.763671875,
|
|
"rewards/rejected": 0.87139892578125,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.6109510086455331,
|
|
"grad_norm": 141.0,
|
|
"learning_rate": 7.744680851063829e-07,
|
|
"logits/chosen": -3.787109375,
|
|
"logits/rejected": -3.93359375,
|
|
"logps/chosen": -437.75,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.5537,
|
|
"nll_loss": 0.89453125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.953125,
|
|
"rewards/margins": 1.92724609375,
|
|
"rewards/rejected": 1.03125,
|
|
"step": 159
|
|
},
|
|
{
|
|
"epoch": 0.6147934678194045,
|
|
"grad_norm": 169.0,
|
|
"learning_rate": 7.723404255319148e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -481.25,
|
|
"logps/rejected": -259.625,
|
|
"loss": 1.4082,
|
|
"nll_loss": 0.78955078125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.7470703125,
|
|
"rewards/margins": 2.177978515625,
|
|
"rewards/rejected": 0.57177734375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.6186359269932757,
|
|
"grad_norm": 197.0,
|
|
"learning_rate": 7.702127659574467e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -438.875,
|
|
"logps/rejected": -212.4375,
|
|
"loss": 1.5586,
|
|
"nll_loss": 0.8818359375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.44921875,
|
|
"rewards/margins": 1.86083984375,
|
|
"rewards/rejected": 0.590545654296875,
|
|
"step": 161
|
|
},
|
|
{
|
|
"epoch": 0.622478386167147,
|
|
"grad_norm": 165.0,
|
|
"learning_rate": 7.680851063829787e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.88671875,
|
|
"logps/chosen": -457.625,
|
|
"logps/rejected": -241.125,
|
|
"loss": 1.6582,
|
|
"nll_loss": 1.0185546875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.80078125,
|
|
"rewards/margins": 2.031982421875,
|
|
"rewards/rejected": 0.771240234375,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.6263208453410183,
|
|
"grad_norm": 236.0,
|
|
"learning_rate": 7.659574468085106e-07,
|
|
"logits/chosen": -3.861328125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -469.0,
|
|
"logps/rejected": -246.6875,
|
|
"loss": 1.7158,
|
|
"nll_loss": 0.99267578125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.564453125,
|
|
"rewards/margins": 1.6572265625,
|
|
"rewards/rejected": 0.90728759765625,
|
|
"step": 163
|
|
},
|
|
{
|
|
"epoch": 0.6301633045148896,
|
|
"grad_norm": 189.0,
|
|
"learning_rate": 7.638297872340426e-07,
|
|
"logits/chosen": -3.783203125,
|
|
"logits/rejected": -3.826171875,
|
|
"logps/chosen": -413.0,
|
|
"logps/rejected": -217.5625,
|
|
"loss": 1.5645,
|
|
"nll_loss": 0.9404296875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 2.8408203125,
|
|
"rewards/margins": 2.01220703125,
|
|
"rewards/rejected": 0.83270263671875,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.6340057636887608,
|
|
"grad_norm": 205.0,
|
|
"learning_rate": 7.617021276595744e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.841796875,
|
|
"logps/chosen": -413.5,
|
|
"logps/rejected": -237.625,
|
|
"loss": 1.5508,
|
|
"nll_loss": 0.89404296875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.5185546875,
|
|
"rewards/margins": 1.78564453125,
|
|
"rewards/rejected": 0.734466552734375,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.6378482228626321,
|
|
"grad_norm": 134.0,
|
|
"learning_rate": 7.595744680851064e-07,
|
|
"logits/chosen": -3.70703125,
|
|
"logits/rejected": -3.740234375,
|
|
"logps/chosen": -388.75,
|
|
"logps/rejected": -207.625,
|
|
"loss": 1.5195,
|
|
"nll_loss": 0.86376953125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.3662109375,
|
|
"rewards/margins": 1.86181640625,
|
|
"rewards/rejected": 0.5078125,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.6416906820365034,
|
|
"grad_norm": 179.0,
|
|
"learning_rate": 7.574468085106383e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -396.5,
|
|
"logps/rejected": -201.0,
|
|
"loss": 1.4912,
|
|
"nll_loss": 0.87060546875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5078125,
|
|
"rewards/margins": 2.03515625,
|
|
"rewards/rejected": 0.47314453125,
|
|
"step": 167
|
|
},
|
|
{
|
|
"epoch": 0.6455331412103746,
|
|
"grad_norm": 115.0,
|
|
"learning_rate": 7.553191489361702e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.83984375,
|
|
"logps/chosen": -441.375,
|
|
"logps/rejected": -247.625,
|
|
"loss": 1.5264,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.04296875,
|
|
"rewards/margins": 2.189453125,
|
|
"rewards/rejected": 0.8546142578125,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.6493756003842459,
|
|
"grad_norm": 145.0,
|
|
"learning_rate": 7.531914893617021e-07,
|
|
"logits/chosen": -3.779296875,
|
|
"logits/rejected": -3.830078125,
|
|
"logps/chosen": -353.125,
|
|
"logps/rejected": -221.5625,
|
|
"loss": 1.4199,
|
|
"nll_loss": 0.734375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7216796875,
|
|
"rewards/margins": 1.8662109375,
|
|
"rewards/rejected": 0.858154296875,
|
|
"step": 169
|
|
},
|
|
{
|
|
"epoch": 0.6532180595581172,
|
|
"grad_norm": 420.0,
|
|
"learning_rate": 7.510638297872341e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.958984375,
|
|
"logps/chosen": -459.625,
|
|
"logps/rejected": -244.3125,
|
|
"loss": 1.5928,
|
|
"nll_loss": 0.97314453125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.11181640625,
|
|
"rewards/margins": 2.3369140625,
|
|
"rewards/rejected": 0.77044677734375,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.6570605187319885,
|
|
"grad_norm": 193.0,
|
|
"learning_rate": 7.489361702127658e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.841796875,
|
|
"logps/chosen": -501.0,
|
|
"logps/rejected": -260.9375,
|
|
"loss": 1.5576,
|
|
"nll_loss": 0.92529296875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.12109375,
|
|
"rewards/margins": 2.169677734375,
|
|
"rewards/rejected": 0.94866943359375,
|
|
"step": 171
|
|
},
|
|
{
|
|
"epoch": 0.6609029779058597,
|
|
"grad_norm": 232.0,
|
|
"learning_rate": 7.468085106382978e-07,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.83984375,
|
|
"logps/chosen": -419.5,
|
|
"logps/rejected": -269.0,
|
|
"loss": 1.5303,
|
|
"nll_loss": 0.8681640625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.69140625,
|
|
"rewards/margins": 1.7353515625,
|
|
"rewards/rejected": 0.9569091796875,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.664745437079731,
|
|
"grad_norm": 159.0,
|
|
"learning_rate": 7.446808510638297e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.91015625,
|
|
"logps/chosen": -473.0,
|
|
"logps/rejected": -190.75,
|
|
"loss": 1.5039,
|
|
"nll_loss": 0.92822265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.083984375,
|
|
"rewards/margins": 2.6005859375,
|
|
"rewards/rejected": 0.48876953125,
|
|
"step": 173
|
|
},
|
|
{
|
|
"epoch": 0.664745437079731,
|
|
"eval_logits/chosen": -3.7869317531585693,
|
|
"eval_logits/rejected": -3.843465805053711,
|
|
"eval_logps/chosen": -452.8636474609375,
|
|
"eval_logps/rejected": -244.22727966308594,
|
|
"eval_loss": 1.534204125404358,
|
|
"eval_nll_loss": 0.8988281488418579,
|
|
"eval_rewards/accuracies": 0.8719696998596191,
|
|
"eval_rewards/chosen": 2.9147372245788574,
|
|
"eval_rewards/margins": 2.097864866256714,
|
|
"eval_rewards/rejected": 0.8157692551612854,
|
|
"eval_runtime": 99.0053,
|
|
"eval_samples_per_second": 4.434,
|
|
"eval_steps_per_second": 1.111,
|
|
"step": 173
|
|
},
|
|
{
|
|
"epoch": 0.6685878962536023,
|
|
"grad_norm": 142.0,
|
|
"learning_rate": 7.425531914893617e-07,
|
|
"logits/chosen": -3.673828125,
|
|
"logits/rejected": -3.712890625,
|
|
"logps/chosen": -340.625,
|
|
"logps/rejected": -205.75,
|
|
"loss": 1.5381,
|
|
"nll_loss": 0.83642578125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.04296875,
|
|
"rewards/margins": 1.58740234375,
|
|
"rewards/rejected": 0.453948974609375,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.6724303554274735,
|
|
"grad_norm": 164.0,
|
|
"learning_rate": 7.404255319148935e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.8515625,
|
|
"logps/chosen": -495.5,
|
|
"logps/rejected": -224.0,
|
|
"loss": 1.4424,
|
|
"nll_loss": 0.8544921875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.958984375,
|
|
"rewards/margins": 2.544921875,
|
|
"rewards/rejected": 0.41546630859375,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.6762728146013448,
|
|
"grad_norm": 124.0,
|
|
"learning_rate": 7.382978723404255e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.939453125,
|
|
"logps/chosen": -448.625,
|
|
"logps/rejected": -227.0,
|
|
"loss": 1.5391,
|
|
"nll_loss": 0.95166015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0302734375,
|
|
"rewards/margins": 2.4423828125,
|
|
"rewards/rejected": 0.58929443359375,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.6801152737752162,
|
|
"grad_norm": 165.0,
|
|
"learning_rate": 7.361702127659574e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.775390625,
|
|
"logps/chosen": -469.5,
|
|
"logps/rejected": -245.125,
|
|
"loss": 1.5039,
|
|
"nll_loss": 0.88037109375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.03125,
|
|
"rewards/margins": 2.251953125,
|
|
"rewards/rejected": 0.782012939453125,
|
|
"step": 177
|
|
},
|
|
{
|
|
"epoch": 0.6839577329490875,
|
|
"grad_norm": 189.0,
|
|
"learning_rate": 7.340425531914893e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.912109375,
|
|
"logps/chosen": -597.25,
|
|
"logps/rejected": -320.625,
|
|
"loss": 1.5342,
|
|
"nll_loss": 0.9814453125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.861328125,
|
|
"rewards/margins": 2.94140625,
|
|
"rewards/rejected": 0.9156494140625,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.6878001921229587,
|
|
"grad_norm": 126.0,
|
|
"learning_rate": 7.319148936170212e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.87890625,
|
|
"logps/chosen": -547.5,
|
|
"logps/rejected": -252.375,
|
|
"loss": 1.46,
|
|
"nll_loss": 0.880859375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.619140625,
|
|
"rewards/margins": 2.7490234375,
|
|
"rewards/rejected": 0.87322998046875,
|
|
"step": 179
|
|
},
|
|
{
|
|
"epoch": 0.69164265129683,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 7.297872340425532e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -368.25,
|
|
"logps/rejected": -194.0625,
|
|
"loss": 1.6357,
|
|
"nll_loss": 0.95263671875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.2880859375,
|
|
"rewards/margins": 1.59130859375,
|
|
"rewards/rejected": 0.69732666015625,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.6954851104707013,
|
|
"grad_norm": 161.0,
|
|
"learning_rate": 7.276595744680852e-07,
|
|
"logits/chosen": -3.8203125,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -583.5,
|
|
"logps/rejected": -281.25,
|
|
"loss": 1.4902,
|
|
"nll_loss": 0.93798828125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.755859375,
|
|
"rewards/margins": 2.79296875,
|
|
"rewards/rejected": 0.96649169921875,
|
|
"step": 181
|
|
},
|
|
{
|
|
"epoch": 0.6993275696445725,
|
|
"grad_norm": 175.0,
|
|
"learning_rate": 7.25531914893617e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -406.5,
|
|
"logps/rejected": -216.3125,
|
|
"loss": 1.6201,
|
|
"nll_loss": 0.98193359375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.076171875,
|
|
"rewards/margins": 2.27734375,
|
|
"rewards/rejected": 0.8037109375,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.7031700288184438,
|
|
"grad_norm": 206.0,
|
|
"learning_rate": 7.23404255319149e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -291.75,
|
|
"loss": 1.5371,
|
|
"nll_loss": 0.85791015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.7001953125,
|
|
"rewards/margins": 1.8291015625,
|
|
"rewards/rejected": 0.871002197265625,
|
|
"step": 183
|
|
},
|
|
{
|
|
"epoch": 0.7070124879923151,
|
|
"grad_norm": 223.0,
|
|
"learning_rate": 7.212765957446808e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -461.0,
|
|
"logps/rejected": -299.375,
|
|
"loss": 1.6543,
|
|
"nll_loss": 0.953125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 2.890625,
|
|
"rewards/margins": 1.75732421875,
|
|
"rewards/rejected": 1.1343994140625,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.7108549471661864,
|
|
"grad_norm": 189.0,
|
|
"learning_rate": 7.191489361702127e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.806640625,
|
|
"logps/chosen": -451.25,
|
|
"logps/rejected": -214.75,
|
|
"loss": 1.7754,
|
|
"nll_loss": 1.09326171875,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 2.876953125,
|
|
"rewards/margins": 2.07373046875,
|
|
"rewards/rejected": 0.803802490234375,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.7146974063400576,
|
|
"grad_norm": 219.0,
|
|
"learning_rate": 7.170212765957446e-07,
|
|
"logits/chosen": -3.716796875,
|
|
"logits/rejected": -3.701171875,
|
|
"logps/chosen": -384.75,
|
|
"logps/rejected": -228.9375,
|
|
"loss": 1.6006,
|
|
"nll_loss": 0.8798828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.453125,
|
|
"rewards/margins": 1.5185546875,
|
|
"rewards/rejected": 0.93548583984375,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.7185398655139289,
|
|
"grad_norm": 242.0,
|
|
"learning_rate": 7.148936170212766e-07,
|
|
"logits/chosen": -3.744140625,
|
|
"logits/rejected": -3.771484375,
|
|
"logps/chosen": -470.25,
|
|
"logps/rejected": -244.875,
|
|
"loss": 1.5479,
|
|
"nll_loss": 0.93505859375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.1416015625,
|
|
"rewards/margins": 2.41015625,
|
|
"rewards/rejected": 0.729248046875,
|
|
"step": 187
|
|
},
|
|
{
|
|
"epoch": 0.7223823246878002,
|
|
"grad_norm": 223.0,
|
|
"learning_rate": 7.127659574468084e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -395.0,
|
|
"logps/rejected": -217.625,
|
|
"loss": 1.5762,
|
|
"nll_loss": 0.9091796875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.541015625,
|
|
"rewards/margins": 1.736328125,
|
|
"rewards/rejected": 0.8046875,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.7262247838616714,
|
|
"grad_norm": 560.0,
|
|
"learning_rate": 7.106382978723404e-07,
|
|
"logits/chosen": -3.76171875,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -377.125,
|
|
"logps/rejected": -193.875,
|
|
"loss": 1.6006,
|
|
"nll_loss": 0.95166015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.66796875,
|
|
"rewards/margins": 1.99267578125,
|
|
"rewards/rejected": 0.68115234375,
|
|
"step": 189
|
|
},
|
|
{
|
|
"epoch": 0.7300672430355427,
|
|
"grad_norm": 148.0,
|
|
"learning_rate": 7.085106382978723e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -476.25,
|
|
"logps/rejected": -241.0,
|
|
"loss": 1.4458,
|
|
"nll_loss": 0.8701171875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.064453125,
|
|
"rewards/margins": 2.40869140625,
|
|
"rewards/rejected": 0.6568603515625,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.733909702209414,
|
|
"grad_norm": 172.0,
|
|
"learning_rate": 7.063829787234043e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -434.5,
|
|
"logps/rejected": -209.5,
|
|
"loss": 1.4482,
|
|
"nll_loss": 0.880859375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.935546875,
|
|
"rewards/margins": 2.3828125,
|
|
"rewards/rejected": 0.554443359375,
|
|
"step": 191
|
|
},
|
|
{
|
|
"epoch": 0.7377521613832853,
|
|
"grad_norm": 164.0,
|
|
"learning_rate": 7.042553191489361e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -439.75,
|
|
"logps/rejected": -231.0625,
|
|
"loss": 1.4863,
|
|
"nll_loss": 0.87353515625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.2080078125,
|
|
"rewards/margins": 2.437255859375,
|
|
"rewards/rejected": 0.7763671875,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.7415946205571565,
|
|
"grad_norm": 145.0,
|
|
"learning_rate": 7.021276595744681e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.71484375,
|
|
"logps/chosen": -447.75,
|
|
"logps/rejected": -249.25,
|
|
"loss": 1.4814,
|
|
"nll_loss": 0.8681640625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.884765625,
|
|
"rewards/margins": 2.1962890625,
|
|
"rewards/rejected": 0.68975830078125,
|
|
"step": 193
|
|
},
|
|
{
|
|
"epoch": 0.7454370797310279,
|
|
"grad_norm": 159.0,
|
|
"learning_rate": 7e-07,
|
|
"logits/chosen": -3.732421875,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -457.25,
|
|
"logps/rejected": -236.25,
|
|
"loss": 1.4473,
|
|
"nll_loss": 0.8447265625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.908203125,
|
|
"rewards/margins": 2.3173828125,
|
|
"rewards/rejected": 0.59228515625,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.7492795389048992,
|
|
"grad_norm": 192.0,
|
|
"learning_rate": 6.978723404255319e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -432.125,
|
|
"logps/rejected": -235.875,
|
|
"loss": 1.5566,
|
|
"nll_loss": 0.87109375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.7744140625,
|
|
"rewards/margins": 1.86865234375,
|
|
"rewards/rejected": 0.9033203125,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.7531219980787704,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 6.957446808510637e-07,
|
|
"logits/chosen": -3.833984375,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -503.5,
|
|
"logps/rejected": -281.0,
|
|
"loss": 1.4258,
|
|
"nll_loss": 0.869140625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.8203125,
|
|
"rewards/margins": 2.818359375,
|
|
"rewards/rejected": 1.002685546875,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.7569644572526417,
|
|
"grad_norm": 135.0,
|
|
"learning_rate": 6.936170212765957e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -487.0,
|
|
"logps/rejected": -233.875,
|
|
"loss": 1.3809,
|
|
"nll_loss": 0.7783203125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.23828125,
|
|
"rewards/margins": 2.65234375,
|
|
"rewards/rejected": 0.58453369140625,
|
|
"step": 197
|
|
},
|
|
{
|
|
"epoch": 0.760806916426513,
|
|
"grad_norm": 179.0,
|
|
"learning_rate": 6.914893617021277e-07,
|
|
"logits/chosen": -3.822265625,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -527.125,
|
|
"logps/rejected": -236.375,
|
|
"loss": 1.5029,
|
|
"nll_loss": 0.93115234375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.384765625,
|
|
"rewards/margins": 2.87188720703125,
|
|
"rewards/rejected": 0.515869140625,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.7646493756003843,
|
|
"grad_norm": 372.0,
|
|
"learning_rate": 6.893617021276595e-07,
|
|
"logits/chosen": -3.72265625,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -544.25,
|
|
"logps/rejected": -271.875,
|
|
"loss": 1.4502,
|
|
"nll_loss": 0.8876953125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.447265625,
|
|
"rewards/margins": 2.6640625,
|
|
"rewards/rejected": 0.782470703125,
|
|
"step": 199
|
|
},
|
|
{
|
|
"epoch": 0.7684918347742555,
|
|
"grad_norm": 150.0,
|
|
"learning_rate": 6.872340425531915e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.841796875,
|
|
"logps/chosen": -413.75,
|
|
"logps/rejected": -238.0625,
|
|
"loss": 1.5127,
|
|
"nll_loss": 0.90869140625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.22265625,
|
|
"rewards/margins": 2.31689453125,
|
|
"rewards/rejected": 0.90435791015625,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.7723342939481268,
|
|
"grad_norm": 175.0,
|
|
"learning_rate": 6.851063829787234e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.7578125,
|
|
"logps/chosen": -374.25,
|
|
"logps/rejected": -212.125,
|
|
"loss": 1.4268,
|
|
"nll_loss": 0.8056640625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.40234375,
|
|
"rewards/margins": 1.900390625,
|
|
"rewards/rejected": 0.500244140625,
|
|
"step": 201
|
|
},
|
|
{
|
|
"epoch": 0.7761767531219981,
|
|
"grad_norm": 178.0,
|
|
"learning_rate": 6.829787234042553e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -609.25,
|
|
"logps/rejected": -293.25,
|
|
"loss": 1.3906,
|
|
"nll_loss": 0.890625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.078125,
|
|
"rewards/margins": 3.3359375,
|
|
"rewards/rejected": 0.7420654296875,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.7800192122958693,
|
|
"grad_norm": 1944.0,
|
|
"learning_rate": 6.808510638297872e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -476.25,
|
|
"logps/rejected": -289.875,
|
|
"loss": 1.5664,
|
|
"nll_loss": 0.92919921875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.255859375,
|
|
"rewards/margins": 2.330078125,
|
|
"rewards/rejected": 0.927581787109375,
|
|
"step": 203
|
|
},
|
|
{
|
|
"epoch": 0.7838616714697406,
|
|
"grad_norm": 110.5,
|
|
"learning_rate": 6.787234042553192e-07,
|
|
"logits/chosen": -3.712890625,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -457.125,
|
|
"logps/rejected": -239.375,
|
|
"loss": 1.3916,
|
|
"nll_loss": 0.81103515625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.8671875,
|
|
"rewards/margins": 2.27734375,
|
|
"rewards/rejected": 0.592254638671875,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.7877041306436119,
|
|
"grad_norm": 238.0,
|
|
"learning_rate": 6.76595744680851e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -440.875,
|
|
"logps/rejected": -240.625,
|
|
"loss": 1.6504,
|
|
"nll_loss": 0.99853515625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.037109375,
|
|
"rewards/margins": 2.1580810546875,
|
|
"rewards/rejected": 0.881072998046875,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.7915465898174832,
|
|
"grad_norm": 700.0,
|
|
"learning_rate": 6.74468085106383e-07,
|
|
"logits/chosen": -3.755859375,
|
|
"logits/rejected": -3.865234375,
|
|
"logps/chosen": -335.0,
|
|
"logps/rejected": -204.375,
|
|
"loss": 1.5771,
|
|
"nll_loss": 0.798095703125,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": 1.99755859375,
|
|
"rewards/margins": 1.284423828125,
|
|
"rewards/rejected": 0.71728515625,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.7953890489913544,
|
|
"grad_norm": 236.0,
|
|
"learning_rate": 6.723404255319149e-07,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -444.5,
|
|
"logps/rejected": -288.0,
|
|
"loss": 1.5977,
|
|
"nll_loss": 0.90478515625,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 3.228515625,
|
|
"rewards/margins": 1.951171875,
|
|
"rewards/rejected": 1.279541015625,
|
|
"step": 207
|
|
},
|
|
{
|
|
"epoch": 0.7992315081652257,
|
|
"grad_norm": 210.0,
|
|
"learning_rate": 6.702127659574468e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.8984375,
|
|
"logps/chosen": -431.25,
|
|
"logps/rejected": -227.875,
|
|
"loss": 1.3926,
|
|
"nll_loss": 0.84521484375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 2.9912109375,
|
|
"rewards/margins": 2.4765625,
|
|
"rewards/rejected": 0.5184326171875,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.803073967339097,
|
|
"grad_norm": 288.0,
|
|
"learning_rate": 6.680851063829786e-07,
|
|
"logits/chosen": -3.798828125,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -355.5,
|
|
"logps/rejected": -223.0,
|
|
"loss": 1.4746,
|
|
"nll_loss": 0.852294921875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.63671875,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": 0.664794921875,
|
|
"step": 209
|
|
},
|
|
{
|
|
"epoch": 0.8069164265129684,
|
|
"grad_norm": 280.0,
|
|
"learning_rate": 6.659574468085106e-07,
|
|
"logits/chosen": -3.849609375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -453.875,
|
|
"logps/rejected": -241.125,
|
|
"loss": 1.5791,
|
|
"nll_loss": 0.94970703125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.060546875,
|
|
"rewards/margins": 2.365234375,
|
|
"rewards/rejected": 0.692626953125,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.8107588856868396,
|
|
"grad_norm": 189.0,
|
|
"learning_rate": 6.638297872340425e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -242.25,
|
|
"loss": 1.4805,
|
|
"nll_loss": 0.8671875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.93017578125,
|
|
"rewards/margins": 2.37841796875,
|
|
"rewards/rejected": 0.5509033203125,
|
|
"step": 211
|
|
},
|
|
{
|
|
"epoch": 0.8146013448607109,
|
|
"grad_norm": 374.0,
|
|
"learning_rate": 6.617021276595744e-07,
|
|
"logits/chosen": -3.80859375,
|
|
"logits/rejected": -3.837890625,
|
|
"logps/chosen": -417.75,
|
|
"logps/rejected": -231.75,
|
|
"loss": 1.5596,
|
|
"nll_loss": 0.888671875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.0,
|
|
"rewards/margins": 2.12109375,
|
|
"rewards/rejected": 0.8765869140625,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.8184438040345822,
|
|
"grad_norm": 2320.0,
|
|
"learning_rate": 6.595744680851063e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.9140625,
|
|
"logps/chosen": -352.375,
|
|
"logps/rejected": -208.5625,
|
|
"loss": 1.498,
|
|
"nll_loss": 0.82275390625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.630859375,
|
|
"rewards/margins": 1.8463134765625,
|
|
"rewards/rejected": 0.7886962890625,
|
|
"step": 213
|
|
},
|
|
{
|
|
"epoch": 0.8222862632084534,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 6.574468085106383e-07,
|
|
"logits/chosen": -3.75390625,
|
|
"logits/rejected": -3.853515625,
|
|
"logps/chosen": -433.0,
|
|
"logps/rejected": -261.25,
|
|
"loss": 1.5693,
|
|
"nll_loss": 0.9521484375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.052734375,
|
|
"rewards/margins": 2.2099609375,
|
|
"rewards/rejected": 0.843505859375,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.8261287223823247,
|
|
"grad_norm": 264.0,
|
|
"learning_rate": 6.553191489361701e-07,
|
|
"logits/chosen": -3.8359375,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -412.25,
|
|
"logps/rejected": -240.25,
|
|
"loss": 1.5322,
|
|
"nll_loss": 0.84814453125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 3.07421875,
|
|
"rewards/margins": 1.99176025390625,
|
|
"rewards/rejected": 1.083740234375,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.829971181556196,
|
|
"grad_norm": 124.5,
|
|
"learning_rate": 6.531914893617021e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -383.25,
|
|
"logps/rejected": -229.625,
|
|
"loss": 1.4883,
|
|
"nll_loss": 0.82568359375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.2421875,
|
|
"rewards/margins": 1.88623046875,
|
|
"rewards/rejected": 0.35101318359375,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.8338136407300673,
|
|
"grad_norm": 214.0,
|
|
"learning_rate": 6.510638297872341e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -447.875,
|
|
"logps/rejected": -252.5,
|
|
"loss": 1.5195,
|
|
"nll_loss": 0.833984375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.1201171875,
|
|
"rewards/margins": 2.15576171875,
|
|
"rewards/rejected": 0.96490478515625,
|
|
"step": 217
|
|
},
|
|
{
|
|
"epoch": 0.8376560999039385,
|
|
"grad_norm": 201.0,
|
|
"learning_rate": 6.48936170212766e-07,
|
|
"logits/chosen": -3.708984375,
|
|
"logits/rejected": -3.767578125,
|
|
"logps/chosen": -447.0,
|
|
"logps/rejected": -226.75,
|
|
"loss": 1.5039,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.3291015625,
|
|
"rewards/margins": 2.4716796875,
|
|
"rewards/rejected": 0.85400390625,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.8414985590778098,
|
|
"grad_norm": 312.0,
|
|
"learning_rate": 6.468085106382979e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.791015625,
|
|
"logps/chosen": -484.5,
|
|
"logps/rejected": -235.625,
|
|
"loss": 1.3857,
|
|
"nll_loss": 0.87060546875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.521484375,
|
|
"rewards/margins": 3.06884765625,
|
|
"rewards/rejected": 0.447265625,
|
|
"step": 219
|
|
},
|
|
{
|
|
"epoch": 0.8453410182516811,
|
|
"grad_norm": 193.0,
|
|
"learning_rate": 6.446808510638297e-07,
|
|
"logits/chosen": -3.88671875,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -442.875,
|
|
"logps/rejected": -253.125,
|
|
"loss": 1.4668,
|
|
"nll_loss": 0.84228515625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.2177734375,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": 0.99041748046875,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.8491834774255523,
|
|
"grad_norm": 109.5,
|
|
"learning_rate": 6.425531914893617e-07,
|
|
"logits/chosen": -3.69921875,
|
|
"logits/rejected": -3.748046875,
|
|
"logps/chosen": -456.25,
|
|
"logps/rejected": -241.75,
|
|
"loss": 1.3232,
|
|
"nll_loss": 0.8154296875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.26171875,
|
|
"rewards/margins": 2.8974609375,
|
|
"rewards/rejected": 0.36376953125,
|
|
"step": 221
|
|
},
|
|
{
|
|
"epoch": 0.8530259365994236,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 6.404255319148935e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.83984375,
|
|
"logps/chosen": -518.25,
|
|
"logps/rejected": -247.0,
|
|
"loss": 1.3779,
|
|
"nll_loss": 0.83251953125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.48828125,
|
|
"rewards/margins": 2.8662109375,
|
|
"rewards/rejected": 0.62738037109375,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.8568683957732949,
|
|
"grad_norm": 406.0,
|
|
"learning_rate": 6.382978723404255e-07,
|
|
"logits/chosen": -3.787109375,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -402.875,
|
|
"logps/rejected": -212.375,
|
|
"loss": 1.4863,
|
|
"nll_loss": 0.88330078125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.34130859375,
|
|
"rewards/margins": 2.467529296875,
|
|
"rewards/rejected": 0.874755859375,
|
|
"step": 223
|
|
},
|
|
{
|
|
"epoch": 0.8607108549471661,
|
|
"grad_norm": 310.0,
|
|
"learning_rate": 6.361702127659574e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -348.125,
|
|
"logps/rejected": -229.125,
|
|
"loss": 1.5693,
|
|
"nll_loss": 0.9150390625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.8916015625,
|
|
"rewards/margins": 1.9091796875,
|
|
"rewards/rejected": 0.98388671875,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.8645533141210374,
|
|
"grad_norm": 219.0,
|
|
"learning_rate": 6.340425531914893e-07,
|
|
"logits/chosen": -3.6953125,
|
|
"logits/rejected": -3.779296875,
|
|
"logps/chosen": -457.25,
|
|
"logps/rejected": -216.625,
|
|
"loss": 1.4912,
|
|
"nll_loss": 0.93798828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.94140625,
|
|
"rewards/margins": 2.544921875,
|
|
"rewards/rejected": 0.3935546875,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.8683957732949087,
|
|
"grad_norm": 212.0,
|
|
"learning_rate": 6.319148936170212e-07,
|
|
"logits/chosen": -3.853515625,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -398.625,
|
|
"logps/rejected": -239.0,
|
|
"loss": 1.5586,
|
|
"nll_loss": 0.92822265625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.2861328125,
|
|
"rewards/margins": 2.166015625,
|
|
"rewards/rejected": 1.119873046875,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.8722382324687801,
|
|
"grad_norm": 214.0,
|
|
"learning_rate": 6.297872340425532e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.830078125,
|
|
"logps/chosen": -536.75,
|
|
"logps/rejected": -261.375,
|
|
"loss": 1.4053,
|
|
"nll_loss": 0.83740234375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.732421875,
|
|
"rewards/margins": 3.10546875,
|
|
"rewards/rejected": 0.6224365234375,
|
|
"step": 227
|
|
},
|
|
{
|
|
"epoch": 0.8760806916426513,
|
|
"grad_norm": 236.0,
|
|
"learning_rate": 6.276595744680851e-07,
|
|
"logits/chosen": -3.6796875,
|
|
"logits/rejected": -3.748046875,
|
|
"logps/chosen": -400.25,
|
|
"logps/rejected": -227.5,
|
|
"loss": 1.5186,
|
|
"nll_loss": 0.96240234375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8203125,
|
|
"rewards/margins": 2.5546875,
|
|
"rewards/rejected": 0.267059326171875,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.8799231508165226,
|
|
"grad_norm": 179.0,
|
|
"learning_rate": 6.25531914893617e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -481.5,
|
|
"logps/rejected": -249.5,
|
|
"loss": 1.5068,
|
|
"nll_loss": 0.90625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.5986328125,
|
|
"rewards/margins": 2.740234375,
|
|
"rewards/rejected": 0.86077880859375,
|
|
"step": 229
|
|
},
|
|
{
|
|
"epoch": 0.8837656099903939,
|
|
"grad_norm": 237.0,
|
|
"learning_rate": 6.234042553191489e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.8984375,
|
|
"logps/chosen": -448.625,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.6172,
|
|
"nll_loss": 0.94677734375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3154296875,
|
|
"rewards/margins": 2.23095703125,
|
|
"rewards/rejected": 1.08544921875,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.8876080691642652,
|
|
"grad_norm": 268.0,
|
|
"learning_rate": 6.212765957446809e-07,
|
|
"logits/chosen": -3.80859375,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -636.25,
|
|
"logps/rejected": -313.625,
|
|
"loss": 1.4727,
|
|
"nll_loss": 0.94677734375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.453125,
|
|
"rewards/margins": 3.326416015625,
|
|
"rewards/rejected": 1.132080078125,
|
|
"step": 231
|
|
},
|
|
{
|
|
"epoch": 0.8914505283381364,
|
|
"grad_norm": 502.0,
|
|
"learning_rate": 6.191489361702127e-07,
|
|
"logits/chosen": -3.822265625,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -427.25,
|
|
"logps/rejected": -233.25,
|
|
"loss": 1.5098,
|
|
"nll_loss": 0.869140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0439453125,
|
|
"rewards/margins": 2.1865234375,
|
|
"rewards/rejected": 0.8575439453125,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.8952929875120077,
|
|
"grad_norm": 168.0,
|
|
"learning_rate": 6.170212765957446e-07,
|
|
"logits/chosen": -3.7265625,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -488.5,
|
|
"logps/rejected": -262.25,
|
|
"loss": 1.5352,
|
|
"nll_loss": 0.92724609375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.283203125,
|
|
"rewards/margins": 2.29248046875,
|
|
"rewards/rejected": 0.9892578125,
|
|
"step": 233
|
|
},
|
|
{
|
|
"epoch": 0.899135446685879,
|
|
"grad_norm": 172.0,
|
|
"learning_rate": 6.148936170212766e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.8671875,
|
|
"logps/chosen": -495.75,
|
|
"logps/rejected": -245.75,
|
|
"loss": 1.6201,
|
|
"nll_loss": 0.95849609375,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.51171875,
|
|
"rewards/margins": 2.415283203125,
|
|
"rewards/rejected": 1.10302734375,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.9029779058597502,
|
|
"grad_norm": 180.0,
|
|
"learning_rate": 6.127659574468084e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -412.75,
|
|
"logps/rejected": -279.5625,
|
|
"loss": 1.4961,
|
|
"nll_loss": 0.83837890625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.83203125,
|
|
"rewards/margins": 2.001708984375,
|
|
"rewards/rejected": 0.83062744140625,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.9068203650336215,
|
|
"grad_norm": 222.0,
|
|
"learning_rate": 6.106382978723404e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.7734375,
|
|
"logps/chosen": -469.5,
|
|
"logps/rejected": -232.125,
|
|
"loss": 1.375,
|
|
"nll_loss": 0.8193359375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.0224609375,
|
|
"rewards/margins": 2.7265625,
|
|
"rewards/rejected": 0.3011474609375,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.9106628242074928,
|
|
"grad_norm": 223.0,
|
|
"learning_rate": 6.085106382978723e-07,
|
|
"logits/chosen": -3.78515625,
|
|
"logits/rejected": -3.7578125,
|
|
"logps/chosen": -414.625,
|
|
"logps/rejected": -197.75,
|
|
"loss": 1.6211,
|
|
"nll_loss": 1.01904296875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.046875,
|
|
"rewards/margins": 2.3349609375,
|
|
"rewards/rejected": 0.7138671875,
|
|
"step": 237
|
|
},
|
|
{
|
|
"epoch": 0.914505283381364,
|
|
"grad_norm": 150.0,
|
|
"learning_rate": 6.063829787234043e-07,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -415.625,
|
|
"logps/rejected": -228.9375,
|
|
"loss": 1.4229,
|
|
"nll_loss": 0.84912109375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.2373046875,
|
|
"rewards/margins": 2.6064453125,
|
|
"rewards/rejected": 0.629852294921875,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.9183477425552353,
|
|
"grad_norm": 364.0,
|
|
"learning_rate": 6.042553191489361e-07,
|
|
"logits/chosen": -3.748046875,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -426.75,
|
|
"logps/rejected": -213.9375,
|
|
"loss": 1.5977,
|
|
"nll_loss": 0.921875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.125,
|
|
"rewards/margins": 2.17919921875,
|
|
"rewards/rejected": 0.9476318359375,
|
|
"step": 239
|
|
},
|
|
{
|
|
"epoch": 0.9221902017291066,
|
|
"grad_norm": 126.5,
|
|
"learning_rate": 6.021276595744681e-07,
|
|
"logits/chosen": -3.748046875,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -417.75,
|
|
"logps/rejected": -238.0,
|
|
"loss": 1.3906,
|
|
"nll_loss": 0.80126953125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.173828125,
|
|
"rewards/margins": 2.333984375,
|
|
"rewards/rejected": 0.8408203125,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.9260326609029779,
|
|
"grad_norm": 170.0,
|
|
"learning_rate": 6e-07,
|
|
"logits/chosen": -3.857421875,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -419.875,
|
|
"logps/rejected": -261.5,
|
|
"loss": 1.5698,
|
|
"nll_loss": 0.9013671875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.12890625,
|
|
"rewards/margins": 2.24072265625,
|
|
"rewards/rejected": 0.89031982421875,
|
|
"step": 241
|
|
},
|
|
{
|
|
"epoch": 0.9298751200768491,
|
|
"grad_norm": 173.0,
|
|
"learning_rate": 5.978723404255319e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.716796875,
|
|
"logps/chosen": -546.0,
|
|
"logps/rejected": -295.25,
|
|
"loss": 1.4941,
|
|
"nll_loss": 0.91845703125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.43359375,
|
|
"rewards/margins": 2.650390625,
|
|
"rewards/rejected": 0.77935791015625,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 0.9337175792507204,
|
|
"grad_norm": 252.0,
|
|
"learning_rate": 5.957446808510638e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -439.75,
|
|
"logps/rejected": -273.0,
|
|
"loss": 1.4941,
|
|
"nll_loss": 0.892822265625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.123046875,
|
|
"rewards/margins": 2.2939453125,
|
|
"rewards/rejected": 0.830169677734375,
|
|
"step": 243
|
|
},
|
|
{
|
|
"epoch": 0.9375600384245918,
|
|
"grad_norm": 348.0,
|
|
"learning_rate": 5.936170212765958e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.81640625,
|
|
"logps/chosen": -383.75,
|
|
"logps/rejected": -236.625,
|
|
"loss": 1.5947,
|
|
"nll_loss": 0.88916015625,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 2.6728515625,
|
|
"rewards/margins": 1.78857421875,
|
|
"rewards/rejected": 0.884765625,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 0.9414024975984631,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 5.914893617021275e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -418.375,
|
|
"logps/rejected": -207.25,
|
|
"loss": 1.4395,
|
|
"nll_loss": 0.78662109375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.0009765625,
|
|
"rewards/margins": 2.140625,
|
|
"rewards/rejected": 0.861083984375,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.9452449567723343,
|
|
"grad_norm": 166.0,
|
|
"learning_rate": 5.893617021276595e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.91796875,
|
|
"logps/chosen": -445.625,
|
|
"logps/rejected": -226.5,
|
|
"loss": 1.5156,
|
|
"nll_loss": 0.92138671875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.5859375,
|
|
"rewards/margins": 2.66552734375,
|
|
"rewards/rejected": 0.921875,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 0.9490874159462056,
|
|
"grad_norm": 114.0,
|
|
"learning_rate": 5.872340425531914e-07,
|
|
"logits/chosen": -3.70703125,
|
|
"logits/rejected": -3.748046875,
|
|
"logps/chosen": -413.75,
|
|
"logps/rejected": -219.25,
|
|
"loss": 1.3955,
|
|
"nll_loss": 0.7978515625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.80859375,
|
|
"rewards/margins": 2.337158203125,
|
|
"rewards/rejected": 0.47113037109375,
|
|
"step": 247
|
|
},
|
|
{
|
|
"epoch": 0.9529298751200769,
|
|
"grad_norm": 173.0,
|
|
"learning_rate": 5.851063829787234e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.81640625,
|
|
"logps/chosen": -382.5,
|
|
"logps/rejected": -208.0,
|
|
"loss": 1.4805,
|
|
"nll_loss": 0.8876953125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3291015625,
|
|
"rewards/margins": 2.2783203125,
|
|
"rewards/rejected": 1.05224609375,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 0.9567723342939481,
|
|
"grad_norm": 350.0,
|
|
"learning_rate": 5.829787234042552e-07,
|
|
"logits/chosen": -3.771484375,
|
|
"logits/rejected": -3.759765625,
|
|
"logps/chosen": -426.375,
|
|
"logps/rejected": -256.25,
|
|
"loss": 1.5254,
|
|
"nll_loss": 0.85986328125,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": 3.10546875,
|
|
"rewards/margins": 2.10986328125,
|
|
"rewards/rejected": 0.995849609375,
|
|
"step": 249
|
|
},
|
|
{
|
|
"epoch": 0.9606147934678194,
|
|
"grad_norm": 239.0,
|
|
"learning_rate": 5.808510638297872e-07,
|
|
"logits/chosen": -3.8125,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -463.25,
|
|
"logps/rejected": -259.5,
|
|
"loss": 1.4238,
|
|
"nll_loss": 0.85400390625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.490234375,
|
|
"rewards/margins": 2.708984375,
|
|
"rewards/rejected": 0.7867431640625,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.9644572526416907,
|
|
"grad_norm": 88.5,
|
|
"learning_rate": 5.787234042553191e-07,
|
|
"logits/chosen": -3.72265625,
|
|
"logits/rejected": -3.763671875,
|
|
"logps/chosen": -334.3125,
|
|
"logps/rejected": -179.5625,
|
|
"loss": 1.4014,
|
|
"nll_loss": 0.8017578125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6845703125,
|
|
"rewards/margins": 2.279541015625,
|
|
"rewards/rejected": 0.408447265625,
|
|
"step": 251
|
|
},
|
|
{
|
|
"epoch": 0.968299711815562,
|
|
"grad_norm": 288.0,
|
|
"learning_rate": 5.76595744680851e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -394.5,
|
|
"logps/rejected": -216.125,
|
|
"loss": 1.665,
|
|
"nll_loss": 0.9833984375,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 3.4580078125,
|
|
"rewards/margins": 2.388671875,
|
|
"rewards/rejected": 1.0673828125,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 0.9721421709894332,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 5.74468085106383e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.90625,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -223.125,
|
|
"loss": 1.4834,
|
|
"nll_loss": 0.916015625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.2734375,
|
|
"rewards/margins": 2.6962890625,
|
|
"rewards/rejected": 0.58251953125,
|
|
"step": 253
|
|
},
|
|
{
|
|
"epoch": 0.9759846301633045,
|
|
"grad_norm": 147.0,
|
|
"learning_rate": 5.723404255319149e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -508.875,
|
|
"logps/rejected": -231.0,
|
|
"loss": 1.5293,
|
|
"nll_loss": 0.95458984375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.048828125,
|
|
"rewards/margins": 2.52734375,
|
|
"rewards/rejected": 0.5234375,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 0.9798270893371758,
|
|
"grad_norm": 106.0,
|
|
"learning_rate": 5.702127659574469e-07,
|
|
"logits/chosen": -3.701171875,
|
|
"logits/rejected": -3.787109375,
|
|
"logps/chosen": -416.0,
|
|
"logps/rejected": -214.75,
|
|
"loss": 1.2871,
|
|
"nll_loss": 0.78125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.185546875,
|
|
"rewards/margins": 2.8427734375,
|
|
"rewards/rejected": 0.345458984375,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.983669548511047,
|
|
"grad_norm": 246.0,
|
|
"learning_rate": 5.680851063829787e-07,
|
|
"logits/chosen": -3.72265625,
|
|
"logits/rejected": -3.794921875,
|
|
"logps/chosen": -353.875,
|
|
"logps/rejected": -213.625,
|
|
"loss": 1.4785,
|
|
"nll_loss": 0.84130859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.7578125,
|
|
"rewards/margins": 1.93896484375,
|
|
"rewards/rejected": 0.821075439453125,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 0.9875120076849183,
|
|
"grad_norm": 231.0,
|
|
"learning_rate": 5.659574468085107e-07,
|
|
"logits/chosen": -3.76171875,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -402.375,
|
|
"logps/rejected": -225.375,
|
|
"loss": 1.4424,
|
|
"nll_loss": 0.84765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1884765625,
|
|
"rewards/margins": 2.45263671875,
|
|
"rewards/rejected": 0.73870849609375,
|
|
"step": 257
|
|
},
|
|
{
|
|
"epoch": 0.9913544668587896,
|
|
"grad_norm": 172.0,
|
|
"learning_rate": 5.638297872340425e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.748046875,
|
|
"logps/chosen": -381.875,
|
|
"logps/rejected": -219.5,
|
|
"loss": 1.5361,
|
|
"nll_loss": 0.88671875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.556640625,
|
|
"rewards/margins": 1.82421875,
|
|
"rewards/rejected": 0.732421875,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 0.9951969260326609,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 5.617021276595744e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -449.25,
|
|
"logps/rejected": -246.25,
|
|
"loss": 1.4092,
|
|
"nll_loss": 0.84765625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.162109375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": 0.70648193359375,
|
|
"step": 259
|
|
},
|
|
{
|
|
"epoch": 0.9990393852065321,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 5.595744680851063e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.861328125,
|
|
"logps/chosen": -514.375,
|
|
"logps/rejected": -217.25,
|
|
"loss": 1.4355,
|
|
"nll_loss": 0.92822265625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.310546875,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": 0.46923828125,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"grad_norm": 231.0,
|
|
"learning_rate": 5.574468085106383e-07,
|
|
"logits/chosen": -3.9609375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -335.0,
|
|
"loss": 1.4102,
|
|
"nll_loss": 0.9296875,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.40625,
|
|
"rewards/margins": 3.0859375,
|
|
"rewards/rejected": 1.322265625,
|
|
"step": 261
|
|
},
|
|
{
|
|
"epoch": 1.0038424591738713,
|
|
"grad_norm": 252.0,
|
|
"learning_rate": 5.553191489361701e-07,
|
|
"logits/chosen": -3.818359375,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -325.625,
|
|
"logps/rejected": -180.375,
|
|
"loss": 1.5205,
|
|
"nll_loss": 0.88525390625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 2.5703125,
|
|
"rewards/margins": 1.873046875,
|
|
"rewards/rejected": 0.699951171875,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 1.0076849183477425,
|
|
"grad_norm": 176.0,
|
|
"learning_rate": 5.531914893617021e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.86328125,
|
|
"logps/chosen": -518.25,
|
|
"logps/rejected": -283.75,
|
|
"loss": 1.293,
|
|
"nll_loss": 0.7685546875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.5771484375,
|
|
"rewards/margins": 3.1064453125,
|
|
"rewards/rejected": 0.469970703125,
|
|
"step": 263
|
|
},
|
|
{
|
|
"epoch": 1.0115273775216138,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 5.51063829787234e-07,
|
|
"logits/chosen": -3.787109375,
|
|
"logits/rejected": -3.806640625,
|
|
"logps/chosen": -367.125,
|
|
"logps/rejected": -232.75,
|
|
"loss": 1.5107,
|
|
"nll_loss": 0.869384765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.53515625,
|
|
"rewards/margins": 1.85888671875,
|
|
"rewards/rejected": 0.681884765625,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 1.015369836695485,
|
|
"grad_norm": 142.0,
|
|
"learning_rate": 5.48936170212766e-07,
|
|
"logits/chosen": -3.841796875,
|
|
"logits/rejected": -3.826171875,
|
|
"logps/chosen": -461.75,
|
|
"logps/rejected": -248.125,
|
|
"loss": 1.4277,
|
|
"nll_loss": 0.88818359375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.869140625,
|
|
"rewards/margins": 2.9560546875,
|
|
"rewards/rejected": 0.91015625,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 1.0192122958693564,
|
|
"grad_norm": 302.0,
|
|
"learning_rate": 5.468085106382978e-07,
|
|
"logits/chosen": -3.740234375,
|
|
"logits/rejected": -3.771484375,
|
|
"logps/chosen": -358.0,
|
|
"logps/rejected": -237.75,
|
|
"loss": 1.6299,
|
|
"nll_loss": 0.90478515625,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": 2.98828125,
|
|
"rewards/margins": 1.845947265625,
|
|
"rewards/rejected": 1.139373779296875,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 1.0230547550432276,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 5.446808510638298e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -513.75,
|
|
"logps/rejected": -263.625,
|
|
"loss": 1.4443,
|
|
"nll_loss": 0.884765625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.13671875,
|
|
"rewards/margins": 3.052734375,
|
|
"rewards/rejected": 1.07733154296875,
|
|
"step": 267
|
|
},
|
|
{
|
|
"epoch": 1.026897214217099,
|
|
"grad_norm": 215.0,
|
|
"learning_rate": 5.425531914893617e-07,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -425.75,
|
|
"logps/rejected": -242.625,
|
|
"loss": 1.4424,
|
|
"nll_loss": 0.88037109375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.7275390625,
|
|
"rewards/margins": 2.9228515625,
|
|
"rewards/rejected": 0.81048583984375,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 1.0307396733909702,
|
|
"grad_norm": 151.0,
|
|
"learning_rate": 5.404255319148936e-07,
|
|
"logits/chosen": -3.76171875,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -392.75,
|
|
"logps/rejected": -261.0625,
|
|
"loss": 1.6299,
|
|
"nll_loss": 0.9404296875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.9375,
|
|
"rewards/margins": 1.89678955078125,
|
|
"rewards/rejected": 1.040771484375,
|
|
"step": 269
|
|
},
|
|
{
|
|
"epoch": 1.0345821325648414,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 5.382978723404255e-07,
|
|
"logits/chosen": -3.806640625,
|
|
"logits/rejected": -3.814453125,
|
|
"logps/chosen": -400.625,
|
|
"logps/rejected": -227.125,
|
|
"loss": 1.3994,
|
|
"nll_loss": 0.8740234375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.451171875,
|
|
"rewards/margins": 2.810546875,
|
|
"rewards/rejected": 0.63812255859375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 1.0384245917387127,
|
|
"grad_norm": 179.0,
|
|
"learning_rate": 5.361702127659574e-07,
|
|
"logits/chosen": -3.783203125,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -460.0,
|
|
"logps/rejected": -234.4375,
|
|
"loss": 1.4453,
|
|
"nll_loss": 0.86962890625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.400390625,
|
|
"rewards/margins": 2.62744140625,
|
|
"rewards/rejected": 0.77667236328125,
|
|
"step": 271
|
|
},
|
|
{
|
|
"epoch": 1.042267050912584,
|
|
"grad_norm": 216.0,
|
|
"learning_rate": 5.340425531914894e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -401.75,
|
|
"logps/rejected": -246.375,
|
|
"loss": 1.5518,
|
|
"nll_loss": 0.92236328125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.27734375,
|
|
"rewards/margins": 2.30322265625,
|
|
"rewards/rejected": 0.97509765625,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 1.0461095100864553,
|
|
"grad_norm": 144.0,
|
|
"learning_rate": 5.319148936170212e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -377.0,
|
|
"logps/rejected": -213.125,
|
|
"loss": 1.5278,
|
|
"nll_loss": 0.89501953125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.064453125,
|
|
"rewards/margins": 2.4580078125,
|
|
"rewards/rejected": 0.606781005859375,
|
|
"step": 273
|
|
},
|
|
{
|
|
"epoch": 1.0499519692603265,
|
|
"grad_norm": 138.0,
|
|
"learning_rate": 5.297872340425532e-07,
|
|
"logits/chosen": -3.69921875,
|
|
"logits/rejected": -3.751953125,
|
|
"logps/chosen": -431.0,
|
|
"logps/rejected": -229.625,
|
|
"loss": 1.4463,
|
|
"nll_loss": 0.91943359375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.4638671875,
|
|
"rewards/margins": 2.994140625,
|
|
"rewards/rejected": 0.4691162109375,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 1.0537944284341978,
|
|
"grad_norm": 151.0,
|
|
"learning_rate": 5.276595744680851e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.705078125,
|
|
"logps/chosen": -467.625,
|
|
"logps/rejected": -252.125,
|
|
"loss": 1.5674,
|
|
"nll_loss": 0.96826171875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.7138671875,
|
|
"rewards/margins": 2.79833984375,
|
|
"rewards/rejected": 0.9154052734375,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 1.057636887608069,
|
|
"grad_norm": 328.0,
|
|
"learning_rate": 5.25531914893617e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.89453125,
|
|
"logps/chosen": -366.75,
|
|
"logps/rejected": -191.0625,
|
|
"loss": 1.4707,
|
|
"nll_loss": 0.91357421875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.318359375,
|
|
"rewards/margins": 2.5791015625,
|
|
"rewards/rejected": 0.736785888671875,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 1.0614793467819403,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 5.234042553191489e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.7890625,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -240.75,
|
|
"loss": 1.2998,
|
|
"nll_loss": 0.759033203125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.421875,
|
|
"rewards/margins": 2.904296875,
|
|
"rewards/rejected": 0.518829345703125,
|
|
"step": 277
|
|
},
|
|
{
|
|
"epoch": 1.0653218059558118,
|
|
"grad_norm": 112.0,
|
|
"learning_rate": 5.212765957446809e-07,
|
|
"logits/chosen": -3.744140625,
|
|
"logits/rejected": -3.705078125,
|
|
"logps/chosen": -320.375,
|
|
"logps/rejected": -189.875,
|
|
"loss": 1.5957,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.3212890625,
|
|
"rewards/margins": 1.68212890625,
|
|
"rewards/rejected": 0.639892578125,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 1.069164265129683,
|
|
"grad_norm": 225.0,
|
|
"learning_rate": 5.191489361702127e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -415.75,
|
|
"logps/rejected": -224.375,
|
|
"loss": 1.5049,
|
|
"nll_loss": 0.93310546875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.71484375,
|
|
"rewards/margins": 2.912109375,
|
|
"rewards/rejected": 0.80029296875,
|
|
"step": 279
|
|
},
|
|
{
|
|
"epoch": 1.0730067243035544,
|
|
"grad_norm": 143.0,
|
|
"learning_rate": 5.170212765957447e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.91015625,
|
|
"logps/chosen": -426.5,
|
|
"logps/rejected": -226.1875,
|
|
"loss": 1.6201,
|
|
"nll_loss": 1.0205078125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0986328125,
|
|
"rewards/margins": 2.354736328125,
|
|
"rewards/rejected": 0.746337890625,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 1.0768491834774256,
|
|
"grad_norm": 129.0,
|
|
"learning_rate": 5.148936170212766e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.810546875,
|
|
"logps/chosen": -347.0,
|
|
"logps/rejected": -220.6875,
|
|
"loss": 1.4082,
|
|
"nll_loss": 0.7880859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.8203125,
|
|
"rewards/margins": 1.935546875,
|
|
"rewards/rejected": 0.8857421875,
|
|
"step": 281
|
|
},
|
|
{
|
|
"epoch": 1.080691642651297,
|
|
"grad_norm": 175.0,
|
|
"learning_rate": 5.127659574468085e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -439.625,
|
|
"logps/rejected": -226.625,
|
|
"loss": 1.3604,
|
|
"nll_loss": 0.845703125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.6162109375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": 0.4779052734375,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 1.0845341018251682,
|
|
"grad_norm": 100.0,
|
|
"learning_rate": 5.106382978723403e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -408.0,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.4941,
|
|
"nll_loss": 0.9453125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.333984375,
|
|
"rewards/margins": 2.759765625,
|
|
"rewards/rejected": 0.5758056640625,
|
|
"step": 283
|
|
},
|
|
{
|
|
"epoch": 1.0883765609990395,
|
|
"grad_norm": 122.0,
|
|
"learning_rate": 5.085106382978723e-07,
|
|
"logits/chosen": -3.8828125,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -377.125,
|
|
"logps/rejected": -226.125,
|
|
"loss": 1.6309,
|
|
"nll_loss": 0.9638671875,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 3.1318359375,
|
|
"rewards/margins": 2.3363037109375,
|
|
"rewards/rejected": 0.794921875,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 1.0922190201729107,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 5.063829787234042e-07,
|
|
"logits/chosen": -3.69140625,
|
|
"logits/rejected": -3.6875,
|
|
"logps/chosen": -486.375,
|
|
"logps/rejected": -264.25,
|
|
"loss": 1.4033,
|
|
"nll_loss": 0.90283203125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.88671875,
|
|
"rewards/margins": 3.4814453125,
|
|
"rewards/rejected": 0.39849853515625,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 1.096061479346782,
|
|
"grad_norm": 142.0,
|
|
"learning_rate": 5.042553191489361e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.783203125,
|
|
"logps/chosen": -444.625,
|
|
"logps/rejected": -213.0,
|
|
"loss": 1.4658,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.30078125,
|
|
"rewards/margins": 2.635498046875,
|
|
"rewards/rejected": 0.66351318359375,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 1.0999039385206533,
|
|
"grad_norm": 104.5,
|
|
"learning_rate": 5.02127659574468e-07,
|
|
"logits/chosen": -3.798828125,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -346.5,
|
|
"logps/rejected": -181.8125,
|
|
"loss": 1.4375,
|
|
"nll_loss": 0.8544921875,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 2.7783203125,
|
|
"rewards/margins": 2.35302734375,
|
|
"rewards/rejected": 0.423004150390625,
|
|
"step": 287
|
|
},
|
|
{
|
|
"epoch": 1.1037463976945245,
|
|
"grad_norm": 136.0,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -3.701171875,
|
|
"logits/rejected": -3.70703125,
|
|
"logps/chosen": -324.25,
|
|
"logps/rejected": -179.0,
|
|
"loss": 1.6572,
|
|
"nll_loss": 0.99755859375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.390625,
|
|
"rewards/margins": 1.822509765625,
|
|
"rewards/rejected": 0.57208251953125,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 1.1075888568683958,
|
|
"grad_norm": 326.0,
|
|
"learning_rate": 4.978723404255318e-07,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.912109375,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -219.625,
|
|
"loss": 1.4521,
|
|
"nll_loss": 0.88525390625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.0458984375,
|
|
"rewards/margins": 2.4677734375,
|
|
"rewards/rejected": 0.5817413330078125,
|
|
"step": 289
|
|
},
|
|
{
|
|
"epoch": 1.111431316042267,
|
|
"grad_norm": 126.0,
|
|
"learning_rate": 4.957446808510638e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -462.25,
|
|
"logps/rejected": -241.125,
|
|
"loss": 1.4785,
|
|
"nll_loss": 0.9462890625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.751953125,
|
|
"rewards/margins": 3.093994140625,
|
|
"rewards/rejected": 0.6513671875,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 1.1152737752161384,
|
|
"grad_norm": 197.0,
|
|
"learning_rate": 4.936170212765957e-07,
|
|
"logits/chosen": -3.7109375,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -421.125,
|
|
"logps/rejected": -228.1875,
|
|
"loss": 1.4873,
|
|
"nll_loss": 0.87939453125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.64794921875,
|
|
"rewards/margins": 2.419921875,
|
|
"rewards/rejected": 0.2265625,
|
|
"step": 291
|
|
},
|
|
{
|
|
"epoch": 1.1191162343900096,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 4.914893617021277e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.869140625,
|
|
"logps/chosen": -441.5,
|
|
"logps/rejected": -235.6875,
|
|
"loss": 1.4844,
|
|
"nll_loss": 0.9443359375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.609375,
|
|
"rewards/margins": 3.1533203125,
|
|
"rewards/rejected": 0.4490509033203125,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 1.122958693563881,
|
|
"grad_norm": 199.0,
|
|
"learning_rate": 4.893617021276595e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.7734375,
|
|
"logps/chosen": -433.75,
|
|
"logps/rejected": -292.5,
|
|
"loss": 1.5029,
|
|
"nll_loss": 0.90087890625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.3173828125,
|
|
"rewards/margins": 2.44921875,
|
|
"rewards/rejected": 0.86474609375,
|
|
"step": 293
|
|
},
|
|
{
|
|
"epoch": 1.1268011527377522,
|
|
"grad_norm": 318.0,
|
|
"learning_rate": 4.872340425531915e-07,
|
|
"logits/chosen": -3.75390625,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -474.5,
|
|
"logps/rejected": -275.5,
|
|
"loss": 1.4531,
|
|
"nll_loss": 0.91064453125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.7109375,
|
|
"rewards/margins": 2.7900390625,
|
|
"rewards/rejected": 0.9208984375,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 1.1306436119116234,
|
|
"grad_norm": 202.0,
|
|
"learning_rate": 4.851063829787234e-07,
|
|
"logits/chosen": -3.861328125,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -272.625,
|
|
"loss": 1.5283,
|
|
"nll_loss": 0.927734375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.861328125,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": 1.096923828125,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 1.1344860710854947,
|
|
"grad_norm": 446.0,
|
|
"learning_rate": 4.829787234042552e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -384.5,
|
|
"logps/rejected": -193.375,
|
|
"loss": 1.2949,
|
|
"nll_loss": 0.80615234375,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.546875,
|
|
"rewards/margins": 3.169921875,
|
|
"rewards/rejected": 0.371612548828125,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 1.138328530259366,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 4.808510638297872e-07,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.82421875,
|
|
"logps/chosen": -470.5,
|
|
"logps/rejected": -258.25,
|
|
"loss": 1.4336,
|
|
"nll_loss": 0.9345703125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.0390625,
|
|
"rewards/margins": 3.1181640625,
|
|
"rewards/rejected": 0.919677734375,
|
|
"step": 297
|
|
},
|
|
{
|
|
"epoch": 1.1421709894332372,
|
|
"grad_norm": 147.0,
|
|
"learning_rate": 4.787234042553192e-07,
|
|
"logits/chosen": -3.751953125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -396.25,
|
|
"logps/rejected": -219.8125,
|
|
"loss": 1.4688,
|
|
"nll_loss": 0.890625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9970703125,
|
|
"rewards/margins": 2.521484375,
|
|
"rewards/rejected": 0.4781494140625,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 1.1460134486071085,
|
|
"grad_norm": 149.0,
|
|
"learning_rate": 4.7659574468085105e-07,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.751953125,
|
|
"logps/chosen": -413.5,
|
|
"logps/rejected": -253.875,
|
|
"loss": 1.4736,
|
|
"nll_loss": 0.8662109375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.8896484375,
|
|
"rewards/margins": 2.1279296875,
|
|
"rewards/rejected": 0.763916015625,
|
|
"step": 299
|
|
},
|
|
{
|
|
"epoch": 1.1498559077809798,
|
|
"grad_norm": 125.0,
|
|
"learning_rate": 4.7446808510638297e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.921875,
|
|
"logps/chosen": -456.75,
|
|
"logps/rejected": -270.5,
|
|
"loss": 1.416,
|
|
"nll_loss": 0.85400390625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.8818359375,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": 0.945556640625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 1.153698366954851,
|
|
"grad_norm": 516.0,
|
|
"learning_rate": 4.723404255319149e-07,
|
|
"logits/chosen": -3.8203125,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -398.25,
|
|
"logps/rejected": -243.25,
|
|
"loss": 1.6289,
|
|
"nll_loss": 0.96875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.3984375,
|
|
"rewards/margins": 2.277099609375,
|
|
"rewards/rejected": 1.1224365234375,
|
|
"step": 301
|
|
},
|
|
{
|
|
"epoch": 1.1575408261287223,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 4.702127659574468e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.810546875,
|
|
"logps/chosen": -525.25,
|
|
"logps/rejected": -239.125,
|
|
"loss": 1.3584,
|
|
"nll_loss": 0.94287109375,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.474609375,
|
|
"rewards/margins": 3.923828125,
|
|
"rewards/rejected": 0.553863525390625,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 1.1613832853025936,
|
|
"grad_norm": 147.0,
|
|
"learning_rate": 4.6808510638297873e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -424.375,
|
|
"logps/rejected": -250.375,
|
|
"loss": 1.4697,
|
|
"nll_loss": 0.88232421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6875,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": 1.04754638671875,
|
|
"step": 303
|
|
},
|
|
{
|
|
"epoch": 1.1652257444764649,
|
|
"grad_norm": 157.0,
|
|
"learning_rate": 4.659574468085106e-07,
|
|
"logits/chosen": -3.740234375,
|
|
"logits/rejected": -3.77734375,
|
|
"logps/chosen": -481.0,
|
|
"logps/rejected": -247.9375,
|
|
"loss": 1.3711,
|
|
"nll_loss": 0.873046875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.05078125,
|
|
"rewards/margins": 3.162109375,
|
|
"rewards/rejected": 0.8905029296875,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 1.1690682036503361,
|
|
"grad_norm": 166.0,
|
|
"learning_rate": 4.638297872340425e-07,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.77734375,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -236.875,
|
|
"loss": 1.3345,
|
|
"nll_loss": 0.81884765625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.701171875,
|
|
"rewards/margins": 2.9482421875,
|
|
"rewards/rejected": 0.755615234375,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 1.1729106628242074,
|
|
"grad_norm": 118.0,
|
|
"learning_rate": 4.6170212765957444e-07,
|
|
"logits/chosen": -3.751953125,
|
|
"logits/rejected": -3.7578125,
|
|
"logps/chosen": -398.75,
|
|
"logps/rejected": -220.75,
|
|
"loss": 1.4619,
|
|
"nll_loss": 0.88427734375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.2724609375,
|
|
"rewards/margins": 2.5166015625,
|
|
"rewards/rejected": 0.7598876953125,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 1.1767531219980787,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 4.5957446808510636e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -352.125,
|
|
"logps/rejected": -216.625,
|
|
"loss": 1.4014,
|
|
"nll_loss": 0.80712890625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.912109375,
|
|
"rewards/margins": 2.2841796875,
|
|
"rewards/rejected": 0.632568359375,
|
|
"step": 307
|
|
},
|
|
{
|
|
"epoch": 1.18059558117195,
|
|
"grad_norm": 111.5,
|
|
"learning_rate": 4.574468085106383e-07,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.89453125,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -210.5,
|
|
"loss": 1.3701,
|
|
"nll_loss": 0.880859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.4189453125,
|
|
"rewards/margins": 3.3046875,
|
|
"rewards/rejected": 0.1109619140625,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 1.1844380403458212,
|
|
"grad_norm": 152.0,
|
|
"learning_rate": 4.553191489361702e-07,
|
|
"logits/chosen": -3.751953125,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -375.125,
|
|
"logps/rejected": -230.75,
|
|
"loss": 1.5977,
|
|
"nll_loss": 0.93408203125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.8759765625,
|
|
"rewards/margins": 1.8955078125,
|
|
"rewards/rejected": 0.98260498046875,
|
|
"step": 309
|
|
},
|
|
{
|
|
"epoch": 1.1882804995196925,
|
|
"grad_norm": 652.0,
|
|
"learning_rate": 4.5319148936170207e-07,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.876953125,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -240.5,
|
|
"loss": 1.5557,
|
|
"nll_loss": 1.03759765625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.943359375,
|
|
"rewards/margins": 3.06640625,
|
|
"rewards/rejected": 0.87890625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 1.192122958693564,
|
|
"grad_norm": 242.0,
|
|
"learning_rate": 4.51063829787234e-07,
|
|
"logits/chosen": -3.716796875,
|
|
"logits/rejected": -3.759765625,
|
|
"logps/chosen": -466.625,
|
|
"logps/rejected": -230.0,
|
|
"loss": 1.3955,
|
|
"nll_loss": 0.87255859375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6015625,
|
|
"rewards/margins": 3.0654296875,
|
|
"rewards/rejected": 0.536376953125,
|
|
"step": 311
|
|
},
|
|
{
|
|
"epoch": 1.195965417867435,
|
|
"grad_norm": 390.0,
|
|
"learning_rate": 4.489361702127659e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.900390625,
|
|
"logps/chosen": -566.5,
|
|
"logps/rejected": -309.625,
|
|
"loss": 1.4814,
|
|
"nll_loss": 0.86474609375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 5.751953125,
|
|
"rewards/margins": 3.49951171875,
|
|
"rewards/rejected": 2.24951171875,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 1.1998078770413065,
|
|
"grad_norm": 452.0,
|
|
"learning_rate": 4.4680851063829783e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -522.75,
|
|
"logps/rejected": -299.75,
|
|
"loss": 1.6963,
|
|
"nll_loss": 1.01513671875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.76171875,
|
|
"rewards/margins": 2.6083984375,
|
|
"rewards/rejected": 1.155517578125,
|
|
"step": 313
|
|
},
|
|
{
|
|
"epoch": 1.2036503362151778,
|
|
"grad_norm": 1576.0,
|
|
"learning_rate": 4.4468085106382975e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -430.875,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.4121,
|
|
"nll_loss": 0.85986328125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.1748046875,
|
|
"rewards/margins": 2.5478515625,
|
|
"rewards/rejected": 0.6265869140625,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 1.207492795389049,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 4.425531914893617e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.923828125,
|
|
"logps/chosen": -390.125,
|
|
"logps/rejected": -284.0,
|
|
"loss": 1.5303,
|
|
"nll_loss": 0.92431640625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.013671875,
|
|
"rewards/margins": 3.7646484375,
|
|
"rewards/rejected": -0.7373046875,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 1.2113352545629203,
|
|
"grad_norm": 824.0,
|
|
"learning_rate": 4.404255319148936e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.712890625,
|
|
"logps/chosen": -399.375,
|
|
"logps/rejected": -222.125,
|
|
"loss": 1.6523,
|
|
"nll_loss": 1.03857421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6875,
|
|
"rewards/margins": 1.97265625,
|
|
"rewards/rejected": 0.71649169921875,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 1.2151777137367916,
|
|
"grad_norm": 158.0,
|
|
"learning_rate": 4.382978723404255e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.720703125,
|
|
"logps/chosen": -334.375,
|
|
"logps/rejected": -171.75,
|
|
"loss": 1.3071,
|
|
"nll_loss": 0.75,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.6494140625,
|
|
"rewards/margins": 2.4697265625,
|
|
"rewards/rejected": 0.17755126953125,
|
|
"step": 317
|
|
},
|
|
{
|
|
"epoch": 1.219020172910663,
|
|
"grad_norm": 100.0,
|
|
"learning_rate": 4.3617021276595744e-07,
|
|
"logits/chosen": -3.712890625,
|
|
"logits/rejected": -3.814453125,
|
|
"logps/chosen": -425.75,
|
|
"logps/rejected": -225.375,
|
|
"loss": 1.4238,
|
|
"nll_loss": 0.91650390625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.310546875,
|
|
"rewards/margins": 2.978515625,
|
|
"rewards/rejected": 0.3358154296875,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 1.2228626320845342,
|
|
"grad_norm": 172.0,
|
|
"learning_rate": 4.3404255319148936e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -497.375,
|
|
"logps/rejected": -220.5625,
|
|
"loss": 1.5586,
|
|
"nll_loss": 1.03662109375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.90625,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": 0.800506591796875,
|
|
"step": 319
|
|
},
|
|
{
|
|
"epoch": 1.2267050912584054,
|
|
"grad_norm": 210.0,
|
|
"learning_rate": 4.319148936170213e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -265.125,
|
|
"loss": 1.3574,
|
|
"nll_loss": 0.853515625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.751953125,
|
|
"rewards/margins": 3.07421875,
|
|
"rewards/rejected": 0.68212890625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 1.2305475504322767,
|
|
"grad_norm": 232.0,
|
|
"learning_rate": 4.297872340425532e-07,
|
|
"logits/chosen": -3.830078125,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -427.75,
|
|
"logps/rejected": -252.625,
|
|
"loss": 1.4141,
|
|
"nll_loss": 0.83203125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.677734375,
|
|
"rewards/margins": 2.798828125,
|
|
"rewards/rejected": 0.8818359375,
|
|
"step": 321
|
|
},
|
|
{
|
|
"epoch": 1.234390009606148,
|
|
"grad_norm": 768.0,
|
|
"learning_rate": 4.276595744680851e-07,
|
|
"logits/chosen": -3.78515625,
|
|
"logits/rejected": -3.6796875,
|
|
"logps/chosen": -448.75,
|
|
"logps/rejected": -296.375,
|
|
"loss": 1.4736,
|
|
"nll_loss": 0.94091796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.634765625,
|
|
"rewards/margins": 3.001953125,
|
|
"rewards/rejected": 0.62884521484375,
|
|
"step": 322
|
|
},
|
|
{
|
|
"epoch": 1.2382324687800192,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 4.25531914893617e-07,
|
|
"logits/chosen": -3.771484375,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -463.125,
|
|
"logps/rejected": -242.5,
|
|
"loss": 1.3281,
|
|
"nll_loss": 0.830078125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.7841796875,
|
|
"rewards/margins": 3.16796875,
|
|
"rewards/rejected": 0.6173095703125,
|
|
"step": 323
|
|
},
|
|
{
|
|
"epoch": 1.2420749279538905,
|
|
"grad_norm": 199.0,
|
|
"learning_rate": 4.234042553191489e-07,
|
|
"logits/chosen": -3.84765625,
|
|
"logits/rejected": -3.900390625,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -233.0,
|
|
"loss": 1.501,
|
|
"nll_loss": 0.9853515625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.41015625,
|
|
"rewards/margins": 2.8115234375,
|
|
"rewards/rejected": 0.5987548828125,
|
|
"step": 324
|
|
},
|
|
{
|
|
"epoch": 1.2459173871277618,
|
|
"grad_norm": 117.0,
|
|
"learning_rate": 4.2127659574468083e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.806640625,
|
|
"logps/chosen": -437.75,
|
|
"logps/rejected": -218.125,
|
|
"loss": 1.3887,
|
|
"nll_loss": 0.86279296875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.91015625,
|
|
"rewards/margins": 3.0966796875,
|
|
"rewards/rejected": 0.812255859375,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 1.249759846301633,
|
|
"grad_norm": 205.0,
|
|
"learning_rate": 4.1914893617021275e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -397.125,
|
|
"logps/rejected": -258.375,
|
|
"loss": 1.5479,
|
|
"nll_loss": 0.93994140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.140625,
|
|
"rewards/margins": 2.1328125,
|
|
"rewards/rejected": 1.0125732421875,
|
|
"step": 326
|
|
},
|
|
{
|
|
"epoch": 1.2536023054755043,
|
|
"grad_norm": 384.0,
|
|
"learning_rate": 4.1702127659574467e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.92578125,
|
|
"logps/chosen": -506.25,
|
|
"logps/rejected": -238.0625,
|
|
"loss": 1.4082,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.923828125,
|
|
"rewards/margins": 3.4775390625,
|
|
"rewards/rejected": 0.450531005859375,
|
|
"step": 327
|
|
},
|
|
{
|
|
"epoch": 1.2574447646493756,
|
|
"grad_norm": 254.0,
|
|
"learning_rate": 4.148936170212766e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -413.0,
|
|
"logps/rejected": -200.6875,
|
|
"loss": 1.3945,
|
|
"nll_loss": 0.85400390625,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.18359375,
|
|
"rewards/margins": 2.630859375,
|
|
"rewards/rejected": 0.551025390625,
|
|
"step": 328
|
|
},
|
|
{
|
|
"epoch": 1.2612872238232469,
|
|
"grad_norm": 131.0,
|
|
"learning_rate": 4.1276595744680846e-07,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.787109375,
|
|
"logps/chosen": -409.0,
|
|
"logps/rejected": -221.375,
|
|
"loss": 1.2715,
|
|
"nll_loss": 0.8203125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.5869140625,
|
|
"rewards/margins": 3.2490234375,
|
|
"rewards/rejected": 0.33172607421875,
|
|
"step": 329
|
|
},
|
|
{
|
|
"epoch": 1.2651296829971181,
|
|
"grad_norm": 174.0,
|
|
"learning_rate": 4.106382978723404e-07,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.791015625,
|
|
"logps/chosen": -462.5,
|
|
"logps/rejected": -227.375,
|
|
"loss": 1.3652,
|
|
"nll_loss": 0.873046875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.701171875,
|
|
"rewards/margins": 3.0458984375,
|
|
"rewards/rejected": 0.65032958984375,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 1.2689721421709894,
|
|
"grad_norm": 187.0,
|
|
"learning_rate": 4.085106382978723e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.767578125,
|
|
"logps/chosen": -365.125,
|
|
"logps/rejected": -272.75,
|
|
"loss": 1.7578,
|
|
"nll_loss": 1.02001953125,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 3.068359375,
|
|
"rewards/margins": 1.67138671875,
|
|
"rewards/rejected": 1.393798828125,
|
|
"step": 331
|
|
},
|
|
{
|
|
"epoch": 1.2728146013448607,
|
|
"grad_norm": 500.0,
|
|
"learning_rate": 4.063829787234042e-07,
|
|
"logits/chosen": -3.75390625,
|
|
"logits/rejected": -3.857421875,
|
|
"logps/chosen": -374.75,
|
|
"logps/rejected": -262.375,
|
|
"loss": 1.6846,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.4794921875,
|
|
"rewards/margins": 1.3084716796875,
|
|
"rewards/rejected": 1.16796875,
|
|
"step": 332
|
|
},
|
|
{
|
|
"epoch": 1.276657060518732,
|
|
"grad_norm": 131.0,
|
|
"learning_rate": 4.0425531914893614e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -398.25,
|
|
"logps/rejected": -251.25,
|
|
"loss": 1.4033,
|
|
"nll_loss": 0.7705078125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.2783203125,
|
|
"rewards/margins": 2.5927734375,
|
|
"rewards/rejected": 0.68408203125,
|
|
"step": 333
|
|
},
|
|
{
|
|
"epoch": 1.2804995196926032,
|
|
"grad_norm": 157.0,
|
|
"learning_rate": 4.021276595744681e-07,
|
|
"logits/chosen": -3.697265625,
|
|
"logits/rejected": -3.716796875,
|
|
"logps/chosen": -422.75,
|
|
"logps/rejected": -260.375,
|
|
"loss": 1.4355,
|
|
"nll_loss": 0.8720703125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.359375,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": 0.765869140625,
|
|
"step": 334
|
|
},
|
|
{
|
|
"epoch": 1.2843419788664745,
|
|
"grad_norm": 322.0,
|
|
"learning_rate": 4e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.83984375,
|
|
"logps/chosen": -420.5,
|
|
"logps/rejected": -257.5,
|
|
"loss": 1.5049,
|
|
"nll_loss": 0.91650390625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.6259765625,
|
|
"rewards/margins": 2.541015625,
|
|
"rewards/rejected": 1.08428955078125,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 1.2881844380403458,
|
|
"grad_norm": 163.0,
|
|
"learning_rate": 3.978723404255319e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.759765625,
|
|
"logps/chosen": -469.75,
|
|
"logps/rejected": -242.875,
|
|
"loss": 1.5342,
|
|
"nll_loss": 0.97216796875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.38671875,
|
|
"rewards/margins": 3.173828125,
|
|
"rewards/rejected": 1.216796875,
|
|
"step": 336
|
|
},
|
|
{
|
|
"epoch": 1.292026897214217,
|
|
"grad_norm": 109.0,
|
|
"learning_rate": 3.957446808510638e-07,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -375.375,
|
|
"logps/rejected": -231.0,
|
|
"loss": 1.4824,
|
|
"nll_loss": 0.90478515625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.17578125,
|
|
"rewards/margins": 2.517578125,
|
|
"rewards/rejected": 0.65863037109375,
|
|
"step": 337
|
|
},
|
|
{
|
|
"epoch": 1.2958693563880883,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 3.9361702127659574e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.787109375,
|
|
"logps/chosen": -381.875,
|
|
"logps/rejected": -210.375,
|
|
"loss": 1.499,
|
|
"nll_loss": 0.916015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.8798828125,
|
|
"rewards/margins": 2.272705078125,
|
|
"rewards/rejected": 0.607666015625,
|
|
"step": 338
|
|
},
|
|
{
|
|
"epoch": 1.2997118155619596,
|
|
"grad_norm": 226.0,
|
|
"learning_rate": 3.9148936170212766e-07,
|
|
"logits/chosen": -3.705078125,
|
|
"logits/rejected": -3.705078125,
|
|
"logps/chosen": -353.75,
|
|
"logps/rejected": -218.375,
|
|
"loss": 1.5723,
|
|
"nll_loss": 0.89892578125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.970703125,
|
|
"rewards/margins": 1.94580078125,
|
|
"rewards/rejected": 1.0252685546875,
|
|
"step": 339
|
|
},
|
|
{
|
|
"epoch": 1.3035542747358309,
|
|
"grad_norm": 107.0,
|
|
"learning_rate": 3.893617021276596e-07,
|
|
"logits/chosen": -3.703125,
|
|
"logits/rejected": -3.791015625,
|
|
"logps/chosen": -460.5,
|
|
"logps/rejected": -256.0,
|
|
"loss": 1.231,
|
|
"nll_loss": 0.775390625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.919921875,
|
|
"rewards/margins": 3.4453125,
|
|
"rewards/rejected": 0.4830322265625,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 1.3073967339097021,
|
|
"grad_norm": 124.5,
|
|
"learning_rate": 3.8723404255319145e-07,
|
|
"logits/chosen": -3.755859375,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -546.25,
|
|
"logps/rejected": -261.375,
|
|
"loss": 1.4619,
|
|
"nll_loss": 0.95166015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.07421875,
|
|
"rewards/margins": 3.35546875,
|
|
"rewards/rejected": 0.7132568359375,
|
|
"step": 341
|
|
},
|
|
{
|
|
"epoch": 1.3112391930835736,
|
|
"grad_norm": 1004.0,
|
|
"learning_rate": 3.8510638297872337e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.826171875,
|
|
"logps/chosen": -302.75,
|
|
"logps/rejected": -217.3125,
|
|
"loss": 1.8906,
|
|
"nll_loss": 1.08544921875,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.9658203125,
|
|
"rewards/margins": 1.1220703125,
|
|
"rewards/rejected": 0.84307861328125,
|
|
"step": 342
|
|
},
|
|
{
|
|
"epoch": 1.3150816522574447,
|
|
"grad_norm": 266.0,
|
|
"learning_rate": 3.829787234042553e-07,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -408.5,
|
|
"logps/rejected": -242.125,
|
|
"loss": 1.499,
|
|
"nll_loss": 0.9228515625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.435546875,
|
|
"rewards/margins": 2.6708984375,
|
|
"rewards/rejected": 0.76068115234375,
|
|
"step": 343
|
|
},
|
|
{
|
|
"epoch": 1.3189241114313162,
|
|
"grad_norm": 138.0,
|
|
"learning_rate": 3.808510638297872e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.884765625,
|
|
"logps/chosen": -417.75,
|
|
"logps/rejected": -219.0,
|
|
"loss": 1.3809,
|
|
"nll_loss": 0.82666015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.3515625,
|
|
"rewards/margins": 2.716796875,
|
|
"rewards/rejected": 0.635009765625,
|
|
"step": 344
|
|
},
|
|
{
|
|
"epoch": 1.3227665706051872,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 3.7872340425531914e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -424.875,
|
|
"logps/rejected": -196.875,
|
|
"loss": 1.4971,
|
|
"nll_loss": 0.94189453125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4873046875,
|
|
"rewards/margins": 2.918212890625,
|
|
"rewards/rejected": 0.56536865234375,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 1.3266090297790587,
|
|
"grad_norm": 264.0,
|
|
"learning_rate": 3.7659574468085106e-07,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.81640625,
|
|
"logps/chosen": -509.875,
|
|
"logps/rejected": -279.875,
|
|
"loss": 1.3369,
|
|
"nll_loss": 0.80810546875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.484375,
|
|
"rewards/margins": 3.0107421875,
|
|
"rewards/rejected": 0.4742431640625,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 1.3266090297790587,
|
|
"eval_logits/chosen": -3.7707386016845703,
|
|
"eval_logits/rejected": -3.823153495788574,
|
|
"eval_logps/chosen": -449.9454650878906,
|
|
"eval_logps/rejected": -244.11817932128906,
|
|
"eval_loss": 1.467628836631775,
|
|
"eval_nll_loss": 0.8924893736839294,
|
|
"eval_rewards/accuracies": 0.8901515007019043,
|
|
"eval_rewards/chosen": 3.547159194946289,
|
|
"eval_rewards/margins": 2.7079367637634277,
|
|
"eval_rewards/rejected": 0.8406782746315002,
|
|
"eval_runtime": 105.568,
|
|
"eval_samples_per_second": 4.158,
|
|
"eval_steps_per_second": 1.042,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 1.3304514889529298,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 3.744680851063829e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -434.625,
|
|
"logps/rejected": -233.4375,
|
|
"loss": 1.5273,
|
|
"nll_loss": 0.97607421875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.611328125,
|
|
"rewards/margins": 2.88330078125,
|
|
"rewards/rejected": 0.726318359375,
|
|
"step": 347
|
|
},
|
|
{
|
|
"epoch": 1.3342939481268012,
|
|
"grad_norm": 212.0,
|
|
"learning_rate": 3.7234042553191484e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -254.375,
|
|
"loss": 1.4951,
|
|
"nll_loss": 0.9189453125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.2900390625,
|
|
"rewards/margins": 2.79833984375,
|
|
"rewards/rejected": 0.493408203125,
|
|
"step": 348
|
|
},
|
|
{
|
|
"epoch": 1.3381364073006723,
|
|
"grad_norm": 172.0,
|
|
"learning_rate": 3.7021276595744676e-07,
|
|
"logits/chosen": -3.7265625,
|
|
"logits/rejected": -3.75,
|
|
"logps/chosen": -403.75,
|
|
"logps/rejected": -216.75,
|
|
"loss": 1.3994,
|
|
"nll_loss": 0.82177734375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.111328125,
|
|
"rewards/margins": 2.52880859375,
|
|
"rewards/rejected": 0.5870361328125,
|
|
"step": 349
|
|
},
|
|
{
|
|
"epoch": 1.3419788664745438,
|
|
"grad_norm": 158.0,
|
|
"learning_rate": 3.680851063829787e-07,
|
|
"logits/chosen": -3.73046875,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -388.75,
|
|
"logps/rejected": -224.875,
|
|
"loss": 1.5547,
|
|
"nll_loss": 0.9658203125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.193359375,
|
|
"rewards/margins": 2.294921875,
|
|
"rewards/rejected": 0.89874267578125,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 1.345821325648415,
|
|
"grad_norm": 199.0,
|
|
"learning_rate": 3.659574468085106e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.912109375,
|
|
"logps/chosen": -346.375,
|
|
"logps/rejected": -180.6875,
|
|
"loss": 1.4014,
|
|
"nll_loss": 0.7705078125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.30078125,
|
|
"rewards/margins": 2.39794921875,
|
|
"rewards/rejected": 0.9044189453125,
|
|
"step": 351
|
|
},
|
|
{
|
|
"epoch": 1.3496637848222863,
|
|
"grad_norm": 126.5,
|
|
"learning_rate": 3.638297872340426e-07,
|
|
"logits/chosen": -3.67578125,
|
|
"logits/rejected": -3.73046875,
|
|
"logps/chosen": -337.25,
|
|
"logps/rejected": -189.5,
|
|
"loss": 1.3066,
|
|
"nll_loss": 0.759765625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.6025390625,
|
|
"rewards/margins": 2.5107421875,
|
|
"rewards/rejected": 0.0919189453125,
|
|
"step": 352
|
|
},
|
|
{
|
|
"epoch": 1.3535062439961576,
|
|
"grad_norm": 235.0,
|
|
"learning_rate": 3.617021276595745e-07,
|
|
"logits/chosen": -3.794921875,
|
|
"logits/rejected": -3.84765625,
|
|
"logps/chosen": -466.25,
|
|
"logps/rejected": -270.625,
|
|
"loss": 1.3955,
|
|
"nll_loss": 0.86181640625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.29296875,
|
|
"rewards/margins": 2.8525390625,
|
|
"rewards/rejected": 0.4505615234375,
|
|
"step": 353
|
|
},
|
|
{
|
|
"epoch": 1.3573487031700289,
|
|
"grad_norm": 1568.0,
|
|
"learning_rate": 3.5957446808510637e-07,
|
|
"logits/chosen": -3.70703125,
|
|
"logits/rejected": -3.78515625,
|
|
"logps/chosen": -518.5,
|
|
"logps/rejected": -246.375,
|
|
"loss": 1.4619,
|
|
"nll_loss": 0.8994140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.501953125,
|
|
"rewards/margins": 2.8076171875,
|
|
"rewards/rejected": 0.6917724609375,
|
|
"step": 354
|
|
},
|
|
{
|
|
"epoch": 1.3611911623439001,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 3.574468085106383e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.869140625,
|
|
"logps/chosen": -482.875,
|
|
"logps/rejected": -281.5,
|
|
"loss": 1.4849,
|
|
"nll_loss": 0.91357421875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.796875,
|
|
"rewards/margins": 2.937255859375,
|
|
"rewards/rejected": 0.86181640625,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 1.3650336215177714,
|
|
"grad_norm": 114.0,
|
|
"learning_rate": 3.553191489361702e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -576.5,
|
|
"logps/rejected": -302.5,
|
|
"loss": 1.3799,
|
|
"nll_loss": 0.89990234375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 4.92578125,
|
|
"rewards/margins": 4.138671875,
|
|
"rewards/rejected": 0.7935791015625,
|
|
"step": 356
|
|
},
|
|
{
|
|
"epoch": 1.3688760806916427,
|
|
"grad_norm": 134.0,
|
|
"learning_rate": 3.5319148936170213e-07,
|
|
"logits/chosen": -3.708984375,
|
|
"logits/rejected": -3.791015625,
|
|
"logps/chosen": -375.875,
|
|
"logps/rejected": -212.25,
|
|
"loss": 1.5215,
|
|
"nll_loss": 0.90869140625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.8046875,
|
|
"rewards/margins": 2.072265625,
|
|
"rewards/rejected": 0.736572265625,
|
|
"step": 357
|
|
},
|
|
{
|
|
"epoch": 1.372718539865514,
|
|
"grad_norm": 88.0,
|
|
"learning_rate": 3.5106382978723405e-07,
|
|
"logits/chosen": -3.78515625,
|
|
"logits/rejected": -3.78515625,
|
|
"logps/chosen": -546.25,
|
|
"logps/rejected": -249.125,
|
|
"loss": 1.1406,
|
|
"nll_loss": 0.724853515625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 4.2509765625,
|
|
"rewards/margins": 4.0986328125,
|
|
"rewards/rejected": 0.158203125,
|
|
"step": 358
|
|
},
|
|
{
|
|
"epoch": 1.3765609990393852,
|
|
"grad_norm": 226.0,
|
|
"learning_rate": 3.4893617021276597e-07,
|
|
"logits/chosen": -3.748046875,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -464.625,
|
|
"logps/rejected": -224.75,
|
|
"loss": 1.2598,
|
|
"nll_loss": 0.81201171875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.9619140625,
|
|
"rewards/margins": 3.470703125,
|
|
"rewards/rejected": 0.49176025390625,
|
|
"step": 359
|
|
},
|
|
{
|
|
"epoch": 1.3804034582132565,
|
|
"grad_norm": 121.0,
|
|
"learning_rate": 3.4680851063829784e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -352.875,
|
|
"logps/rejected": -189.375,
|
|
"loss": 1.502,
|
|
"nll_loss": 0.904296875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.603515625,
|
|
"rewards/margins": 2.2353515625,
|
|
"rewards/rejected": 0.3668060302734375,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 1.3842459173871278,
|
|
"grad_norm": 106.5,
|
|
"learning_rate": 3.4468085106382976e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.7890625,
|
|
"logps/chosen": -417.5,
|
|
"logps/rejected": -204.5,
|
|
"loss": 1.3105,
|
|
"nll_loss": 0.8134765625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.7421875,
|
|
"rewards/margins": 3.220703125,
|
|
"rewards/rejected": 0.51953125,
|
|
"step": 361
|
|
},
|
|
{
|
|
"epoch": 1.388088376560999,
|
|
"grad_norm": 151.0,
|
|
"learning_rate": 3.425531914893617e-07,
|
|
"logits/chosen": -3.720703125,
|
|
"logits/rejected": -3.794921875,
|
|
"logps/chosen": -367.25,
|
|
"logps/rejected": -226.25,
|
|
"loss": 1.4775,
|
|
"nll_loss": 0.90966796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.19140625,
|
|
"rewards/margins": 2.513671875,
|
|
"rewards/rejected": 0.6826171875,
|
|
"step": 362
|
|
},
|
|
{
|
|
"epoch": 1.3919308357348703,
|
|
"grad_norm": 136.0,
|
|
"learning_rate": 3.404255319148936e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.853515625,
|
|
"logps/chosen": -420.875,
|
|
"logps/rejected": -216.25,
|
|
"loss": 1.3384,
|
|
"nll_loss": 0.84130859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.783203125,
|
|
"rewards/margins": 3.1240234375,
|
|
"rewards/rejected": 0.6591796875,
|
|
"step": 363
|
|
},
|
|
{
|
|
"epoch": 1.3957732949087416,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 3.382978723404255e-07,
|
|
"logits/chosen": -3.708984375,
|
|
"logits/rejected": -3.72265625,
|
|
"logps/chosen": -426.125,
|
|
"logps/rejected": -247.0,
|
|
"loss": 1.4961,
|
|
"nll_loss": 0.96923828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.18359375,
|
|
"rewards/margins": 2.9052734375,
|
|
"rewards/rejected": 0.276580810546875,
|
|
"step": 364
|
|
},
|
|
{
|
|
"epoch": 1.3996157540826129,
|
|
"grad_norm": 117.5,
|
|
"learning_rate": 3.3617021276595744e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.78515625,
|
|
"logps/chosen": -398.0,
|
|
"logps/rejected": -222.5,
|
|
"loss": 1.4443,
|
|
"nll_loss": 0.8564453125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.376953125,
|
|
"rewards/margins": 2.5869140625,
|
|
"rewards/rejected": 0.78985595703125,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 1.4034582132564841,
|
|
"grad_norm": 168.0,
|
|
"learning_rate": 3.340425531914893e-07,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -377.0,
|
|
"logps/rejected": -217.0,
|
|
"loss": 1.6006,
|
|
"nll_loss": 0.962890625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.234375,
|
|
"rewards/margins": 2.22802734375,
|
|
"rewards/rejected": 1.00927734375,
|
|
"step": 366
|
|
},
|
|
{
|
|
"epoch": 1.4073006724303554,
|
|
"grad_norm": 163.0,
|
|
"learning_rate": 3.3191489361702123e-07,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.87890625,
|
|
"logps/chosen": -439.125,
|
|
"logps/rejected": -221.0,
|
|
"loss": 1.4766,
|
|
"nll_loss": 0.92626953125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.61328125,
|
|
"rewards/margins": 2.892578125,
|
|
"rewards/rejected": 0.722320556640625,
|
|
"step": 367
|
|
},
|
|
{
|
|
"epoch": 1.4111431316042267,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 3.2978723404255315e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.837890625,
|
|
"logps/chosen": -416.5,
|
|
"logps/rejected": -222.0,
|
|
"loss": 1.3984,
|
|
"nll_loss": 0.890625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.490234375,
|
|
"rewards/margins": 3.509765625,
|
|
"rewards/rejected": 0.98284912109375,
|
|
"step": 368
|
|
},
|
|
{
|
|
"epoch": 1.414985590778098,
|
|
"grad_norm": 212.0,
|
|
"learning_rate": 3.2765957446808507e-07,
|
|
"logits/chosen": -3.8203125,
|
|
"logits/rejected": -3.8671875,
|
|
"logps/chosen": -520.5,
|
|
"logps/rejected": -225.5,
|
|
"loss": 1.3779,
|
|
"nll_loss": 0.9453125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.25390625,
|
|
"rewards/margins": 3.818359375,
|
|
"rewards/rejected": 0.4375,
|
|
"step": 369
|
|
},
|
|
{
|
|
"epoch": 1.4188280499519692,
|
|
"grad_norm": 150.0,
|
|
"learning_rate": 3.2553191489361704e-07,
|
|
"logits/chosen": -3.814453125,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -382.875,
|
|
"logps/rejected": -210.625,
|
|
"loss": 1.3022,
|
|
"nll_loss": 0.78564453125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.205078125,
|
|
"rewards/margins": 2.7626953125,
|
|
"rewards/rejected": 0.44091796875,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 1.4226705091258405,
|
|
"grad_norm": 108.0,
|
|
"learning_rate": 3.2340425531914897e-07,
|
|
"logits/chosen": -3.798828125,
|
|
"logits/rejected": -3.8125,
|
|
"logps/chosen": -347.375,
|
|
"logps/rejected": -177.4375,
|
|
"loss": 1.3652,
|
|
"nll_loss": 0.85693359375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9873046875,
|
|
"rewards/margins": 2.8896484375,
|
|
"rewards/rejected": 0.1068115234375,
|
|
"step": 371
|
|
},
|
|
{
|
|
"epoch": 1.4265129682997117,
|
|
"grad_norm": 136.0,
|
|
"learning_rate": 3.2127659574468083e-07,
|
|
"logits/chosen": -3.87109375,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -476.75,
|
|
"logps/rejected": -257.375,
|
|
"loss": 1.3311,
|
|
"nll_loss": 0.8154296875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.337890625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": 0.81591796875,
|
|
"step": 372
|
|
},
|
|
{
|
|
"epoch": 1.430355427473583,
|
|
"grad_norm": 209.0,
|
|
"learning_rate": 3.1914893617021275e-07,
|
|
"logits/chosen": -3.806640625,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -459.0,
|
|
"logps/rejected": -273.25,
|
|
"loss": 1.5186,
|
|
"nll_loss": 0.870361328125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 4.2255859375,
|
|
"rewards/margins": 2.943359375,
|
|
"rewards/rejected": 1.2890625,
|
|
"step": 373
|
|
},
|
|
{
|
|
"epoch": 1.4341978866474543,
|
|
"grad_norm": 117.0,
|
|
"learning_rate": 3.170212765957447e-07,
|
|
"logits/chosen": -3.783203125,
|
|
"logits/rejected": -3.8515625,
|
|
"logps/chosen": -410.75,
|
|
"logps/rejected": -221.25,
|
|
"loss": 1.5859,
|
|
"nll_loss": 1.03955078125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.107421875,
|
|
"rewards/margins": 2.560546875,
|
|
"rewards/rejected": 0.5440673828125,
|
|
"step": 374
|
|
},
|
|
{
|
|
"epoch": 1.4380403458213258,
|
|
"grad_norm": 110.5,
|
|
"learning_rate": 3.148936170212766e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.76953125,
|
|
"logps/chosen": -471.375,
|
|
"logps/rejected": -225.125,
|
|
"loss": 1.3662,
|
|
"nll_loss": 0.8955078125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.7314453125,
|
|
"rewards/margins": 3.283203125,
|
|
"rewards/rejected": 0.4498291015625,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 1.4418828049951968,
|
|
"grad_norm": 161.0,
|
|
"learning_rate": 3.127659574468085e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.7265625,
|
|
"logps/chosen": -422.875,
|
|
"logps/rejected": -218.0,
|
|
"loss": 1.3506,
|
|
"nll_loss": 0.8134765625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.15625,
|
|
"rewards/margins": 2.7646484375,
|
|
"rewards/rejected": 0.390472412109375,
|
|
"step": 376
|
|
},
|
|
{
|
|
"epoch": 1.4457252641690683,
|
|
"grad_norm": 362.0,
|
|
"learning_rate": 3.1063829787234044e-07,
|
|
"logits/chosen": -3.693359375,
|
|
"logits/rejected": -3.728515625,
|
|
"logps/chosen": -411.25,
|
|
"logps/rejected": -279.125,
|
|
"loss": 1.457,
|
|
"nll_loss": 0.78955078125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.998046875,
|
|
"rewards/margins": 1.990234375,
|
|
"rewards/rejected": 1.0076904296875,
|
|
"step": 377
|
|
},
|
|
{
|
|
"epoch": 1.4495677233429394,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 3.085106382978723e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.798828125,
|
|
"logps/chosen": -391.5,
|
|
"logps/rejected": -253.25,
|
|
"loss": 1.4443,
|
|
"nll_loss": 0.93017578125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.3154296875,
|
|
"rewards/margins": 2.7861328125,
|
|
"rewards/rejected": 0.5325927734375,
|
|
"step": 378
|
|
},
|
|
{
|
|
"epoch": 1.4534101825168109,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 3.063829787234042e-07,
|
|
"logits/chosen": -3.833984375,
|
|
"logits/rejected": -3.849609375,
|
|
"logps/chosen": -511.0,
|
|
"logps/rejected": -270.875,
|
|
"loss": 1.4028,
|
|
"nll_loss": 0.830078125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.9521484375,
|
|
"rewards/margins": 2.808349609375,
|
|
"rewards/rejected": 1.154541015625,
|
|
"step": 379
|
|
},
|
|
{
|
|
"epoch": 1.457252641690682,
|
|
"grad_norm": 144.0,
|
|
"learning_rate": 3.0425531914893615e-07,
|
|
"logits/chosen": -3.708984375,
|
|
"logits/rejected": -3.724609375,
|
|
"logps/chosen": -330.125,
|
|
"logps/rejected": -178.5,
|
|
"loss": 1.5957,
|
|
"nll_loss": 0.94384765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.669921875,
|
|
"rewards/margins": 2.0262451171875,
|
|
"rewards/rejected": 0.6431884765625,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 1.4610951008645534,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 3.0212765957446807e-07,
|
|
"logits/chosen": -3.66015625,
|
|
"logits/rejected": -3.693359375,
|
|
"logps/chosen": -444.25,
|
|
"logps/rejected": -227.0625,
|
|
"loss": 1.501,
|
|
"nll_loss": 0.9794921875,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.435546875,
|
|
"rewards/margins": 2.986328125,
|
|
"rewards/rejected": 0.45013427734375,
|
|
"step": 381
|
|
},
|
|
{
|
|
"epoch": 1.4649375600384245,
|
|
"grad_norm": 350.0,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.716796875,
|
|
"logps/chosen": -413.125,
|
|
"logps/rejected": -245.625,
|
|
"loss": 1.5527,
|
|
"nll_loss": 0.8955078125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.140625,
|
|
"rewards/margins": 2.13623046875,
|
|
"rewards/rejected": 1.0045166015625,
|
|
"step": 382
|
|
},
|
|
{
|
|
"epoch": 1.468780019212296,
|
|
"grad_norm": 122.5,
|
|
"learning_rate": 2.978723404255319e-07,
|
|
"logits/chosen": -3.73046875,
|
|
"logits/rejected": -3.7421875,
|
|
"logps/chosen": -371.5,
|
|
"logps/rejected": -210.5,
|
|
"loss": 1.4717,
|
|
"nll_loss": 0.8662109375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.955078125,
|
|
"rewards/margins": 2.3955078125,
|
|
"rewards/rejected": 0.56134033203125,
|
|
"step": 383
|
|
},
|
|
{
|
|
"epoch": 1.4726224783861672,
|
|
"grad_norm": 332.0,
|
|
"learning_rate": 2.957446808510638e-07,
|
|
"logits/chosen": -3.78125,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -358.125,
|
|
"logps/rejected": -257.875,
|
|
"loss": 1.4961,
|
|
"nll_loss": 0.828125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.185546875,
|
|
"rewards/margins": 1.892578125,
|
|
"rewards/rejected": 1.294189453125,
|
|
"step": 384
|
|
},
|
|
{
|
|
"epoch": 1.4764649375600385,
|
|
"grad_norm": 144.0,
|
|
"learning_rate": 2.936170212765957e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.830078125,
|
|
"logps/chosen": -359.25,
|
|
"logps/rejected": -192.875,
|
|
"loss": 1.3262,
|
|
"nll_loss": 0.7802734375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.283203125,
|
|
"rewards/margins": 2.6064453125,
|
|
"rewards/rejected": 0.679779052734375,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 1.4803073967339098,
|
|
"grad_norm": 131.0,
|
|
"learning_rate": 2.914893617021276e-07,
|
|
"logits/chosen": -3.8671875,
|
|
"logits/rejected": -3.953125,
|
|
"logps/chosen": -456.125,
|
|
"logps/rejected": -230.25,
|
|
"loss": 1.417,
|
|
"nll_loss": 0.91162109375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.0986328125,
|
|
"rewards/margins": 3.38134765625,
|
|
"rewards/rejected": 0.721435546875,
|
|
"step": 386
|
|
},
|
|
{
|
|
"epoch": 1.484149855907781,
|
|
"grad_norm": 204.0,
|
|
"learning_rate": 2.8936170212765954e-07,
|
|
"logits/chosen": -3.73828125,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -488.125,
|
|
"logps/rejected": -247.75,
|
|
"loss": 1.4922,
|
|
"nll_loss": 0.93603515625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.921875,
|
|
"rewards/margins": 3.092041015625,
|
|
"rewards/rejected": 0.8319091796875,
|
|
"step": 387
|
|
},
|
|
{
|
|
"epoch": 1.4879923150816523,
|
|
"grad_norm": 270.0,
|
|
"learning_rate": 2.872340425531915e-07,
|
|
"logits/chosen": -3.826171875,
|
|
"logits/rejected": -3.8515625,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -229.625,
|
|
"loss": 1.4746,
|
|
"nll_loss": 0.986328125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.97265625,
|
|
"rewards/margins": 3.47265625,
|
|
"rewards/rejected": 0.50445556640625,
|
|
"step": 388
|
|
},
|
|
{
|
|
"epoch": 1.4918347742555236,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 2.8510638297872343e-07,
|
|
"logits/chosen": -3.751953125,
|
|
"logits/rejected": -3.88671875,
|
|
"logps/chosen": -349.75,
|
|
"logps/rejected": -197.875,
|
|
"loss": 1.6855,
|
|
"nll_loss": 0.98388671875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.8115234375,
|
|
"rewards/margins": 1.86767578125,
|
|
"rewards/rejected": 0.94500732421875,
|
|
"step": 389
|
|
},
|
|
{
|
|
"epoch": 1.4956772334293948,
|
|
"grad_norm": 109.0,
|
|
"learning_rate": 2.8297872340425535e-07,
|
|
"logits/chosen": -3.783203125,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -486.25,
|
|
"logps/rejected": -247.75,
|
|
"loss": 1.5098,
|
|
"nll_loss": 0.96826171875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.1826171875,
|
|
"rewards/margins": 3.294921875,
|
|
"rewards/rejected": 0.888427734375,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 1.4995196926032661,
|
|
"grad_norm": 572.0,
|
|
"learning_rate": 2.808510638297872e-07,
|
|
"logits/chosen": -3.859375,
|
|
"logits/rejected": -3.8515625,
|
|
"logps/chosen": -421.25,
|
|
"logps/rejected": -251.375,
|
|
"loss": 1.6006,
|
|
"nll_loss": 0.93798828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6220703125,
|
|
"rewards/margins": 2.410400390625,
|
|
"rewards/rejected": 1.2132568359375,
|
|
"step": 391
|
|
},
|
|
{
|
|
"epoch": 1.5033621517771374,
|
|
"grad_norm": 122.5,
|
|
"learning_rate": 2.7872340425531914e-07,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.783203125,
|
|
"logps/chosen": -467.25,
|
|
"logps/rejected": -236.0,
|
|
"loss": 1.3867,
|
|
"nll_loss": 0.8779296875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.9296875,
|
|
"rewards/margins": 3.3173828125,
|
|
"rewards/rejected": 0.610107421875,
|
|
"step": 392
|
|
},
|
|
{
|
|
"epoch": 1.5072046109510087,
|
|
"grad_norm": 167.0,
|
|
"learning_rate": 2.7659574468085106e-07,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -426.5,
|
|
"logps/rejected": -215.125,
|
|
"loss": 1.3301,
|
|
"nll_loss": 0.8291015625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.544921875,
|
|
"rewards/margins": 3.0537109375,
|
|
"rewards/rejected": 0.49139404296875,
|
|
"step": 393
|
|
},
|
|
{
|
|
"epoch": 1.51104707012488,
|
|
"grad_norm": 126.0,
|
|
"learning_rate": 2.74468085106383e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.759765625,
|
|
"logps/chosen": -493.75,
|
|
"logps/rejected": -209.15625,
|
|
"loss": 1.3164,
|
|
"nll_loss": 0.85693359375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.9072265625,
|
|
"rewards/margins": 3.3427734375,
|
|
"rewards/rejected": 0.5655517578125,
|
|
"step": 394
|
|
},
|
|
{
|
|
"epoch": 1.5148895292987512,
|
|
"grad_norm": 192.0,
|
|
"learning_rate": 2.723404255319149e-07,
|
|
"logits/chosen": -3.88671875,
|
|
"logits/rejected": -3.94140625,
|
|
"logps/chosen": -427.625,
|
|
"logps/rejected": -220.4375,
|
|
"loss": 1.583,
|
|
"nll_loss": 1.02294921875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.814453125,
|
|
"rewards/margins": 2.814453125,
|
|
"rewards/rejected": 1.00390625,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 1.5187319884726225,
|
|
"grad_norm": 254.0,
|
|
"learning_rate": 2.702127659574468e-07,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.8671875,
|
|
"logps/chosen": -444.75,
|
|
"logps/rejected": -249.25,
|
|
"loss": 1.396,
|
|
"nll_loss": 0.92919921875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.712890625,
|
|
"rewards/margins": 3.16015625,
|
|
"rewards/rejected": 0.54913330078125,
|
|
"step": 396
|
|
},
|
|
{
|
|
"epoch": 1.5225744476464937,
|
|
"grad_norm": 161.0,
|
|
"learning_rate": 2.680851063829787e-07,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -389.875,
|
|
"logps/rejected": -191.25,
|
|
"loss": 1.4697,
|
|
"nll_loss": 0.9091796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9658203125,
|
|
"rewards/margins": 2.46484375,
|
|
"rewards/rejected": 0.49749755859375,
|
|
"step": 397
|
|
},
|
|
{
|
|
"epoch": 1.526416906820365,
|
|
"grad_norm": 104.5,
|
|
"learning_rate": 2.659574468085106e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.802734375,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -252.3125,
|
|
"loss": 1.4785,
|
|
"nll_loss": 0.94140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.009765625,
|
|
"rewards/margins": 3.05517578125,
|
|
"rewards/rejected": 0.9501953125,
|
|
"step": 398
|
|
},
|
|
{
|
|
"epoch": 1.5302593659942363,
|
|
"grad_norm": 118.0,
|
|
"learning_rate": 2.6382978723404253e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.841796875,
|
|
"logps/chosen": -473.875,
|
|
"logps/rejected": -284.0,
|
|
"loss": 1.3027,
|
|
"nll_loss": 0.7841796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.810546875,
|
|
"rewards/margins": 3.0439453125,
|
|
"rewards/rejected": 0.768798828125,
|
|
"step": 399
|
|
},
|
|
{
|
|
"epoch": 1.5341018251681076,
|
|
"grad_norm": 105.5,
|
|
"learning_rate": 2.6170212765957445e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -444.375,
|
|
"logps/rejected": -214.0,
|
|
"loss": 1.3525,
|
|
"nll_loss": 0.84814453125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.3603515625,
|
|
"rewards/margins": 2.880859375,
|
|
"rewards/rejected": 0.48095703125,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 1.5379442843419788,
|
|
"grad_norm": 91.0,
|
|
"learning_rate": 2.5957446808510637e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.7890625,
|
|
"logps/chosen": -502.75,
|
|
"logps/rejected": -252.375,
|
|
"loss": 1.3247,
|
|
"nll_loss": 0.83984375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.89453125,
|
|
"rewards/margins": 3.4228515625,
|
|
"rewards/rejected": 0.4725341796875,
|
|
"step": 401
|
|
},
|
|
{
|
|
"epoch": 1.54178674351585,
|
|
"grad_norm": 114.5,
|
|
"learning_rate": 2.574468085106383e-07,
|
|
"logits/chosen": -3.83203125,
|
|
"logits/rejected": -3.830078125,
|
|
"logps/chosen": -409.875,
|
|
"logps/rejected": -215.75,
|
|
"loss": 1.4785,
|
|
"nll_loss": 0.958984375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.716796875,
|
|
"rewards/margins": 3.0224609375,
|
|
"rewards/rejected": 0.7022705078125,
|
|
"step": 402
|
|
},
|
|
{
|
|
"epoch": 1.5456292026897214,
|
|
"grad_norm": 176.0,
|
|
"learning_rate": 2.5531914893617016e-07,
|
|
"logits/chosen": -3.8359375,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -387.0,
|
|
"logps/rejected": -284.75,
|
|
"loss": 1.4336,
|
|
"nll_loss": 0.8095703125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.2197265625,
|
|
"rewards/margins": 2.4893798828125,
|
|
"rewards/rejected": 0.728759765625,
|
|
"step": 403
|
|
},
|
|
{
|
|
"epoch": 1.5494716618635929,
|
|
"grad_norm": 116.0,
|
|
"learning_rate": 2.531914893617021e-07,
|
|
"logits/chosen": -3.6953125,
|
|
"logits/rejected": -3.720703125,
|
|
"logps/chosen": -375.5,
|
|
"logps/rejected": -208.125,
|
|
"loss": 1.4541,
|
|
"nll_loss": 0.8828125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 2.6044921875,
|
|
"rewards/margins": 2.2109375,
|
|
"rewards/rejected": 0.39691162109375,
|
|
"step": 404
|
|
},
|
|
{
|
|
"epoch": 1.553314121037464,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 2.51063829787234e-07,
|
|
"logits/chosen": -3.744140625,
|
|
"logits/rejected": -3.79296875,
|
|
"logps/chosen": -507.25,
|
|
"logps/rejected": -275.375,
|
|
"loss": 1.3979,
|
|
"nll_loss": 0.90673828125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.98046875,
|
|
"rewards/margins": 3.3427734375,
|
|
"rewards/rejected": 0.6416015625,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 1.5571565802113354,
|
|
"grad_norm": 139.0,
|
|
"learning_rate": 2.489361702127659e-07,
|
|
"logits/chosen": -3.783203125,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -443.0,
|
|
"logps/rejected": -254.25,
|
|
"loss": 1.3877,
|
|
"nll_loss": 0.87646484375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.060546875,
|
|
"rewards/margins": 3.326171875,
|
|
"rewards/rejected": 0.732757568359375,
|
|
"step": 406
|
|
},
|
|
{
|
|
"epoch": 1.5609990393852065,
|
|
"grad_norm": 113.5,
|
|
"learning_rate": 2.4680851063829784e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.814453125,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -203.1875,
|
|
"loss": 1.4102,
|
|
"nll_loss": 0.87890625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.3876953125,
|
|
"rewards/margins": 2.7890625,
|
|
"rewards/rejected": 0.596923828125,
|
|
"step": 407
|
|
},
|
|
{
|
|
"epoch": 1.564841498559078,
|
|
"grad_norm": 105.5,
|
|
"learning_rate": 2.4468085106382976e-07,
|
|
"logits/chosen": -3.669921875,
|
|
"logits/rejected": -3.6953125,
|
|
"logps/chosen": -444.25,
|
|
"logps/rejected": -219.375,
|
|
"loss": 1.4209,
|
|
"nll_loss": 0.87841796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.486328125,
|
|
"rewards/margins": 2.830078125,
|
|
"rewards/rejected": 0.65350341796875,
|
|
"step": 408
|
|
},
|
|
{
|
|
"epoch": 1.568683957732949,
|
|
"grad_norm": 166.0,
|
|
"learning_rate": 2.425531914893617e-07,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -502.75,
|
|
"logps/rejected": -230.125,
|
|
"loss": 1.4014,
|
|
"nll_loss": 0.892578125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 4.05078125,
|
|
"rewards/margins": 3.19140625,
|
|
"rewards/rejected": 0.860107421875,
|
|
"step": 409
|
|
},
|
|
{
|
|
"epoch": 1.5725264169068205,
|
|
"grad_norm": 126.0,
|
|
"learning_rate": 2.404255319148936e-07,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -519.0,
|
|
"logps/rejected": -268.25,
|
|
"loss": 1.4619,
|
|
"nll_loss": 0.92529296875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.51953125,
|
|
"rewards/margins": 3.6162109375,
|
|
"rewards/rejected": 0.906494140625,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 1.5763688760806915,
|
|
"grad_norm": 126.5,
|
|
"learning_rate": 2.3829787234042553e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.77734375,
|
|
"logps/chosen": -493.0,
|
|
"logps/rejected": -262.625,
|
|
"loss": 1.3809,
|
|
"nll_loss": 0.89208984375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.509765625,
|
|
"rewards/margins": 3.623046875,
|
|
"rewards/rejected": 0.885009765625,
|
|
"step": 411
|
|
},
|
|
{
|
|
"epoch": 1.580211335254563,
|
|
"grad_norm": 123.5,
|
|
"learning_rate": 2.3617021276595745e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.77734375,
|
|
"logps/chosen": -434.875,
|
|
"logps/rejected": -198.8125,
|
|
"loss": 1.374,
|
|
"nll_loss": 0.8359375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.552734375,
|
|
"rewards/margins": 3.3173828125,
|
|
"rewards/rejected": 0.236572265625,
|
|
"step": 412
|
|
},
|
|
{
|
|
"epoch": 1.584053794428434,
|
|
"grad_norm": 264.0,
|
|
"learning_rate": 2.3404255319148937e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.798828125,
|
|
"logps/chosen": -519.5,
|
|
"logps/rejected": -236.625,
|
|
"loss": 1.3545,
|
|
"nll_loss": 0.88330078125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.28125,
|
|
"rewards/margins": 3.822265625,
|
|
"rewards/rejected": 0.458251953125,
|
|
"step": 413
|
|
},
|
|
{
|
|
"epoch": 1.5878962536023056,
|
|
"grad_norm": 113.5,
|
|
"learning_rate": 2.3191489361702126e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.861328125,
|
|
"logps/chosen": -427.25,
|
|
"logps/rejected": -206.75,
|
|
"loss": 1.418,
|
|
"nll_loss": 0.89794921875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.3095703125,
|
|
"rewards/margins": 2.966796875,
|
|
"rewards/rejected": 0.3388671875,
|
|
"step": 414
|
|
},
|
|
{
|
|
"epoch": 1.5917387127761766,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 2.2978723404255318e-07,
|
|
"logits/chosen": -3.779296875,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -387.375,
|
|
"logps/rejected": -233.25,
|
|
"loss": 1.4678,
|
|
"nll_loss": 0.8544921875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.294921875,
|
|
"rewards/margins": 2.33642578125,
|
|
"rewards/rejected": 0.9560546875,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 1.5955811719500481,
|
|
"grad_norm": 182.0,
|
|
"learning_rate": 2.276595744680851e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.78515625,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -240.125,
|
|
"loss": 1.4258,
|
|
"nll_loss": 0.8779296875,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.7041015625,
|
|
"rewards/margins": 3.115234375,
|
|
"rewards/rejected": 0.5924072265625,
|
|
"step": 416
|
|
},
|
|
{
|
|
"epoch": 1.5994236311239192,
|
|
"grad_norm": 125.5,
|
|
"learning_rate": 2.25531914893617e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.802734375,
|
|
"logps/chosen": -442.75,
|
|
"logps/rejected": -203.625,
|
|
"loss": 1.4106,
|
|
"nll_loss": 0.87109375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.4873046875,
|
|
"rewards/margins": 2.9384765625,
|
|
"rewards/rejected": 0.547119140625,
|
|
"step": 417
|
|
},
|
|
{
|
|
"epoch": 1.6032660902977907,
|
|
"grad_norm": 182.0,
|
|
"learning_rate": 2.2340425531914892e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.830078125,
|
|
"logps/chosen": -511.125,
|
|
"logps/rejected": -250.125,
|
|
"loss": 1.4727,
|
|
"nll_loss": 0.9560546875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.84765625,
|
|
"rewards/margins": 3.044921875,
|
|
"rewards/rejected": 0.7982177734375,
|
|
"step": 418
|
|
},
|
|
{
|
|
"epoch": 1.6071085494716617,
|
|
"grad_norm": 108.0,
|
|
"learning_rate": 2.2127659574468084e-07,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.759765625,
|
|
"logps/chosen": -366.5,
|
|
"logps/rejected": -200.4375,
|
|
"loss": 1.4609,
|
|
"nll_loss": 0.9072265625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.322265625,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": 0.63739013671875,
|
|
"step": 419
|
|
},
|
|
{
|
|
"epoch": 1.6109510086455332,
|
|
"grad_norm": 704.0,
|
|
"learning_rate": 2.1914893617021276e-07,
|
|
"logits/chosen": -3.748046875,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -446.75,
|
|
"logps/rejected": -241.125,
|
|
"loss": 1.417,
|
|
"nll_loss": 0.8505859375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.6181640625,
|
|
"rewards/margins": 2.95849609375,
|
|
"rewards/rejected": 0.6580810546875,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 1.6147934678194045,
|
|
"grad_norm": 348.0,
|
|
"learning_rate": 2.1702127659574468e-07,
|
|
"logits/chosen": -3.73828125,
|
|
"logits/rejected": -3.7734375,
|
|
"logps/chosen": -422.875,
|
|
"logps/rejected": -272.125,
|
|
"loss": 1.5107,
|
|
"nll_loss": 0.86865234375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.462890625,
|
|
"rewards/margins": 2.4482421875,
|
|
"rewards/rejected": 1.01708984375,
|
|
"step": 421
|
|
},
|
|
{
|
|
"epoch": 1.6186359269932757,
|
|
"grad_norm": 147.0,
|
|
"learning_rate": 2.148936170212766e-07,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.861328125,
|
|
"logps/chosen": -464.5,
|
|
"logps/rejected": -246.875,
|
|
"loss": 1.4219,
|
|
"nll_loss": 0.8720703125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.146484375,
|
|
"rewards/margins": 3.17578125,
|
|
"rewards/rejected": 0.97412109375,
|
|
"step": 422
|
|
},
|
|
{
|
|
"epoch": 1.622478386167147,
|
|
"grad_norm": 255.0,
|
|
"learning_rate": 2.127659574468085e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.859375,
|
|
"logps/chosen": -459.75,
|
|
"logps/rejected": -260.25,
|
|
"loss": 1.5791,
|
|
"nll_loss": 0.9384765625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.189453125,
|
|
"rewards/margins": 2.90283203125,
|
|
"rewards/rejected": 1.284423828125,
|
|
"step": 423
|
|
},
|
|
{
|
|
"epoch": 1.6263208453410183,
|
|
"grad_norm": 98.5,
|
|
"learning_rate": 2.1063829787234041e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.7890625,
|
|
"logps/chosen": -460.375,
|
|
"logps/rejected": -275.625,
|
|
"loss": 1.3672,
|
|
"nll_loss": 0.85009765625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.361328125,
|
|
"rewards/margins": 3.3310546875,
|
|
"rewards/rejected": 1.035400390625,
|
|
"step": 424
|
|
},
|
|
{
|
|
"epoch": 1.6301633045148896,
|
|
"grad_norm": 195.0,
|
|
"learning_rate": 2.0851063829787233e-07,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.86328125,
|
|
"logps/chosen": -345.25,
|
|
"logps/rejected": -220.75,
|
|
"loss": 1.5742,
|
|
"nll_loss": 0.96875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 2.91796875,
|
|
"rewards/margins": 2.271484375,
|
|
"rewards/rejected": 0.649169921875,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 1.6340057636887608,
|
|
"grad_norm": 118.0,
|
|
"learning_rate": 2.0638297872340423e-07,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.814453125,
|
|
"logps/chosen": -524.5,
|
|
"logps/rejected": -233.5625,
|
|
"loss": 1.4639,
|
|
"nll_loss": 0.97705078125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.1357421875,
|
|
"rewards/margins": 3.30859375,
|
|
"rewards/rejected": 0.828369140625,
|
|
"step": 426
|
|
},
|
|
{
|
|
"epoch": 1.637848222862632,
|
|
"grad_norm": 183.0,
|
|
"learning_rate": 2.0425531914893615e-07,
|
|
"logits/chosen": -3.66015625,
|
|
"logits/rejected": -3.634765625,
|
|
"logps/chosen": -435.75,
|
|
"logps/rejected": -208.75,
|
|
"loss": 1.4189,
|
|
"nll_loss": 0.8916015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.3037109375,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": 0.383575439453125,
|
|
"step": 427
|
|
},
|
|
{
|
|
"epoch": 1.6416906820365034,
|
|
"grad_norm": 92.5,
|
|
"learning_rate": 2.0212765957446807e-07,
|
|
"logits/chosen": -3.740234375,
|
|
"logits/rejected": -3.818359375,
|
|
"logps/chosen": -566.0,
|
|
"logps/rejected": -280.875,
|
|
"loss": 1.3535,
|
|
"nll_loss": 0.900390625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.9228515625,
|
|
"rewards/margins": 4.2615966796875,
|
|
"rewards/rejected": 0.66845703125,
|
|
"step": 428
|
|
},
|
|
{
|
|
"epoch": 1.6455331412103746,
|
|
"grad_norm": 117.0,
|
|
"learning_rate": 2e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -435.875,
|
|
"logps/rejected": -245.0,
|
|
"loss": 1.4727,
|
|
"nll_loss": 0.91259765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6484375,
|
|
"rewards/margins": 2.8193359375,
|
|
"rewards/rejected": 0.8323974609375,
|
|
"step": 429
|
|
},
|
|
{
|
|
"epoch": 1.649375600384246,
|
|
"grad_norm": 106.5,
|
|
"learning_rate": 1.978723404255319e-07,
|
|
"logits/chosen": -3.861328125,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -390.5,
|
|
"logps/rejected": -205.125,
|
|
"loss": 1.3311,
|
|
"nll_loss": 0.7919921875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.353515625,
|
|
"rewards/margins": 2.81640625,
|
|
"rewards/rejected": 0.538330078125,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 1.6532180595581172,
|
|
"grad_norm": 266.0,
|
|
"learning_rate": 1.9574468085106383e-07,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -454.25,
|
|
"logps/rejected": -245.625,
|
|
"loss": 1.3535,
|
|
"nll_loss": 0.78759765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.447265625,
|
|
"rewards/margins": 2.88671875,
|
|
"rewards/rejected": 0.5560302734375,
|
|
"step": 431
|
|
},
|
|
{
|
|
"epoch": 1.6570605187319885,
|
|
"grad_norm": 231.0,
|
|
"learning_rate": 1.9361702127659573e-07,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -448.5,
|
|
"logps/rejected": -250.25,
|
|
"loss": 1.3496,
|
|
"nll_loss": 0.82373046875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.68359375,
|
|
"rewards/margins": 2.9345703125,
|
|
"rewards/rejected": 0.756591796875,
|
|
"step": 432
|
|
},
|
|
{
|
|
"epoch": 1.6609029779058597,
|
|
"grad_norm": 255.0,
|
|
"learning_rate": 1.9148936170212765e-07,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -449.0,
|
|
"logps/rejected": -261.75,
|
|
"loss": 1.6406,
|
|
"nll_loss": 0.9970703125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.998046875,
|
|
"rewards/margins": 2.28662109375,
|
|
"rewards/rejected": 0.714599609375,
|
|
"step": 433
|
|
},
|
|
{
|
|
"epoch": 1.664745437079731,
|
|
"grad_norm": 104.0,
|
|
"learning_rate": 1.8936170212765957e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -403.375,
|
|
"logps/rejected": -184.4375,
|
|
"loss": 1.4404,
|
|
"nll_loss": 0.9072265625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.490234375,
|
|
"rewards/margins": 3.00634765625,
|
|
"rewards/rejected": 0.489044189453125,
|
|
"step": 434
|
|
},
|
|
{
|
|
"epoch": 1.6685878962536023,
|
|
"grad_norm": 632.0,
|
|
"learning_rate": 1.8723404255319146e-07,
|
|
"logits/chosen": -3.91015625,
|
|
"logits/rejected": -3.8984375,
|
|
"logps/chosen": -469.25,
|
|
"logps/rejected": -237.625,
|
|
"loss": 1.4795,
|
|
"nll_loss": 0.86474609375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.806640625,
|
|
"rewards/margins": 2.94580078125,
|
|
"rewards/rejected": 0.86248779296875,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 1.6724303554274735,
|
|
"grad_norm": 404.0,
|
|
"learning_rate": 1.8510638297872338e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.728515625,
|
|
"logps/chosen": -412.75,
|
|
"logps/rejected": -227.75,
|
|
"loss": 1.4463,
|
|
"nll_loss": 0.8984375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.6640625,
|
|
"rewards/margins": 3.0322265625,
|
|
"rewards/rejected": 0.635986328125,
|
|
"step": 436
|
|
},
|
|
{
|
|
"epoch": 1.6762728146013448,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 1.829787234042553e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -395.875,
|
|
"logps/rejected": -232.9375,
|
|
"loss": 1.3989,
|
|
"nll_loss": 0.8173828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7410888671875,
|
|
"rewards/margins": 2.42333984375,
|
|
"rewards/rejected": 0.31817626953125,
|
|
"step": 437
|
|
},
|
|
{
|
|
"epoch": 1.680115273775216,
|
|
"grad_norm": 136.0,
|
|
"learning_rate": 1.8085106382978725e-07,
|
|
"logits/chosen": -3.853515625,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -562.25,
|
|
"logps/rejected": -276.5,
|
|
"loss": 1.3408,
|
|
"nll_loss": 0.8935546875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 5.400390625,
|
|
"rewards/margins": 4.265625,
|
|
"rewards/rejected": 1.1341552734375,
|
|
"step": 438
|
|
},
|
|
{
|
|
"epoch": 1.6839577329490876,
|
|
"grad_norm": 145.0,
|
|
"learning_rate": 1.7872340425531914e-07,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -390.625,
|
|
"logps/rejected": -229.25,
|
|
"loss": 1.4639,
|
|
"nll_loss": 0.9052734375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.6982421875,
|
|
"rewards/margins": 2.9183349609375,
|
|
"rewards/rejected": 0.778564453125,
|
|
"step": 439
|
|
},
|
|
{
|
|
"epoch": 1.6878001921229586,
|
|
"grad_norm": 434.0,
|
|
"learning_rate": 1.7659574468085106e-07,
|
|
"logits/chosen": -3.6953125,
|
|
"logits/rejected": -3.77734375,
|
|
"logps/chosen": -410.25,
|
|
"logps/rejected": -219.25,
|
|
"loss": 1.3936,
|
|
"nll_loss": 0.82861328125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.3662109375,
|
|
"rewards/margins": 2.666015625,
|
|
"rewards/rejected": 0.7016754150390625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 1.6916426512968301,
|
|
"grad_norm": 96.0,
|
|
"learning_rate": 1.7446808510638299e-07,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -532.25,
|
|
"logps/rejected": -249.875,
|
|
"loss": 1.3936,
|
|
"nll_loss": 0.892578125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.458984375,
|
|
"rewards/margins": 3.53515625,
|
|
"rewards/rejected": 0.92401123046875,
|
|
"step": 441
|
|
},
|
|
{
|
|
"epoch": 1.6954851104707012,
|
|
"grad_norm": 288.0,
|
|
"learning_rate": 1.7234042553191488e-07,
|
|
"logits/chosen": -3.787109375,
|
|
"logits/rejected": -3.736328125,
|
|
"logps/chosen": -457.0,
|
|
"logps/rejected": -248.0,
|
|
"loss": 1.3965,
|
|
"nll_loss": 0.83056640625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.712890625,
|
|
"rewards/margins": 2.9560546875,
|
|
"rewards/rejected": 0.7581787109375,
|
|
"step": 442
|
|
},
|
|
{
|
|
"epoch": 1.6993275696445727,
|
|
"grad_norm": 290.0,
|
|
"learning_rate": 1.702127659574468e-07,
|
|
"logits/chosen": -3.751953125,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -462.25,
|
|
"logps/rejected": -236.125,
|
|
"loss": 1.4922,
|
|
"nll_loss": 0.873046875,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.8671875,
|
|
"rewards/margins": 2.802734375,
|
|
"rewards/rejected": 1.0596923828125,
|
|
"step": 443
|
|
},
|
|
{
|
|
"epoch": 1.7031700288184437,
|
|
"grad_norm": 163.0,
|
|
"learning_rate": 1.6808510638297872e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.84375,
|
|
"logps/chosen": -464.5,
|
|
"logps/rejected": -263.125,
|
|
"loss": 1.3379,
|
|
"nll_loss": 0.83837890625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.6171875,
|
|
"rewards/margins": 3.0029296875,
|
|
"rewards/rejected": 0.61822509765625,
|
|
"step": 444
|
|
},
|
|
{
|
|
"epoch": 1.7070124879923152,
|
|
"grad_norm": 152.0,
|
|
"learning_rate": 1.6595744680851062e-07,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.7734375,
|
|
"logps/chosen": -417.75,
|
|
"logps/rejected": -248.875,
|
|
"loss": 1.4473,
|
|
"nll_loss": 0.83740234375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.736328125,
|
|
"rewards/margins": 2.5439453125,
|
|
"rewards/rejected": 1.193359375,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 1.7108549471661862,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 1.6382978723404254e-07,
|
|
"logits/chosen": -3.76953125,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -650.5,
|
|
"logps/rejected": -293.75,
|
|
"loss": 1.2124,
|
|
"nll_loss": 0.798095703125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.978515625,
|
|
"rewards/margins": 4.69140625,
|
|
"rewards/rejected": 0.29046630859375,
|
|
"step": 446
|
|
},
|
|
{
|
|
"epoch": 1.7146974063400577,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 1.6170212765957448e-07,
|
|
"logits/chosen": -3.796875,
|
|
"logits/rejected": -3.939453125,
|
|
"logps/chosen": -354.375,
|
|
"logps/rejected": -213.4375,
|
|
"loss": 1.4648,
|
|
"nll_loss": 0.82568359375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.193359375,
|
|
"rewards/margins": 2.33349609375,
|
|
"rewards/rejected": 0.85638427734375,
|
|
"step": 447
|
|
},
|
|
{
|
|
"epoch": 1.7185398655139288,
|
|
"grad_norm": 175.0,
|
|
"learning_rate": 1.5957446808510638e-07,
|
|
"logits/chosen": -3.68359375,
|
|
"logits/rejected": -3.76171875,
|
|
"logps/chosen": -371.5,
|
|
"logps/rejected": -201.125,
|
|
"loss": 1.5176,
|
|
"nll_loss": 0.93505859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.92578125,
|
|
"rewards/margins": 2.5030517578125,
|
|
"rewards/rejected": 0.420318603515625,
|
|
"step": 448
|
|
},
|
|
{
|
|
"epoch": 1.7223823246878003,
|
|
"grad_norm": 152.0,
|
|
"learning_rate": 1.574468085106383e-07,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.7421875,
|
|
"logps/chosen": -489.75,
|
|
"logps/rejected": -257.3125,
|
|
"loss": 1.2979,
|
|
"nll_loss": 0.841796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.248046875,
|
|
"rewards/margins": 3.7919921875,
|
|
"rewards/rejected": 0.45709228515625,
|
|
"step": 449
|
|
},
|
|
{
|
|
"epoch": 1.7262247838616713,
|
|
"grad_norm": 135.0,
|
|
"learning_rate": 1.5531914893617022e-07,
|
|
"logits/chosen": -3.69921875,
|
|
"logits/rejected": -3.763671875,
|
|
"logps/chosen": -430.5,
|
|
"logps/rejected": -193.5,
|
|
"loss": 1.5215,
|
|
"nll_loss": 0.9658203125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.150390625,
|
|
"rewards/margins": 2.7919921875,
|
|
"rewards/rejected": 0.361328125,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 1.7300672430355428,
|
|
"grad_norm": 324.0,
|
|
"learning_rate": 1.531914893617021e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.74609375,
|
|
"logps/chosen": -422.5,
|
|
"logps/rejected": -272.625,
|
|
"loss": 1.4307,
|
|
"nll_loss": 0.8271484375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.3984375,
|
|
"rewards/margins": 2.58251953125,
|
|
"rewards/rejected": 0.81787109375,
|
|
"step": 451
|
|
},
|
|
{
|
|
"epoch": 1.7339097022094139,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 1.5106382978723403e-07,
|
|
"logits/chosen": -3.837890625,
|
|
"logits/rejected": -3.833984375,
|
|
"logps/chosen": -509.75,
|
|
"logps/rejected": -284.125,
|
|
"loss": 1.46,
|
|
"nll_loss": 0.978515625,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.427734375,
|
|
"rewards/margins": 3.361328125,
|
|
"rewards/rejected": 1.060791015625,
|
|
"step": 452
|
|
},
|
|
{
|
|
"epoch": 1.7377521613832854,
|
|
"grad_norm": 140.0,
|
|
"learning_rate": 1.4893617021276595e-07,
|
|
"logits/chosen": -3.779296875,
|
|
"logits/rejected": -3.77734375,
|
|
"logps/chosen": -505.75,
|
|
"logps/rejected": -230.5,
|
|
"loss": 1.4531,
|
|
"nll_loss": 0.97119140625,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 4.05859375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": 0.545867919921875,
|
|
"step": 453
|
|
},
|
|
{
|
|
"epoch": 1.7415946205571564,
|
|
"grad_norm": 892.0,
|
|
"learning_rate": 1.4680851063829785e-07,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.845703125,
|
|
"logps/chosen": -372.5,
|
|
"logps/rejected": -209.3125,
|
|
"loss": 1.4697,
|
|
"nll_loss": 0.84423828125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 2.947265625,
|
|
"rewards/margins": 2.34814453125,
|
|
"rewards/rejected": 0.596588134765625,
|
|
"step": 454
|
|
},
|
|
{
|
|
"epoch": 1.745437079731028,
|
|
"grad_norm": 245.0,
|
|
"learning_rate": 1.4468085106382977e-07,
|
|
"logits/chosen": -3.78515625,
|
|
"logits/rejected": -3.8203125,
|
|
"logps/chosen": -448.75,
|
|
"logps/rejected": -226.25,
|
|
"loss": 1.3838,
|
|
"nll_loss": 0.8740234375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.39453125,
|
|
"rewards/margins": 2.931640625,
|
|
"rewards/rejected": 0.4647216796875,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 1.7492795389048992,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 1.4255319148936172e-07,
|
|
"logits/chosen": -3.84375,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -502.25,
|
|
"logps/rejected": -262.875,
|
|
"loss": 1.417,
|
|
"nll_loss": 0.94287109375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 4.548828125,
|
|
"rewards/margins": 3.626953125,
|
|
"rewards/rejected": 0.9281005859375,
|
|
"step": 456
|
|
},
|
|
{
|
|
"epoch": 1.7531219980787704,
|
|
"grad_norm": 113.5,
|
|
"learning_rate": 1.404255319148936e-07,
|
|
"logits/chosen": -3.8515625,
|
|
"logits/rejected": -3.890625,
|
|
"logps/chosen": -519.5,
|
|
"logps/rejected": -276.25,
|
|
"loss": 1.2754,
|
|
"nll_loss": 0.8447265625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.46484375,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": 0.489501953125,
|
|
"step": 457
|
|
},
|
|
{
|
|
"epoch": 1.7569644572526417,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 1.3829787234042553e-07,
|
|
"logits/chosen": -3.85546875,
|
|
"logits/rejected": -3.86328125,
|
|
"logps/chosen": -446.625,
|
|
"logps/rejected": -261.375,
|
|
"loss": 1.4126,
|
|
"nll_loss": 0.896484375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.3984375,
|
|
"rewards/margins": 3.4736328125,
|
|
"rewards/rejected": 0.9244384765625,
|
|
"step": 458
|
|
},
|
|
{
|
|
"epoch": 1.760806916426513,
|
|
"grad_norm": 87.0,
|
|
"learning_rate": 1.3617021276595745e-07,
|
|
"logits/chosen": -3.802734375,
|
|
"logits/rejected": -3.828125,
|
|
"logps/chosen": -530.0,
|
|
"logps/rejected": -234.8125,
|
|
"loss": 1.2871,
|
|
"nll_loss": 0.869140625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.158203125,
|
|
"rewards/margins": 3.876953125,
|
|
"rewards/rejected": 0.2840576171875,
|
|
"step": 459
|
|
},
|
|
{
|
|
"epoch": 1.7646493756003843,
|
|
"grad_norm": 112.5,
|
|
"learning_rate": 1.3404255319148934e-07,
|
|
"logits/chosen": -3.7265625,
|
|
"logits/rejected": -3.732421875,
|
|
"logps/chosen": -423.875,
|
|
"logps/rejected": -199.625,
|
|
"loss": 1.2959,
|
|
"nll_loss": 0.78173828125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.28515625,
|
|
"rewards/margins": 2.9287109375,
|
|
"rewards/rejected": 0.3538818359375,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 1.7684918347742555,
|
|
"grad_norm": 227.0,
|
|
"learning_rate": 1.3191489361702127e-07,
|
|
"logits/chosen": -3.880859375,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -495.5,
|
|
"logps/rejected": -284.625,
|
|
"loss": 1.4492,
|
|
"nll_loss": 0.890625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.224609375,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": 0.869873046875,
|
|
"step": 461
|
|
},
|
|
{
|
|
"epoch": 1.7723342939481268,
|
|
"grad_norm": 107.0,
|
|
"learning_rate": 1.2978723404255319e-07,
|
|
"logits/chosen": -3.732421875,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -339.375,
|
|
"logps/rejected": -181.75,
|
|
"loss": 1.457,
|
|
"nll_loss": 0.89404296875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.748046875,
|
|
"rewards/margins": 2.4453125,
|
|
"rewards/rejected": 0.30487060546875,
|
|
"step": 462
|
|
},
|
|
{
|
|
"epoch": 1.776176753121998,
|
|
"grad_norm": 129.0,
|
|
"learning_rate": 1.2765957446808508e-07,
|
|
"logits/chosen": -3.724609375,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -460.0,
|
|
"logps/rejected": -212.1875,
|
|
"loss": 1.3037,
|
|
"nll_loss": 0.84130859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.96484375,
|
|
"rewards/margins": 3.5810546875,
|
|
"rewards/rejected": 0.381103515625,
|
|
"step": 463
|
|
},
|
|
{
|
|
"epoch": 1.7800192122958693,
|
|
"grad_norm": 168.0,
|
|
"learning_rate": 1.25531914893617e-07,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.85546875,
|
|
"logps/chosen": -432.75,
|
|
"logps/rejected": -242.375,
|
|
"loss": 1.4512,
|
|
"nll_loss": 0.90283203125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.46875,
|
|
"rewards/margins": 2.7900390625,
|
|
"rewards/rejected": 0.67926025390625,
|
|
"step": 464
|
|
},
|
|
{
|
|
"epoch": 1.7838616714697406,
|
|
"grad_norm": 77.0,
|
|
"learning_rate": 1.2340425531914892e-07,
|
|
"logits/chosen": -3.7421875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -430.75,
|
|
"logps/rejected": -235.0,
|
|
"loss": 1.2861,
|
|
"nll_loss": 0.78955078125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.5029296875,
|
|
"rewards/margins": 3.24609375,
|
|
"rewards/rejected": 0.2626953125,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 1.7877041306436119,
|
|
"grad_norm": 249.0,
|
|
"learning_rate": 1.2127659574468084e-07,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.80859375,
|
|
"logps/chosen": -519.125,
|
|
"logps/rejected": -268.375,
|
|
"loss": 1.4189,
|
|
"nll_loss": 0.923828125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.498046875,
|
|
"rewards/margins": 3.6123046875,
|
|
"rewards/rejected": 0.8900146484375,
|
|
"step": 466
|
|
},
|
|
{
|
|
"epoch": 1.7915465898174832,
|
|
"grad_norm": 208.0,
|
|
"learning_rate": 1.1914893617021276e-07,
|
|
"logits/chosen": -3.685546875,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -342.875,
|
|
"logps/rejected": -181.375,
|
|
"loss": 1.4404,
|
|
"nll_loss": 0.853515625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9697265625,
|
|
"rewards/margins": 2.3056640625,
|
|
"rewards/rejected": 0.6676025390625,
|
|
"step": 467
|
|
},
|
|
{
|
|
"epoch": 1.7953890489913544,
|
|
"grad_norm": 128.0,
|
|
"learning_rate": 1.1702127659574468e-07,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -390.25,
|
|
"logps/rejected": -225.125,
|
|
"loss": 1.457,
|
|
"nll_loss": 0.89990234375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.28515625,
|
|
"rewards/margins": 2.5126953125,
|
|
"rewards/rejected": 0.7784423828125,
|
|
"step": 468
|
|
},
|
|
{
|
|
"epoch": 1.7992315081652257,
|
|
"grad_norm": 162.0,
|
|
"learning_rate": 1.1489361702127659e-07,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.8515625,
|
|
"logps/chosen": -440.25,
|
|
"logps/rejected": -191.125,
|
|
"loss": 1.5098,
|
|
"nll_loss": 0.95751953125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.453125,
|
|
"rewards/margins": 2.94384765625,
|
|
"rewards/rejected": 0.51129150390625,
|
|
"step": 469
|
|
},
|
|
{
|
|
"epoch": 1.803073967339097,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 1.127659574468085e-07,
|
|
"logits/chosen": -3.740234375,
|
|
"logits/rejected": -3.837890625,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -231.25,
|
|
"loss": 1.5537,
|
|
"nll_loss": 0.95263671875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.44140625,
|
|
"rewards/margins": 2.662109375,
|
|
"rewards/rejected": 0.77899169921875,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 1.8069164265129682,
|
|
"grad_norm": 121.0,
|
|
"learning_rate": 1.1063829787234042e-07,
|
|
"logits/chosen": -3.720703125,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -406.25,
|
|
"logps/rejected": -240.25,
|
|
"loss": 1.3994,
|
|
"nll_loss": 0.78662109375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.345703125,
|
|
"rewards/margins": 2.4853515625,
|
|
"rewards/rejected": 0.85931396484375,
|
|
"step": 471
|
|
},
|
|
{
|
|
"epoch": 1.8107588856868397,
|
|
"grad_norm": 187.0,
|
|
"learning_rate": 1.0851063829787234e-07,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -390.875,
|
|
"logps/rejected": -219.875,
|
|
"loss": 1.4443,
|
|
"nll_loss": 0.8671875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.37109375,
|
|
"rewards/margins": 2.4462890625,
|
|
"rewards/rejected": 0.92083740234375,
|
|
"step": 472
|
|
},
|
|
{
|
|
"epoch": 1.8146013448607108,
|
|
"grad_norm": 101.0,
|
|
"learning_rate": 1.0638297872340425e-07,
|
|
"logits/chosen": -3.69921875,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -501.25,
|
|
"logps/rejected": -236.125,
|
|
"loss": 1.3604,
|
|
"nll_loss": 0.880859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.71484375,
|
|
"rewards/margins": 3.1484375,
|
|
"rewards/rejected": 0.564971923828125,
|
|
"step": 473
|
|
},
|
|
{
|
|
"epoch": 1.8184438040345823,
|
|
"grad_norm": 167.0,
|
|
"learning_rate": 1.0425531914893617e-07,
|
|
"logits/chosen": -3.7890625,
|
|
"logits/rejected": -3.8828125,
|
|
"logps/chosen": -413.75,
|
|
"logps/rejected": -231.0,
|
|
"loss": 1.5225,
|
|
"nll_loss": 1.00146484375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.38671875,
|
|
"rewards/margins": 2.8740234375,
|
|
"rewards/rejected": 0.513763427734375,
|
|
"step": 474
|
|
},
|
|
{
|
|
"epoch": 1.8222862632084533,
|
|
"grad_norm": 107.0,
|
|
"learning_rate": 1.0212765957446807e-07,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -471.0,
|
|
"logps/rejected": -206.0,
|
|
"loss": 1.5391,
|
|
"nll_loss": 0.92529296875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.119140625,
|
|
"rewards/margins": 2.5977783203125,
|
|
"rewards/rejected": 0.522705078125,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 1.8261287223823248,
|
|
"grad_norm": 116.0,
|
|
"learning_rate": 1e-07,
|
|
"logits/chosen": -3.80859375,
|
|
"logits/rejected": -3.8984375,
|
|
"logps/chosen": -426.5,
|
|
"logps/rejected": -230.625,
|
|
"loss": 1.4375,
|
|
"nll_loss": 0.91796875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.9765625,
|
|
"rewards/margins": 3.22265625,
|
|
"rewards/rejected": 0.7489013671875,
|
|
"step": 476
|
|
},
|
|
{
|
|
"epoch": 1.8299711815561959,
|
|
"grad_norm": 165.0,
|
|
"learning_rate": 9.787234042553192e-08,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.798828125,
|
|
"logps/chosen": -464.25,
|
|
"logps/rejected": -220.875,
|
|
"loss": 1.4404,
|
|
"nll_loss": 0.9189453125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.859375,
|
|
"rewards/margins": 3.111328125,
|
|
"rewards/rejected": 0.740814208984375,
|
|
"step": 477
|
|
},
|
|
{
|
|
"epoch": 1.8338136407300674,
|
|
"grad_norm": 152.0,
|
|
"learning_rate": 9.574468085106382e-08,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.775390625,
|
|
"logps/chosen": -381.75,
|
|
"logps/rejected": -247.5,
|
|
"loss": 1.4512,
|
|
"nll_loss": 0.84912109375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6162109375,
|
|
"rewards/margins": 2.6474609375,
|
|
"rewards/rejected": 0.97265625,
|
|
"step": 478
|
|
},
|
|
{
|
|
"epoch": 1.8376560999039384,
|
|
"grad_norm": 179.0,
|
|
"learning_rate": 9.361702127659573e-08,
|
|
"logits/chosen": -3.8046875,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -439.5,
|
|
"logps/rejected": -258.625,
|
|
"loss": 1.5684,
|
|
"nll_loss": 0.99267578125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.7119140625,
|
|
"rewards/margins": 2.69140625,
|
|
"rewards/rejected": 1.019287109375,
|
|
"step": 479
|
|
},
|
|
{
|
|
"epoch": 1.84149855907781,
|
|
"grad_norm": 112.5,
|
|
"learning_rate": 9.148936170212765e-08,
|
|
"logits/chosen": -3.70703125,
|
|
"logits/rejected": -3.763671875,
|
|
"logps/chosen": -427.875,
|
|
"logps/rejected": -241.875,
|
|
"loss": 1.3809,
|
|
"nll_loss": 0.865234375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.013671875,
|
|
"rewards/margins": 2.7744140625,
|
|
"rewards/rejected": 0.24261474609375,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 1.845341018251681,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 8.936170212765957e-08,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.8359375,
|
|
"logps/chosen": -555.25,
|
|
"logps/rejected": -283.0,
|
|
"loss": 1.4697,
|
|
"nll_loss": 0.96728515625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.291015625,
|
|
"rewards/margins": 3.6708984375,
|
|
"rewards/rejected": 0.61669921875,
|
|
"step": 481
|
|
},
|
|
{
|
|
"epoch": 1.8491834774255524,
|
|
"grad_norm": 276.0,
|
|
"learning_rate": 8.723404255319149e-08,
|
|
"logits/chosen": -3.771484375,
|
|
"logits/rejected": -3.822265625,
|
|
"logps/chosen": -403.25,
|
|
"logps/rejected": -208.75,
|
|
"loss": 1.3125,
|
|
"nll_loss": 0.80517578125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.693359375,
|
|
"rewards/margins": 3.2197265625,
|
|
"rewards/rejected": 0.4735107421875,
|
|
"step": 482
|
|
},
|
|
{
|
|
"epoch": 1.8530259365994235,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 8.51063829787234e-08,
|
|
"logits/chosen": -3.767578125,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -450.75,
|
|
"logps/rejected": -239.5,
|
|
"loss": 1.5518,
|
|
"nll_loss": 0.93603515625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.365234375,
|
|
"rewards/margins": 2.6416015625,
|
|
"rewards/rejected": 0.72613525390625,
|
|
"step": 483
|
|
},
|
|
{
|
|
"epoch": 1.856868395773295,
|
|
"grad_norm": 127.5,
|
|
"learning_rate": 8.297872340425531e-08,
|
|
"logits/chosen": -3.81640625,
|
|
"logits/rejected": -3.86328125,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -202.75,
|
|
"loss": 1.3896,
|
|
"nll_loss": 0.84228515625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.205078125,
|
|
"rewards/margins": 2.86572265625,
|
|
"rewards/rejected": 0.341094970703125,
|
|
"step": 484
|
|
},
|
|
{
|
|
"epoch": 1.860710854947166,
|
|
"grad_norm": 116.0,
|
|
"learning_rate": 8.085106382978724e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.888671875,
|
|
"logps/chosen": -384.375,
|
|
"logps/rejected": -160.8125,
|
|
"loss": 1.4707,
|
|
"nll_loss": 0.9296875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.486328125,
|
|
"rewards/margins": 2.9892578125,
|
|
"rewards/rejected": 0.4937744140625,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 1.8645533141210375,
|
|
"grad_norm": 151.0,
|
|
"learning_rate": 7.872340425531915e-08,
|
|
"logits/chosen": -3.87109375,
|
|
"logits/rejected": -3.83203125,
|
|
"logps/chosen": -479.75,
|
|
"logps/rejected": -285.25,
|
|
"loss": 1.4648,
|
|
"nll_loss": 0.9111328125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.103515625,
|
|
"rewards/margins": 3.06982421875,
|
|
"rewards/rejected": 1.033447265625,
|
|
"step": 486
|
|
},
|
|
{
|
|
"epoch": 1.8683957732949086,
|
|
"grad_norm": 110.0,
|
|
"learning_rate": 7.659574468085106e-08,
|
|
"logits/chosen": -3.77734375,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -472.5,
|
|
"logps/rejected": -253.125,
|
|
"loss": 1.3262,
|
|
"nll_loss": 0.83544921875,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 4.3857421875,
|
|
"rewards/margins": 3.591796875,
|
|
"rewards/rejected": 0.7962646484375,
|
|
"step": 487
|
|
},
|
|
{
|
|
"epoch": 1.87223823246878,
|
|
"grad_norm": 572.0,
|
|
"learning_rate": 7.446808510638298e-08,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.775390625,
|
|
"logps/chosen": -498.75,
|
|
"logps/rejected": -256.875,
|
|
"loss": 1.5732,
|
|
"nll_loss": 1.0615234375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.078125,
|
|
"rewards/margins": 3.443359375,
|
|
"rewards/rejected": 0.63232421875,
|
|
"step": 488
|
|
},
|
|
{
|
|
"epoch": 1.8760806916426513,
|
|
"grad_norm": 121.5,
|
|
"learning_rate": 7.234042553191488e-08,
|
|
"logits/chosen": -3.759765625,
|
|
"logits/rejected": -3.810546875,
|
|
"logps/chosen": -494.5,
|
|
"logps/rejected": -272.25,
|
|
"loss": 1.3965,
|
|
"nll_loss": 0.8525390625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6328125,
|
|
"rewards/margins": 3.0595703125,
|
|
"rewards/rejected": 0.57086181640625,
|
|
"step": 489
|
|
},
|
|
{
|
|
"epoch": 1.8799231508165226,
|
|
"grad_norm": 144.0,
|
|
"learning_rate": 7.02127659574468e-08,
|
|
"logits/chosen": -3.71484375,
|
|
"logits/rejected": -3.755859375,
|
|
"logps/chosen": -434.25,
|
|
"logps/rejected": -219.875,
|
|
"loss": 1.3643,
|
|
"nll_loss": 0.89892578125,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.408203125,
|
|
"rewards/margins": 3.1494140625,
|
|
"rewards/rejected": 0.25634765625,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 1.8837656099903939,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 6.808510638297873e-08,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.7265625,
|
|
"logps/chosen": -401.0,
|
|
"logps/rejected": -235.875,
|
|
"loss": 1.5518,
|
|
"nll_loss": 0.97900390625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.5810546875,
|
|
"rewards/margins": 2.81103515625,
|
|
"rewards/rejected": 0.772674560546875,
|
|
"step": 491
|
|
},
|
|
{
|
|
"epoch": 1.8876080691642652,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 6.595744680851063e-08,
|
|
"logits/chosen": -3.71875,
|
|
"logits/rejected": -3.712890625,
|
|
"logps/chosen": -372.625,
|
|
"logps/rejected": -220.1875,
|
|
"loss": 1.4111,
|
|
"nll_loss": 0.8154296875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.021484375,
|
|
"rewards/margins": 2.31884765625,
|
|
"rewards/rejected": 0.7039794921875,
|
|
"step": 492
|
|
},
|
|
{
|
|
"epoch": 1.8914505283381364,
|
|
"grad_norm": 129.0,
|
|
"learning_rate": 6.382978723404254e-08,
|
|
"logits/chosen": -3.775390625,
|
|
"logits/rejected": -3.8046875,
|
|
"logps/chosen": -414.25,
|
|
"logps/rejected": -208.125,
|
|
"loss": 1.4883,
|
|
"nll_loss": 0.91552734375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.505859375,
|
|
"rewards/margins": 2.7958984375,
|
|
"rewards/rejected": 0.71142578125,
|
|
"step": 493
|
|
},
|
|
{
|
|
"epoch": 1.8952929875120077,
|
|
"grad_norm": 276.0,
|
|
"learning_rate": 6.170212765957446e-08,
|
|
"logits/chosen": -3.765625,
|
|
"logits/rejected": -3.736328125,
|
|
"logps/chosen": -425.875,
|
|
"logps/rejected": -236.3125,
|
|
"loss": 1.5273,
|
|
"nll_loss": 0.978515625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.591796875,
|
|
"rewards/margins": 2.935546875,
|
|
"rewards/rejected": 0.6568603515625,
|
|
"step": 494
|
|
},
|
|
{
|
|
"epoch": 1.899135446685879,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 5.957446808510638e-08,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.78515625,
|
|
"logps/chosen": -411.75,
|
|
"logps/rejected": -224.375,
|
|
"loss": 1.5166,
|
|
"nll_loss": 0.966796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.357421875,
|
|
"rewards/margins": 2.6630859375,
|
|
"rewards/rejected": 0.69488525390625,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 1.9029779058597502,
|
|
"grad_norm": 235.0,
|
|
"learning_rate": 5.7446808510638295e-08,
|
|
"logits/chosen": -3.75,
|
|
"logits/rejected": -3.810546875,
|
|
"logps/chosen": -496.5,
|
|
"logps/rejected": -243.25,
|
|
"loss": 1.5244,
|
|
"nll_loss": 0.861328125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.9189453125,
|
|
"rewards/margins": 2.9736328125,
|
|
"rewards/rejected": 0.94476318359375,
|
|
"step": 496
|
|
},
|
|
{
|
|
"epoch": 1.9068203650336215,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 5.531914893617021e-08,
|
|
"logits/chosen": -3.828125,
|
|
"logits/rejected": -3.86328125,
|
|
"logps/chosen": -495.0,
|
|
"logps/rejected": -250.0,
|
|
"loss": 1.5557,
|
|
"nll_loss": 0.96728515625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.5859375,
|
|
"rewards/margins": 2.80078125,
|
|
"rewards/rejected": 0.785888671875,
|
|
"step": 497
|
|
},
|
|
{
|
|
"epoch": 1.9106628242074928,
|
|
"grad_norm": 122.0,
|
|
"learning_rate": 5.3191489361702123e-08,
|
|
"logits/chosen": -3.82421875,
|
|
"logits/rejected": -3.802734375,
|
|
"logps/chosen": -473.5,
|
|
"logps/rejected": -208.375,
|
|
"loss": 1.375,
|
|
"nll_loss": 0.93212890625,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 4.396484375,
|
|
"rewards/margins": 3.728515625,
|
|
"rewards/rejected": 0.66510009765625,
|
|
"step": 498
|
|
},
|
|
{
|
|
"epoch": 1.914505283381364,
|
|
"grad_norm": 251.0,
|
|
"learning_rate": 5.106382978723404e-08,
|
|
"logits/chosen": -3.849609375,
|
|
"logits/rejected": -3.806640625,
|
|
"logps/chosen": -409.125,
|
|
"logps/rejected": -269.75,
|
|
"loss": 1.5361,
|
|
"nll_loss": 0.93505859375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.517578125,
|
|
"rewards/margins": 2.5595703125,
|
|
"rewards/rejected": 0.9552001953125,
|
|
"step": 499
|
|
},
|
|
{
|
|
"epoch": 1.9183477425552353,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 4.893617021276596e-08,
|
|
"logits/chosen": -3.810546875,
|
|
"logits/rejected": -3.873046875,
|
|
"logps/chosen": -470.75,
|
|
"logps/rejected": -261.375,
|
|
"loss": 1.4512,
|
|
"nll_loss": 0.94775390625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.91796875,
|
|
"rewards/margins": 3.1650390625,
|
|
"rewards/rejected": 0.7562255859375,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 1.9221902017291066,
|
|
"grad_norm": 156.0,
|
|
"learning_rate": 4.6808510638297865e-08,
|
|
"logits/chosen": -3.720703125,
|
|
"logits/rejected": -3.74609375,
|
|
"logps/chosen": -512.75,
|
|
"logps/rejected": -279.5,
|
|
"loss": 1.4082,
|
|
"nll_loss": 0.8359375,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.15625,
|
|
"rewards/margins": 2.8798828125,
|
|
"rewards/rejected": 1.271240234375,
|
|
"step": 501
|
|
},
|
|
{
|
|
"epoch": 1.9260326609029779,
|
|
"grad_norm": 186.0,
|
|
"learning_rate": 4.4680851063829786e-08,
|
|
"logits/chosen": -3.6796875,
|
|
"logits/rejected": -3.775390625,
|
|
"logps/chosen": -445.75,
|
|
"logps/rejected": -249.375,
|
|
"loss": 1.4902,
|
|
"nll_loss": 0.88330078125,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.6357421875,
|
|
"rewards/margins": 2.69189453125,
|
|
"rewards/rejected": 0.951171875,
|
|
"step": 502
|
|
},
|
|
{
|
|
"epoch": 1.9298751200768491,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 4.25531914893617e-08,
|
|
"logits/chosen": -3.734375,
|
|
"logits/rejected": -3.736328125,
|
|
"logps/chosen": -462.0,
|
|
"logps/rejected": -271.5,
|
|
"loss": 1.5371,
|
|
"nll_loss": 0.96630859375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.30859375,
|
|
"rewards/margins": 2.5009765625,
|
|
"rewards/rejected": 0.80517578125,
|
|
"step": 503
|
|
},
|
|
{
|
|
"epoch": 1.9337175792507204,
|
|
"grad_norm": 130.0,
|
|
"learning_rate": 4.042553191489362e-08,
|
|
"logits/chosen": -3.85546875,
|
|
"logits/rejected": -3.900390625,
|
|
"logps/chosen": -439.25,
|
|
"logps/rejected": -240.875,
|
|
"loss": 1.4482,
|
|
"nll_loss": 0.95166015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.130859375,
|
|
"rewards/margins": 3.28515625,
|
|
"rewards/rejected": 0.84967041015625,
|
|
"step": 504
|
|
},
|
|
{
|
|
"epoch": 1.937560038424592,
|
|
"grad_norm": 158.0,
|
|
"learning_rate": 3.829787234042553e-08,
|
|
"logits/chosen": -3.7578125,
|
|
"logits/rejected": -3.796875,
|
|
"logps/chosen": -439.5,
|
|
"logps/rejected": -217.875,
|
|
"loss": 1.377,
|
|
"nll_loss": 0.873046875,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.59375,
|
|
"rewards/margins": 3.232421875,
|
|
"rewards/rejected": 0.361328125,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 1.941402497598463,
|
|
"grad_norm": 476.0,
|
|
"learning_rate": 3.617021276595744e-08,
|
|
"logits/chosen": -3.7265625,
|
|
"logits/rejected": -3.755859375,
|
|
"logps/chosen": -435.375,
|
|
"logps/rejected": -211.875,
|
|
"loss": 1.3193,
|
|
"nll_loss": 0.84814453125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.798828125,
|
|
"rewards/margins": 3.4111328125,
|
|
"rewards/rejected": 0.382598876953125,
|
|
"step": 506
|
|
},
|
|
{
|
|
"epoch": 1.9452449567723344,
|
|
"grad_norm": 123.5,
|
|
"learning_rate": 3.404255319148936e-08,
|
|
"logits/chosen": -3.763671875,
|
|
"logits/rejected": -3.748046875,
|
|
"logps/chosen": -521.5,
|
|
"logps/rejected": -264.9375,
|
|
"loss": 1.3398,
|
|
"nll_loss": 0.85009765625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.505859375,
|
|
"rewards/margins": 3.365234375,
|
|
"rewards/rejected": 1.144287109375,
|
|
"step": 507
|
|
},
|
|
{
|
|
"epoch": 1.9490874159462055,
|
|
"grad_norm": 124.5,
|
|
"learning_rate": 3.191489361702127e-08,
|
|
"logits/chosen": -3.791015625,
|
|
"logits/rejected": -3.875,
|
|
"logps/chosen": -390.125,
|
|
"logps/rejected": -251.25,
|
|
"loss": 1.4521,
|
|
"nll_loss": 0.8115234375,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.06640625,
|
|
"rewards/margins": 2.1075439453125,
|
|
"rewards/rejected": 0.957763671875,
|
|
"step": 508
|
|
},
|
|
{
|
|
"epoch": 1.952929875120077,
|
|
"grad_norm": 114.5,
|
|
"learning_rate": 2.978723404255319e-08,
|
|
"logits/chosen": -3.74609375,
|
|
"logits/rejected": -3.765625,
|
|
"logps/chosen": -433.875,
|
|
"logps/rejected": -222.75,
|
|
"loss": 1.5059,
|
|
"nll_loss": 0.95361328125,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.55859375,
|
|
"rewards/margins": 2.755859375,
|
|
"rewards/rejected": 0.8037109375,
|
|
"step": 509
|
|
},
|
|
{
|
|
"epoch": 1.956772334293948,
|
|
"grad_norm": 104.0,
|
|
"learning_rate": 2.7659574468085105e-08,
|
|
"logits/chosen": -3.712890625,
|
|
"logits/rejected": -3.732421875,
|
|
"logps/chosen": -438.375,
|
|
"logps/rejected": -220.6875,
|
|
"loss": 1.2871,
|
|
"nll_loss": 0.85888671875,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": 3.62890625,
|
|
"rewards/margins": 3.4130859375,
|
|
"rewards/rejected": 0.2108154296875,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 1.9606147934678195,
|
|
"grad_norm": 177.0,
|
|
"learning_rate": 2.553191489361702e-08,
|
|
"logits/chosen": -3.79296875,
|
|
"logits/rejected": -3.806640625,
|
|
"logps/chosen": -467.75,
|
|
"logps/rejected": -241.875,
|
|
"loss": 1.4146,
|
|
"nll_loss": 0.83935546875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.15625,
|
|
"rewards/margins": 3.06103515625,
|
|
"rewards/rejected": 1.0931396484375,
|
|
"step": 511
|
|
},
|
|
{
|
|
"epoch": 1.9644572526416906,
|
|
"grad_norm": 204.0,
|
|
"learning_rate": 2.3404255319148933e-08,
|
|
"logits/chosen": -3.875,
|
|
"logits/rejected": -3.876953125,
|
|
"logps/chosen": -413.25,
|
|
"logps/rejected": -264.75,
|
|
"loss": 1.5928,
|
|
"nll_loss": 0.9541015625,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 3.19140625,
|
|
"rewards/margins": 2.431640625,
|
|
"rewards/rejected": 0.75982666015625,
|
|
"step": 512
|
|
},
|
|
{
|
|
"epoch": 1.968299711815562,
|
|
"grad_norm": 128.0,
|
|
"learning_rate": 2.127659574468085e-08,
|
|
"logits/chosen": -3.7734375,
|
|
"logits/rejected": -3.759765625,
|
|
"logps/chosen": -503.875,
|
|
"logps/rejected": -236.25,
|
|
"loss": 1.3477,
|
|
"nll_loss": 0.888671875,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.8955078125,
|
|
"rewards/margins": 3.55712890625,
|
|
"rewards/rejected": 0.3360595703125,
|
|
"step": 513
|
|
},
|
|
{
|
|
"epoch": 1.9721421709894331,
|
|
"grad_norm": 115.5,
|
|
"learning_rate": 1.9148936170212764e-08,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.90234375,
|
|
"logps/chosen": -488.75,
|
|
"logps/rejected": -244.5,
|
|
"loss": 1.3525,
|
|
"nll_loss": 0.8916015625,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 4.470703125,
|
|
"rewards/margins": 3.541015625,
|
|
"rewards/rejected": 0.92620849609375,
|
|
"step": 514
|
|
},
|
|
{
|
|
"epoch": 1.9759846301633046,
|
|
"grad_norm": 119.0,
|
|
"learning_rate": 1.702127659574468e-08,
|
|
"logits/chosen": -3.80078125,
|
|
"logits/rejected": -3.779296875,
|
|
"logps/chosen": -426.5,
|
|
"logps/rejected": -195.0,
|
|
"loss": 1.3643,
|
|
"nll_loss": 0.865234375,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": 3.828125,
|
|
"rewards/margins": 3.23828125,
|
|
"rewards/rejected": 0.5927734375,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 1.9798270893371757,
|
|
"grad_norm": 153.0,
|
|
"learning_rate": 1.4893617021276595e-08,
|
|
"logits/chosen": -3.720703125,
|
|
"logits/rejected": -3.76171875,
|
|
"logps/chosen": -404.5,
|
|
"logps/rejected": -220.375,
|
|
"loss": 1.4365,
|
|
"nll_loss": 0.8349609375,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.24609375,
|
|
"rewards/margins": 2.525390625,
|
|
"rewards/rejected": 0.72021484375,
|
|
"step": 516
|
|
},
|
|
{
|
|
"epoch": 1.9836695485110472,
|
|
"grad_norm": 232.0,
|
|
"learning_rate": 1.276595744680851e-08,
|
|
"logits/chosen": -3.728515625,
|
|
"logits/rejected": -3.80078125,
|
|
"logps/chosen": -491.0,
|
|
"logps/rejected": -248.5,
|
|
"loss": 1.2461,
|
|
"nll_loss": 0.7548828125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.2080078125,
|
|
"rewards/margins": 3.5234375,
|
|
"rewards/rejected": 0.68084716796875,
|
|
"step": 517
|
|
},
|
|
{
|
|
"epoch": 1.9875120076849182,
|
|
"grad_norm": 127.5,
|
|
"learning_rate": 1.0638297872340425e-08,
|
|
"logits/chosen": -3.736328125,
|
|
"logits/rejected": -3.75390625,
|
|
"logps/chosen": -444.75,
|
|
"logps/rejected": -236.25,
|
|
"loss": 1.3828,
|
|
"nll_loss": 0.85986328125,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 3.755859375,
|
|
"rewards/margins": 2.978515625,
|
|
"rewards/rejected": 0.780731201171875,
|
|
"step": 518
|
|
},
|
|
{
|
|
"epoch": 1.9913544668587897,
|
|
"grad_norm": 166.0,
|
|
"learning_rate": 8.51063829787234e-09,
|
|
"logits/chosen": -3.693359375,
|
|
"logits/rejected": -3.78125,
|
|
"logps/chosen": -397.5,
|
|
"logps/rejected": -233.25,
|
|
"loss": 1.6104,
|
|
"nll_loss": 0.9541015625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.8798828125,
|
|
"rewards/margins": 2.2728271484375,
|
|
"rewards/rejected": 0.59698486328125,
|
|
"step": 519
|
|
},
|
|
{
|
|
"epoch": 1.9913544668587897,
|
|
"eval_logits/chosen": -3.7707386016845703,
|
|
"eval_logits/rejected": -3.8210227489471436,
|
|
"eval_logps/chosen": -449.6363525390625,
|
|
"eval_logps/rejected": -244.23635864257812,
|
|
"eval_loss": 1.4559190273284912,
|
|
"eval_nll_loss": 0.8917436003684998,
|
|
"eval_rewards/accuracies": 0.8901515007019043,
|
|
"eval_rewards/chosen": 3.636150598526001,
|
|
"eval_rewards/margins": 2.8020241260528564,
|
|
"eval_rewards/rejected": 0.8355379700660706,
|
|
"eval_runtime": 105.5363,
|
|
"eval_samples_per_second": 4.16,
|
|
"eval_steps_per_second": 1.042,
|
|
"step": 519
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 522,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 173,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|