7565 lines
224 KiB
JSON
7565 lines
224 KiB
JSON
{
|
|
"best_metric": 0.25248709321022034,
|
|
"best_model_checkpoint": "models/qwen2.5-3b-dpo-coarse/checkpoint-5000",
|
|
"epoch": 0.37619441727484765,
|
|
"eval_steps": 5000,
|
|
"global_step": 5000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 7.523888345496953e-05,
|
|
"grad_norm": 2.0655642984991194,
|
|
"learning_rate": 3.7593984962406016e-10,
|
|
"logits/chosen": -2.03125,
|
|
"logits/rejected": -1.53125,
|
|
"logps/chosen": -121.0,
|
|
"logps/rejected": -146.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.0007523888345496953,
|
|
"grad_norm": 1.9538445459115994,
|
|
"learning_rate": 3.759398496240601e-09,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -149.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.2986111044883728,
|
|
"rewards/chosen": 0.0013580322265625,
|
|
"rewards/margins": 0.00238037109375,
|
|
"rewards/rejected": -0.0010223388671875,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0015047776690993906,
|
|
"grad_norm": 1.7482239733475604,
|
|
"learning_rate": 7.518796992481202e-09,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -178.0,
|
|
"logps/rejected": -159.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.00087738037109375,
|
|
"rewards/margins": 0.0001277923583984375,
|
|
"rewards/rejected": 0.000751495361328125,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.0022571665036490857,
|
|
"grad_norm": 1.6937945346625471,
|
|
"learning_rate": 1.1278195488721804e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -173.0,
|
|
"logps/rejected": -171.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.40625,
|
|
"rewards/chosen": 0.00151824951171875,
|
|
"rewards/margins": 0.0020599365234375,
|
|
"rewards/rejected": -0.000545501708984375,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.003009555338198781,
|
|
"grad_norm": 1.8569983758522628,
|
|
"learning_rate": 1.5037593984962404e-08,
|
|
"logits/chosen": -2.34375,
|
|
"logits/rejected": -2.078125,
|
|
"logps/chosen": -161.0,
|
|
"logps/rejected": -161.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": 0.000705718994140625,
|
|
"rewards/margins": -0.000499725341796875,
|
|
"rewards/rejected": 0.0012054443359375,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.0037619441727484763,
|
|
"grad_norm": 1.955956889927452,
|
|
"learning_rate": 1.8796992481203004e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": -0.00151824951171875,
|
|
"rewards/margins": 0.00048828125,
|
|
"rewards/rejected": -0.0019989013671875,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.004514333007298171,
|
|
"grad_norm": 1.8154990127537094,
|
|
"learning_rate": 2.2556390977443608e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -161.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.32499998807907104,
|
|
"rewards/chosen": -0.000499725341796875,
|
|
"rewards/margins": -0.000720977783203125,
|
|
"rewards/rejected": 0.00021839141845703125,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.005266721841847867,
|
|
"grad_norm": 1.987488235998793,
|
|
"learning_rate": 2.6315789473684208e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -1.890625,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -171.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.00022125244140625,
|
|
"rewards/margins": 0.00021839141845703125,
|
|
"rewards/rejected": -1.5273690223693848e-06,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.006019110676397562,
|
|
"grad_norm": 1.7715247772447917,
|
|
"learning_rate": 3.007518796992481e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -148.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 0.00032806396484375,
|
|
"rewards/margins": 0.0004253387451171875,
|
|
"rewards/rejected": -9.059906005859375e-05,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.006771499510947258,
|
|
"grad_norm": 1.5425426980374812,
|
|
"learning_rate": 3.383458646616541e-08,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": -0.000644683837890625,
|
|
"rewards/margins": 7.82012939453125e-05,
|
|
"rewards/rejected": -0.000720977783203125,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.0075238883454969525,
|
|
"grad_norm": 1.8781269521393504,
|
|
"learning_rate": 3.759398496240601e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6922,
|
|
"rewards/accuracies": 0.29374998807907104,
|
|
"rewards/chosen": -0.00089263916015625,
|
|
"rewards/margins": -0.00010251998901367188,
|
|
"rewards/rejected": -0.000789642333984375,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.008276277180046648,
|
|
"grad_norm": 1.8425671870697207,
|
|
"learning_rate": 4.1353383458646615e-08,
|
|
"logits/chosen": -2.21875,
|
|
"logits/rejected": -1.953125,
|
|
"logps/chosen": -159.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6927,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 0.00018787384033203125,
|
|
"rewards/margins": -0.000438690185546875,
|
|
"rewards/rejected": 0.0006256103515625,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.009028666014596343,
|
|
"grad_norm": 1.8232396339646215,
|
|
"learning_rate": 4.5112781954887216e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -1.953125,
|
|
"logps/chosen": -158.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.3125,
|
|
"rewards/chosen": 0.00189208984375,
|
|
"rewards/margins": -0.0004062652587890625,
|
|
"rewards/rejected": 0.0023040771484375,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.009781054849146039,
|
|
"grad_norm": 2.0676564160851814,
|
|
"learning_rate": 4.8872180451127816e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -137.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 0.000797271728515625,
|
|
"rewards/margins": -3.0994415283203125e-05,
|
|
"rewards/rejected": 0.000827789306640625,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.010533443683695734,
|
|
"grad_norm": 1.9395451342559986,
|
|
"learning_rate": 5.2631578947368416e-08,
|
|
"logits/chosen": -2.3125,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -159.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": 0.001190185546875,
|
|
"rewards/margins": -1.5974044799804688e-05,
|
|
"rewards/rejected": 0.0012054443359375,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.01128583251824543,
|
|
"grad_norm": 1.9021298256381785,
|
|
"learning_rate": 5.6390977443609016e-08,
|
|
"logits/chosen": -2.375,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -175.0,
|
|
"logps/rejected": -148.0,
|
|
"loss": 0.6921,
|
|
"rewards/accuracies": 0.42500001192092896,
|
|
"rewards/chosen": 0.00177001953125,
|
|
"rewards/margins": 0.00213623046875,
|
|
"rewards/rejected": -0.000370025634765625,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.012038221352795125,
|
|
"grad_norm": 1.9054050326686525,
|
|
"learning_rate": 6.015037593984962e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -160.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6917,
|
|
"rewards/accuracies": 0.4375,
|
|
"rewards/chosen": 0.0025482177734375,
|
|
"rewards/margins": 0.0022430419921875,
|
|
"rewards/rejected": 0.0003108978271484375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.01279061018734482,
|
|
"grad_norm": 1.926853611550257,
|
|
"learning_rate": 6.390977443609022e-08,
|
|
"logits/chosen": -2.296875,
|
|
"logits/rejected": -1.859375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6912,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.0035552978515625,
|
|
"rewards/margins": 0.003875732421875,
|
|
"rewards/rejected": -0.0003299713134765625,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.013542999021894516,
|
|
"grad_norm": 1.9234881642608486,
|
|
"learning_rate": 6.766917293233082e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -145.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6915,
|
|
"rewards/accuracies": 0.4937500059604645,
|
|
"rewards/chosen": 0.00628662109375,
|
|
"rewards/margins": 0.00213623046875,
|
|
"rewards/rejected": 0.004150390625,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.01429538785644421,
|
|
"grad_norm": 1.9784096184868571,
|
|
"learning_rate": 7.142857142857142e-08,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.109375,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6911,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.006256103515625,
|
|
"rewards/margins": 0.0028839111328125,
|
|
"rewards/rejected": 0.0033721923828125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.015047776690993905,
|
|
"grad_norm": 1.631724548633527,
|
|
"learning_rate": 7.518796992481202e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -162.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.691,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.00616455078125,
|
|
"rewards/margins": 0.0031280517578125,
|
|
"rewards/rejected": 0.003021240234375,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.0158001655255436,
|
|
"grad_norm": 1.7460592334442049,
|
|
"learning_rate": 7.894736842105262e-08,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -158.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6906,
|
|
"rewards/accuracies": 0.4749999940395355,
|
|
"rewards/chosen": 0.00665283203125,
|
|
"rewards/margins": 0.002532958984375,
|
|
"rewards/rejected": 0.004119873046875,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.016552554360093296,
|
|
"grad_norm": 1.864652158743688,
|
|
"learning_rate": 8.270676691729323e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.6898,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": 0.010498046875,
|
|
"rewards/margins": 0.00445556640625,
|
|
"rewards/rejected": 0.00604248046875,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.017304943194642992,
|
|
"grad_norm": 1.8807523922343017,
|
|
"learning_rate": 8.646616541353382e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -166.0,
|
|
"loss": 0.6892,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": 0.01275634765625,
|
|
"rewards/margins": 0.007049560546875,
|
|
"rewards/rejected": 0.005706787109375,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.018057332029192685,
|
|
"grad_norm": 1.8543547888579985,
|
|
"learning_rate": 9.022556390977443e-08,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -160.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6882,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 0.016357421875,
|
|
"rewards/margins": 0.0087890625,
|
|
"rewards/rejected": 0.007537841796875,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.01880972086374238,
|
|
"grad_norm": 2.002695069307674,
|
|
"learning_rate": 9.398496240601502e-08,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -1.953125,
|
|
"logps/chosen": -138.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6882,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 0.017822265625,
|
|
"rewards/margins": 0.01043701171875,
|
|
"rewards/rejected": 0.00738525390625,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.019562109698292078,
|
|
"grad_norm": 1.694938582256644,
|
|
"learning_rate": 9.774436090225563e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -143.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6874,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": 0.0152587890625,
|
|
"rewards/margins": 0.00787353515625,
|
|
"rewards/rejected": 0.007354736328125,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.02031449853284177,
|
|
"grad_norm": 1.6267392659283901,
|
|
"learning_rate": 1.0150375939849622e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -173.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.6872,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 0.0145263671875,
|
|
"rewards/margins": 0.0120849609375,
|
|
"rewards/rejected": 0.0024261474609375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.021066887367391467,
|
|
"grad_norm": 1.8009833530928583,
|
|
"learning_rate": 1.0526315789473683e-07,
|
|
"logits/chosen": -2.359375,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6859,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": 0.01953125,
|
|
"rewards/margins": 0.0123291015625,
|
|
"rewards/rejected": 0.00714111328125,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.021819276201941164,
|
|
"grad_norm": 1.751273107274032,
|
|
"learning_rate": 1.0902255639097744e-07,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -156.0,
|
|
"loss": 0.686,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 0.01953125,
|
|
"rewards/margins": 0.01397705078125,
|
|
"rewards/rejected": 0.005462646484375,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.02257166503649086,
|
|
"grad_norm": 1.8215393575640535,
|
|
"learning_rate": 1.1278195488721803e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -148.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6845,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": 0.02099609375,
|
|
"rewards/margins": 0.0157470703125,
|
|
"rewards/rejected": 0.005279541015625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.023324053871040553,
|
|
"grad_norm": 1.7387394745988194,
|
|
"learning_rate": 1.1654135338345864e-07,
|
|
"logits/chosen": -2.328125,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -127.0,
|
|
"loss": 0.6844,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": 0.017333984375,
|
|
"rewards/margins": 0.0174560546875,
|
|
"rewards/rejected": -0.00016117095947265625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.02407644270559025,
|
|
"grad_norm": 1.7127796622071032,
|
|
"learning_rate": 1.2030075187969923e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -158.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6838,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": 0.0218505859375,
|
|
"rewards/margins": 0.016357421875,
|
|
"rewards/rejected": 0.005523681640625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.024828831540139946,
|
|
"grad_norm": 1.8326169180769538,
|
|
"learning_rate": 1.2406015037593983e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -159.0,
|
|
"logps/rejected": -174.0,
|
|
"loss": 0.6806,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": 0.0296630859375,
|
|
"rewards/margins": 0.030029296875,
|
|
"rewards/rejected": -0.0003986358642578125,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.02558122037468964,
|
|
"grad_norm": 1.7066967555396413,
|
|
"learning_rate": 1.2781954887218045e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6797,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": 0.0230712890625,
|
|
"rewards/margins": 0.0279541015625,
|
|
"rewards/rejected": -0.0048828125,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.026333609209239335,
|
|
"grad_norm": 1.735716628750598,
|
|
"learning_rate": 1.3157894736842104e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -170.0,
|
|
"logps/rejected": -168.0,
|
|
"loss": 0.6781,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": 0.029541015625,
|
|
"rewards/margins": 0.03857421875,
|
|
"rewards/rejected": -0.00909423828125,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.02708599804378903,
|
|
"grad_norm": 2.113635572426321,
|
|
"learning_rate": 1.3533834586466163e-07,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6773,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": 0.0218505859375,
|
|
"rewards/margins": 0.02783203125,
|
|
"rewards/rejected": -0.005889892578125,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.027838386878338724,
|
|
"grad_norm": 1.7305039899173027,
|
|
"learning_rate": 1.3909774436090225e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.078125,
|
|
"logps/chosen": -147.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6743,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 0.0341796875,
|
|
"rewards/margins": 0.039794921875,
|
|
"rewards/rejected": -0.005645751953125,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.02859077571288842,
|
|
"grad_norm": 1.7424950093564118,
|
|
"learning_rate": 1.4285714285714285e-07,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.109375,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6739,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": 0.0274658203125,
|
|
"rewards/margins": 0.050048828125,
|
|
"rewards/rejected": -0.0225830078125,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.029343164547438117,
|
|
"grad_norm": 1.8220095077196594,
|
|
"learning_rate": 1.4661654135338344e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -177.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.6736,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": 0.016357421875,
|
|
"rewards/margins": 0.04541015625,
|
|
"rewards/rejected": -0.0291748046875,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.03009555338198781,
|
|
"grad_norm": 1.9063850709586252,
|
|
"learning_rate": 1.5037593984962403e-07,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.109375,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6705,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": 0.01025390625,
|
|
"rewards/margins": 0.0380859375,
|
|
"rewards/rejected": -0.02783203125,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.030847942216537506,
|
|
"grad_norm": 1.6795215046369691,
|
|
"learning_rate": 1.5413533834586465e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -162.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6681,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": 0.01104736328125,
|
|
"rewards/margins": 0.056396484375,
|
|
"rewards/rejected": -0.045166015625,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.0316003310510872,
|
|
"grad_norm": 2.091341380928474,
|
|
"learning_rate": 1.5789473684210525e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6638,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.00799560546875,
|
|
"rewards/margins": 0.056396484375,
|
|
"rewards/rejected": -0.064453125,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.0323527198856369,
|
|
"grad_norm": 2.083815342302955,
|
|
"learning_rate": 1.6165413533834584e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -159.0,
|
|
"loss": 0.6609,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.004791259765625,
|
|
"rewards/margins": 0.07470703125,
|
|
"rewards/rejected": -0.0791015625,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.03310510872018659,
|
|
"grad_norm": 2.1697428767427125,
|
|
"learning_rate": 1.6541353383458646e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -197.0,
|
|
"logps/rejected": -177.0,
|
|
"loss": 0.6559,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.0189208984375,
|
|
"rewards/margins": 0.0888671875,
|
|
"rewards/rejected": -0.10791015625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.033857497554736285,
|
|
"grad_norm": 2.1298018090255297,
|
|
"learning_rate": 1.6917293233082705e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -179.0,
|
|
"logps/rejected": -157.0,
|
|
"loss": 0.6531,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.0400390625,
|
|
"rewards/margins": 0.07080078125,
|
|
"rewards/rejected": -0.11083984375,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.034609886389285985,
|
|
"grad_norm": 2.1836244309606507,
|
|
"learning_rate": 1.7293233082706765e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -167.0,
|
|
"logps/rejected": -182.0,
|
|
"loss": 0.6425,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.0615234375,
|
|
"rewards/margins": 0.11376953125,
|
|
"rewards/rejected": -0.17578125,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.03536227522383568,
|
|
"grad_norm": 2.4213655673685968,
|
|
"learning_rate": 1.7669172932330824e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -192.0,
|
|
"loss": 0.6384,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.0830078125,
|
|
"rewards/margins": 0.130859375,
|
|
"rewards/rejected": -0.2138671875,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.03611466405838537,
|
|
"grad_norm": 1.944004930995184,
|
|
"learning_rate": 1.8045112781954886e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -176.0,
|
|
"logps/rejected": -174.0,
|
|
"loss": 0.6312,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.12451171875,
|
|
"rewards/margins": 0.1494140625,
|
|
"rewards/rejected": -0.2734375,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.03686705289293507,
|
|
"grad_norm": 2.191837059576488,
|
|
"learning_rate": 1.8421052631578946e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -188.0,
|
|
"logps/rejected": -201.0,
|
|
"loss": 0.63,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.1689453125,
|
|
"rewards/margins": 0.1669921875,
|
|
"rewards/rejected": -0.3359375,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.03761944172748476,
|
|
"grad_norm": 2.2196701470400395,
|
|
"learning_rate": 1.8796992481203005e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -187.0,
|
|
"logps/rejected": -185.0,
|
|
"loss": 0.6251,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -0.2314453125,
|
|
"rewards/margins": 0.138671875,
|
|
"rewards/rejected": -0.369140625,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.038371830562034456,
|
|
"grad_norm": 2.3001313944863644,
|
|
"learning_rate": 1.9172932330827067e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -174.0,
|
|
"logps/rejected": -182.0,
|
|
"loss": 0.6189,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -0.255859375,
|
|
"rewards/margins": 0.2001953125,
|
|
"rewards/rejected": -0.45703125,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.039124219396584156,
|
|
"grad_norm": 2.1503105403846376,
|
|
"learning_rate": 1.9548872180451126e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -185.0,
|
|
"logps/rejected": -190.0,
|
|
"loss": 0.6124,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.30859375,
|
|
"rewards/margins": 0.1826171875,
|
|
"rewards/rejected": -0.4921875,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.03987660823113385,
|
|
"grad_norm": 2.330665502460398,
|
|
"learning_rate": 1.9924812030075186e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -185.0,
|
|
"logps/rejected": -208.0,
|
|
"loss": 0.6012,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -0.333984375,
|
|
"rewards/margins": 0.25390625,
|
|
"rewards/rejected": -0.58984375,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.04062899706568354,
|
|
"grad_norm": 3.5479816270273523,
|
|
"learning_rate": 2.0300751879699245e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.234375,
|
|
"logps/chosen": -197.0,
|
|
"logps/rejected": -206.0,
|
|
"loss": 0.5986,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.416015625,
|
|
"rewards/margins": 0.2353515625,
|
|
"rewards/rejected": -0.65234375,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.04138138590023324,
|
|
"grad_norm": 2.227973578985842,
|
|
"learning_rate": 2.0676691729323307e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -194.0,
|
|
"logps/rejected": -204.0,
|
|
"loss": 0.6006,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.439453125,
|
|
"rewards/margins": 0.2275390625,
|
|
"rewards/rejected": -0.66796875,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.042133774734782935,
|
|
"grad_norm": 2.3382178358022494,
|
|
"learning_rate": 2.1052631578947366e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -231.0,
|
|
"loss": 0.5962,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.451171875,
|
|
"rewards/margins": 0.25390625,
|
|
"rewards/rejected": -0.703125,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.042886163569332635,
|
|
"grad_norm": 2.3431494680035843,
|
|
"learning_rate": 2.1428571428571426e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -216.0,
|
|
"logps/rejected": -224.0,
|
|
"loss": 0.5836,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.51171875,
|
|
"rewards/margins": 0.27734375,
|
|
"rewards/rejected": -0.7890625,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.04363855240388233,
|
|
"grad_norm": 2.5522490577618884,
|
|
"learning_rate": 2.1804511278195488e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -223.0,
|
|
"logps/rejected": -230.0,
|
|
"loss": 0.5791,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.546875,
|
|
"rewards/margins": 0.306640625,
|
|
"rewards/rejected": -0.85546875,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.04439094123843202,
|
|
"grad_norm": 2.8512898992256646,
|
|
"learning_rate": 2.2180451127819547e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -217.0,
|
|
"logps/rejected": -237.0,
|
|
"loss": 0.5741,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.62890625,
|
|
"rewards/margins": 0.36328125,
|
|
"rewards/rejected": -0.9921875,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.04514333007298172,
|
|
"grad_norm": 4.011523561431844,
|
|
"learning_rate": 2.2556390977443606e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -245.0,
|
|
"logps/rejected": -266.0,
|
|
"loss": 0.5703,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.703125,
|
|
"rewards/margins": 0.369140625,
|
|
"rewards/rejected": -1.0703125,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.04589571890753141,
|
|
"grad_norm": 3.41561545490801,
|
|
"learning_rate": 2.2932330827067666e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -208.0,
|
|
"logps/rejected": -260.0,
|
|
"loss": 0.548,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.6953125,
|
|
"rewards/margins": 0.484375,
|
|
"rewards/rejected": -1.1796875,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.046648107742081106,
|
|
"grad_norm": 4.4201440287646205,
|
|
"learning_rate": 2.3308270676691728e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -224.0,
|
|
"logps/rejected": -278.0,
|
|
"loss": 0.5474,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.78515625,
|
|
"rewards/margins": 0.478515625,
|
|
"rewards/rejected": -1.265625,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.047400496576630806,
|
|
"grad_norm": 4.695382679355146,
|
|
"learning_rate": 2.3684210526315787e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -249.0,
|
|
"logps/rejected": -278.0,
|
|
"loss": 0.5397,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -0.80078125,
|
|
"rewards/margins": 0.458984375,
|
|
"rewards/rejected": -1.2578125,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.0481528854111805,
|
|
"grad_norm": 4.146268880636602,
|
|
"learning_rate": 2.4060150375939847e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -234.0,
|
|
"logps/rejected": -290.0,
|
|
"loss": 0.5382,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -0.76171875,
|
|
"rewards/margins": 0.59765625,
|
|
"rewards/rejected": -1.359375,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.04890527424573019,
|
|
"grad_norm": 5.032383387570057,
|
|
"learning_rate": 2.443609022556391e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -237.0,
|
|
"logps/rejected": -300.0,
|
|
"loss": 0.548,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.85546875,
|
|
"rewards/margins": 0.55078125,
|
|
"rewards/rejected": -1.40625,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.04965766308027989,
|
|
"grad_norm": 3.68940996433093,
|
|
"learning_rate": 2.4812030075187965e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -228.0,
|
|
"logps/rejected": -280.0,
|
|
"loss": 0.5468,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -0.83984375,
|
|
"rewards/margins": 0.4921875,
|
|
"rewards/rejected": -1.328125,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.050410051914829584,
|
|
"grad_norm": 3.876718495059869,
|
|
"learning_rate": 2.518796992481203e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -258.0,
|
|
"logps/rejected": -294.0,
|
|
"loss": 0.5255,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.8828125,
|
|
"rewards/margins": 0.5,
|
|
"rewards/rejected": -1.3828125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.05116244074937928,
|
|
"grad_norm": 4.75321383421562,
|
|
"learning_rate": 2.556390977443609e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -260.0,
|
|
"logps/rejected": -320.0,
|
|
"loss": 0.5196,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.875,
|
|
"rewards/margins": 0.7421875,
|
|
"rewards/rejected": -1.6171875,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.05191482958392898,
|
|
"grad_norm": 7.793668826360735,
|
|
"learning_rate": 2.593984962406015e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -266.0,
|
|
"logps/rejected": -318.0,
|
|
"loss": 0.5229,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.9609375,
|
|
"rewards/margins": 0.63671875,
|
|
"rewards/rejected": -1.6015625,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.05266721841847867,
|
|
"grad_norm": 4.6452917151128785,
|
|
"learning_rate": 2.631578947368421e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -266.0,
|
|
"logps/rejected": -324.0,
|
|
"loss": 0.5161,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -0.96875,
|
|
"rewards/margins": 0.69921875,
|
|
"rewards/rejected": -1.6640625,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.05341960725302836,
|
|
"grad_norm": 7.056235535697858,
|
|
"learning_rate": 2.669172932330827e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -276.0,
|
|
"logps/rejected": -348.0,
|
|
"loss": 0.5158,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -0.99609375,
|
|
"rewards/margins": 0.69921875,
|
|
"rewards/rejected": -1.6953125,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.05417199608757806,
|
|
"grad_norm": 7.272525055737868,
|
|
"learning_rate": 2.7067669172932327e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -348.0,
|
|
"loss": 0.4925,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.046875,
|
|
"rewards/margins": 0.8046875,
|
|
"rewards/rejected": -1.8515625,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.054924384922127756,
|
|
"grad_norm": 8.479125069828605,
|
|
"learning_rate": 2.744360902255639e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -326.0,
|
|
"loss": 0.5078,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -1.1640625,
|
|
"rewards/margins": 0.7890625,
|
|
"rewards/rejected": -1.953125,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.05567677375667745,
|
|
"grad_norm": 7.1392220553921,
|
|
"learning_rate": 2.781954887218045e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -264.0,
|
|
"logps/rejected": -320.0,
|
|
"loss": 0.5066,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.015625,
|
|
"rewards/margins": 0.7109375,
|
|
"rewards/rejected": -1.7265625,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.05642916259122715,
|
|
"grad_norm": 8.584081832995,
|
|
"learning_rate": 2.8195488721804513e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.4837,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.15625,
|
|
"rewards/margins": 0.82421875,
|
|
"rewards/rejected": -1.984375,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.05718155142577684,
|
|
"grad_norm": 7.646861159193735,
|
|
"learning_rate": 2.857142857142857e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -340.0,
|
|
"loss": 0.4889,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -1.2109375,
|
|
"rewards/margins": 0.72265625,
|
|
"rewards/rejected": -1.9375,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.057933940260326534,
|
|
"grad_norm": 6.103871181309778,
|
|
"learning_rate": 2.894736842105263e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -368.0,
|
|
"loss": 0.4781,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.21875,
|
|
"rewards/margins": 0.94140625,
|
|
"rewards/rejected": -2.15625,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.058686329094876234,
|
|
"grad_norm": 7.815641392476333,
|
|
"learning_rate": 2.932330827067669e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -358.0,
|
|
"loss": 0.4766,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.2109375,
|
|
"rewards/margins": 0.91796875,
|
|
"rewards/rejected": -2.125,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.05943871792942593,
|
|
"grad_norm": 13.713331235769063,
|
|
"learning_rate": 2.969924812030075e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -304.0,
|
|
"logps/rejected": -356.0,
|
|
"loss": 0.4681,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.2890625,
|
|
"rewards/margins": 0.80078125,
|
|
"rewards/rejected": -2.09375,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.06019110676397562,
|
|
"grad_norm": 6.54533749931702,
|
|
"learning_rate": 3.0075187969924807e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -358.0,
|
|
"loss": 0.4805,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.1640625,
|
|
"rewards/margins": 0.78515625,
|
|
"rewards/rejected": -1.9453125,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.06094349559852532,
|
|
"grad_norm": 7.465470155476823,
|
|
"learning_rate": 3.0451127819548874e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -312.0,
|
|
"logps/rejected": -394.0,
|
|
"loss": 0.4693,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -1.3984375,
|
|
"rewards/margins": 0.9140625,
|
|
"rewards/rejected": -2.3125,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.06169588443307501,
|
|
"grad_norm": 9.535546928152705,
|
|
"learning_rate": 3.082706766917293e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -396.0,
|
|
"loss": 0.4545,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.3125,
|
|
"rewards/margins": 1.0390625,
|
|
"rewards/rejected": -2.359375,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.062448273267624706,
|
|
"grad_norm": 9.078214464965928,
|
|
"learning_rate": 3.1203007518796993e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -394.0,
|
|
"loss": 0.4864,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -1.2890625,
|
|
"rewards/margins": 0.828125,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.0632006621021744,
|
|
"grad_norm": 8.079147706507966,
|
|
"learning_rate": 3.157894736842105e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -292.0,
|
|
"logps/rejected": -404.0,
|
|
"loss": 0.4627,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.3125,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.484375,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.0639530509367241,
|
|
"grad_norm": 6.623406076716393,
|
|
"learning_rate": 3.195488721804511e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -384.0,
|
|
"loss": 0.463,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.3125,
|
|
"rewards/margins": 1.0078125,
|
|
"rewards/rejected": -2.3125,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.0647054397712738,
|
|
"grad_norm": 10.955204890181086,
|
|
"learning_rate": 3.233082706766917e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -414.0,
|
|
"loss": 0.4563,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.40625,
|
|
"rewards/margins": 1.1171875,
|
|
"rewards/rejected": -2.53125,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.06545782860582348,
|
|
"grad_norm": 7.434705610127428,
|
|
"learning_rate": 3.270676691729323e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -388.0,
|
|
"loss": 0.4604,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.359375,
|
|
"rewards/margins": 1.0625,
|
|
"rewards/rejected": -2.421875,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.06621021744037318,
|
|
"grad_norm": 8.609722009537244,
|
|
"learning_rate": 3.308270676691729e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.4257,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -1.765625,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -2.9375,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.06696260627492288,
|
|
"grad_norm": 11.999385437912036,
|
|
"learning_rate": 3.3458646616541354e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -426.0,
|
|
"loss": 0.4347,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5625,
|
|
"rewards/margins": 1.1953125,
|
|
"rewards/rejected": -2.75,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.06771499510947257,
|
|
"grad_norm": 9.118722433060201,
|
|
"learning_rate": 3.383458646616541e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -400.0,
|
|
"loss": 0.4424,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 1.1015625,
|
|
"rewards/rejected": -2.625,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.06846738394402227,
|
|
"grad_norm": 8.90229000330093,
|
|
"learning_rate": 3.4210526315789473e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -314.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.4658,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.671875,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -2.84375,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.06921977277857197,
|
|
"grad_norm": 12.740993275502067,
|
|
"learning_rate": 3.458646616541353e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -346.0,
|
|
"logps/rejected": -430.0,
|
|
"loss": 0.4558,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.703125,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.06997216161312166,
|
|
"grad_norm": 9.335082174627217,
|
|
"learning_rate": 3.496240601503759e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -332.0,
|
|
"logps/rejected": -458.0,
|
|
"loss": 0.4054,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -3.171875,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.07072455044767136,
|
|
"grad_norm": 8.239119204332837,
|
|
"learning_rate": 3.533834586466165e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -312.0,
|
|
"logps/rejected": -446.0,
|
|
"loss": 0.4412,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -2.90625,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.07147693928222106,
|
|
"grad_norm": 7.440153278177778,
|
|
"learning_rate": 3.5714285714285716e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -434.0,
|
|
"loss": 0.4392,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.4921875,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -2.75,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.07222932811677074,
|
|
"grad_norm": 9.61612083498033,
|
|
"learning_rate": 3.609022556390977e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -320.0,
|
|
"logps/rejected": -434.0,
|
|
"loss": 0.4324,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.5390625,
|
|
"rewards/margins": 1.2578125,
|
|
"rewards/rejected": -2.796875,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.07298171695132044,
|
|
"grad_norm": 10.276517044122064,
|
|
"learning_rate": 3.6466165413533834e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -448.0,
|
|
"loss": 0.4271,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.6875,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -2.953125,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.07373410578587014,
|
|
"grad_norm": 7.955238750998195,
|
|
"learning_rate": 3.684210526315789e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -350.0,
|
|
"logps/rejected": -476.0,
|
|
"loss": 0.4415,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.8515625,
|
|
"rewards/margins": 1.3046875,
|
|
"rewards/rejected": -3.15625,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.07448649462041983,
|
|
"grad_norm": 9.228975986129836,
|
|
"learning_rate": 3.7218045112781953e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -320.0,
|
|
"logps/rejected": -442.0,
|
|
"loss": 0.4177,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.71875,
|
|
"rewards/margins": 1.265625,
|
|
"rewards/rejected": -2.984375,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.07523888345496953,
|
|
"grad_norm": 7.6300102127807765,
|
|
"learning_rate": 3.759398496240601e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -318.0,
|
|
"logps/rejected": -430.0,
|
|
"loss": 0.4256,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.546875,
|
|
"rewards/margins": 1.125,
|
|
"rewards/rejected": -2.671875,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.07599127228951923,
|
|
"grad_norm": 11.203017463036868,
|
|
"learning_rate": 3.796992481203007e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -468.0,
|
|
"loss": 0.4073,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -1.5859375,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -3.234375,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.07674366112406891,
|
|
"grad_norm": 10.44396520900687,
|
|
"learning_rate": 3.8345864661654134e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.4036,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.71875,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -3.09375,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.07749604995861861,
|
|
"grad_norm": 10.352095048437782,
|
|
"learning_rate": 3.8721804511278196e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -436.0,
|
|
"loss": 0.4089,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -1.484375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -2.953125,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.07824843879316831,
|
|
"grad_norm": 9.027432013217565,
|
|
"learning_rate": 3.909774436090225e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -332.0,
|
|
"logps/rejected": -444.0,
|
|
"loss": 0.422,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.1640625,
|
|
"rewards/rejected": -2.875,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.079000827627718,
|
|
"grad_norm": 8.493987186289845,
|
|
"learning_rate": 3.9473684210526315e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -480.0,
|
|
"loss": 0.4011,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.6484375,
|
|
"rewards/margins": 1.4609375,
|
|
"rewards/rejected": -3.109375,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.0797532164622677,
|
|
"grad_norm": 13.673005824406173,
|
|
"learning_rate": 3.984962406015037e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -334.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.3971,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.6484375,
|
|
"rewards/margins": 1.3828125,
|
|
"rewards/rejected": -3.03125,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.0805056052968174,
|
|
"grad_norm": 11.280443761201498,
|
|
"learning_rate": 4.0225563909774433e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.4124,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.3359375,
|
|
"rewards/rejected": -3.046875,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.08125799413136708,
|
|
"grad_norm": 11.852849318261235,
|
|
"learning_rate": 4.060150375939849e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -388.0,
|
|
"logps/rejected": -544.0,
|
|
"loss": 0.4084,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.078125,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -3.703125,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.08201038296591678,
|
|
"grad_norm": 8.668455285022874,
|
|
"learning_rate": 4.0977443609022557e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -366.0,
|
|
"logps/rejected": -516.0,
|
|
"loss": 0.4122,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.875,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -3.484375,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.08276277180046648,
|
|
"grad_norm": 16.385621268125355,
|
|
"learning_rate": 4.1353383458646614e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3919,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.6640625,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -3.078125,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.08351516063501617,
|
|
"grad_norm": 11.29836657890373,
|
|
"learning_rate": 4.1729323308270676e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -368.0,
|
|
"logps/rejected": -560.0,
|
|
"loss": 0.3525,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.046875,
|
|
"rewards/margins": 2.1875,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.08426754946956587,
|
|
"grad_norm": 14.191366008515217,
|
|
"learning_rate": 4.2105263157894733e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -346.0,
|
|
"logps/rejected": -468.0,
|
|
"loss": 0.4123,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.8203125,
|
|
"rewards/margins": 1.3515625,
|
|
"rewards/rejected": -3.171875,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.08501993830411557,
|
|
"grad_norm": 12.571352956931529,
|
|
"learning_rate": 4.2481203007518795e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -372.0,
|
|
"logps/rejected": -532.0,
|
|
"loss": 0.3899,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -2.0625,
|
|
"rewards/margins": 1.6953125,
|
|
"rewards/rejected": -3.75,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.08577232713866527,
|
|
"grad_norm": 9.584329220428629,
|
|
"learning_rate": 4.285714285714285e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -396.0,
|
|
"logps/rejected": -544.0,
|
|
"loss": 0.3737,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.40625,
|
|
"rewards/margins": 1.421875,
|
|
"rewards/rejected": -3.828125,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.08652471597321496,
|
|
"grad_norm": 9.655313020328894,
|
|
"learning_rate": 4.3233082706766913e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -358.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3823,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.0625,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -3.703125,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.08727710480776465,
|
|
"grad_norm": 8.344975957766751,
|
|
"learning_rate": 4.3609022556390975e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -544.0,
|
|
"loss": 0.4058,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -2.484375,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -3.828125,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.08802949364231435,
|
|
"grad_norm": 17.90436214681741,
|
|
"learning_rate": 4.398496240601504e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.3911,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -2.359375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -3.984375,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.08878188247686404,
|
|
"grad_norm": 10.09502529582343,
|
|
"learning_rate": 4.4360902255639094e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -358.0,
|
|
"logps/rejected": -468.0,
|
|
"loss": 0.3816,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -2.0,
|
|
"rewards/margins": 1.203125,
|
|
"rewards/rejected": -3.203125,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.08953427131141374,
|
|
"grad_norm": 12.723840138089093,
|
|
"learning_rate": 4.4736842105263156e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -532.0,
|
|
"loss": 0.3603,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.15625,
|
|
"rewards/margins": 1.7734375,
|
|
"rewards/rejected": -3.9375,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.09028666014596344,
|
|
"grad_norm": 13.550815678408945,
|
|
"learning_rate": 4.5112781954887213e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -348.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.3987,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.8203125,
|
|
"rewards/margins": 1.5390625,
|
|
"rewards/rejected": -3.359375,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.09103904898051313,
|
|
"grad_norm": 9.008638592876,
|
|
"learning_rate": 4.5488721804511275e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -382.0,
|
|
"logps/rejected": -576.0,
|
|
"loss": 0.3829,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.1875,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -4.15625,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.09179143781506283,
|
|
"grad_norm": 35.08614510436194,
|
|
"learning_rate": 4.586466165413533e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -330.0,
|
|
"logps/rejected": -482.0,
|
|
"loss": 0.3832,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.7734375,
|
|
"rewards/margins": 1.6640625,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.09254382664961253,
|
|
"grad_norm": 11.415330910582554,
|
|
"learning_rate": 4.62406015037594e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -388.0,
|
|
"logps/rejected": -572.0,
|
|
"loss": 0.4032,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.15625,
|
|
"rewards/margins": 1.875,
|
|
"rewards/rejected": -4.03125,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.09329621548416221,
|
|
"grad_norm": 9.726227634850884,
|
|
"learning_rate": 4.6616541353383456e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -360.0,
|
|
"logps/rejected": -490.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.0,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.09404860431871191,
|
|
"grad_norm": 10.938204411889682,
|
|
"learning_rate": 4.699248120300752e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -402.0,
|
|
"logps/rejected": -524.0,
|
|
"loss": 0.3515,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.140625,
|
|
"rewards/margins": 1.5703125,
|
|
"rewards/rejected": -3.71875,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.09480099315326161,
|
|
"grad_norm": 11.359648859339549,
|
|
"learning_rate": 4.7368421052631574e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -388.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3956,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.265625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -4.15625,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.0955533819878113,
|
|
"grad_norm": 9.557735064723317,
|
|
"learning_rate": 4.774436090225564e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -548.0,
|
|
"loss": 0.3486,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.28125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -3.953125,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.096305770822361,
|
|
"grad_norm": 12.586703006940098,
|
|
"learning_rate": 4.812030075187969e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -348.0,
|
|
"logps/rejected": -516.0,
|
|
"loss": 0.3639,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -1.84375,
|
|
"rewards/margins": 1.78125,
|
|
"rewards/rejected": -3.609375,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.0970581596569107,
|
|
"grad_norm": 11.590852438383495,
|
|
"learning_rate": 4.849624060150376e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -396.0,
|
|
"logps/rejected": -556.0,
|
|
"loss": 0.363,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -4.1875,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.09781054849146038,
|
|
"grad_norm": 9.666020281314843,
|
|
"learning_rate": 4.887218045112782e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -580.0,
|
|
"loss": 0.3747,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -2.296875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.09856293732601008,
|
|
"grad_norm": 17.089677017414804,
|
|
"learning_rate": 4.924812030075187e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.3642,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.546875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -4.5,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.09931532616055978,
|
|
"grad_norm": 14.284746492926356,
|
|
"learning_rate": 4.962406015037593e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -394.0,
|
|
"logps/rejected": -536.0,
|
|
"loss": 0.356,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.34375,
|
|
"rewards/margins": 1.6484375,
|
|
"rewards/rejected": -3.984375,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.10006771499510947,
|
|
"grad_norm": 10.0309975081672,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3597,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -2.78125,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.10082010382965917,
|
|
"grad_norm": 12.11681402357444,
|
|
"learning_rate": 4.999991376679495e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.3456,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -2.6875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -4.59375,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.10157249266420887,
|
|
"grad_norm": 9.241869652249655,
|
|
"learning_rate": 4.999965506777466e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.3523,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.875,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.10232488149875855,
|
|
"grad_norm": 10.700012765427152,
|
|
"learning_rate": 4.999922390472384e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -572.0,
|
|
"loss": 0.3516,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.484375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -4.1875,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.10307727033330825,
|
|
"grad_norm": 10.778973183136214,
|
|
"learning_rate": 4.999862028061692e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.3608,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 1.609375,
|
|
"rewards/rejected": -4.3125,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.10382965916785795,
|
|
"grad_norm": 10.942848875930892,
|
|
"learning_rate": 4.99978441996181e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3596,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.515625,
|
|
"rewards/margins": 1.953125,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.10458204800240764,
|
|
"grad_norm": 13.27627553772671,
|
|
"learning_rate": 4.999689566708128e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -428.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.3523,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 1.7421875,
|
|
"rewards/rejected": -4.34375,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.10533443683695734,
|
|
"grad_norm": 9.279591495260535,
|
|
"learning_rate": 4.999577468955008e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3669,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.890625,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.10608682567150704,
|
|
"grad_norm": 15.338794234291706,
|
|
"learning_rate": 4.999448127475773e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3322,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.125,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.10683921450605673,
|
|
"grad_norm": 11.07525013808053,
|
|
"learning_rate": 4.999301543162706e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.3323,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.10759160334060643,
|
|
"grad_norm": 14.47067636759482,
|
|
"learning_rate": 4.999137717027041e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3365,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.046875,
|
|
"rewards/margins": 2.03125,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.10834399217515613,
|
|
"grad_norm": 48.895026311812195,
|
|
"learning_rate": 4.998956650198959e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 0.3544,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -2.703125,
|
|
"rewards/margins": 1.8828125,
|
|
"rewards/rejected": -4.5625,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.10909638100970581,
|
|
"grad_norm": 12.33907739866595,
|
|
"learning_rate": 4.998758343927576e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3275,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.9375,
|
|
"rewards/margins": 1.765625,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.10984876984425551,
|
|
"grad_norm": 10.624130772065378,
|
|
"learning_rate": 4.998542799580941e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3492,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -2.734375,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.11060115867880521,
|
|
"grad_norm": 15.790847710499458,
|
|
"learning_rate": 4.998310018646021e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 0.3503,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.1113535475133549,
|
|
"grad_norm": 13.668416666518036,
|
|
"learning_rate": 4.998060002728689e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.339,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 2.125,
|
|
"rewards/rejected": -5.375,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.1121059363479046,
|
|
"grad_norm": 13.367505264457792,
|
|
"learning_rate": 4.99779275355372e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3514,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.640625,
|
|
"rewards/margins": 1.8203125,
|
|
"rewards/rejected": -4.46875,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.1128583251824543,
|
|
"grad_norm": 9.043259055103826,
|
|
"learning_rate": 4.997508272964775e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.3178,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -5.125,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.11361071401700398,
|
|
"grad_norm": 11.016220353679143,
|
|
"learning_rate": 4.997206562924387e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.3126,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.171875,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.11436310285155368,
|
|
"grad_norm": 9.506781512625706,
|
|
"learning_rate": 4.99688762551395e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -508.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.3293,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.11511549168610338,
|
|
"grad_norm": 9.87913772768346,
|
|
"learning_rate": 4.996551462933705e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.3387,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.03125,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.11586788052065307,
|
|
"grad_norm": 9.143562630664047,
|
|
"learning_rate": 4.996198077502719e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.323,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -5.03125,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.11662026935520277,
|
|
"grad_norm": 10.38817675345434,
|
|
"learning_rate": 4.99582747165888e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.319,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 2.0625,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.11737265818975247,
|
|
"grad_norm": 11.492302524993693,
|
|
"learning_rate": 4.995439647958869e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.3247,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.0,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.11812504702430215,
|
|
"grad_norm": 11.709867997960567,
|
|
"learning_rate": 4.995034609078148e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3327,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.890625,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.11887743585885185,
|
|
"grad_norm": 12.146034299644466,
|
|
"learning_rate": 4.994612357810942e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.3127,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.11962982469340155,
|
|
"grad_norm": 37.73933841606365,
|
|
"learning_rate": 4.994172897070217e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -486.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.359,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.171875,
|
|
"rewards/margins": 2.046875,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.12038221352795124,
|
|
"grad_norm": 8.414319765873188,
|
|
"learning_rate": 4.99371622988766e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3201,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.12113460236250094,
|
|
"grad_norm": 9.915553480092923,
|
|
"learning_rate": 4.993242359413664e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.3145,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.203125,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.12188699119705064,
|
|
"grad_norm": 12.120046031370594,
|
|
"learning_rate": 4.992751288917297e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -688.0,
|
|
"loss": 0.3246,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.875,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.12263938003160033,
|
|
"grad_norm": 10.565988140666803,
|
|
"learning_rate": 4.992243021786285e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.3196,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -5.375,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.12339176886615003,
|
|
"grad_norm": 9.873827794682715,
|
|
"learning_rate": 4.99171756152699e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -688.0,
|
|
"loss": 0.3253,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.12414415770069973,
|
|
"grad_norm": 11.240991564793916,
|
|
"learning_rate": 4.991174911764381e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.3067,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.359375,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.12489654653524941,
|
|
"grad_norm": 10.750766844675857,
|
|
"learning_rate": 4.990615076242011e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.3395,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.75,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.12564893536979913,
|
|
"grad_norm": 20.008574618115496,
|
|
"learning_rate": 4.990038058821995e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.3142,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.625,
|
|
"rewards/margins": 1.890625,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.1264013242043488,
|
|
"grad_norm": 12.148202383381813,
|
|
"learning_rate": 4.989443863484976e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3193,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -5.03125,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.1271537130388985,
|
|
"grad_norm": 15.287954256111135,
|
|
"learning_rate": 4.988832494330106e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.3325,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -5.1875,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.1279061018734482,
|
|
"grad_norm": 11.463559327588364,
|
|
"learning_rate": 4.988203955575006e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2976,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.640625,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.1286584907079979,
|
|
"grad_norm": 12.056679797446584,
|
|
"learning_rate": 4.987558251555755e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.317,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.921875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.1294108795425476,
|
|
"grad_norm": 9.484624994057103,
|
|
"learning_rate": 4.98689538672684e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.3096,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.1301632683770973,
|
|
"grad_norm": 14.197927584727962,
|
|
"learning_rate": 4.986215365661137e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.3089,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.015625,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.13091565721164697,
|
|
"grad_norm": 8.583390608989294,
|
|
"learning_rate": 4.985518193049879e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -652.0,
|
|
"loss": 0.3193,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 2.15625,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.13166804604619667,
|
|
"grad_norm": 15.994335121139475,
|
|
"learning_rate": 4.984803873702619e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.3372,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.78125,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -5.03125,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.13242043488074637,
|
|
"grad_norm": 21.914213241176697,
|
|
"learning_rate": 4.984072412547202e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -640.0,
|
|
"loss": 0.3436,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.13317282371529607,
|
|
"grad_norm": 11.643305728816616,
|
|
"learning_rate": 4.983323814629727e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -612.0,
|
|
"loss": 0.3335,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.5625,
|
|
"rewards/margins": 1.9921875,
|
|
"rewards/rejected": -4.5625,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.13392521254984577,
|
|
"grad_norm": 9.9183144070942,
|
|
"learning_rate": 4.982558085114515e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -460.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.349,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.015625,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -5.0,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.13467760138439547,
|
|
"grad_norm": 12.622299854386242,
|
|
"learning_rate": 4.981775229284068e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.32,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.015625,
|
|
"rewards/margins": 1.9140625,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.13542999021894514,
|
|
"grad_norm": 13.726359537170527,
|
|
"learning_rate": 4.98097525253904e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -652.0,
|
|
"loss": 0.3353,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.015625,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.13618237905349484,
|
|
"grad_norm": 8.533957196814246,
|
|
"learning_rate": 4.980158160398198e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.287,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -2.84375,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.13693476788804454,
|
|
"grad_norm": 9.66875724969042,
|
|
"learning_rate": 4.979323958498378e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -466.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3176,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.13768715672259424,
|
|
"grad_norm": 11.673654825211232,
|
|
"learning_rate": 4.978472652594453e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.3245,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.53125,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.13843954555714394,
|
|
"grad_norm": 9.459008565137387,
|
|
"learning_rate": 4.977604248559289e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.3099,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.9375,
|
|
"rewards/margins": 2.265625,
|
|
"rewards/rejected": -5.1875,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.13919193439169364,
|
|
"grad_norm": 10.074612458953805,
|
|
"learning_rate": 4.976718752383709e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.3083,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.1399443232262433,
|
|
"grad_norm": 9.111393234918845,
|
|
"learning_rate": 4.975816170176445e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.289,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.140696712060793,
|
|
"grad_norm": 11.023262933345384,
|
|
"learning_rate": 4.974896508164101e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.3058,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.453125,
|
|
"rewards/margins": 2.34375,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.1414491008953427,
|
|
"grad_norm": 9.465827793956825,
|
|
"learning_rate": 4.973959772691112e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.3245,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.1422014897298924,
|
|
"grad_norm": 9.001035578417973,
|
|
"learning_rate": 4.973005970219692e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.294,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -5.53125,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.1429538785644421,
|
|
"grad_norm": 10.221159879257584,
|
|
"learning_rate": 4.972035107329796e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.3002,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.1437062673989918,
|
|
"grad_norm": 9.557554250860324,
|
|
"learning_rate": 4.971047190719076e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -688.0,
|
|
"loss": 0.2758,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.78125,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.14445865623354148,
|
|
"grad_norm": 24.124984900386067,
|
|
"learning_rate": 4.970042227202828e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2653,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.75,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.14521104506809118,
|
|
"grad_norm": 9.428412327971445,
|
|
"learning_rate": 4.969020223713948e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.3094,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.14596343390264088,
|
|
"grad_norm": 9.244732989145508,
|
|
"learning_rate": 4.967981187302889e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2917,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.34375,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.14671582273719058,
|
|
"grad_norm": 10.135613942587367,
|
|
"learning_rate": 4.966925125137607e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2774,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.14746821157174028,
|
|
"grad_norm": 15.320900000215662,
|
|
"learning_rate": 4.965852044503512e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.31,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.046875,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.14822060040628998,
|
|
"grad_norm": 9.157953298675203,
|
|
"learning_rate": 4.964761952803416e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -450.0,
|
|
"logps/rejected": -660.0,
|
|
"loss": 0.3058,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.09375,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.14897298924083965,
|
|
"grad_norm": 9.500153891031406,
|
|
"learning_rate": 4.963654857557488e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -652.0,
|
|
"loss": 0.3027,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.03125,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.14972537807538935,
|
|
"grad_norm": 7.779021609791956,
|
|
"learning_rate": 4.962530766403199e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2961,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.15047776690993905,
|
|
"grad_norm": 10.341517042412354,
|
|
"learning_rate": 4.961389687095267e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2755,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.203125,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.15123015574448875,
|
|
"grad_norm": 10.775662545927513,
|
|
"learning_rate": 4.960231627505606e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2881,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.625,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.15198254457903845,
|
|
"grad_norm": 10.755771111022861,
|
|
"learning_rate": 4.959056595623271e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.2946,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.265625,
|
|
"rewards/rejected": -5.5,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.15273493341358815,
|
|
"grad_norm": 8.940161113038451,
|
|
"learning_rate": 4.957864599554404e-07,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3193,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.796875,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -5.1875,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.15348732224813783,
|
|
"grad_norm": 10.0217606927789,
|
|
"learning_rate": 4.956655647522176e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2761,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.15423971108268753,
|
|
"grad_norm": 8.538140666623187,
|
|
"learning_rate": 4.95542974786673e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2786,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.15499209991723722,
|
|
"grad_norm": 11.092066592354108,
|
|
"learning_rate": 4.954186909045129e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -692.0,
|
|
"loss": 0.2978,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.15574448875178692,
|
|
"grad_norm": 8.810015279238543,
|
|
"learning_rate": 4.95292713963129e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.3037,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -2.984375,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.15649687758633662,
|
|
"grad_norm": 10.304083443176951,
|
|
"learning_rate": 4.951650448315929e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.2742,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.359375,
|
|
"rewards/margins": 2.34375,
|
|
"rewards/rejected": -5.6875,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.15724926642088632,
|
|
"grad_norm": 11.038842624924929,
|
|
"learning_rate": 4.950356843906501e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.3043,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.158001655255436,
|
|
"grad_norm": 16.086119459510908,
|
|
"learning_rate": 4.949046335327138e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -466.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2645,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.953125,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -5.25,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.1587540440899857,
|
|
"grad_norm": 10.440581591351142,
|
|
"learning_rate": 4.94771893161859e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.3073,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.1595064329245354,
|
|
"grad_norm": 8.118566889874467,
|
|
"learning_rate": 4.946374641938157e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.3307,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.203125,
|
|
"rewards/margins": 2.1875,
|
|
"rewards/rejected": -5.375,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 0.1602588217590851,
|
|
"grad_norm": 12.613630152927428,
|
|
"learning_rate": 4.945013475559632e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2839,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.359375,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -5.65625,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 0.1610112105936348,
|
|
"grad_norm": 10.910342487887005,
|
|
"learning_rate": 4.943635441873235e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2975,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 0.1617635994281845,
|
|
"grad_norm": 7.615912452628361,
|
|
"learning_rate": 4.942240550385547e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2741,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.16251598826273417,
|
|
"grad_norm": 12.776860043962529,
|
|
"learning_rate": 4.940828810719444e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2956,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 0.16326837709728387,
|
|
"grad_norm": 7.715958635368018,
|
|
"learning_rate": 4.939400232614033e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.3187,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.78125,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"epoch": 0.16402076593183357,
|
|
"grad_norm": 10.909720452071825,
|
|
"learning_rate": 4.937954825924585e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2798,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.015625,
|
|
"rewards/margins": 2.140625,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 0.16477315476638327,
|
|
"grad_norm": 11.056113501762782,
|
|
"learning_rate": 4.936492600622464e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.3351,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"epoch": 0.16552554360093297,
|
|
"grad_norm": 9.181288831576246,
|
|
"learning_rate": 4.935013566795058e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -494.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2665,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.16627793243548267,
|
|
"grad_norm": 15.421194070464846,
|
|
"learning_rate": 4.933517734645714e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2725,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"epoch": 0.16703032127003234,
|
|
"grad_norm": 7.732850963579106,
|
|
"learning_rate": 4.932005114493665e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2775,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.703125,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 0.16778271010458204,
|
|
"grad_norm": 8.73231439791769,
|
|
"learning_rate": 4.930475716773956e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2924,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"epoch": 0.16853509893913174,
|
|
"grad_norm": 9.123008878161743,
|
|
"learning_rate": 4.928929552037378e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -492.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2833,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 0.16928748777368144,
|
|
"grad_norm": 12.07063346888074,
|
|
"learning_rate": 4.927366630950389e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.2693,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.5625,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.17003987660823114,
|
|
"grad_norm": 9.58671682334943,
|
|
"learning_rate": 4.925786964295046e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.3029,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 0.17079226544278084,
|
|
"grad_norm": 9.336994205772083,
|
|
"learning_rate": 4.924190562968926e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2713,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -5.75,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"epoch": 0.17154465427733054,
|
|
"grad_norm": 11.646865344527923,
|
|
"learning_rate": 4.922577437985052e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -498.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 0.1722970431118802,
|
|
"grad_norm": 12.116174831993108,
|
|
"learning_rate": 4.920947600471821e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2828,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"epoch": 0.1730494319464299,
|
|
"grad_norm": 9.542246025337626,
|
|
"learning_rate": 4.919301061672919e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2701,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.1738018207809796,
|
|
"grad_norm": 10.606436086492515,
|
|
"learning_rate": 4.917637832947256e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2781,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"epoch": 0.1745542096155293,
|
|
"grad_norm": 10.611161178350386,
|
|
"learning_rate": 4.915957925768871e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2981,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.265625,
|
|
"rewards/rejected": -6.0,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"epoch": 0.175306598450079,
|
|
"grad_norm": 12.829898396212654,
|
|
"learning_rate": 4.914261351726868e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2684,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.78125,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"epoch": 0.1760589872846287,
|
|
"grad_norm": 10.222444110704744,
|
|
"learning_rate": 4.91254812252533e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2601,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"epoch": 0.17681137611917838,
|
|
"grad_norm": 9.955713067057467,
|
|
"learning_rate": 4.910818249983235e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.283,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.17756376495372808,
|
|
"grad_norm": 9.899985953459382,
|
|
"learning_rate": 4.909071746034379e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2834,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"epoch": 0.17831615378827778,
|
|
"grad_norm": 9.734416734362911,
|
|
"learning_rate": 4.907308622727293e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -494.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2868,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"epoch": 0.17906854262282748,
|
|
"grad_norm": 8.463086487753493,
|
|
"learning_rate": 4.90552889222516e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2605,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"epoch": 0.17982093145737718,
|
|
"grad_norm": 13.146535023342294,
|
|
"learning_rate": 4.903732566805727e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2833,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.890625,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"epoch": 0.18057332029192688,
|
|
"grad_norm": 9.711136432827589,
|
|
"learning_rate": 4.901919658861228e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3008,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.296875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -5.375,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.18132570912647655,
|
|
"grad_norm": 12.52993235918333,
|
|
"learning_rate": 4.900090180898292e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -792.0,
|
|
"loss": 0.2695,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"epoch": 0.18207809796102625,
|
|
"grad_norm": 7.896403216710635,
|
|
"learning_rate": 4.898244145537857e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2626,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"epoch": 0.18283048679557595,
|
|
"grad_norm": 8.144329513203688,
|
|
"learning_rate": 4.896381565515087e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2379,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"epoch": 0.18358287563012565,
|
|
"grad_norm": 9.329496514618313,
|
|
"learning_rate": 4.894502453679284e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2785,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.203125,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"epoch": 0.18433526446467535,
|
|
"grad_norm": 12.10939641210324,
|
|
"learning_rate": 4.892606822993793e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -708.0,
|
|
"loss": 0.299,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.18508765329922505,
|
|
"grad_norm": 9.857872781155606,
|
|
"learning_rate": 4.890694686535918e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2701,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.28125,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"epoch": 0.18584004213377472,
|
|
"grad_norm": 17.101449726474733,
|
|
"learning_rate": 4.888766057496833e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2916,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.203125,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"epoch": 0.18659243096832442,
|
|
"grad_norm": 8.347297641682713,
|
|
"learning_rate": 4.886820949181486e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -5.875,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"epoch": 0.18734481980287412,
|
|
"grad_norm": 9.678661384787706,
|
|
"learning_rate": 4.884859375008512e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -486.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2875,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"epoch": 0.18809720863742382,
|
|
"grad_norm": 11.145929450555997,
|
|
"learning_rate": 4.882881348510136e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2611,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.84375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.18884959747197352,
|
|
"grad_norm": 7.1935463269752065,
|
|
"learning_rate": 4.880886883332083e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2421,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"epoch": 0.18960198630652322,
|
|
"grad_norm": 10.65990633403162,
|
|
"learning_rate": 4.878875993233486e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2631,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -3.703125,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"epoch": 0.1903543751410729,
|
|
"grad_norm": 10.814709273380368,
|
|
"learning_rate": 4.876848692086782e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2662,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.25,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"epoch": 0.1911067639756226,
|
|
"grad_norm": 8.274113340101351,
|
|
"learning_rate": 4.874804993877625e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2679,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"epoch": 0.1918591528101723,
|
|
"grad_norm": 7.8585668700799465,
|
|
"learning_rate": 4.872744912704788e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2575,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.192611541644722,
|
|
"grad_norm": 9.359705035781545,
|
|
"learning_rate": 4.870668462780062e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -492.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2643,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.34375,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"epoch": 0.1933639304792717,
|
|
"grad_norm": 10.260726382149574,
|
|
"learning_rate": 4.868575658428163e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -474.0,
|
|
"logps/rejected": -704.0,
|
|
"loss": 0.2763,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.1875,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.625,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"epoch": 0.1941163193138214,
|
|
"grad_norm": 14.078164696272687,
|
|
"learning_rate": 4.866466514086628e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -480.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2753,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.03125,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"epoch": 0.19486870814837107,
|
|
"grad_norm": 8.453767523010452,
|
|
"learning_rate": 4.864341044305719e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -720.0,
|
|
"loss": 0.2626,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"epoch": 0.19562109698292077,
|
|
"grad_norm": 10.18587974692239,
|
|
"learning_rate": 4.862199263748323e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2657,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.78125,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.19637348581747047,
|
|
"grad_norm": 11.079570317722963,
|
|
"learning_rate": 4.860041187189846e-07,
|
|
"logits/chosen": -2.328125,
|
|
"logits/rejected": -2.171875,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.2942,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"epoch": 0.19712587465202017,
|
|
"grad_norm": 7.9462869418389745,
|
|
"learning_rate": 4.857866829518118e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -494.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2422,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.4375,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"epoch": 0.19787826348656987,
|
|
"grad_norm": 10.695129742443964,
|
|
"learning_rate": 4.855676205733287e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2931,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"epoch": 0.19863065232111957,
|
|
"grad_norm": 13.909715185144693,
|
|
"learning_rate": 4.853469330947712e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2719,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"epoch": 0.19938304115566924,
|
|
"grad_norm": 7.977293712578769,
|
|
"learning_rate": 4.851246220385862e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2662,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.20013542999021894,
|
|
"grad_norm": 7.690656776892857,
|
|
"learning_rate": 4.849006889384217e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2768,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"epoch": 0.20088781882476864,
|
|
"grad_norm": 8.564652490755712,
|
|
"learning_rate": 4.84675135339115e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -494.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2813,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"epoch": 0.20164020765931834,
|
|
"grad_norm": 6.442833431737132,
|
|
"learning_rate": 4.84447962796683e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2707,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"epoch": 0.20239259649386804,
|
|
"grad_norm": 11.556397527239499,
|
|
"learning_rate": 4.842191728783107e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2938,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.578125,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"epoch": 0.20314498532841774,
|
|
"grad_norm": 10.766048526372968,
|
|
"learning_rate": 4.839887671623414e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -478.0,
|
|
"logps/rejected": -768.0,
|
|
"loss": 0.2471,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.2038973741629674,
|
|
"grad_norm": 13.5244223811314,
|
|
"learning_rate": 4.837567472382651e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2909,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"epoch": 0.2046497629975171,
|
|
"grad_norm": 9.738583048600992,
|
|
"learning_rate": 4.835231147067072e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2766,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"epoch": 0.2054021518320668,
|
|
"grad_norm": 10.261504032860017,
|
|
"learning_rate": 4.832878711794185e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2551,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -3.578125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"epoch": 0.2061545406666165,
|
|
"grad_norm": 6.5383786738976415,
|
|
"learning_rate": 4.830510182792632e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2445,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"epoch": 0.2069069295011662,
|
|
"grad_norm": 10.447397489817225,
|
|
"learning_rate": 4.82812557640208e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2563,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.2076593183357159,
|
|
"grad_norm": 9.348554817009415,
|
|
"learning_rate": 4.82572490907311e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2576,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.765625,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"epoch": 0.20841170717026558,
|
|
"grad_norm": 8.70381283312396,
|
|
"learning_rate": 4.8233081973671e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2484,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"epoch": 0.20916409600481528,
|
|
"grad_norm": 7.4097108054434955,
|
|
"learning_rate": 4.820875457956115e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2774,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"epoch": 0.20991648483936498,
|
|
"grad_norm": 8.929321008061232,
|
|
"learning_rate": 4.818426707622788e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -494.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.257,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"epoch": 0.21066887367391468,
|
|
"grad_norm": 8.981128088011406,
|
|
"learning_rate": 4.815961963260207e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2419,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.21142126250846438,
|
|
"grad_norm": 10.05509938002628,
|
|
"learning_rate": 4.813481241871794e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2477,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"epoch": 0.21217365134301408,
|
|
"grad_norm": 11.72370585695034,
|
|
"learning_rate": 4.810984560571195e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2559,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"epoch": 0.21292604017756375,
|
|
"grad_norm": 9.647642410986215,
|
|
"learning_rate": 4.808471936582156e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.2374,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.140625,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"epoch": 0.21367842901211345,
|
|
"grad_norm": 11.420752468431145,
|
|
"learning_rate": 4.805943387238409e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.252,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.0,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"epoch": 0.21443081784666315,
|
|
"grad_norm": 9.785252828322875,
|
|
"learning_rate": 4.803398929983543e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2856,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.21518320668121285,
|
|
"grad_norm": 11.971762302414026,
|
|
"learning_rate": 4.800838582370898e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2718,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -5.3125,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"epoch": 0.21593559551576255,
|
|
"grad_norm": 8.477706468059246,
|
|
"learning_rate": 4.79826236206343e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.171875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2513,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.25,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -5.625,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"epoch": 0.21668798435031225,
|
|
"grad_norm": 11.869962119536272,
|
|
"learning_rate": 4.795670286833599e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2611,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.34375,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"epoch": 0.21744037318486192,
|
|
"grad_norm": 8.487625004734987,
|
|
"learning_rate": 4.79306237456324e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -764.0,
|
|
"loss": 0.265,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"epoch": 0.21819276201941162,
|
|
"grad_norm": 11.615793395248453,
|
|
"learning_rate": 4.790438643243447e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2533,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -3.140625,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.21894515085396132,
|
|
"grad_norm": 10.318914547631199,
|
|
"learning_rate": 4.787799110974436e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2414,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"epoch": 0.21969753968851102,
|
|
"grad_norm": 12.904513672408855,
|
|
"learning_rate": 4.785143795965437e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2577,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.640625,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"epoch": 0.22044992852306072,
|
|
"grad_norm": 8.687685973219471,
|
|
"learning_rate": 4.782472716534554e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.25,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"epoch": 0.22120231735761042,
|
|
"grad_norm": 8.425580278247608,
|
|
"learning_rate": 4.779785891108647e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2682,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"epoch": 0.2219547061921601,
|
|
"grad_norm": 8.92649200504024,
|
|
"learning_rate": 4.777083338223202e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2475,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.2227070950267098,
|
|
"grad_norm": 9.22059724524348,
|
|
"learning_rate": 4.774365076522202e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2469,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.765625,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"epoch": 0.2234594838612595,
|
|
"grad_norm": 9.454530660849164,
|
|
"learning_rate": 4.771631124758e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2499,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"epoch": 0.2242118726958092,
|
|
"grad_norm": 9.480975232151959,
|
|
"learning_rate": 4.76888150179119e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"epoch": 0.2249642615303589,
|
|
"grad_norm": 7.916418722194274,
|
|
"learning_rate": 4.7661162265904773e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2273,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"epoch": 0.2257166503649086,
|
|
"grad_norm": 8.229870586292975,
|
|
"learning_rate": 4.7633353182325443e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2373,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.84375,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.22646903919945827,
|
|
"grad_norm": 10.226118184963164,
|
|
"learning_rate": 4.760538795901923e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2677,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"epoch": 0.22722142803400797,
|
|
"grad_norm": 8.49693960463905,
|
|
"learning_rate": 4.757726678890859e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.254,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"epoch": 0.22797381686855767,
|
|
"grad_norm": 9.5155953716937,
|
|
"learning_rate": 4.754898986599182e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2568,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.234375,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"epoch": 0.22872620570310737,
|
|
"grad_norm": 9.050451730524532,
|
|
"learning_rate": 4.75205573853417e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2664,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"epoch": 0.22947859453765707,
|
|
"grad_norm": 11.230060400251109,
|
|
"learning_rate": 4.749196954310414e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.25,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.23023098337220677,
|
|
"grad_norm": 8.780783249418834,
|
|
"learning_rate": 4.746322653649683e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.252,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"epoch": 0.23098337220675647,
|
|
"grad_norm": 9.12154565614575,
|
|
"learning_rate": 4.7434328563807913e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2577,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"epoch": 0.23173576104130614,
|
|
"grad_norm": 14.056073227705715,
|
|
"learning_rate": 4.7405275824394574e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2442,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"epoch": 0.23248814987585584,
|
|
"grad_norm": 9.397850408648406,
|
|
"learning_rate": 4.737606851868167e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2361,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.546875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -6.75,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"epoch": 0.23324053871040554,
|
|
"grad_norm": 8.73361335302379,
|
|
"learning_rate": 4.734670684816037e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.2494,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.203125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.23399292754495524,
|
|
"grad_norm": 10.750693728458632,
|
|
"learning_rate": 4.7317191015386744e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2487,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.625,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"epoch": 0.23474531637950494,
|
|
"grad_norm": 7.196445794814489,
|
|
"learning_rate": 4.7287521223980395e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.269,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.28125,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -6.0,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"epoch": 0.23549770521405464,
|
|
"grad_norm": 8.778673968279726,
|
|
"learning_rate": 4.725769767862301e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2344,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -6.375,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"epoch": 0.2362500940486043,
|
|
"grad_norm": 9.278296782454238,
|
|
"learning_rate": 4.7227720585056986e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2251,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"epoch": 0.237002482883154,
|
|
"grad_norm": 11.778051631603764,
|
|
"learning_rate": 4.7197590150083986e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2454,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.2377548717177037,
|
|
"grad_norm": 10.909061617481207,
|
|
"learning_rate": 4.716730658156354e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2329,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"epoch": 0.2385072605522534,
|
|
"grad_norm": 9.24366980179484,
|
|
"learning_rate": 4.7136870088411564e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2298,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"epoch": 0.2392596493868031,
|
|
"grad_norm": 10.949962350431266,
|
|
"learning_rate": 4.710628088059898e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2271,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"epoch": 0.2400120382213528,
|
|
"grad_norm": 11.993941852243937,
|
|
"learning_rate": 4.707553916915022e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2549,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.78125,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"epoch": 0.24076442705590248,
|
|
"grad_norm": 10.660695498977127,
|
|
"learning_rate": 4.704464516614178e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2583,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.24151681589045218,
|
|
"grad_norm": 8.141720298029627,
|
|
"learning_rate": 4.7013599084700787e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2528,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.75,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"epoch": 0.24226920472500188,
|
|
"grad_norm": 12.508230218250487,
|
|
"learning_rate": 4.698240113900348e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2322,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"epoch": 0.24302159355955158,
|
|
"grad_norm": 8.835584611496607,
|
|
"learning_rate": 4.6951051544273765e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2373,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"epoch": 0.24377398239410128,
|
|
"grad_norm": 8.181209169447907,
|
|
"learning_rate": 4.6919550516781723e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2355,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.765625,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"epoch": 0.24452637122865098,
|
|
"grad_norm": 9.207106143916054,
|
|
"learning_rate": 4.688789827384213e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2627,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.24527876006320065,
|
|
"grad_norm": 12.041595328496067,
|
|
"learning_rate": 4.6856095033812927e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2378,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.546875,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"epoch": 0.24603114889775035,
|
|
"grad_norm": 8.162952877767816,
|
|
"learning_rate": 4.682414101609374e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2397,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"epoch": 0.24678353773230005,
|
|
"grad_norm": 7.661645701725578,
|
|
"learning_rate": 4.679203644112436e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2426,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"epoch": 0.24753592656684975,
|
|
"grad_norm": 9.39559412819907,
|
|
"learning_rate": 4.6759781530383214e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2572,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"epoch": 0.24828831540139945,
|
|
"grad_norm": 8.664281653841188,
|
|
"learning_rate": 4.672737650638586e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2428,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.24904070423594915,
|
|
"grad_norm": 9.341469048911085,
|
|
"learning_rate": 4.669482159268341e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2357,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"epoch": 0.24979309307049882,
|
|
"grad_norm": 8.73158517562442,
|
|
"learning_rate": 4.666211701386103e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2537,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"epoch": 0.25054548190504855,
|
|
"grad_norm": 9.193942328899496,
|
|
"learning_rate": 4.662926299553638e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2391,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"epoch": 0.25129787073959825,
|
|
"grad_norm": 9.383374978565644,
|
|
"learning_rate": 4.6596259764358037e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"epoch": 0.2520502595741479,
|
|
"grad_norm": 8.09815946238127,
|
|
"learning_rate": 4.6563107548003967e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2575,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.2528026484086976,
|
|
"grad_norm": 9.28524948295606,
|
|
"learning_rate": 4.6529806575179895e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2532,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"epoch": 0.2535550372432473,
|
|
"grad_norm": 8.372693128057215,
|
|
"learning_rate": 4.6496357075617816e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2312,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"epoch": 0.254307426077797,
|
|
"grad_norm": 8.973796350635121,
|
|
"learning_rate": 4.646275928007431e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"epoch": 0.2550598149123467,
|
|
"grad_norm": 15.043288873372585,
|
|
"learning_rate": 4.642901342032905e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2644,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"epoch": 0.2558122037468964,
|
|
"grad_norm": 8.60829166534939,
|
|
"learning_rate": 4.639511972918311e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2515,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.2565645925814461,
|
|
"grad_norm": 9.286255016844617,
|
|
"learning_rate": 4.636107844045742e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2498,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"epoch": 0.2573169814159958,
|
|
"grad_norm": 7.683783123714642,
|
|
"learning_rate": 4.632688978899114e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2387,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"epoch": 0.2580693702505455,
|
|
"grad_norm": 10.260751299912629,
|
|
"learning_rate": 4.629255401064004e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2323,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"epoch": 0.2588217590850952,
|
|
"grad_norm": 7.726491373550179,
|
|
"learning_rate": 4.625807134227483e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2666,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"epoch": 0.2595741479196449,
|
|
"grad_norm": 9.175160897527569,
|
|
"learning_rate": 4.622344202177961e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2426,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -3.59375,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.40625,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.2603265367541946,
|
|
"grad_norm": 10.620774123065072,
|
|
"learning_rate": 4.6188666288050163e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2542,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.375,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"epoch": 0.26107892558874424,
|
|
"grad_norm": 8.48922670158365,
|
|
"learning_rate": 4.615374438099232e-07,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2499,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"epoch": 0.26183131442329394,
|
|
"grad_norm": 9.380206108680339,
|
|
"learning_rate": 4.611867654152033e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2377,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"epoch": 0.26258370325784364,
|
|
"grad_norm": 8.630455909850124,
|
|
"learning_rate": 4.608346301155516e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2446,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"epoch": 0.26333609209239334,
|
|
"grad_norm": 8.010936942529533,
|
|
"learning_rate": 4.604810403402285e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.26408848092694304,
|
|
"grad_norm": 8.594226664704044,
|
|
"learning_rate": 4.601259985285284e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2225,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"epoch": 0.26484086976149274,
|
|
"grad_norm": 7.3686969444976,
|
|
"learning_rate": 4.5976950712976286e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2473,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"epoch": 0.26559325859604244,
|
|
"grad_norm": 9.006304280429053,
|
|
"learning_rate": 4.5941156860324345e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2532,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"epoch": 0.26634564743059214,
|
|
"grad_norm": 8.731170425293595,
|
|
"learning_rate": 4.590521854182651e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2361,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"epoch": 0.26709803626514184,
|
|
"grad_norm": 10.35512022707896,
|
|
"learning_rate": 4.5869136005408893e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.255,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.26785042509969154,
|
|
"grad_norm": 8.593676807068588,
|
|
"learning_rate": 4.5832909499992514e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2537,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"epoch": 0.26860281393424124,
|
|
"grad_norm": 7.229452609142653,
|
|
"learning_rate": 4.579653927549159e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2514,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"epoch": 0.26935520276879094,
|
|
"grad_norm": 9.484733316900101,
|
|
"learning_rate": 4.57600255828118e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2488,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"epoch": 0.2701075916033406,
|
|
"grad_norm": 7.616886332053109,
|
|
"learning_rate": 4.572336867384856e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2532,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"epoch": 0.2708599804378903,
|
|
"grad_norm": 9.424387046076045,
|
|
"learning_rate": 4.5686568801485306e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.219,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.25,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.27161236927244,
|
|
"grad_norm": 12.547475129287186,
|
|
"learning_rate": 4.5649626219591685e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2504,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"epoch": 0.2723647581069897,
|
|
"grad_norm": 10.17739824717397,
|
|
"learning_rate": 4.5612541183021905e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2157,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"epoch": 0.2731171469415394,
|
|
"grad_norm": 12.055309990415822,
|
|
"learning_rate": 4.5575313947612875e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2546,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"epoch": 0.2738695357760891,
|
|
"grad_norm": 9.381452162285955,
|
|
"learning_rate": 4.553794477018251e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2405,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"epoch": 0.2746219246106388,
|
|
"grad_norm": 9.717179832719665,
|
|
"learning_rate": 4.550043390852791e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2198,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.2753743134451885,
|
|
"grad_norm": 9.596874231784692,
|
|
"learning_rate": 4.546278162142364e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2173,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"epoch": 0.2761267022797382,
|
|
"grad_norm": 7.719511977487069,
|
|
"learning_rate": 4.542498816861988e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2265,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"epoch": 0.2768790911142879,
|
|
"grad_norm": 11.043176918667355,
|
|
"learning_rate": 4.538705381084067e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2592,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"epoch": 0.2776314799488376,
|
|
"grad_norm": 11.332378533299988,
|
|
"learning_rate": 4.5348978809782123e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2477,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"epoch": 0.2783838687833873,
|
|
"grad_norm": 9.797098424450917,
|
|
"learning_rate": 4.531076342811059e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2161,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.2791362576179369,
|
|
"grad_norm": 12.677047667665043,
|
|
"learning_rate": 4.5272407929460846e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2478,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"epoch": 0.2798886464524866,
|
|
"grad_norm": 7.7470550829514995,
|
|
"learning_rate": 4.5233912578434297e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2311,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"epoch": 0.2806410352870363,
|
|
"grad_norm": 9.903990733335885,
|
|
"learning_rate": 4.5195277640597165e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2546,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"epoch": 0.281393424121586,
|
|
"grad_norm": 14.923608246368913,
|
|
"learning_rate": 4.5156503382478583e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2426,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"epoch": 0.2821458129561357,
|
|
"grad_norm": 10.290171086499644,
|
|
"learning_rate": 4.511759007156886e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2252,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.2828982017906854,
|
|
"grad_norm": 9.12773732569798,
|
|
"learning_rate": 4.507853797631753e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"epoch": 0.2836505906252351,
|
|
"grad_norm": 8.54967309370874,
|
|
"learning_rate": 4.503934736613161e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2154,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"epoch": 0.2844029794597848,
|
|
"grad_norm": 8.658423249282029,
|
|
"learning_rate": 4.500001851137363e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2313,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"epoch": 0.2851553682943345,
|
|
"grad_norm": 9.32583793890318,
|
|
"learning_rate": 4.496055168335986e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2481,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"epoch": 0.2859077571288842,
|
|
"grad_norm": 12.386535228494425,
|
|
"learning_rate": 4.4920947154358384e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2265,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.2866601459634339,
|
|
"grad_norm": 7.1901796846029296,
|
|
"learning_rate": 4.4881205197587236e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2304,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.375,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"epoch": 0.2874125347979836,
|
|
"grad_norm": 8.02806199551109,
|
|
"learning_rate": 4.484132608721252e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2235,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"epoch": 0.2881649236325333,
|
|
"grad_norm": 10.603106664851868,
|
|
"learning_rate": 4.480131009834651e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2457,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"epoch": 0.28891731246708297,
|
|
"grad_norm": 6.45000355010823,
|
|
"learning_rate": 4.476115750704578e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2465,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"epoch": 0.28966970130163266,
|
|
"grad_norm": 8.929005375270732,
|
|
"learning_rate": 4.472086859030926e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2149,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.29042209013618236,
|
|
"grad_norm": 8.024874951607805,
|
|
"learning_rate": 4.468044362607633e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2153,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"epoch": 0.29117447897073206,
|
|
"grad_norm": 9.5901959856907,
|
|
"learning_rate": 4.4639882893224924e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2361,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"epoch": 0.29192686780528176,
|
|
"grad_norm": 9.101424498045278,
|
|
"learning_rate": 4.4599186671569623e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2144,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"epoch": 0.29267925663983146,
|
|
"grad_norm": 8.652186114303964,
|
|
"learning_rate": 4.4558355241859655e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2362,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"epoch": 0.29343164547438116,
|
|
"grad_norm": 11.241719704363485,
|
|
"learning_rate": 4.451738888577705e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2483,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.29418403430893086,
|
|
"grad_norm": 7.151893594323524,
|
|
"learning_rate": 4.44762878859346e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2198,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -7.125,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"epoch": 0.29493642314348056,
|
|
"grad_norm": 6.694485106549714,
|
|
"learning_rate": 4.443505252587399e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2236,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"epoch": 0.29568881197803026,
|
|
"grad_norm": 9.938005153509135,
|
|
"learning_rate": 4.43936830900638e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"epoch": 0.29644120081257996,
|
|
"grad_norm": 16.682092053114292,
|
|
"learning_rate": 4.4352179863897556e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.241,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"epoch": 0.29719358964712966,
|
|
"grad_norm": 7.512938135048488,
|
|
"learning_rate": 4.4310543133691746e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2316,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.2979459784816793,
|
|
"grad_norm": 12.509486795634867,
|
|
"learning_rate": 4.426877318668387e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2155,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"epoch": 0.298698367316229,
|
|
"grad_norm": 7.987377416432433,
|
|
"learning_rate": 4.422687031103045e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.262,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"epoch": 0.2994507561507787,
|
|
"grad_norm": 8.497924735104682,
|
|
"learning_rate": 4.4184834795805016e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2233,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"epoch": 0.3002031449853284,
|
|
"grad_norm": 12.602114991629081,
|
|
"learning_rate": 4.414266693099615e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"epoch": 0.3009555338198781,
|
|
"grad_norm": 8.14636861622685,
|
|
"learning_rate": 4.4100367007505455e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2228,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.3017079226544278,
|
|
"grad_norm": 8.71715153360338,
|
|
"learning_rate": 4.405793531714558e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.233,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"epoch": 0.3024603114889775,
|
|
"grad_norm": 7.675931534560127,
|
|
"learning_rate": 4.401537215263816e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2029,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"epoch": 0.3032127003235272,
|
|
"grad_norm": 10.17405003846414,
|
|
"learning_rate": 4.3972677807611866e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"epoch": 0.3039650891580769,
|
|
"grad_norm": 7.09818779533062,
|
|
"learning_rate": 4.39298525766003e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.211,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"epoch": 0.3047174779926266,
|
|
"grad_norm": 9.918785165797798,
|
|
"learning_rate": 4.388689675504002e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.3054698668271763,
|
|
"grad_norm": 8.835187248857086,
|
|
"learning_rate": 4.3843810639268475e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2353,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"epoch": 0.306222255661726,
|
|
"grad_norm": 8.367596510251957,
|
|
"learning_rate": 4.3800594526521983e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"epoch": 0.30697464449627565,
|
|
"grad_norm": 9.414826271664367,
|
|
"learning_rate": 4.375724871493365e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2373,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"epoch": 0.30772703333082535,
|
|
"grad_norm": 8.162028844684041,
|
|
"learning_rate": 4.371377350353134e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.229,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"epoch": 0.30847942216537505,
|
|
"grad_norm": 7.801467771167628,
|
|
"learning_rate": 4.3670169192235586e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2394,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.30923181099992475,
|
|
"grad_norm": 10.277704427965585,
|
|
"learning_rate": 4.362643608185757e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2344,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"epoch": 0.30998419983447445,
|
|
"grad_norm": 13.33467825685964,
|
|
"learning_rate": 4.358257447409698e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"epoch": 0.31073658866902415,
|
|
"grad_norm": 8.16160167838467,
|
|
"learning_rate": 4.353858467153998e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2297,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"epoch": 0.31148897750357385,
|
|
"grad_norm": 7.879945152353835,
|
|
"learning_rate": 4.349446697765711e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.227,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"epoch": 0.31224136633812355,
|
|
"grad_norm": 8.616338325133528,
|
|
"learning_rate": 4.345022169680117e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.31299375517267325,
|
|
"grad_norm": 9.359562218439928,
|
|
"learning_rate": 4.3405849134205164e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2257,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.4375,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"epoch": 0.31374614400722295,
|
|
"grad_norm": 8.969523285372976,
|
|
"learning_rate": 4.3361349595980136e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.2182,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"epoch": 0.31449853284177265,
|
|
"grad_norm": 10.138740882360356,
|
|
"learning_rate": 4.331672338911313e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2201,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"epoch": 0.31525092167632235,
|
|
"grad_norm": 9.223656893908142,
|
|
"learning_rate": 4.3271970821465e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2257,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"epoch": 0.316003310510872,
|
|
"grad_norm": 11.12549439598949,
|
|
"learning_rate": 4.3227092201768346e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2265,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.3167556993454217,
|
|
"grad_norm": 9.207865303317924,
|
|
"learning_rate": 4.3182087839625326e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2244,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"epoch": 0.3175080881799714,
|
|
"grad_norm": 8.426670829083685,
|
|
"learning_rate": 4.313695804550559e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2203,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"epoch": 0.3182604770145211,
|
|
"grad_norm": 9.764940312491921,
|
|
"learning_rate": 4.309170313074408e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2608,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"epoch": 0.3190128658490708,
|
|
"grad_norm": 7.952069517630966,
|
|
"learning_rate": 4.304632340753889e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2279,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"epoch": 0.3197652546836205,
|
|
"grad_norm": 8.83343001948812,
|
|
"learning_rate": 4.3000819188949145e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2347,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.3205176435181702,
|
|
"grad_norm": 8.597957828748827,
|
|
"learning_rate": 4.2955190788892827e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"epoch": 0.3212700323527199,
|
|
"grad_norm": 7.6696314669829775,
|
|
"learning_rate": 4.2909438522144566e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2319,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"epoch": 0.3220224211872696,
|
|
"grad_norm": 10.007443872180758,
|
|
"learning_rate": 4.2863562704333545e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2052,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"epoch": 0.3227748100218193,
|
|
"grad_norm": 11.118831248083465,
|
|
"learning_rate": 4.2817563651941263e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2277,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"epoch": 0.323527198856369,
|
|
"grad_norm": 10.345024663416677,
|
|
"learning_rate": 4.277144168229939e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2453,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.3242795876909187,
|
|
"grad_norm": 7.051363685111973,
|
|
"learning_rate": 4.272519711358753e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"epoch": 0.32503197652546834,
|
|
"grad_norm": 9.476688840579149,
|
|
"learning_rate": 4.2678830264831077e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2251,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"epoch": 0.32578436536001804,
|
|
"grad_norm": 7.95698757416428,
|
|
"learning_rate": 4.2632341455899e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2135,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"epoch": 0.32653675419456774,
|
|
"grad_norm": 6.006840590992643,
|
|
"learning_rate": 4.25857310075016e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2293,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"epoch": 0.32728914302911744,
|
|
"grad_norm": 10.148748285674634,
|
|
"learning_rate": 4.2538999241188374e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.1906,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.32804153186366714,
|
|
"grad_norm": 11.129120168982654,
|
|
"learning_rate": 4.2492146479345693e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.22,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"epoch": 0.32879392069821683,
|
|
"grad_norm": 9.910453441221708,
|
|
"learning_rate": 4.244517304519467e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2234,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"epoch": 0.32954630953276653,
|
|
"grad_norm": 11.052178846558816,
|
|
"learning_rate": 4.2398079262788893e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.222,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"epoch": 0.33029869836731623,
|
|
"grad_norm": 9.642152534654413,
|
|
"learning_rate": 4.2350865457012184e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2332,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"epoch": 0.33105108720186593,
|
|
"grad_norm": 9.60330425948754,
|
|
"learning_rate": 4.2303531953576366e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2225,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.33180347603641563,
|
|
"grad_norm": 7.6600645560452785,
|
|
"learning_rate": 4.2256079079019015e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2416,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"epoch": 0.33255586487096533,
|
|
"grad_norm": 7.850742770509769,
|
|
"learning_rate": 4.220850716070121e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2238,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.75,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"epoch": 0.33330825370551503,
|
|
"grad_norm": 7.909302118744176,
|
|
"learning_rate": 4.2160816526805267e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2501,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"epoch": 0.3340606425400647,
|
|
"grad_norm": 8.94738188436282,
|
|
"learning_rate": 4.211300750633248e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"epoch": 0.3348130313746144,
|
|
"grad_norm": 8.676073044947644,
|
|
"learning_rate": 4.2065080429100865e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2225,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.625,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.3355654202091641,
|
|
"grad_norm": 9.806473078557882,
|
|
"learning_rate": 4.2017035625742846e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2098,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"epoch": 0.3363178090437138,
|
|
"grad_norm": 7.522444333584003,
|
|
"learning_rate": 4.196887342770302e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2095,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"epoch": 0.3370701978782635,
|
|
"grad_norm": 8.728501659564188,
|
|
"learning_rate": 4.1920594167235845e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.234,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"epoch": 0.3378225867128132,
|
|
"grad_norm": 9.465075398652212,
|
|
"learning_rate": 4.187219817740336e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.390625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2285,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"epoch": 0.3385749755473629,
|
|
"grad_norm": 7.109319264397478,
|
|
"learning_rate": 4.182368579207285e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.3393273643819126,
|
|
"grad_norm": 10.507873292947329,
|
|
"learning_rate": 4.177505734591461e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.75,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"epoch": 0.3400797532164623,
|
|
"grad_norm": 7.3959641126703435,
|
|
"learning_rate": 4.172631317439956e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2317,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"epoch": 0.340832142051012,
|
|
"grad_norm": 8.87302820276429,
|
|
"learning_rate": 4.167745361379702e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2207,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"epoch": 0.3415845308855617,
|
|
"grad_norm": 9.758571350415783,
|
|
"learning_rate": 4.162847900117229e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2177,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.25,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"epoch": 0.3423369197201114,
|
|
"grad_norm": 8.896439051201618,
|
|
"learning_rate": 4.1579389674384394e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.234375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2145,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.3430893085546611,
|
|
"grad_norm": 7.711661820543145,
|
|
"learning_rate": 4.153018597208374e-07,
|
|
"logits/chosen": -2.640625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2203,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"epoch": 0.3438416973892107,
|
|
"grad_norm": 8.21240547755215,
|
|
"learning_rate": 4.1480868233709765e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.219,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"epoch": 0.3445940862237604,
|
|
"grad_norm": 11.90114190092608,
|
|
"learning_rate": 4.1431436799488606e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2127,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"epoch": 0.3453464750583101,
|
|
"grad_norm": 6.8696936160082265,
|
|
"learning_rate": 4.1381892010430717e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2498,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"epoch": 0.3460988638928598,
|
|
"grad_norm": 10.171026913882173,
|
|
"learning_rate": 4.13322342083286e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2332,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.3468512527274095,
|
|
"grad_norm": 8.63453504033253,
|
|
"learning_rate": 4.1282463735754356e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2289,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"epoch": 0.3476036415619592,
|
|
"grad_norm": 9.580783720183787,
|
|
"learning_rate": 4.123258093605739e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2302,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"epoch": 0.3483560303965089,
|
|
"grad_norm": 7.120063150875206,
|
|
"learning_rate": 4.118258615336199e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2366,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"epoch": 0.3491084192310586,
|
|
"grad_norm": 5.827687137113672,
|
|
"learning_rate": 4.1132479732564964e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2027,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"epoch": 0.3498608080656083,
|
|
"grad_norm": 10.67088810537178,
|
|
"learning_rate": 4.1082262019333315e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2132,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.350613196900158,
|
|
"grad_norm": 10.094322081691317,
|
|
"learning_rate": 4.103193336010179e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.215,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"epoch": 0.3513655857347077,
|
|
"grad_norm": 8.374810404276547,
|
|
"learning_rate": 4.098149410207051e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"epoch": 0.3521179745692574,
|
|
"grad_norm": 8.392131172676132,
|
|
"learning_rate": 4.09309445932026e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.226,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"epoch": 0.35287036340380706,
|
|
"grad_norm": 9.468666629268613,
|
|
"learning_rate": 4.088028518222173e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2211,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"epoch": 0.35362275223835676,
|
|
"grad_norm": 9.71697535829911,
|
|
"learning_rate": 4.0829516218609785e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2035,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.35437514107290646,
|
|
"grad_norm": 8.282815293843191,
|
|
"learning_rate": 4.077863805260439e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2176,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"epoch": 0.35512752990745616,
|
|
"grad_norm": 7.901474744472959,
|
|
"learning_rate": 4.0727651035196545e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"epoch": 0.35587991874200586,
|
|
"grad_norm": 10.371105232687958,
|
|
"learning_rate": 4.0676555518128157e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2551,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"epoch": 0.35663230757655556,
|
|
"grad_norm": 7.284751062931884,
|
|
"learning_rate": 4.062535185388964e-07,
|
|
"logits/chosen": -2.609375,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2446,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.375,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"epoch": 0.35738469641110526,
|
|
"grad_norm": 7.798212343706078,
|
|
"learning_rate": 4.0574040395717493e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.199,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.35813708524565496,
|
|
"grad_norm": 7.738938259994802,
|
|
"learning_rate": 4.0522621497591813e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2369,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"epoch": 0.35888947408020466,
|
|
"grad_norm": 8.30773676851323,
|
|
"learning_rate": 4.0471095514233933e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2093,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"epoch": 0.35964186291475436,
|
|
"grad_norm": 8.691676803479753,
|
|
"learning_rate": 4.041946280110389e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2141,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"epoch": 0.36039425174930406,
|
|
"grad_norm": 8.607549823897239,
|
|
"learning_rate": 4.0367723714398047e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.46875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1978,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"epoch": 0.36114664058385376,
|
|
"grad_norm": 8.771643431392596,
|
|
"learning_rate": 4.031587861104657e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2288,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.3618990294184034,
|
|
"grad_norm": 7.347767311708758,
|
|
"learning_rate": 4.0263927848711026e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2333,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"epoch": 0.3626514182529531,
|
|
"grad_norm": 8.949761500130862,
|
|
"learning_rate": 4.021187178578187e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2175,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"epoch": 0.3634038070875028,
|
|
"grad_norm": 10.64662225641983,
|
|
"learning_rate": 4.0159710781375994e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2273,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"epoch": 0.3641561959220525,
|
|
"grad_norm": 10.534790254504463,
|
|
"learning_rate": 4.0107445195334244e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"epoch": 0.3649085847566022,
|
|
"grad_norm": 7.871168672929195,
|
|
"learning_rate": 4.005507538821894e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.1875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2123,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"epoch": 0.3656609735911519,
|
|
"grad_norm": 8.593711676069107,
|
|
"learning_rate": 4.0002601721311393e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1999,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"epoch": 0.3664133624257016,
|
|
"grad_norm": 8.635194837292705,
|
|
"learning_rate": 3.995002455660939e-07,
|
|
"logits/chosen": -2.671875,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2006,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"epoch": 0.3671657512602513,
|
|
"grad_norm": 9.553334243561997,
|
|
"learning_rate": 3.989734425682473e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.140625,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2232,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"epoch": 0.367918140094801,
|
|
"grad_norm": 9.264445545978198,
|
|
"learning_rate": 3.984456118538071e-07,
|
|
"logits/chosen": -2.5625,
|
|
"logits/rejected": -2.359375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2264,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"epoch": 0.3686705289293507,
|
|
"grad_norm": 7.532750216761137,
|
|
"learning_rate": 3.9791675706409593e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"epoch": 0.3694229177639004,
|
|
"grad_norm": 6.92523330567057,
|
|
"learning_rate": 3.9738688184750125e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"epoch": 0.3701753065984501,
|
|
"grad_norm": 10.657909024859954,
|
|
"learning_rate": 3.968559898594502e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.28125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2223,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"epoch": 0.37092769543299975,
|
|
"grad_norm": 6.720765504539305,
|
|
"learning_rate": 3.9632408476238416e-07,
|
|
"logits/chosen": -2.59375,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2112,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"epoch": 0.37168008426754945,
|
|
"grad_norm": 7.963706047021695,
|
|
"learning_rate": 3.957911702257337e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2229,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"epoch": 0.37243247310209915,
|
|
"grad_norm": 9.314200881993933,
|
|
"learning_rate": 3.952572499258929e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2239,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"epoch": 0.37318486193664885,
|
|
"grad_norm": 8.115661280449446,
|
|
"learning_rate": 3.9472232754619474e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"epoch": 0.37393725077119855,
|
|
"grad_norm": 7.013722897295247,
|
|
"learning_rate": 3.9418640677688464e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.3125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2489,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"epoch": 0.37468963960574825,
|
|
"grad_norm": 9.184692139331514,
|
|
"learning_rate": 3.936494913150961e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.328125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.198,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"epoch": 0.37544202844029795,
|
|
"grad_norm": 9.392069646399577,
|
|
"learning_rate": 3.931115848648242e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2036,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"epoch": 0.37619441727484765,
|
|
"grad_norm": 9.776594081515965,
|
|
"learning_rate": 3.925726911369008e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.37619441727484765,
|
|
"eval_logits/chosen": -2.84375,
|
|
"eval_logits/rejected": -2.5625,
|
|
"eval_logps/chosen": -688.0,
|
|
"eval_logps/rejected": -1032.0,
|
|
"eval_loss": 0.25248709321022034,
|
|
"eval_rewards/accuracies": 0.888202428817749,
|
|
"eval_rewards/chosen": -5.21875,
|
|
"eval_rewards/margins": 3.578125,
|
|
"eval_rewards/rejected": -8.8125,
|
|
"eval_runtime": 3417.1912,
|
|
"eval_samples_per_second": 27.652,
|
|
"eval_steps_per_second": 0.432,
|
|
"step": 5000
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 13291,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 5000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|