11779 lines
347 KiB
JSON
11779 lines
347 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9999360981532366,
|
|
"eval_steps": 10000,
|
|
"global_step": 7824,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.00012780369352674293,
|
|
"grad_norm": 1.583091404664238,
|
|
"learning_rate": 6.385696040868454e-10,
|
|
"logits/chosen": -2.140625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -141.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.0012780369352674292,
|
|
"grad_norm": 1.831193607031671,
|
|
"learning_rate": 6.3856960408684546e-09,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -1.9375,
|
|
"logps/chosen": -156.0,
|
|
"logps/rejected": -136.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.2361111044883728,
|
|
"rewards/chosen": 8.869171142578125e-05,
|
|
"rewards/margins": 8.726119995117188e-05,
|
|
"rewards/rejected": -4.246830940246582e-07,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0025560738705348585,
|
|
"grad_norm": 1.9051082450552785,
|
|
"learning_rate": 1.2771392081736909e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -167.0,
|
|
"logps/rejected": -134.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.33125001192092896,
|
|
"rewards/chosen": -0.00012493133544921875,
|
|
"rewards/margins": -0.000560760498046875,
|
|
"rewards/rejected": 0.000438690185546875,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.0038341108058022877,
|
|
"grad_norm": 2.0951178941868793,
|
|
"learning_rate": 1.915708812260536e-08,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": -0.00020313262939453125,
|
|
"rewards/margins": 0.000377655029296875,
|
|
"rewards/rejected": -0.000579833984375,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.005112147741069717,
|
|
"grad_norm": 1.8582175367673581,
|
|
"learning_rate": 2.5542784163473818e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -133.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": -0.00213623046875,
|
|
"rewards/margins": -0.0004711151123046875,
|
|
"rewards/rejected": -0.00165557861328125,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.006390184676337146,
|
|
"grad_norm": 1.8191369002557825,
|
|
"learning_rate": 3.192848020434227e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -135.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": 6.29425048828125e-05,
|
|
"rewards/margins": -0.0002498626708984375,
|
|
"rewards/rejected": 0.0003147125244140625,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.007668221611604575,
|
|
"grad_norm": 1.8104389933900382,
|
|
"learning_rate": 3.831417624521072e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -126.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": -7.82012939453125e-05,
|
|
"rewards/margins": -0.00019168853759765625,
|
|
"rewards/rejected": 0.00010967254638671875,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.008946258546872005,
|
|
"grad_norm": 1.9080659587821094,
|
|
"learning_rate": 4.469987228607918e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -164.0,
|
|
"logps/rejected": -141.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": 0.000797271728515625,
|
|
"rewards/margins": 0.0012969970703125,
|
|
"rewards/rejected": -0.000499725341796875,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.010224295482139434,
|
|
"grad_norm": 1.7284420631380606,
|
|
"learning_rate": 5.1085568326947637e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -135.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.42500001192092896,
|
|
"rewards/chosen": 0.0013580322265625,
|
|
"rewards/margins": 0.0010833740234375,
|
|
"rewards/rejected": 0.00028228759765625,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.011502332417406863,
|
|
"grad_norm": 1.9985804873605706,
|
|
"learning_rate": 5.747126436781609e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 7.867813110351562e-05,
|
|
"rewards/margins": 0.0002193450927734375,
|
|
"rewards/rejected": -0.00014019012451171875,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.012780369352674292,
|
|
"grad_norm": 1.7673732788906307,
|
|
"learning_rate": 6.385696040868454e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -1.9296875,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -138.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": -0.000423431396484375,
|
|
"rewards/margins": -0.0003070831298828125,
|
|
"rewards/rejected": -0.00011539459228515625,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.014058406287941722,
|
|
"grad_norm": 1.8348538098786262,
|
|
"learning_rate": 7.024265644955301e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -1.96875,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -132.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.4124999940395355,
|
|
"rewards/chosen": 0.001129150390625,
|
|
"rewards/margins": 0.000797271728515625,
|
|
"rewards/rejected": 0.00032806396484375,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.01533644322320915,
|
|
"grad_norm": 2.0218021664474373,
|
|
"learning_rate": 7.662835249042144e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -1.859375,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6918,
|
|
"rewards/accuracies": 0.48124998807907104,
|
|
"rewards/chosen": 0.002960205078125,
|
|
"rewards/margins": 0.0024261474609375,
|
|
"rewards/rejected": 0.000530242919921875,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.01661448015847658,
|
|
"grad_norm": 1.82922758998825,
|
|
"learning_rate": 8.301404853128991e-08,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -144.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6915,
|
|
"rewards/accuracies": 0.543749988079071,
|
|
"rewards/chosen": 0.006317138671875,
|
|
"rewards/margins": 0.00396728515625,
|
|
"rewards/rejected": 0.002349853515625,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.01789251709374401,
|
|
"grad_norm": 1.7861618345539088,
|
|
"learning_rate": 8.939974457215836e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -141.0,
|
|
"loss": 0.6909,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.0064697265625,
|
|
"rewards/margins": 0.0040283203125,
|
|
"rewards/rejected": 0.0024261474609375,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.01917055402901144,
|
|
"grad_norm": 1.8958781010879984,
|
|
"learning_rate": 9.578544061302682e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -132.0,
|
|
"loss": 0.6903,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.0076904296875,
|
|
"rewards/margins": 0.0042724609375,
|
|
"rewards/rejected": 0.00341796875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.020448590964278868,
|
|
"grad_norm": 1.689453237922255,
|
|
"learning_rate": 1.0217113665389527e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6903,
|
|
"rewards/accuracies": 0.4937500059604645,
|
|
"rewards/chosen": 0.005950927734375,
|
|
"rewards/margins": 0.0031585693359375,
|
|
"rewards/rejected": 0.0027923583984375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.021726627899546297,
|
|
"grad_norm": 1.6908724447283379,
|
|
"learning_rate": 1.0855683269476372e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -1.890625,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6894,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": 0.01123046875,
|
|
"rewards/margins": 0.00665283203125,
|
|
"rewards/rejected": 0.004608154296875,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.023004664834813726,
|
|
"grad_norm": 1.9217126730360887,
|
|
"learning_rate": 1.1494252873563217e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -1.8828125,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -146.0,
|
|
"loss": 0.6875,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.01275634765625,
|
|
"rewards/margins": 0.00927734375,
|
|
"rewards/rejected": 0.003509521484375,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.024282701770081155,
|
|
"grad_norm": 1.7796346205464415,
|
|
"learning_rate": 1.2132822477650062e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6877,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": 0.016845703125,
|
|
"rewards/margins": 0.00970458984375,
|
|
"rewards/rejected": 0.007080078125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.025560738705348585,
|
|
"grad_norm": 1.8062803830724468,
|
|
"learning_rate": 1.277139208173691e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -128.0,
|
|
"logps/rejected": -117.0,
|
|
"loss": 0.6868,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": 0.0159912109375,
|
|
"rewards/margins": 0.010986328125,
|
|
"rewards/rejected": 0.0050048828125,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.026838775640616014,
|
|
"grad_norm": 1.8109623854322845,
|
|
"learning_rate": 1.3409961685823753e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -135.0,
|
|
"loss": 0.6859,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": 0.0263671875,
|
|
"rewards/margins": 0.01507568359375,
|
|
"rewards/rejected": 0.01129150390625,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.028116812575883443,
|
|
"grad_norm": 1.7555569281404872,
|
|
"learning_rate": 1.4048531289910602e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -1.9609375,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6846,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 0.0260009765625,
|
|
"rewards/margins": 0.0228271484375,
|
|
"rewards/rejected": 0.003173828125,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.029394849511150872,
|
|
"grad_norm": 1.6784346166574386,
|
|
"learning_rate": 1.4687100893997445e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -156.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6833,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 0.024658203125,
|
|
"rewards/margins": 0.0225830078125,
|
|
"rewards/rejected": 0.002105712890625,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.0306728864464183,
|
|
"grad_norm": 2.017021630939051,
|
|
"learning_rate": 1.532567049808429e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6819,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.0257568359375,
|
|
"rewards/margins": 0.0235595703125,
|
|
"rewards/rejected": 0.0020751953125,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.03195092338168573,
|
|
"grad_norm": 1.7084660802314153,
|
|
"learning_rate": 1.5964240102171135e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6796,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 0.0361328125,
|
|
"rewards/margins": 0.0294189453125,
|
|
"rewards/rejected": 0.006744384765625,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.03322896031695316,
|
|
"grad_norm": 1.8365627743934905,
|
|
"learning_rate": 1.6602809706257982e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6764,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 0.032470703125,
|
|
"rewards/margins": 0.036865234375,
|
|
"rewards/rejected": -0.004364013671875,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.03450699725222059,
|
|
"grad_norm": 1.7335796846774567,
|
|
"learning_rate": 1.7241379310344828e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -1.9765625,
|
|
"logps/chosen": -158.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6742,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": 0.028564453125,
|
|
"rewards/margins": 0.046630859375,
|
|
"rewards/rejected": -0.0179443359375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.03578503418748802,
|
|
"grad_norm": 1.6763039372574249,
|
|
"learning_rate": 1.7879948914431672e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6716,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": 0.03759765625,
|
|
"rewards/margins": 0.05615234375,
|
|
"rewards/rejected": -0.0184326171875,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.03706307112275545,
|
|
"grad_norm": 1.7476768718236924,
|
|
"learning_rate": 1.8518518518518516e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -131.0,
|
|
"loss": 0.6693,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 0.0230712890625,
|
|
"rewards/margins": 0.03955078125,
|
|
"rewards/rejected": -0.016357421875,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.03834110805802288,
|
|
"grad_norm": 1.9904155949768438,
|
|
"learning_rate": 1.9157088122605365e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6654,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": 0.01300048828125,
|
|
"rewards/margins": 0.05810546875,
|
|
"rewards/rejected": -0.045166015625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.039619144993290306,
|
|
"grad_norm": 2.1919340946635275,
|
|
"learning_rate": 1.9795657726692208e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6579,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": 0.00860595703125,
|
|
"rewards/margins": 0.08154296875,
|
|
"rewards/rejected": -0.07275390625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.040897181928557735,
|
|
"grad_norm": 2.050188028710248,
|
|
"learning_rate": 2.0434227330779055e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.654,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 0.0015411376953125,
|
|
"rewards/margins": 0.08837890625,
|
|
"rewards/rejected": -0.0869140625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.042175218863825165,
|
|
"grad_norm": 2.016463993421717,
|
|
"learning_rate": 2.1072796934865898e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -164.0,
|
|
"logps/rejected": -165.0,
|
|
"loss": 0.6475,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.03125,
|
|
"rewards/margins": 0.10693359375,
|
|
"rewards/rejected": -0.1376953125,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.043453255799092594,
|
|
"grad_norm": 2.4323566534644714,
|
|
"learning_rate": 2.1711366538952745e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.171875,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6386,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -0.05078125,
|
|
"rewards/margins": 0.1279296875,
|
|
"rewards/rejected": -0.177734375,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.04473129273436002,
|
|
"grad_norm": 2.202903045169722,
|
|
"learning_rate": 2.234993614303959e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6285,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -0.0888671875,
|
|
"rewards/margins": 0.1650390625,
|
|
"rewards/rejected": -0.25390625,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.04600932966962745,
|
|
"grad_norm": 2.206602901584152,
|
|
"learning_rate": 2.2988505747126435e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -181.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6227,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.1357421875,
|
|
"rewards/margins": 0.1533203125,
|
|
"rewards/rejected": -0.2890625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.04728736660489488,
|
|
"grad_norm": 2.077125523450064,
|
|
"learning_rate": 2.3627075351213284e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -181.0,
|
|
"logps/rejected": -185.0,
|
|
"loss": 0.6089,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.1953125,
|
|
"rewards/margins": 0.2177734375,
|
|
"rewards/rejected": -0.4140625,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.04856540354016231,
|
|
"grad_norm": 2.286616738197532,
|
|
"learning_rate": 2.4265644955300125e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -192.0,
|
|
"loss": 0.5919,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -0.2578125,
|
|
"rewards/margins": 0.2734375,
|
|
"rewards/rejected": -0.53125,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.04984344047542974,
|
|
"grad_norm": 2.3378312139893755,
|
|
"learning_rate": 2.490421455938697e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -205.0,
|
|
"logps/rejected": -184.0,
|
|
"loss": 0.5987,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -0.38671875,
|
|
"rewards/margins": 0.19921875,
|
|
"rewards/rejected": -0.5859375,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.05112147741069717,
|
|
"grad_norm": 2.0376352660492505,
|
|
"learning_rate": 2.554278416347382e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -205.0,
|
|
"logps/rejected": -225.0,
|
|
"loss": 0.5894,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.423828125,
|
|
"rewards/margins": 0.25,
|
|
"rewards/rejected": -0.671875,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.0523995143459646,
|
|
"grad_norm": 2.198540810606561,
|
|
"learning_rate": 2.6181353767560667e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -228.0,
|
|
"loss": 0.5725,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.423828125,
|
|
"rewards/margins": 0.341796875,
|
|
"rewards/rejected": -0.765625,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.05367755128123203,
|
|
"grad_norm": 2.5064207498697306,
|
|
"learning_rate": 2.6819923371647505e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -231.0,
|
|
"loss": 0.5589,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.45703125,
|
|
"rewards/margins": 0.408203125,
|
|
"rewards/rejected": -0.86328125,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.05495558821649946,
|
|
"grad_norm": 2.7332223174524604,
|
|
"learning_rate": 2.7458492975734354e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -207.0,
|
|
"logps/rejected": -217.0,
|
|
"loss": 0.5532,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.53515625,
|
|
"rewards/margins": 0.384765625,
|
|
"rewards/rejected": -0.91796875,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.056233625151766886,
|
|
"grad_norm": 3.160820809244754,
|
|
"learning_rate": 2.8097062579821203e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -224.0,
|
|
"logps/rejected": -276.0,
|
|
"loss": 0.5351,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.63671875,
|
|
"rewards/margins": 0.609375,
|
|
"rewards/rejected": -1.25,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.057511662087034315,
|
|
"grad_norm": 3.1305847824157005,
|
|
"learning_rate": 2.873563218390804e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -225.0,
|
|
"logps/rejected": -266.0,
|
|
"loss": 0.5226,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.69921875,
|
|
"rewards/margins": 0.5859375,
|
|
"rewards/rejected": -1.2890625,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.058789699022301745,
|
|
"grad_norm": 3.2458132009733975,
|
|
"learning_rate": 2.937420178799489e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -212.0,
|
|
"logps/rejected": -284.0,
|
|
"loss": 0.5138,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.734375,
|
|
"rewards/margins": 0.7421875,
|
|
"rewards/rejected": -1.4765625,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.060067735957569174,
|
|
"grad_norm": 3.590934206396528,
|
|
"learning_rate": 3.001277139208174e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -227.0,
|
|
"logps/rejected": -286.0,
|
|
"loss": 0.5261,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.78515625,
|
|
"rewards/margins": 0.65234375,
|
|
"rewards/rejected": -1.4375,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.0613457728928366,
|
|
"grad_norm": 4.090558879807313,
|
|
"learning_rate": 3.065134099616858e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -241.0,
|
|
"logps/rejected": -296.0,
|
|
"loss": 0.5063,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.71484375,
|
|
"rewards/margins": 0.80078125,
|
|
"rewards/rejected": -1.515625,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.06262380982810403,
|
|
"grad_norm": 3.738095666639755,
|
|
"learning_rate": 3.1289910600255427e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -232.0,
|
|
"logps/rejected": -306.0,
|
|
"loss": 0.5051,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -0.78125,
|
|
"rewards/margins": 0.87890625,
|
|
"rewards/rejected": -1.6640625,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.06390184676337146,
|
|
"grad_norm": 4.03887506227773,
|
|
"learning_rate": 3.192848020434227e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -227.0,
|
|
"logps/rejected": -286.0,
|
|
"loss": 0.5208,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.8046875,
|
|
"rewards/margins": 0.63671875,
|
|
"rewards/rejected": -1.4453125,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.06517988369863889,
|
|
"grad_norm": 3.7784530781629444,
|
|
"learning_rate": 3.2567049808429114e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -240.0,
|
|
"logps/rejected": -296.0,
|
|
"loss": 0.4996,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.8515625,
|
|
"rewards/margins": 0.80078125,
|
|
"rewards/rejected": -1.6484375,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.06645792063390632,
|
|
"grad_norm": 4.907395725764965,
|
|
"learning_rate": 3.3205619412515963e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -236.0,
|
|
"logps/rejected": -312.0,
|
|
"loss": 0.4853,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.9765625,
|
|
"rewards/margins": 0.734375,
|
|
"rewards/rejected": -1.7109375,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.06773595756917375,
|
|
"grad_norm": 6.328166419969473,
|
|
"learning_rate": 3.3844189016602807e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -344.0,
|
|
"loss": 0.4789,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.1015625,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -2.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.06901399450444118,
|
|
"grad_norm": 7.41169890134317,
|
|
"learning_rate": 3.4482758620689656e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -264.0,
|
|
"logps/rejected": -350.0,
|
|
"loss": 0.4664,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.9921875,
|
|
"rewards/margins": 0.97265625,
|
|
"rewards/rejected": -1.96875,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.07029203143970861,
|
|
"grad_norm": 6.329307259348903,
|
|
"learning_rate": 3.51213282247765e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -242.0,
|
|
"logps/rejected": -334.0,
|
|
"loss": 0.4756,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -0.97265625,
|
|
"rewards/margins": 0.92578125,
|
|
"rewards/rejected": -1.8984375,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.07157006837497604,
|
|
"grad_norm": 5.5382587319010605,
|
|
"learning_rate": 3.5759897828863344e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.4816,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.21875,
|
|
"rewards/margins": 0.94140625,
|
|
"rewards/rejected": -2.15625,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.07284810531024347,
|
|
"grad_norm": 5.59071348439989,
|
|
"learning_rate": 3.639846743295019e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -336.0,
|
|
"loss": 0.4814,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.1953125,
|
|
"rewards/margins": 0.87109375,
|
|
"rewards/rejected": -2.0625,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.0741261422455109,
|
|
"grad_norm": 7.500668748088703,
|
|
"learning_rate": 3.703703703703703e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -278.0,
|
|
"logps/rejected": -346.0,
|
|
"loss": 0.4581,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 0.87890625,
|
|
"rewards/rejected": -2.125,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.07540417918077832,
|
|
"grad_norm": 7.815381329656392,
|
|
"learning_rate": 3.767560664112388e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -366.0,
|
|
"loss": 0.4516,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.15625,
|
|
"rewards/margins": 0.953125,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.07668221611604575,
|
|
"grad_norm": 7.560002445698683,
|
|
"learning_rate": 3.831417624521073e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -414.0,
|
|
"loss": 0.4415,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.1796875,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -2.5,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.07796025305131318,
|
|
"grad_norm": 7.597044745493579,
|
|
"learning_rate": 3.895274584929757e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -404.0,
|
|
"loss": 0.4573,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.28125,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.453125,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.07923828998658061,
|
|
"grad_norm": 6.0294238622536085,
|
|
"learning_rate": 3.9591315453384417e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.433,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.171875,
|
|
"rewards/margins": 0.93359375,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.08051632692184804,
|
|
"grad_norm": 6.57580363708218,
|
|
"learning_rate": 4.0229885057471266e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -370.0,
|
|
"loss": 0.4397,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.34375,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -2.40625,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.08179436385711547,
|
|
"grad_norm": 8.126812055550822,
|
|
"learning_rate": 4.086845466155811e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -396.0,
|
|
"loss": 0.4337,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.2421875,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -2.390625,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.0830724007923829,
|
|
"grad_norm": 7.662460966992449,
|
|
"learning_rate": 4.1507024265644953e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -436.0,
|
|
"loss": 0.4231,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.4140625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.08435043772765033,
|
|
"grad_norm": 6.26765621154057,
|
|
"learning_rate": 4.2145593869731797e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -368.0,
|
|
"loss": 0.4428,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.1796875,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.34375,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.08562847466291776,
|
|
"grad_norm": 10.966780568835159,
|
|
"learning_rate": 4.2784163473818646e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -384.0,
|
|
"loss": 0.4744,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -1.1484375,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.328125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.08690651159818519,
|
|
"grad_norm": 7.995403962637887,
|
|
"learning_rate": 4.342273307790549e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -406.0,
|
|
"loss": 0.446,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.328125,
|
|
"rewards/margins": 1.1328125,
|
|
"rewards/rejected": -2.46875,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.08818454853345262,
|
|
"grad_norm": 7.767114033697263,
|
|
"learning_rate": 4.4061302681992333e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -386.0,
|
|
"loss": 0.4252,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.359375,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -2.453125,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.08946258546872005,
|
|
"grad_norm": 10.782647668236367,
|
|
"learning_rate": 4.469987228607918e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -266.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.4396,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.1484375,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -2.328125,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.09074062240398748,
|
|
"grad_norm": 13.262314383907762,
|
|
"learning_rate": 4.5338441890166026e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -296.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.412,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.4140625,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -2.6875,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.0920186593392549,
|
|
"grad_norm": 8.452988029757332,
|
|
"learning_rate": 4.597701149425287e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -428.0,
|
|
"loss": 0.4053,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.390625,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -2.8125,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.09329669627452233,
|
|
"grad_norm": 10.451528356808828,
|
|
"learning_rate": 4.661558109833972e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -418.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -1.3671875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.09457473320978976,
|
|
"grad_norm": 11.00358677164108,
|
|
"learning_rate": 4.725415070242657e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -432.0,
|
|
"loss": 0.387,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.515625,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.09585277014505719,
|
|
"grad_norm": 13.575988451351572,
|
|
"learning_rate": 4.789272030651341e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.4079,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.4609375,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -2.78125,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.09713080708032462,
|
|
"grad_norm": 9.325954891766974,
|
|
"learning_rate": 4.853128991060025e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.4176,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.34375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -2.90625,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.09840884401559205,
|
|
"grad_norm": 7.52371689034163,
|
|
"learning_rate": 4.91698595146871e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -330.0,
|
|
"logps/rejected": -448.0,
|
|
"loss": 0.4036,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -1.7265625,
|
|
"rewards/margins": 1.21875,
|
|
"rewards/rejected": -2.953125,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.09968688095085948,
|
|
"grad_norm": 7.847386893700331,
|
|
"learning_rate": 4.980842911877394e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.374,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.5,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -3.03125,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.10096491788612691,
|
|
"grad_norm": 8.270767140969047,
|
|
"learning_rate": 4.999987806263758e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -418.0,
|
|
"loss": 0.4074,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.390625,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -2.734375,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.10224295482139434,
|
|
"grad_norm": 9.313330271911113,
|
|
"learning_rate": 4.999928082127696e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -446.0,
|
|
"loss": 0.4058,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.484375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -3.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.10352099175666177,
|
|
"grad_norm": 10.98658746507854,
|
|
"learning_rate": 4.999818589113488e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -398.0,
|
|
"loss": 0.3954,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -2.65625,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.1047990286919292,
|
|
"grad_norm": 10.59961166689991,
|
|
"learning_rate": 4.99965932940093e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.3963,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.10607706562719663,
|
|
"grad_norm": 13.077749744872387,
|
|
"learning_rate": 4.999450306160585e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.5546875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -3.390625,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.10735510256246406,
|
|
"grad_norm": 10.551892493562612,
|
|
"learning_rate": 4.999191523553715e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -304.0,
|
|
"logps/rejected": -468.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.3984375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -3.125,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.10863313949773148,
|
|
"grad_norm": 12.356141382859514,
|
|
"learning_rate": 4.998882986732195e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -482.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.5390625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -3.109375,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.10991117643299891,
|
|
"grad_norm": 15.20620984625719,
|
|
"learning_rate": 4.998524701838414e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -498.0,
|
|
"loss": 0.3599,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.515625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.11118921336826634,
|
|
"grad_norm": 14.444326022124622,
|
|
"learning_rate": 4.998116676005154e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.4075,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -3.171875,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.11246725030353377,
|
|
"grad_norm": 7.828143338550613,
|
|
"learning_rate": 4.997658917355441e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -500.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.1137452872388012,
|
|
"grad_norm": 18.788227330963323,
|
|
"learning_rate": 4.997151435002394e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.3749,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -3.234375,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.11502332417406863,
|
|
"grad_norm": 11.597065629717148,
|
|
"learning_rate": 4.996594239049032e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.4765625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.11630136110933606,
|
|
"grad_norm": 10.476293056540518,
|
|
"learning_rate": 4.995987340588082e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -486.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.53125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -3.3125,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.11757939804460349,
|
|
"grad_norm": 9.355309673884802,
|
|
"learning_rate": 4.995330751701755e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -406.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.359375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.11885743497987092,
|
|
"grad_norm": 10.550972596004549,
|
|
"learning_rate": 4.994624485461503e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.5859375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -3.390625,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.12013547191513835,
|
|
"grad_norm": 7.584312388304975,
|
|
"learning_rate": 4.99386855592776e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3669,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.46875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.12141350885040578,
|
|
"grad_norm": 14.923871887916482,
|
|
"learning_rate": 4.993062978149668e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -508.0,
|
|
"loss": 0.3733,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.4921875,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.1226915457856732,
|
|
"grad_norm": 12.368485690167017,
|
|
"learning_rate": 4.992207768164769e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -396.0,
|
|
"loss": 0.4143,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.3125,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -2.609375,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.12396958272094064,
|
|
"grad_norm": 12.872066642334858,
|
|
"learning_rate": 4.991302942998686e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.3478,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.71875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.12524761965620806,
|
|
"grad_norm": 8.76843030472801,
|
|
"learning_rate": 4.990348520664794e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -438.0,
|
|
"loss": 0.3752,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.421875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.1265256565914755,
|
|
"grad_norm": 10.846462610025974,
|
|
"learning_rate": 4.989344520163849e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.3621,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -3.171875,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.12780369352674292,
|
|
"grad_norm": 24.815238960975694,
|
|
"learning_rate": 4.98829096148362e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -494.0,
|
|
"loss": 0.3789,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -3.421875,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.12908173046201035,
|
|
"grad_norm": 12.469684746555005,
|
|
"learning_rate": 4.987187865598482e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -312.0,
|
|
"logps/rejected": -480.0,
|
|
"loss": 0.354,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.578125,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -3.546875,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.13035976739727778,
|
|
"grad_norm": 10.651703453569814,
|
|
"learning_rate": 4.986035254469005e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -460.0,
|
|
"loss": 0.3777,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.3671875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.1316378043325452,
|
|
"grad_norm": 10.169153192152455,
|
|
"learning_rate": 4.984833151041512e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -524.0,
|
|
"loss": 0.3529,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.734375,
|
|
"rewards/margins": 2.0625,
|
|
"rewards/rejected": -3.796875,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.13291584126781264,
|
|
"grad_norm": 17.70616690114393,
|
|
"learning_rate": 4.98358157924763e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.344,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.9921875,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -4.3125,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.13419387820308007,
|
|
"grad_norm": 14.910681564668302,
|
|
"learning_rate": 4.982280564003806e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -342.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3676,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.1354719151383475,
|
|
"grad_norm": 9.401234339612973,
|
|
"learning_rate": 4.98093013121081e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -336.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.3497,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -3.453125,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.13674995207361493,
|
|
"grad_norm": 11.021445735908344,
|
|
"learning_rate": 4.979530307753227e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.3518,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.6328125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -3.3125,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.13802798900888236,
|
|
"grad_norm": 13.943751456778854,
|
|
"learning_rate": 4.978081121498916e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -516.0,
|
|
"loss": 0.3489,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.9609375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -3.71875,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.13930602594414979,
|
|
"grad_norm": 10.338244132399032,
|
|
"learning_rate": 4.976582601298456e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -332.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.3353,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.14058406287941722,
|
|
"grad_norm": 8.71888191275971,
|
|
"learning_rate": 4.975034776984573e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -486.0,
|
|
"loss": 0.3553,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.8203125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.14186209981468464,
|
|
"grad_norm": 12.216205581090218,
|
|
"learning_rate": 4.973437679371546e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -292.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.3496,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.5,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.14314013674995207,
|
|
"grad_norm": 12.423722309973517,
|
|
"learning_rate": 4.971791340254592e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -320.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.3644,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -1.6171875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -3.25,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.1444181736852195,
|
|
"grad_norm": 11.14194864610794,
|
|
"learning_rate": 4.970095792409233e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3456,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.5859375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.14569621062048693,
|
|
"grad_norm": 11.464691538377055,
|
|
"learning_rate": 4.968351069590646e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -506.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.9453125,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.14697424755575436,
|
|
"grad_norm": 9.505788032958687,
|
|
"learning_rate": 4.966557206532992e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -352.0,
|
|
"logps/rejected": -474.0,
|
|
"loss": 0.3805,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.8515625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -3.265625,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.1482522844910218,
|
|
"grad_norm": 9.689853960676805,
|
|
"learning_rate": 4.964714238948715e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3429,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.7265625,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -3.6875,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.14953032142628922,
|
|
"grad_norm": 8.325164199298303,
|
|
"learning_rate": 4.962822203527845e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3277,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -4.25,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.15080835836155665,
|
|
"grad_norm": 15.52096274687861,
|
|
"learning_rate": 4.960881137937256e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -312.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.3235,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.546875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.15208639529682408,
|
|
"grad_norm": 12.639275691252218,
|
|
"learning_rate": 4.958891080819923e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -552.0,
|
|
"loss": 0.3541,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.125,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -4.125,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.1533644322320915,
|
|
"grad_norm": 8.869328760760212,
|
|
"learning_rate": 4.956852071794151e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.3544,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -3.28125,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.15464246916735894,
|
|
"grad_norm": 10.63346925167416,
|
|
"learning_rate": 4.954764151452782e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -332.0,
|
|
"logps/rejected": -512.0,
|
|
"loss": 0.3265,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.828125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -3.640625,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.15592050610262637,
|
|
"grad_norm": 11.334624897642897,
|
|
"learning_rate": 4.952627361362395e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -540.0,
|
|
"loss": 0.3435,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.9765625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -3.90625,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.1571985430378938,
|
|
"grad_norm": 11.849129116829852,
|
|
"learning_rate": 4.950441744062471e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.28125,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3435,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.046875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -3.984375,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.15847657997316122,
|
|
"grad_norm": 9.577666300992249,
|
|
"learning_rate": 4.948207343064551e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -374.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.3173,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.203125,
|
|
"rewards/margins": 2.34375,
|
|
"rewards/rejected": -4.5625,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.15975461690842865,
|
|
"grad_norm": 10.910196783813214,
|
|
"learning_rate": 4.945924202851366e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -398.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.3307,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.16103265384369608,
|
|
"grad_norm": 10.57466378686154,
|
|
"learning_rate": 4.943592368875955e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -414.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 0.3491,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.59375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.1623106907789635,
|
|
"grad_norm": 11.230589436335967,
|
|
"learning_rate": 4.941211887560757e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.3023,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.16358872771423094,
|
|
"grad_norm": 11.961477038376033,
|
|
"learning_rate": 4.938782806296691e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3028,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -2.875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.16486676464949837,
|
|
"grad_norm": 9.972580283679875,
|
|
"learning_rate": 4.936305173442206e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -608.0,
|
|
"loss": 0.3068,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 2.203125,
|
|
"rewards/rejected": -4.71875,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.1661448015847658,
|
|
"grad_norm": 10.211364589134622,
|
|
"learning_rate": 4.933779038322324e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.3251,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.59375,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.16742283852003323,
|
|
"grad_norm": 8.909898711500084,
|
|
"learning_rate": 4.931204451227658e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -572.0,
|
|
"loss": 0.3113,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -4.25,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.16870087545530066,
|
|
"grad_norm": 13.917027005009777,
|
|
"learning_rate": 4.928581463413405e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -354.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.3264,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.140625,
|
|
"rewards/margins": 2.265625,
|
|
"rewards/rejected": -4.40625,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.1699789123905681,
|
|
"grad_norm": 9.075174861427325,
|
|
"learning_rate": 4.925910127098333e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -342.0,
|
|
"logps/rejected": -560.0,
|
|
"loss": 0.3096,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.90625,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -4.28125,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.17125694932583552,
|
|
"grad_norm": 10.492320216709347,
|
|
"learning_rate": 4.923190495463736e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.2988,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -2.296875,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -4.75,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.17253498626110295,
|
|
"grad_norm": 11.104892250574451,
|
|
"learning_rate": 4.920422622652377e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -376.0,
|
|
"logps/rejected": -576.0,
|
|
"loss": 0.3202,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.078125,
|
|
"rewards/margins": 2.140625,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.17381302319637038,
|
|
"grad_norm": 8.174183970674996,
|
|
"learning_rate": 4.917606563767411e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.3258,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -2.359375,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.1750910601316378,
|
|
"grad_norm": 9.822967964793463,
|
|
"learning_rate": 4.914742374871289e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.2794,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -5.125,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.17636909706690523,
|
|
"grad_norm": 13.046396762006776,
|
|
"learning_rate": 4.911830112984638e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.28125,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.3158,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.515625,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -4.75,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.17764713400217266,
|
|
"grad_norm": 9.634796199540869,
|
|
"learning_rate": 4.908869836085127e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.109375,
|
|
"logps/chosen": -400.0,
|
|
"logps/rejected": -620.0,
|
|
"loss": 0.2978,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.453125,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.1789251709374401,
|
|
"grad_norm": 10.036265985004444,
|
|
"learning_rate": 4.905861603106318e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3018,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.296875,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.18020320787270752,
|
|
"grad_norm": 18.756659281595155,
|
|
"learning_rate": 4.902805473936483e-07,
|
|
"logits/chosen": -3.4375,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.5625,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.18148124480797495,
|
|
"grad_norm": 11.083399006007372,
|
|
"learning_rate": 4.899701509417423e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.2897,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.18275928174324238,
|
|
"grad_norm": 9.072585092087222,
|
|
"learning_rate": 4.896549771343247e-07,
|
|
"logits/chosen": -3.390625,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.2857,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.1840373186785098,
|
|
"grad_norm": 7.519453357476137,
|
|
"learning_rate": 4.893350322459149e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2972,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.18531535561377724,
|
|
"grad_norm": 10.926070436917199,
|
|
"learning_rate": 4.890103226460152e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3027,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.18659339254904467,
|
|
"grad_norm": 9.889333734788899,
|
|
"learning_rate": 4.886808547989846e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.3162,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.1878714294843121,
|
|
"grad_norm": 11.48343624861157,
|
|
"learning_rate": 4.883466352639099e-07,
|
|
"logits/chosen": -3.296875,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3164,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.18914946641957953,
|
|
"grad_norm": 11.028607347761977,
|
|
"learning_rate": 4.88007670694475e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.292,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.734375,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.19042750335484696,
|
|
"grad_norm": 9.137715141189384,
|
|
"learning_rate": 4.876639678388285e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3172,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.515625,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.19170554029011438,
|
|
"grad_norm": 8.392316890933923,
|
|
"learning_rate": 4.873155335394497e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2895,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.1929835772253818,
|
|
"grad_norm": 9.776765970586137,
|
|
"learning_rate": 4.869623747330116e-07,
|
|
"logits/chosen": -3.265625,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -474.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.3101,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.19426161416064924,
|
|
"grad_norm": 10.113772712494775,
|
|
"learning_rate": 4.866044984502436e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2819,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.19553965109591667,
|
|
"grad_norm": 12.373484208276247,
|
|
"learning_rate": 4.862419118157909e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.2809,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.75,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.1968176880311841,
|
|
"grad_norm": 11.78683291379729,
|
|
"learning_rate": 4.858746220480734e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -632.0,
|
|
"loss": 0.3016,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.453125,
|
|
"rewards/margins": 2.4375,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.19809572496645153,
|
|
"grad_norm": 12.160856594312754,
|
|
"learning_rate": 4.855026364591413e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2759,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.19937376190171896,
|
|
"grad_norm": 7.635186752817528,
|
|
"learning_rate": 4.851259624545297e-07,
|
|
"logits/chosen": -3.296875,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.2999,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.46875,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.2006517988369864,
|
|
"grad_norm": 10.580854781397509,
|
|
"learning_rate": 4.847446075331114e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.298,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.20192983577225382,
|
|
"grad_norm": 15.508397767776305,
|
|
"learning_rate": 4.843585792869476e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.3035,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.625,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.20320787270752125,
|
|
"grad_norm": 12.29502011282903,
|
|
"learning_rate": 4.839678854011362e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3106,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.20448590964278868,
|
|
"grad_norm": 9.221266332248716,
|
|
"learning_rate": 4.835725336536598e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2795,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -5.875,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.2057639465780561,
|
|
"grad_norm": 21.949201663234852,
|
|
"learning_rate": 4.831725319152298e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2822,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.20704198351332354,
|
|
"grad_norm": 8.390379977679709,
|
|
"learning_rate": 4.827678881491305e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2692,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.359375,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.20832002044859096,
|
|
"grad_norm": 9.085937257418248,
|
|
"learning_rate": 4.8235861041106e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2757,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.2095980573838584,
|
|
"grad_norm": 10.915204034593046,
|
|
"learning_rate": 4.819447068489705e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2838,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.21087609431912582,
|
|
"grad_norm": 10.623276883304097,
|
|
"learning_rate": 4.815261857029052e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -462.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2661,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.21215413125439325,
|
|
"grad_norm": 15.529572832655484,
|
|
"learning_rate": 4.811030553048351e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2895,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.21343216818966068,
|
|
"grad_norm": 8.986111656832815,
|
|
"learning_rate": 4.806753240784924e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2822,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.796875,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -5.59375,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.2147102051249281,
|
|
"grad_norm": 10.662265185336627,
|
|
"learning_rate": 4.802430005392037e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.3085,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.21598824206019554,
|
|
"grad_norm": 18.32782419922393,
|
|
"learning_rate": 4.798060932937194e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2825,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.0,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.46875,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.21726627899546297,
|
|
"grad_norm": 8.37402052085016,
|
|
"learning_rate": 4.79364611040043e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2776,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.0,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.2185443159307304,
|
|
"grad_norm": 7.388938797100764,
|
|
"learning_rate": 4.789185625672583e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2688,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.21982235286599783,
|
|
"grad_norm": 11.18964791369844,
|
|
"learning_rate": 4.784679567553531e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2807,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -6.375,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.22110038980126526,
|
|
"grad_norm": 12.788195666651463,
|
|
"learning_rate": 4.780128025750442e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -466.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2667,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.22237842673653269,
|
|
"grad_norm": 13.452879203275968,
|
|
"learning_rate": 4.775531090875971e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2936,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.34375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.22365646367180012,
|
|
"grad_norm": 13.978917804241819,
|
|
"learning_rate": 4.770888854446471e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -462.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2665,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.22493450060706754,
|
|
"grad_norm": 14.827300418751848,
|
|
"learning_rate": 4.766201408880156e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.3005,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.22621253754233497,
|
|
"grad_norm": 8.780665034348093,
|
|
"learning_rate": 4.761468847495277e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.268,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.2274905744776024,
|
|
"grad_norm": 8.913295529168444,
|
|
"learning_rate": 4.756691264508251e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2597,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.140625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.22876861141286983,
|
|
"grad_norm": 9.149720146692632,
|
|
"learning_rate": 4.751868755031793e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2978,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.23004664834813726,
|
|
"grad_norm": 9.864161436507205,
|
|
"learning_rate": 4.747001415073018e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2801,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.171875,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.2313246852834047,
|
|
"grad_norm": 12.049630916813516,
|
|
"learning_rate": 4.742089341531535e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.109375,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2725,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.23260272221867212,
|
|
"grad_norm": 9.245650817902728,
|
|
"learning_rate": 4.7371326321975127e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2838,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.23388075915393955,
|
|
"grad_norm": 10.565374823511835,
|
|
"learning_rate": 4.7321313857497336e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.3001,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.23515879608920698,
|
|
"grad_norm": 11.728663046788858,
|
|
"learning_rate": 4.7270857017536335e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2755,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.2364368330244744,
|
|
"grad_norm": 13.400694062249938,
|
|
"learning_rate": 4.7219956806593143e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2901,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.23771486995974184,
|
|
"grad_norm": 8.737541772532301,
|
|
"learning_rate": 4.716861423799546e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2871,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.23899290689500927,
|
|
"grad_norm": 11.454317171770168,
|
|
"learning_rate": 4.711683033387752e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2612,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.2402709438302767,
|
|
"grad_norm": 8.019796544220595,
|
|
"learning_rate": 4.70646061251597e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2603,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.24154898076554412,
|
|
"grad_norm": 9.374363546593047,
|
|
"learning_rate": 4.701194265152802e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2726,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.24282701770081155,
|
|
"grad_norm": 13.860826584264853,
|
|
"learning_rate": 4.6958840961413447e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2811,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.546875,
|
|
"rewards/margins": 2.4375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.24410505463607898,
|
|
"grad_norm": 10.57554715572998,
|
|
"learning_rate": 4.6905302111971004e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -428.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2829,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -5.59375,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.2453830915713464,
|
|
"grad_norm": 14.259221382970665,
|
|
"learning_rate": 4.6851327169058735e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.2912,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.875,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.24666112850661384,
|
|
"grad_norm": 7.366111663404775,
|
|
"learning_rate": 4.679691720721651e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2693,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.28125,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.24793916544188127,
|
|
"grad_norm": 11.096803393586505,
|
|
"learning_rate": 4.674207330964458e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2789,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.2492172023771487,
|
|
"grad_norm": 8.956111080301984,
|
|
"learning_rate": 4.668679656818207e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2746,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.25049523931241613,
|
|
"grad_norm": 9.513508988130903,
|
|
"learning_rate": 4.663108808328519e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2819,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.25177327624768353,
|
|
"grad_norm": 10.234008316257968,
|
|
"learning_rate": 4.6574948964005367e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2871,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.253051313182951,
|
|
"grad_norm": 8.01274333751014,
|
|
"learning_rate": 4.6518380327967145e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.247,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.2543293501182184,
|
|
"grad_norm": 12.33132352378727,
|
|
"learning_rate": 4.6461383301345947e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2821,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.25560738705348585,
|
|
"grad_norm": 10.395762625615612,
|
|
"learning_rate": 4.6403959018845637e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2645,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.703125,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.25688542398875325,
|
|
"grad_norm": 15.793711134680974,
|
|
"learning_rate": 4.6346108623675954e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.2914,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.2581634609240207,
|
|
"grad_norm": 10.102379635568527,
|
|
"learning_rate": 4.628783326752974e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.261,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.2594414978592881,
|
|
"grad_norm": 11.19917017437885,
|
|
"learning_rate": 4.622913411056e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2563,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.26071953479455556,
|
|
"grad_norm": 9.591252819586105,
|
|
"learning_rate": 4.617001232135684e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2804,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.26199757172982296,
|
|
"grad_norm": 10.927105525570921,
|
|
"learning_rate": 4.6110469076924153e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2661,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.2632756086650904,
|
|
"grad_norm": 8.081281860104301,
|
|
"learning_rate": 4.605050556265624e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2664,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.2645536456003578,
|
|
"grad_norm": 8.719157157796769,
|
|
"learning_rate": 4.599012297231417e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2605,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.2658316825356253,
|
|
"grad_norm": 9.889601643083903,
|
|
"learning_rate": 4.5929322508002045e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.264,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.2671097194708927,
|
|
"grad_norm": 9.01425592242334,
|
|
"learning_rate": 4.586810538014304e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.109375,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.28125,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.26838775640616014,
|
|
"grad_norm": 9.917816992815437,
|
|
"learning_rate": 4.580647280745532e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2651,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.26966579334142754,
|
|
"grad_norm": 9.08237633986445,
|
|
"learning_rate": 4.5744426016927783e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2618,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.270943830276695,
|
|
"grad_norm": 10.374914225435086,
|
|
"learning_rate": 4.5681966243795645e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2563,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.65625,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 0.2722218672119624,
|
|
"grad_norm": 9.39552251607615,
|
|
"learning_rate": 4.5619094731515783e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2488,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 0.27349990414722986,
|
|
"grad_norm": 9.255807709157049,
|
|
"learning_rate": 4.5555812731742085e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2436,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 0.27477794108249726,
|
|
"grad_norm": 13.023248862747112,
|
|
"learning_rate": 4.549212150430042e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2483,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.453125,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -6.375,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.2760559780177647,
|
|
"grad_norm": 10.069695705444243,
|
|
"learning_rate": 4.5428022317163654e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2538,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 0.2773340149530321,
|
|
"grad_norm": 7.905785619069278,
|
|
"learning_rate": 4.5363516446426353e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2553,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"epoch": 0.27861205188829957,
|
|
"grad_norm": 14.110926580452388,
|
|
"learning_rate": 4.5298605176279383e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2546,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 0.279890088823567,
|
|
"grad_norm": 11.437656708234213,
|
|
"learning_rate": 4.5233289798984355e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2515,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"epoch": 0.28116812575883443,
|
|
"grad_norm": 10.92037760467791,
|
|
"learning_rate": 4.51675716148479e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.27,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.28244616269410183,
|
|
"grad_norm": 9.082318419309797,
|
|
"learning_rate": 4.510145193219577e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2611,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"epoch": 0.2837241996293693,
|
|
"grad_norm": 9.696968012792881,
|
|
"learning_rate": 4.503493206734681e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.2665,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -5.875,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 0.2850022365646367,
|
|
"grad_norm": 8.32597970165281,
|
|
"learning_rate": 4.4968013344586723e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2618,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"epoch": 0.28628027349990415,
|
|
"grad_norm": 9.205123244163184,
|
|
"learning_rate": 4.4900697096141754e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2936,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 0.28755831043517155,
|
|
"grad_norm": 8.173644269044605,
|
|
"learning_rate": 4.483298466215211e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2481,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.288836347370439,
|
|
"grad_norm": 13.842959897880784,
|
|
"learning_rate": 4.4764877390645317e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2398,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 0.2901143843057064,
|
|
"grad_norm": 10.298507286958628,
|
|
"learning_rate": 4.469637663750939e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2618,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"epoch": 0.29139242124097386,
|
|
"grad_norm": 11.781378177678372,
|
|
"learning_rate": 4.4627483766465813e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2419,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 0.29267045817624127,
|
|
"grad_norm": 9.975143928231438,
|
|
"learning_rate": 4.4558200149042393e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2498,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"epoch": 0.2939484951115087,
|
|
"grad_norm": 9.81920229020163,
|
|
"learning_rate": 4.4488527164545976e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2174,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.2952265320467761,
|
|
"grad_norm": 8.085865328061837,
|
|
"learning_rate": 4.4418466200034974e-07,
|
|
"logits/chosen": -3.265625,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2624,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"epoch": 0.2965045689820436,
|
|
"grad_norm": 10.263278935222155,
|
|
"learning_rate": 4.4348018650291764e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2542,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"epoch": 0.297782605917311,
|
|
"grad_norm": 8.200991204991244,
|
|
"learning_rate": 4.427718591779489e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2513,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.25,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"epoch": 0.29906064285257844,
|
|
"grad_norm": 9.045676093918619,
|
|
"learning_rate": 4.4205969412691167e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2433,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"epoch": 0.30033867978784584,
|
|
"grad_norm": 9.307362421836325,
|
|
"learning_rate": 4.4134370552767617e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.3016167167231133,
|
|
"grad_norm": 10.633510285123654,
|
|
"learning_rate": 4.406239076342322e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2616,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"epoch": 0.3028947536583807,
|
|
"grad_norm": 7.178556035609511,
|
|
"learning_rate": 4.399003147764053e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -508.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2394,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"epoch": 0.30417279059364816,
|
|
"grad_norm": 8.025880286415548,
|
|
"learning_rate": 4.3917294135957185e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2697,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"epoch": 0.30545082752891556,
|
|
"grad_norm": 9.150701110291598,
|
|
"learning_rate": 4.3844180186437205e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.252,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"epoch": 0.306728864464183,
|
|
"grad_norm": 9.06253052565922,
|
|
"learning_rate": 4.3770691084642153e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2478,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.3080069013994504,
|
|
"grad_norm": 8.361504252843975,
|
|
"learning_rate": 4.3696828293602185e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2583,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"epoch": 0.3092849383347179,
|
|
"grad_norm": 9.674493266614986,
|
|
"learning_rate": 4.3622593283786895e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.235,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"epoch": 0.3105629752699853,
|
|
"grad_norm": 8.99066458935145,
|
|
"learning_rate": 4.354798753307606e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"epoch": 0.31184101220525273,
|
|
"grad_norm": 10.245096178761992,
|
|
"learning_rate": 4.3473012526730216e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.258,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"epoch": 0.31311904914052013,
|
|
"grad_norm": 8.095300746315656,
|
|
"learning_rate": 4.339766975736109e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2535,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.3143970860757876,
|
|
"grad_norm": 9.33334356795753,
|
|
"learning_rate": 4.332196072490185e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.247,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"epoch": 0.315675123011055,
|
|
"grad_norm": 9.68096887344856,
|
|
"learning_rate": 4.324588693657733e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2589,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.625,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"epoch": 0.31695315994632245,
|
|
"grad_norm": 8.74974145611664,
|
|
"learning_rate": 4.3169449906873925e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"epoch": 0.31823119688158985,
|
|
"grad_norm": 6.971622374452891,
|
|
"learning_rate": 4.309265115750949e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2366,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"epoch": 0.3195092338168573,
|
|
"grad_norm": 11.17024488529161,
|
|
"learning_rate": 4.301549221740305e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2874,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.3207872707521247,
|
|
"grad_norm": 10.477310622785978,
|
|
"learning_rate": 4.293797462264436e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2387,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"epoch": 0.32206530768739217,
|
|
"grad_norm": 10.024656692443616,
|
|
"learning_rate": 4.286009991646329e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.253,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"epoch": 0.32334334462265957,
|
|
"grad_norm": 8.009802554248187,
|
|
"learning_rate": 4.2781869649199153e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2492,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"epoch": 0.324621381557927,
|
|
"grad_norm": 8.945237247295104,
|
|
"learning_rate": 4.2703285378269815e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"epoch": 0.3258994184931944,
|
|
"grad_norm": 10.551216216391607,
|
|
"learning_rate": 4.262434866814067e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2183,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.75,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.3271774554284619,
|
|
"grad_norm": 8.688942019994915,
|
|
"learning_rate": 4.254506109029353e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2568,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"epoch": 0.3284554923637293,
|
|
"grad_norm": 6.564420476695321,
|
|
"learning_rate": 4.2465424223195327e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2343,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"epoch": 0.32973352929899674,
|
|
"grad_norm": 9.43421095366614,
|
|
"learning_rate": 4.238543965226668e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2473,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"epoch": 0.33101156623426414,
|
|
"grad_norm": 13.150142319397695,
|
|
"learning_rate": 4.230510896985035e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2324,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"epoch": 0.3322896031695316,
|
|
"grad_norm": 9.050124476173716,
|
|
"learning_rate": 4.2224433775179506e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.268,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.333567640104799,
|
|
"grad_norm": 8.614894031818137,
|
|
"learning_rate": 4.2143415674345937e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2273,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"epoch": 0.33484567704006646,
|
|
"grad_norm": 8.111143191701691,
|
|
"learning_rate": 4.2062056280268033e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2433,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"epoch": 0.33612371397533386,
|
|
"grad_norm": 11.117676038308154,
|
|
"learning_rate": 4.198035721265869e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2518,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"epoch": 0.3374017509106013,
|
|
"grad_norm": 9.258130845121494,
|
|
"learning_rate": 4.1898320097993085e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2846,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"epoch": 0.3386797878458687,
|
|
"grad_norm": 9.194807847235614,
|
|
"learning_rate": 4.181594656947625e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2202,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.3399578247811362,
|
|
"grad_norm": 9.12609259279053,
|
|
"learning_rate": 4.173323826701062e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.235,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"epoch": 0.3412358617164036,
|
|
"grad_norm": 8.558030417957948,
|
|
"learning_rate": 4.165019683716332e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2471,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"epoch": 0.34251389865167103,
|
|
"grad_norm": 7.570195174697268,
|
|
"learning_rate": 4.1566823933133444e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2322,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"epoch": 0.34379193558693844,
|
|
"grad_norm": 12.163376095462333,
|
|
"learning_rate": 4.148312121471908e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2238,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"epoch": 0.3450699725222059,
|
|
"grad_norm": 8.261895776179635,
|
|
"learning_rate": 4.1399090348284336e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.249,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.65625,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.3463480094574733,
|
|
"grad_norm": 7.401313646780075,
|
|
"learning_rate": 4.1314733006726116e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.59375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"epoch": 0.34762604639274075,
|
|
"grad_norm": 8.802407402306804,
|
|
"learning_rate": 4.1230050869440825e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2272,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"epoch": 0.34890408332800815,
|
|
"grad_norm": 8.551492444117791,
|
|
"learning_rate": 4.114504562229096e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"epoch": 0.3501821202632756,
|
|
"grad_norm": 13.237697656186098,
|
|
"learning_rate": 4.105971895757151e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.262,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"epoch": 0.351460157198543,
|
|
"grad_norm": 9.794093445613948,
|
|
"learning_rate": 4.0974072573976295e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2488,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.35273819413381047,
|
|
"grad_norm": 9.723773756216174,
|
|
"learning_rate": 4.088810817656414e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"epoch": 0.35401623106907787,
|
|
"grad_norm": 8.723909251690763,
|
|
"learning_rate": 4.0801827476724923e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2428,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"epoch": 0.3552942680043453,
|
|
"grad_norm": 8.462907156480545,
|
|
"learning_rate": 4.071523219214551e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -7.25,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"epoch": 0.3565723049396127,
|
|
"grad_norm": 9.665834891128702,
|
|
"learning_rate": 4.062832404677556e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"epoch": 0.3578503418748802,
|
|
"grad_norm": 8.989459813679218,
|
|
"learning_rate": 4.054110477079321e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2434,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.3591283788101476,
|
|
"grad_norm": 8.431501557282072,
|
|
"learning_rate": 4.0453576100570606e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -460.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"epoch": 0.36040641574541504,
|
|
"grad_norm": 15.875947653262912,
|
|
"learning_rate": 4.036573977863938e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"epoch": 0.36168445268068244,
|
|
"grad_norm": 11.089723641989362,
|
|
"learning_rate": 4.027759755365591e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2521,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"epoch": 0.3629624896159499,
|
|
"grad_norm": 7.979671614832609,
|
|
"learning_rate": 4.018915118036653e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2409,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"epoch": 0.3642405265512173,
|
|
"grad_norm": 7.47703355045046,
|
|
"learning_rate": 4.0100402419572617e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.36551856348648476,
|
|
"grad_norm": 6.881027994969947,
|
|
"learning_rate": 4.0011353038095497e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2217,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.375,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"epoch": 0.36679660042175216,
|
|
"grad_norm": 8.387929456445715,
|
|
"learning_rate": 3.992200480874128e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2132,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"epoch": 0.3680746373570196,
|
|
"grad_norm": 11.868719055234116,
|
|
"learning_rate": 3.983235951026562e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2397,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"epoch": 0.369352674292287,
|
|
"grad_norm": 6.6165920564533565,
|
|
"learning_rate": 3.974241892733821e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"epoch": 0.3706307112275545,
|
|
"grad_norm": 9.666254997669043,
|
|
"learning_rate": 3.965218485050733e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2361,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.3719087481628219,
|
|
"grad_norm": 8.197217719753612,
|
|
"learning_rate": 3.9561659076164166e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2378,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"epoch": 0.37318678509808934,
|
|
"grad_norm": 7.426775845977119,
|
|
"learning_rate": 3.947084340650706e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2383,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"epoch": 0.37446482203335674,
|
|
"grad_norm": 6.536302581696631,
|
|
"learning_rate": 3.937973964950561e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2321,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.84375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"epoch": 0.3757428589686242,
|
|
"grad_norm": 12.669430664625045,
|
|
"learning_rate": 3.928834961886472e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"epoch": 0.3770208959038916,
|
|
"grad_norm": 10.03412970439382,
|
|
"learning_rate": 3.919667513398843e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2145,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.37829893283915905,
|
|
"grad_norm": 7.385896415249684,
|
|
"learning_rate": 3.910471801994377e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2349,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"epoch": 0.37957696977442645,
|
|
"grad_norm": 9.839499943864618,
|
|
"learning_rate": 3.901248010742435e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"epoch": 0.3808550067096939,
|
|
"grad_norm": 10.643804579819335,
|
|
"learning_rate": 3.891996323271396e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2548,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"epoch": 0.3821330436449613,
|
|
"grad_norm": 6.260351646142021,
|
|
"learning_rate": 3.8827169237650023e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"epoch": 0.38341108058022877,
|
|
"grad_norm": 9.26864693273647,
|
|
"learning_rate": 3.87340999695869e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2277,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.38468911751549617,
|
|
"grad_norm": 11.798346785124874,
|
|
"learning_rate": 3.8640757281359104e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2409,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"epoch": 0.3859671544507636,
|
|
"grad_norm": 9.13276020421161,
|
|
"learning_rate": 3.8547143031244454e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2363,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -7.25,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"epoch": 0.38724519138603103,
|
|
"grad_norm": 11.67031797954755,
|
|
"learning_rate": 3.845325908292704e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2292,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"epoch": 0.3885232283212985,
|
|
"grad_norm": 10.639147098006879,
|
|
"learning_rate": 3.835910730546013e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.245,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"epoch": 0.3898012652565659,
|
|
"grad_norm": 8.44223529175865,
|
|
"learning_rate": 3.8264689573229e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.236,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.39107930219183334,
|
|
"grad_norm": 8.488987566619226,
|
|
"learning_rate": 3.8170007765913563e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2073,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"epoch": 0.39235733912710075,
|
|
"grad_norm": 8.533803292750349,
|
|
"learning_rate": 3.8075063768450977e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2315,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"epoch": 0.3936353760623682,
|
|
"grad_norm": 13.47605711618702,
|
|
"learning_rate": 3.79798594709981e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2244,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"epoch": 0.3949134129976356,
|
|
"grad_norm": 9.088959049871905,
|
|
"learning_rate": 3.78843967688939e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2113,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"epoch": 0.39619144993290306,
|
|
"grad_norm": 6.990268151404954,
|
|
"learning_rate": 3.7788677562621676e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2411,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.39746948686817046,
|
|
"grad_norm": 10.21876475077512,
|
|
"learning_rate": 3.769270375777126e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"epoch": 0.3987475238034379,
|
|
"grad_norm": 8.94493406660772,
|
|
"learning_rate": 3.7596477265001053e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2207,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"epoch": 0.4000255607387053,
|
|
"grad_norm": 8.81703733566685,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2356,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"epoch": 0.4013035976739728,
|
|
"grad_norm": 9.976485922500268,
|
|
"learning_rate": 3.740327388344945e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2087,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"epoch": 0.4025816346092402,
|
|
"grad_norm": 6.8661146681276835,
|
|
"learning_rate": 3.7306300840984927e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2409,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.40385967154450764,
|
|
"grad_norm": 10.705188172801805,
|
|
"learning_rate": 3.720908280315777e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2195,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"epoch": 0.40513770847977504,
|
|
"grad_norm": 7.260979308776354,
|
|
"learning_rate": 3.711162170539673e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2157,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"epoch": 0.4064157454150425,
|
|
"grad_norm": 13.386154459966885,
|
|
"learning_rate": 3.701391948796945e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"epoch": 0.4076937823503099,
|
|
"grad_norm": 10.71858268901349,
|
|
"learning_rate": 3.6915978095943745e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2193,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"epoch": 0.40897181928557735,
|
|
"grad_norm": 7.112292506270725,
|
|
"learning_rate": 3.6817799479149007e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.41024985622084476,
|
|
"grad_norm": 12.88518280843065,
|
|
"learning_rate": 3.6719385592137306e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2387,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"epoch": 0.4115278931561122,
|
|
"grad_norm": 10.14123838967259,
|
|
"learning_rate": 3.662073839414452e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2255,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"epoch": 0.4128059300913796,
|
|
"grad_norm": 9.792291066078825,
|
|
"learning_rate": 3.6521859849051274e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2431,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"epoch": 0.41408396702664707,
|
|
"grad_norm": 8.93660540981452,
|
|
"learning_rate": 3.6422751925343906e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"epoch": 0.41536200396191447,
|
|
"grad_norm": 9.207018470233994,
|
|
"learning_rate": 3.6323416596075244e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2373,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.41664004089718193,
|
|
"grad_norm": 8.798998468661477,
|
|
"learning_rate": 3.622385583882535e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2528,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"epoch": 0.41791807783244933,
|
|
"grad_norm": 7.39070684420073,
|
|
"learning_rate": 3.6124071635662107e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2159,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"epoch": 0.4191961147677168,
|
|
"grad_norm": 10.572409208940522,
|
|
"learning_rate": 3.6024065973101803e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2188,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"epoch": 0.4204741517029842,
|
|
"grad_norm": 11.489752500259677,
|
|
"learning_rate": 3.5923840842069583e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2227,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"epoch": 0.42175218863825165,
|
|
"grad_norm": 8.808785722348407,
|
|
"learning_rate": 3.5823398237859746e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.42303022557351905,
|
|
"grad_norm": 8.69574601913517,
|
|
"learning_rate": 3.572274016009613e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2233,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"epoch": 0.4243082625087865,
|
|
"grad_norm": 9.178583055571378,
|
|
"learning_rate": 3.562186861269223e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2118,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"epoch": 0.4255862994440539,
|
|
"grad_norm": 11.47875876274727,
|
|
"learning_rate": 3.5520785603811296e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2562,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"epoch": 0.42686433637932136,
|
|
"grad_norm": 9.725339945230063,
|
|
"learning_rate": 3.541949314582641e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"epoch": 0.42814237331458876,
|
|
"grad_norm": 8.970902307435473,
|
|
"learning_rate": 3.5317993255280383e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2017,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.4294204102498562,
|
|
"grad_norm": 8.382183001221023,
|
|
"learning_rate": 3.5216287952845614e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1958,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"epoch": 0.4306984471851236,
|
|
"grad_norm": 9.189979249063406,
|
|
"learning_rate": 3.511437926328387e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.246,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"epoch": 0.4319764841203911,
|
|
"grad_norm": 8.228711642134167,
|
|
"learning_rate": 3.501226921540598e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2335,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"epoch": 0.4332545210556585,
|
|
"grad_norm": 9.21386830259457,
|
|
"learning_rate": 3.4909959842031436e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2189,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"epoch": 0.43453255799092594,
|
|
"grad_norm": 6.248955823376214,
|
|
"learning_rate": 3.480745317994793e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2178,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.43581059492619334,
|
|
"grad_norm": 9.020550855639287,
|
|
"learning_rate": 3.4704751269870774e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.1958,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"epoch": 0.4370886318614608,
|
|
"grad_norm": 10.664082807354959,
|
|
"learning_rate": 3.460185615640234e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2194,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"epoch": 0.4383666687967282,
|
|
"grad_norm": 10.56454935423742,
|
|
"learning_rate": 3.4498769887991316e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2377,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"epoch": 0.43964470573199566,
|
|
"grad_norm": 8.491123471406155,
|
|
"learning_rate": 3.4395494516891885e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2178,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"epoch": 0.44092274266726306,
|
|
"grad_norm": 9.201408062848294,
|
|
"learning_rate": 3.4292032099122956e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2088,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.4422007796025305,
|
|
"grad_norm": 7.860836704868255,
|
|
"learning_rate": 3.4188384694427166e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2384,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"epoch": 0.4434788165377979,
|
|
"grad_norm": 13.747547646724954,
|
|
"learning_rate": 3.4084554366229884e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2205,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"epoch": 0.44475685347306537,
|
|
"grad_norm": 8.848263859657068,
|
|
"learning_rate": 3.398054318159816e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2116,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"epoch": 0.4460348904083328,
|
|
"grad_norm": 10.597332854509538,
|
|
"learning_rate": 3.387635321119955e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.228,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"epoch": 0.44731292734360023,
|
|
"grad_norm": 9.674325942945405,
|
|
"learning_rate": 3.3771986529260914e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2123,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.44859096427886763,
|
|
"grad_norm": 12.193772222585018,
|
|
"learning_rate": 3.3667445213527075e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2273,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"epoch": 0.4498690012141351,
|
|
"grad_norm": 11.935477705606276,
|
|
"learning_rate": 3.356273134521951e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2136,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"epoch": 0.4511470381494025,
|
|
"grad_norm": 10.16354356392599,
|
|
"learning_rate": 3.3457847008994895e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2291,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"epoch": 0.45242507508466995,
|
|
"grad_norm": 9.401934005328973,
|
|
"learning_rate": 3.3352794292903584e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2383,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"epoch": 0.45370311201993735,
|
|
"grad_norm": 12.304466333106431,
|
|
"learning_rate": 3.324757528834809e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2339,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.4549811489552048,
|
|
"grad_norm": 10.335666794847032,
|
|
"learning_rate": 3.314219209004139e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2134,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"epoch": 0.4562591858904722,
|
|
"grad_norm": 8.135726139180283,
|
|
"learning_rate": 3.303664679596526e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2045,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"epoch": 0.45753722282573966,
|
|
"grad_norm": 10.36951976978052,
|
|
"learning_rate": 3.293094150732849e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1959,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"epoch": 0.45881525976100707,
|
|
"grad_norm": 12.805385346829228,
|
|
"learning_rate": 3.2825078328525076e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2227,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"epoch": 0.4600932966962745,
|
|
"grad_norm": 8.737872062348318,
|
|
"learning_rate": 3.271905936709231e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2004,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.4613713336315419,
|
|
"grad_norm": 5.90450642703934,
|
|
"learning_rate": 3.261288673366881e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2222,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"epoch": 0.4626493705668094,
|
|
"grad_norm": 10.917396256677707,
|
|
"learning_rate": 3.250656254195252e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2324,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"epoch": 0.4639274075020768,
|
|
"grad_norm": 8.521570669595315,
|
|
"learning_rate": 3.240008890865864e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2087,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"epoch": 0.46520544443734424,
|
|
"grad_norm": 8.202418931720207,
|
|
"learning_rate": 3.2293467953477454e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2127,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"epoch": 0.46648348137261164,
|
|
"grad_norm": 6.5224761862396585,
|
|
"learning_rate": 3.218670179903216e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.4677615183078791,
|
|
"grad_norm": 11.98921345415937,
|
|
"learning_rate": 3.207979257083658e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2271,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"epoch": 0.4690395552431465,
|
|
"grad_norm": 9.018316199665886,
|
|
"learning_rate": 3.19727423972529e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2005,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"epoch": 0.47031759217841396,
|
|
"grad_norm": 7.973706582819973,
|
|
"learning_rate": 3.186555340944923e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2111,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"epoch": 0.47159562911368136,
|
|
"grad_norm": 8.551400923945243,
|
|
"learning_rate": 3.175822774135724e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2167,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"epoch": 0.4728736660489488,
|
|
"grad_norm": 9.50674868300808,
|
|
"learning_rate": 3.165076752962962e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.4741517029842162,
|
|
"grad_norm": 8.941779392618068,
|
|
"learning_rate": 3.15431749135976e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2065,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"epoch": 0.4754297399194837,
|
|
"grad_norm": 13.024991648002365,
|
|
"learning_rate": 3.1435452035228306e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2345,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"epoch": 0.4767077768547511,
|
|
"grad_norm": 9.638702132762546,
|
|
"learning_rate": 3.132760103908216e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.221,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"epoch": 0.47798581379001853,
|
|
"grad_norm": 10.2285096933021,
|
|
"learning_rate": 3.1219624072270166e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2144,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"epoch": 0.47926385072528593,
|
|
"grad_norm": 9.481196208270266,
|
|
"learning_rate": 3.111152328441115e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.4805418876605534,
|
|
"grad_norm": 11.176541898064906,
|
|
"learning_rate": 3.100330082758901e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.193,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"epoch": 0.4818199245958208,
|
|
"grad_norm": 10.520661709895942,
|
|
"learning_rate": 3.089495885630983e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2282,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"epoch": 0.48309796153108825,
|
|
"grad_norm": 12.274708641803729,
|
|
"learning_rate": 3.0786499527459005e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"epoch": 0.48437599846635565,
|
|
"grad_norm": 12.872720663919305,
|
|
"learning_rate": 3.0677925000258285e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2301,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"epoch": 0.4856540354016231,
|
|
"grad_norm": 9.708450732405078,
|
|
"learning_rate": 3.056923743622283e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.4869320723368905,
|
|
"grad_norm": 8.19563346716996,
|
|
"learning_rate": 3.0460438999118144e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.215,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"epoch": 0.48821010927215797,
|
|
"grad_norm": 8.106052628533215,
|
|
"learning_rate": 3.035153185491698e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"epoch": 0.48948814620742537,
|
|
"grad_norm": 13.685997047539583,
|
|
"learning_rate": 3.024251817175628e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"epoch": 0.4907661831426928,
|
|
"grad_norm": 8.488651222369613,
|
|
"learning_rate": 3.0133400119893947e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2136,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"epoch": 0.4920442200779602,
|
|
"grad_norm": 12.500560067856066,
|
|
"learning_rate": 3.0024179871665706e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2082,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.4933222570132277,
|
|
"grad_norm": 9.56821539066649,
|
|
"learning_rate": 2.9914859601441774e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"epoch": 0.4946002939484951,
|
|
"grad_norm": 11.642104475986267,
|
|
"learning_rate": 2.980544148558365e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2271,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"epoch": 0.49587833088376254,
|
|
"grad_norm": 11.975382621959218,
|
|
"learning_rate": 2.9695927702400767e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"epoch": 0.49715636781902994,
|
|
"grad_norm": 8.674175640451184,
|
|
"learning_rate": 2.958632043210706e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2073,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"epoch": 0.4984344047542974,
|
|
"grad_norm": 13.531335502879033,
|
|
"learning_rate": 2.9476621856777686e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2201,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.4997124416895648,
|
|
"grad_norm": 9.753557812476503,
|
|
"learning_rate": 2.9366834160305466e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1996,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"epoch": 0.5009904786248323,
|
|
"grad_norm": 8.623771874932421,
|
|
"learning_rate": 2.925695952835746e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"epoch": 0.5022685155600997,
|
|
"grad_norm": 8.619272047417937,
|
|
"learning_rate": 2.9147000148331483e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1965,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"epoch": 0.5035465524953671,
|
|
"grad_norm": 9.340092455147365,
|
|
"learning_rate": 2.903695820931249e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2062,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"epoch": 0.5048245894306346,
|
|
"grad_norm": 10.45509856513219,
|
|
"learning_rate": 2.8926835902029043e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.506102626365902,
|
|
"grad_norm": 10.115987074829942,
|
|
"learning_rate": 2.8816635418809693e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2186,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"epoch": 0.5073806633011694,
|
|
"grad_norm": 9.859559908143657,
|
|
"learning_rate": 2.870635895353931e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.1997,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"epoch": 0.5086587002364368,
|
|
"grad_norm": 10.87319879727593,
|
|
"learning_rate": 2.8596008701615456e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2066,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"epoch": 0.5099367371717043,
|
|
"grad_norm": 7.8658320285041405,
|
|
"learning_rate": 2.8485586859904596e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2377,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"epoch": 0.5112147741069717,
|
|
"grad_norm": 8.505547152770252,
|
|
"learning_rate": 2.8375095626698464e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2167,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.5124928110422391,
|
|
"grad_norm": 11.91791149654105,
|
|
"learning_rate": 2.826453720167021e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2182,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"epoch": 0.5137708479775065,
|
|
"grad_norm": 9.968567086380052,
|
|
"learning_rate": 2.815391378583069e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2166,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"epoch": 0.515048884912774,
|
|
"grad_norm": 9.35733243554509,
|
|
"learning_rate": 2.8043227581484556e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"epoch": 0.5163269218480414,
|
|
"grad_norm": 12.215635212530254,
|
|
"learning_rate": 2.79324807921865e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.227,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"epoch": 0.5176049587833088,
|
|
"grad_norm": 13.845726328872498,
|
|
"learning_rate": 2.7821675622697317e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2227,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.5188829957185762,
|
|
"grad_norm": 11.50962600886563,
|
|
"learning_rate": 2.7710814278940057e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2251,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"epoch": 0.5201610326538437,
|
|
"grad_norm": 9.136976034548045,
|
|
"learning_rate": 2.759989896795611e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2297,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"epoch": 0.5214390695891111,
|
|
"grad_norm": 8.603249208409274,
|
|
"learning_rate": 2.748893189786122e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"epoch": 0.5227171065243785,
|
|
"grad_norm": 8.84947914185045,
|
|
"learning_rate": 2.7377915277801586e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1985,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"epoch": 0.5239951434596459,
|
|
"grad_norm": 10.135619925874042,
|
|
"learning_rate": 2.726685131790983e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2223,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.5252731803949134,
|
|
"grad_norm": 8.893574056072689,
|
|
"learning_rate": 2.715574222926105e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2135,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"epoch": 0.5265512173301808,
|
|
"grad_norm": 5.695805913492098,
|
|
"learning_rate": 2.704459022382874e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"epoch": 0.5278292542654482,
|
|
"grad_norm": 6.965426261729773,
|
|
"learning_rate": 2.6933397514440796e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"epoch": 0.5291072912007156,
|
|
"grad_norm": 8.527321271713554,
|
|
"learning_rate": 2.6822166314735483e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2099,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"epoch": 0.5303853281359832,
|
|
"grad_norm": 11.360590075652906,
|
|
"learning_rate": 2.67108988391173e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2048,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.5316633650712506,
|
|
"grad_norm": 8.516403048337656,
|
|
"learning_rate": 2.659959730271295e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2067,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"epoch": 0.532941402006518,
|
|
"grad_norm": 8.679204620956797,
|
|
"learning_rate": 2.64882639213272e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"epoch": 0.5342194389417854,
|
|
"grad_norm": 8.214012990646285,
|
|
"learning_rate": 2.6376900911398823e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.208,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"epoch": 0.5354974758770529,
|
|
"grad_norm": 6.735540115827085,
|
|
"learning_rate": 2.626551048995643e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"epoch": 0.5367755128123203,
|
|
"grad_norm": 13.73261695569995,
|
|
"learning_rate": 2.6154094874574326e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2123,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.5380535497475877,
|
|
"grad_norm": 8.6629838322733,
|
|
"learning_rate": 2.60426562833284e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2017,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"epoch": 0.5393315866828551,
|
|
"grad_norm": 8.083638442585006,
|
|
"learning_rate": 2.593119693475193e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"epoch": 0.5406096236181226,
|
|
"grad_norm": 9.85692839498437,
|
|
"learning_rate": 2.5819719047791467e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2145,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"epoch": 0.54188766055339,
|
|
"grad_norm": 8.33356041058765,
|
|
"learning_rate": 2.570822484176257e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.1869,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"epoch": 0.5431656974886574,
|
|
"grad_norm": 10.85907589382582,
|
|
"learning_rate": 2.559671653630574e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1961,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.5444437344239248,
|
|
"grad_norm": 12.760498280651152,
|
|
"learning_rate": 2.5485196351342146e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2171,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"epoch": 0.5457217713591923,
|
|
"grad_norm": 11.083493467932884,
|
|
"learning_rate": 2.537366650702944e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"epoch": 0.5469998082944597,
|
|
"grad_norm": 9.755652576792773,
|
|
"learning_rate": 2.526212922371758e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2014,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"epoch": 0.5482778452297271,
|
|
"grad_norm": 8.621966933478847,
|
|
"learning_rate": 2.5150586721904653e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1909,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"epoch": 0.5495558821649945,
|
|
"grad_norm": 9.93825493293039,
|
|
"learning_rate": 2.50390412221926e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2184,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.550833919100262,
|
|
"grad_norm": 6.4926059090968735,
|
|
"learning_rate": 2.492749494524305e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2104,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"epoch": 0.5521119560355294,
|
|
"grad_norm": 8.178900908424412,
|
|
"learning_rate": 2.481595011173312e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"epoch": 0.5533899929707968,
|
|
"grad_norm": 8.902539400477867,
|
|
"learning_rate": 2.470440894231118e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"epoch": 0.5546680299060642,
|
|
"grad_norm": 10.017993004725309,
|
|
"learning_rate": 2.4592873657552665e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2036,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"epoch": 0.5559460668413317,
|
|
"grad_norm": 9.086190763346574,
|
|
"learning_rate": 2.4481346477915834e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.1972,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.5572241037765991,
|
|
"grad_norm": 8.170239096649071,
|
|
"learning_rate": 2.436982962369761e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"epoch": 0.5585021407118665,
|
|
"grad_norm": 9.91347550249988,
|
|
"learning_rate": 2.4258325314989364e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1968,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"epoch": 0.559780177647134,
|
|
"grad_norm": 12.649043344414686,
|
|
"learning_rate": 2.4146835771632674e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2218,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"epoch": 0.5610582145824015,
|
|
"grad_norm": 10.059811667819446,
|
|
"learning_rate": 2.4035363213175206e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2293,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"epoch": 0.5623362515176689,
|
|
"grad_norm": 7.752963384126441,
|
|
"learning_rate": 2.3923909858826473e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.5636142884529363,
|
|
"grad_norm": 8.121775627856461,
|
|
"learning_rate": 2.3812477927413648e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"epoch": 0.5648923253882037,
|
|
"grad_norm": 10.889500920755431,
|
|
"learning_rate": 2.370106963733746e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"epoch": 0.5661703623234712,
|
|
"grad_norm": 7.787520343491865,
|
|
"learning_rate": 2.358968720652794e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"epoch": 0.5674483992587386,
|
|
"grad_norm": 7.334229409175379,
|
|
"learning_rate": 2.3478332852400337e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1986,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"epoch": 0.568726436194006,
|
|
"grad_norm": 9.306378482430759,
|
|
"learning_rate": 2.3367008791810936e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2612,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.5700044731292734,
|
|
"grad_norm": 7.851220791242363,
|
|
"learning_rate": 2.3255717241012922e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1944,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"epoch": 0.5712825100645409,
|
|
"grad_norm": 8.633601777057988,
|
|
"learning_rate": 2.314446041561231e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1993,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"epoch": 0.5725605469998083,
|
|
"grad_norm": 10.040452596074198,
|
|
"learning_rate": 2.303324053052374e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2164,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"epoch": 0.5738385839350757,
|
|
"grad_norm": 9.442768793474356,
|
|
"learning_rate": 2.29220597999265e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"epoch": 0.5751166208703431,
|
|
"grad_norm": 10.471888355465538,
|
|
"learning_rate": 2.281092043722034e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1983,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.5763946578056106,
|
|
"grad_norm": 10.315495473189406,
|
|
"learning_rate": 2.269982465498147e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.191,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"epoch": 0.577672694740878,
|
|
"grad_norm": 9.421774060820772,
|
|
"learning_rate": 2.2588774664918503e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2269,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"epoch": 0.5789507316761454,
|
|
"grad_norm": 10.653247108656883,
|
|
"learning_rate": 2.2477772677828396e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"epoch": 0.5802287686114128,
|
|
"grad_norm": 12.414812092039039,
|
|
"learning_rate": 2.2366820903552478e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"epoch": 0.5815068055466803,
|
|
"grad_norm": 15.625418407059193,
|
|
"learning_rate": 2.2255921550932418e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1943,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.5827848424819477,
|
|
"grad_norm": 9.748298215017456,
|
|
"learning_rate": 2.214507682776627e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"epoch": 0.5840628794172151,
|
|
"grad_norm": 9.206249587748667,
|
|
"learning_rate": 2.203428894076454e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2063,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"epoch": 0.5853409163524825,
|
|
"grad_norm": 10.742870208744552,
|
|
"learning_rate": 2.1923560095506196e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2239,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"epoch": 0.58661895328775,
|
|
"grad_norm": 9.987652609751306,
|
|
"learning_rate": 2.1812892496394835e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2105,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"epoch": 0.5878969902230174,
|
|
"grad_norm": 10.530610611497671,
|
|
"learning_rate": 2.170228834661472e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.5891750271582848,
|
|
"grad_norm": 9.50246727866108,
|
|
"learning_rate": 2.1591749848086992e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2007,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"epoch": 0.5904530640935522,
|
|
"grad_norm": 8.38828507992591,
|
|
"learning_rate": 2.148127920142577e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"epoch": 0.5917311010288198,
|
|
"grad_norm": 11.01986987749802,
|
|
"learning_rate": 2.1370878605894377e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"epoch": 0.5930091379640872,
|
|
"grad_norm": 10.543139498864132,
|
|
"learning_rate": 2.1260550259361576e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"epoch": 0.5942871748993546,
|
|
"grad_norm": 9.129152697037581,
|
|
"learning_rate": 2.115029635825774e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2175,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.595565211834622,
|
|
"grad_norm": 8.38626724144314,
|
|
"learning_rate": 2.104011909753122e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2239,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"epoch": 0.5968432487698895,
|
|
"grad_norm": 8.949485168979885,
|
|
"learning_rate": 2.093002067060457e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2056,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"epoch": 0.5981212857051569,
|
|
"grad_norm": 10.070633848403004,
|
|
"learning_rate": 2.082000326933092e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2003,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"epoch": 0.5993993226404243,
|
|
"grad_norm": 6.916035441529172,
|
|
"learning_rate": 2.0710069083950342e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2108,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"epoch": 0.6006773595756917,
|
|
"grad_norm": 9.474816951227877,
|
|
"learning_rate": 2.060022030304621e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2323,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.6019553965109592,
|
|
"grad_norm": 6.119960790414473,
|
|
"learning_rate": 2.0490459113501686e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"epoch": 0.6032334334462266,
|
|
"grad_norm": 8.27937601473009,
|
|
"learning_rate": 2.0380787700456132e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"epoch": 0.604511470381494,
|
|
"grad_norm": 8.980086860385386,
|
|
"learning_rate": 2.027120824726162e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2223,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"epoch": 0.6057895073167614,
|
|
"grad_norm": 9.660168225938056,
|
|
"learning_rate": 2.0161722935439513e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"epoch": 0.6070675442520289,
|
|
"grad_norm": 9.627236009510261,
|
|
"learning_rate": 2.0052333944636954e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2116,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.6083455811872963,
|
|
"grad_norm": 11.942856212349632,
|
|
"learning_rate": 1.994304345258354e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1951,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"epoch": 0.6096236181225637,
|
|
"grad_norm": 8.668637933993713,
|
|
"learning_rate": 1.983385363504793e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"epoch": 0.6109016550578311,
|
|
"grad_norm": 8.513667560083928,
|
|
"learning_rate": 1.9724766665794541e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1825,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"epoch": 0.6121796919930986,
|
|
"grad_norm": 12.364360217821515,
|
|
"learning_rate": 1.9615784716540293e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"epoch": 0.613457728928366,
|
|
"grad_norm": 13.291905969721395,
|
|
"learning_rate": 1.950690995691132e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1938,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.6147357658636334,
|
|
"grad_norm": 12.195901066043673,
|
|
"learning_rate": 1.939814455439986e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2181,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"epoch": 0.6160138027989008,
|
|
"grad_norm": 10.210694409079894,
|
|
"learning_rate": 1.9289490674321017e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"epoch": 0.6172918397341683,
|
|
"grad_norm": 11.462328801893005,
|
|
"learning_rate": 1.9180950479769687e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2051,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"epoch": 0.6185698766694357,
|
|
"grad_norm": 11.618759475704962,
|
|
"learning_rate": 1.9072526131577542e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1983,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"epoch": 0.6198479136047031,
|
|
"grad_norm": 8.771287614014419,
|
|
"learning_rate": 1.8964219788269936e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2049,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"epoch": 0.6211259505399706,
|
|
"grad_norm": 8.382633721719383,
|
|
"learning_rate": 1.885603360602298e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"epoch": 0.6224039874752381,
|
|
"grad_norm": 11.874740935105166,
|
|
"learning_rate": 1.8747969738620588e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"epoch": 0.6236820244105055,
|
|
"grad_norm": 10.358959890573601,
|
|
"learning_rate": 1.8640030337411655e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2078,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"epoch": 0.6249600613457729,
|
|
"grad_norm": 6.897393388027196,
|
|
"learning_rate": 1.8532217551267143e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.211,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"epoch": 0.6262380982810403,
|
|
"grad_norm": 8.014943203049432,
|
|
"learning_rate": 1.842453352653735e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1946,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"epoch": 0.6275161352163078,
|
|
"grad_norm": 12.135630997047638,
|
|
"learning_rate": 1.8316980407009213e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2056,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"epoch": 0.6287941721515752,
|
|
"grad_norm": 8.388293855177318,
|
|
"learning_rate": 1.820956033386355e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2067,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"epoch": 0.6300722090868426,
|
|
"grad_norm": 8.19919332061552,
|
|
"learning_rate": 1.8102275445632508e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2086,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"epoch": 0.63135024602211,
|
|
"grad_norm": 11.524825983155498,
|
|
"learning_rate": 1.7995127878156934e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"epoch": 0.6326282829573775,
|
|
"grad_norm": 11.935553547051724,
|
|
"learning_rate": 1.788811976454388e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2033,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"epoch": 0.6339063198926449,
|
|
"grad_norm": 8.72541909787161,
|
|
"learning_rate": 1.7781253235124143e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.217,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"epoch": 0.6351843568279123,
|
|
"grad_norm": 9.174733860455342,
|
|
"learning_rate": 1.7674530417409823e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"epoch": 0.6364623937631797,
|
|
"grad_norm": 8.931555059599443,
|
|
"learning_rate": 1.7567953436052025e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.219,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"epoch": 0.6377404306984472,
|
|
"grad_norm": 14.476630420450562,
|
|
"learning_rate": 1.7461524412798503e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2224,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"epoch": 0.6390184676337146,
|
|
"grad_norm": 9.976279367626784,
|
|
"learning_rate": 1.735524546645142e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.640296504568982,
|
|
"grad_norm": 10.403381451664748,
|
|
"learning_rate": 1.7249118712825246e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 5010
|
|
},
|
|
{
|
|
"epoch": 0.6415745415042494,
|
|
"grad_norm": 10.392003155571185,
|
|
"learning_rate": 1.7143146264704522e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"epoch": 0.6428525784395169,
|
|
"grad_norm": 9.045623892838288,
|
|
"learning_rate": 1.70373302318019e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5030
|
|
},
|
|
{
|
|
"epoch": 0.6441306153747843,
|
|
"grad_norm": 10.532156784041835,
|
|
"learning_rate": 1.6931672720716072e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2078,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"epoch": 0.6454086523100517,
|
|
"grad_norm": 13.016020098769182,
|
|
"learning_rate": 1.6826175834889866e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5050
|
|
},
|
|
{
|
|
"epoch": 0.6466866892453191,
|
|
"grad_norm": 8.620907567456209,
|
|
"learning_rate": 1.6720841674568365e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2049,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"epoch": 0.6479647261805866,
|
|
"grad_norm": 12.006083637867917,
|
|
"learning_rate": 1.6615672336757074e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1859,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5070
|
|
},
|
|
{
|
|
"epoch": 0.649242763115854,
|
|
"grad_norm": 8.58425824054044,
|
|
"learning_rate": 1.6510669915180238e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"epoch": 0.6505208000511215,
|
|
"grad_norm": 10.81312666340287,
|
|
"learning_rate": 1.6405836500239062e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5090
|
|
},
|
|
{
|
|
"epoch": 0.6517988369863889,
|
|
"grad_norm": 10.6110280495848,
|
|
"learning_rate": 1.6301174178970162e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1746,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"epoch": 0.6530768739216564,
|
|
"grad_norm": 9.639168092530126,
|
|
"learning_rate": 1.6196685035004015e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.188,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5110
|
|
},
|
|
{
|
|
"epoch": 0.6543549108569238,
|
|
"grad_norm": 9.11174679070626,
|
|
"learning_rate": 1.6092371148523442e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"epoch": 0.6556329477921912,
|
|
"grad_norm": 13.377523718184259,
|
|
"learning_rate": 1.5988234596222234e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5130
|
|
},
|
|
{
|
|
"epoch": 0.6569109847274586,
|
|
"grad_norm": 7.383701674519595,
|
|
"learning_rate": 1.5884277451263772e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"epoch": 0.6581890216627261,
|
|
"grad_norm": 8.953980108485803,
|
|
"learning_rate": 1.57805017832398e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.1813,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5150
|
|
},
|
|
{
|
|
"epoch": 0.6594670585979935,
|
|
"grad_norm": 9.211324366300165,
|
|
"learning_rate": 1.567690965812918e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"epoch": 0.6607450955332609,
|
|
"grad_norm": 10.95169892297747,
|
|
"learning_rate": 1.5573503138256756e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2315,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5170
|
|
},
|
|
{
|
|
"epoch": 0.6620231324685283,
|
|
"grad_norm": 7.565071174200895,
|
|
"learning_rate": 1.5470284282252367e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2028,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"epoch": 0.6633011694037958,
|
|
"grad_norm": 8.40586608624689,
|
|
"learning_rate": 1.5367255145009784e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1959,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5190
|
|
},
|
|
{
|
|
"epoch": 0.6645792063390632,
|
|
"grad_norm": 8.935789239896653,
|
|
"learning_rate": 1.5264417777645849e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1954,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"epoch": 0.6658572432743306,
|
|
"grad_norm": 10.318660688920453,
|
|
"learning_rate": 1.516177422745962e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2219,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5210
|
|
},
|
|
{
|
|
"epoch": 0.667135280209598,
|
|
"grad_norm": 10.599513311823376,
|
|
"learning_rate": 1.5059326537891602e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"epoch": 0.6684133171448655,
|
|
"grad_norm": 7.096941650990519,
|
|
"learning_rate": 1.4957076748483121e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5230
|
|
},
|
|
{
|
|
"epoch": 0.6696913540801329,
|
|
"grad_norm": 9.134873030915466,
|
|
"learning_rate": 1.4855026894835636e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2057,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"epoch": 0.6709693910154003,
|
|
"grad_norm": 9.184838305556584,
|
|
"learning_rate": 1.4753179008570304e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5250
|
|
},
|
|
{
|
|
"epoch": 0.6722474279506677,
|
|
"grad_norm": 10.491068837200642,
|
|
"learning_rate": 1.4651535117287462e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"epoch": 0.6735254648859352,
|
|
"grad_norm": 13.654106335755474,
|
|
"learning_rate": 1.4550097244526294e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5270
|
|
},
|
|
{
|
|
"epoch": 0.6748035018212026,
|
|
"grad_norm": 7.2043046993448305,
|
|
"learning_rate": 1.444886740972455e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2154,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"epoch": 0.67608153875647,
|
|
"grad_norm": 10.381553344000324,
|
|
"learning_rate": 1.4347847628178315e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.1997,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 5290
|
|
},
|
|
{
|
|
"epoch": 0.6773595756917374,
|
|
"grad_norm": 8.206259981152979,
|
|
"learning_rate": 1.4247039911001952e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2015,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"epoch": 0.678637612627005,
|
|
"grad_norm": 9.733010610608309,
|
|
"learning_rate": 1.4146446265087966e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5310
|
|
},
|
|
{
|
|
"epoch": 0.6799156495622724,
|
|
"grad_norm": 7.818977341913373,
|
|
"learning_rate": 1.4046068693067142e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"epoch": 0.6811936864975398,
|
|
"grad_norm": 11.569940395290942,
|
|
"learning_rate": 1.3945909193268618e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5330
|
|
},
|
|
{
|
|
"epoch": 0.6824717234328072,
|
|
"grad_norm": 7.915367886076844,
|
|
"learning_rate": 1.3845969759680125e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"epoch": 0.6837497603680747,
|
|
"grad_norm": 10.40013069384589,
|
|
"learning_rate": 1.374625238190832e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5350
|
|
},
|
|
{
|
|
"epoch": 0.6850277973033421,
|
|
"grad_norm": 8.748911256197829,
|
|
"learning_rate": 1.36467590451391e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"epoch": 0.6863058342386095,
|
|
"grad_norm": 10.775546531691665,
|
|
"learning_rate": 1.354749173009815e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5370
|
|
},
|
|
{
|
|
"epoch": 0.6875838711738769,
|
|
"grad_norm": 10.000688484948581,
|
|
"learning_rate": 1.3448452413011487e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"epoch": 0.6888619081091444,
|
|
"grad_norm": 8.846111947475448,
|
|
"learning_rate": 1.3349643065566117e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5390
|
|
},
|
|
{
|
|
"epoch": 0.6901399450444118,
|
|
"grad_norm": 6.691267912708803,
|
|
"learning_rate": 1.325106565487077e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.169,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"epoch": 0.6914179819796792,
|
|
"grad_norm": 10.58421750198331,
|
|
"learning_rate": 1.3152722143416748e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2012,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5410
|
|
},
|
|
{
|
|
"epoch": 0.6926960189149466,
|
|
"grad_norm": 10.823577363873508,
|
|
"learning_rate": 1.305461448903889e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1942,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"epoch": 0.6939740558502141,
|
|
"grad_norm": 10.703964915891877,
|
|
"learning_rate": 1.2956744644876527e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1891,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5430
|
|
},
|
|
{
|
|
"epoch": 0.6952520927854815,
|
|
"grad_norm": 11.074215783268322,
|
|
"learning_rate": 1.2859114559334643e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"epoch": 0.6965301297207489,
|
|
"grad_norm": 9.906928540767801,
|
|
"learning_rate": 1.27617261760451e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5450
|
|
},
|
|
{
|
|
"epoch": 0.6978081666560163,
|
|
"grad_norm": 7.989791957700388,
|
|
"learning_rate": 1.26645814338279e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"epoch": 0.6990862035912838,
|
|
"grad_norm": 9.439843209419791,
|
|
"learning_rate": 1.2567682266652603e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1897,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5470
|
|
},
|
|
{
|
|
"epoch": 0.7003642405265512,
|
|
"grad_norm": 12.246737461104036,
|
|
"learning_rate": 1.2471030603599848e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"epoch": 0.7016422774618186,
|
|
"grad_norm": 10.904656119606363,
|
|
"learning_rate": 1.2374628368822914e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1955,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5490
|
|
},
|
|
{
|
|
"epoch": 0.702920314397086,
|
|
"grad_norm": 10.223805884896036,
|
|
"learning_rate": 1.2278477481509462e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"epoch": 0.7041983513323535,
|
|
"grad_norm": 10.18405106135057,
|
|
"learning_rate": 1.2182579855843247e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.1946,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5510
|
|
},
|
|
{
|
|
"epoch": 0.7054763882676209,
|
|
"grad_norm": 9.516374791611842,
|
|
"learning_rate": 1.2086937400966129e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1833,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"epoch": 0.7067544252028883,
|
|
"grad_norm": 9.178417627197618,
|
|
"learning_rate": 1.1991552020939918e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5530
|
|
},
|
|
{
|
|
"epoch": 0.7080324621381557,
|
|
"grad_norm": 7.880000574893539,
|
|
"learning_rate": 1.1896425614708589e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"epoch": 0.7093104990734233,
|
|
"grad_norm": 8.442036728841282,
|
|
"learning_rate": 1.1801560076060444e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5550
|
|
},
|
|
{
|
|
"epoch": 0.7105885360086907,
|
|
"grad_norm": 9.96646161178662,
|
|
"learning_rate": 1.170695729359037e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"epoch": 0.711866572943958,
|
|
"grad_norm": 9.055717430232225,
|
|
"learning_rate": 1.1612619150662303e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1913,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5570
|
|
},
|
|
{
|
|
"epoch": 0.7131446098792255,
|
|
"grad_norm": 10.28494479666354,
|
|
"learning_rate": 1.151854752537168e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"epoch": 0.714422646814493,
|
|
"grad_norm": 32.22289570587893,
|
|
"learning_rate": 1.1424744290508078e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.189,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5590
|
|
},
|
|
{
|
|
"epoch": 0.7157006837497604,
|
|
"grad_norm": 12.269299820154668,
|
|
"learning_rate": 1.1331211313517925e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"epoch": 0.7169787206850278,
|
|
"grad_norm": 9.260295940193947,
|
|
"learning_rate": 1.1237950456467308e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5610
|
|
},
|
|
{
|
|
"epoch": 0.7182567576202952,
|
|
"grad_norm": 10.398934473010499,
|
|
"learning_rate": 1.1144963576004954e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2061,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"epoch": 0.7195347945555627,
|
|
"grad_norm": 12.87874977657243,
|
|
"learning_rate": 1.105225252332519e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5630
|
|
},
|
|
{
|
|
"epoch": 0.7208128314908301,
|
|
"grad_norm": 10.678499925600923,
|
|
"learning_rate": 1.0959819144131144e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.188,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"epoch": 0.7220908684260975,
|
|
"grad_norm": 10.020846983046951,
|
|
"learning_rate": 1.0867665278597999e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5650
|
|
},
|
|
{
|
|
"epoch": 0.7233689053613649,
|
|
"grad_norm": 11.621911848428802,
|
|
"learning_rate": 1.0775792761336324e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1806,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"epoch": 0.7246469422966324,
|
|
"grad_norm": 10.137810011465284,
|
|
"learning_rate": 1.0684203421355587e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1977,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5670
|
|
},
|
|
{
|
|
"epoch": 0.7259249792318998,
|
|
"grad_norm": 7.480549183495816,
|
|
"learning_rate": 1.0592899082027715e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1947,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"epoch": 0.7272030161671672,
|
|
"grad_norm": 8.95863895797164,
|
|
"learning_rate": 1.050188156105081e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2002,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5690
|
|
},
|
|
{
|
|
"epoch": 0.7284810531024346,
|
|
"grad_norm": 10.004556249685614,
|
|
"learning_rate": 1.0411152670412975e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1985,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"epoch": 0.7297590900377021,
|
|
"grad_norm": 9.942783279630579,
|
|
"learning_rate": 1.03207142163562e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1943,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5710
|
|
},
|
|
{
|
|
"epoch": 0.7310371269729695,
|
|
"grad_norm": 11.150110790338827,
|
|
"learning_rate": 1.023056799934045e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2104,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"epoch": 0.7323151639082369,
|
|
"grad_norm": 11.60741920901857,
|
|
"learning_rate": 1.0140715814007783e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2133,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5730
|
|
},
|
|
{
|
|
"epoch": 0.7335932008435043,
|
|
"grad_norm": 11.162758346658364,
|
|
"learning_rate": 1.0051159449146643e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1933,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"epoch": 0.7348712377787718,
|
|
"grad_norm": 8.587527124699529,
|
|
"learning_rate": 9.961900687656242e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1919,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5750
|
|
},
|
|
{
|
|
"epoch": 0.7361492747140392,
|
|
"grad_norm": 13.109963319894257,
|
|
"learning_rate": 9.872941306511056e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"epoch": 0.7374273116493066,
|
|
"grad_norm": 9.579077500243383,
|
|
"learning_rate": 9.784283076725494e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 5770
|
|
},
|
|
{
|
|
"epoch": 0.738705348584574,
|
|
"grad_norm": 10.231998859747685,
|
|
"learning_rate": 9.695927763318568e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"epoch": 0.7399833855198416,
|
|
"grad_norm": 9.12737106467952,
|
|
"learning_rate": 9.607877125278823e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 5790
|
|
},
|
|
{
|
|
"epoch": 0.741261422455109,
|
|
"grad_norm": 10.593757644763386,
|
|
"learning_rate": 9.520132915529269e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"epoch": 0.7425394593903764,
|
|
"grad_norm": 9.210720279818025,
|
|
"learning_rate": 9.432696880892513e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5810
|
|
},
|
|
{
|
|
"epoch": 0.7438174963256438,
|
|
"grad_norm": 7.802302327948419,
|
|
"learning_rate": 9.345570762055968e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2068,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"epoch": 0.7450955332609113,
|
|
"grad_norm": 10.07799824095782,
|
|
"learning_rate": 9.2587562935372e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2027,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5830
|
|
},
|
|
{
|
|
"epoch": 0.7463735701961787,
|
|
"grad_norm": 10.091705801781881,
|
|
"learning_rate": 9.172255203649429e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1918,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"epoch": 0.7476516071314461,
|
|
"grad_norm": 8.580309136599698,
|
|
"learning_rate": 9.086069214467062e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2003,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5850
|
|
},
|
|
{
|
|
"epoch": 0.7489296440667135,
|
|
"grad_norm": 8.566545143121003,
|
|
"learning_rate": 9.00020004179145e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1871,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"epoch": 0.750207681001981,
|
|
"grad_norm": 12.927074227474392,
|
|
"learning_rate": 8.914649395116752e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2015,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5870
|
|
},
|
|
{
|
|
"epoch": 0.7514857179372484,
|
|
"grad_norm": 10.661616736265797,
|
|
"learning_rate": 8.829418977595815e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2289,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"epoch": 0.7527637548725158,
|
|
"grad_norm": 10.401588278458307,
|
|
"learning_rate": 8.744510486006376e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1992,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5890
|
|
},
|
|
{
|
|
"epoch": 0.7540417918077832,
|
|
"grad_norm": 8.981678643474089,
|
|
"learning_rate": 8.659925610717208e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"epoch": 0.7553198287430507,
|
|
"grad_norm": 9.88446374371275,
|
|
"learning_rate": 8.575666035654482e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5910
|
|
},
|
|
{
|
|
"epoch": 0.7565978656783181,
|
|
"grad_norm": 8.026599552996204,
|
|
"learning_rate": 8.491733438268281e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"epoch": 0.7578759026135855,
|
|
"grad_norm": 11.0399148587071,
|
|
"learning_rate": 8.408129489499147e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5930
|
|
},
|
|
{
|
|
"epoch": 0.7591539395488529,
|
|
"grad_norm": 9.850294858942783,
|
|
"learning_rate": 8.324855853744858e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1793,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"epoch": 0.7604319764841204,
|
|
"grad_norm": 8.702468926444062,
|
|
"learning_rate": 8.241914188827278e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1857,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5950
|
|
},
|
|
{
|
|
"epoch": 0.7617100134193878,
|
|
"grad_norm": 13.512408481784156,
|
|
"learning_rate": 8.159306145959346e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1995,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"epoch": 0.7629880503546552,
|
|
"grad_norm": 13.572590071392261,
|
|
"learning_rate": 8.077033369712233e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5970
|
|
},
|
|
{
|
|
"epoch": 0.7642660872899226,
|
|
"grad_norm": 10.395036087264575,
|
|
"learning_rate": 7.99509749798255e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.875,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"epoch": 0.7655441242251901,
|
|
"grad_norm": 11.431047760742594,
|
|
"learning_rate": 7.913500161959808e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1859,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5990
|
|
},
|
|
{
|
|
"epoch": 0.7668221611604575,
|
|
"grad_norm": 12.12350658078121,
|
|
"learning_rate": 7.832242986093877e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 0.7681001980957249,
|
|
"grad_norm": 12.154386070056951,
|
|
"learning_rate": 7.751327588062686e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1784,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6010
|
|
},
|
|
{
|
|
"epoch": 0.7693782350309923,
|
|
"grad_norm": 8.9004131803929,
|
|
"learning_rate": 7.670755578740012e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1882,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"epoch": 0.7706562719662599,
|
|
"grad_norm": 13.277525759657642,
|
|
"learning_rate": 7.590528562163393e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6030
|
|
},
|
|
{
|
|
"epoch": 0.7719343089015273,
|
|
"grad_norm": 10.410706613714614,
|
|
"learning_rate": 7.51064813550224e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.1864,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"epoch": 0.7732123458367947,
|
|
"grad_norm": 9.515020477677249,
|
|
"learning_rate": 7.431115889025969e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1924,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6050
|
|
},
|
|
{
|
|
"epoch": 0.7744903827720621,
|
|
"grad_norm": 13.072675219028618,
|
|
"learning_rate": 7.35193340607242e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"epoch": 0.7757684197073296,
|
|
"grad_norm": 12.782269185266843,
|
|
"learning_rate": 7.27310226301627e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2286,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6070
|
|
},
|
|
{
|
|
"epoch": 0.777046456642597,
|
|
"grad_norm": 9.418209612849605,
|
|
"learning_rate": 7.194624029237686e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"epoch": 0.7783244935778644,
|
|
"grad_norm": 12.973911791232169,
|
|
"learning_rate": 7.116500267091077e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1974,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6090
|
|
},
|
|
{
|
|
"epoch": 0.7796025305131318,
|
|
"grad_norm": 10.172661744116416,
|
|
"learning_rate": 7.038732531873972e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1876,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"epoch": 0.7808805674483993,
|
|
"grad_norm": 11.421418477500511,
|
|
"learning_rate": 6.961322371796105e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.181,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6110
|
|
},
|
|
{
|
|
"epoch": 0.7821586043836667,
|
|
"grad_norm": 11.428398005666034,
|
|
"learning_rate": 6.884271327948524e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"epoch": 0.7834366413189341,
|
|
"grad_norm": 8.869925031073004,
|
|
"learning_rate": 6.807580934272961e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2008,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6130
|
|
},
|
|
{
|
|
"epoch": 0.7847146782542015,
|
|
"grad_norm": 10.839154644975004,
|
|
"learning_rate": 6.731252717531288e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1885,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"epoch": 0.785992715189469,
|
|
"grad_norm": 8.44382605953575,
|
|
"learning_rate": 6.655288197275103e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6150
|
|
},
|
|
{
|
|
"epoch": 0.7872707521247364,
|
|
"grad_norm": 11.179183535261652,
|
|
"learning_rate": 6.579688885815488e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"epoch": 0.7885487890600038,
|
|
"grad_norm": 11.052997512838385,
|
|
"learning_rate": 6.504456288192911e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6170
|
|
},
|
|
{
|
|
"epoch": 0.7898268259952712,
|
|
"grad_norm": 11.267748953593356,
|
|
"learning_rate": 6.429591902147238e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1826,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"epoch": 0.7911048629305387,
|
|
"grad_norm": 8.88100812958174,
|
|
"learning_rate": 6.355097218087968e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1853,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6190
|
|
},
|
|
{
|
|
"epoch": 0.7923828998658061,
|
|
"grad_norm": 13.932434101787049,
|
|
"learning_rate": 6.280973719064486e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.209,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"epoch": 0.7936609368010735,
|
|
"grad_norm": 12.849144357607967,
|
|
"learning_rate": 6.207222880736618e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2326,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6210
|
|
},
|
|
{
|
|
"epoch": 0.7949389737363409,
|
|
"grad_norm": 13.733842838610288,
|
|
"learning_rate": 6.133846171345189e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2189,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"epoch": 0.7962170106716084,
|
|
"grad_norm": 8.758429764508215,
|
|
"learning_rate": 6.060845051682809e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.1997,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 6230
|
|
},
|
|
{
|
|
"epoch": 0.7974950476068758,
|
|
"grad_norm": 10.236369223553389,
|
|
"learning_rate": 5.988220975064839e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"epoch": 0.7987730845421432,
|
|
"grad_norm": 8.991486561608422,
|
|
"learning_rate": 5.915975387300382e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6250
|
|
},
|
|
{
|
|
"epoch": 0.8000511214774106,
|
|
"grad_norm": 10.298101814631558,
|
|
"learning_rate": 5.844109726663568e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1978,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"epoch": 0.8013291584126782,
|
|
"grad_norm": 11.9546027222749,
|
|
"learning_rate": 5.772625423864866e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2097,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 6270
|
|
},
|
|
{
|
|
"epoch": 0.8026071953479456,
|
|
"grad_norm": 10.119304805105461,
|
|
"learning_rate": 5.701523902022659e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"epoch": 0.803885232283213,
|
|
"grad_norm": 11.201139785327099,
|
|
"learning_rate": 5.6308065766348335e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6290
|
|
},
|
|
{
|
|
"epoch": 0.8051632692184804,
|
|
"grad_norm": 9.377393381822943,
|
|
"learning_rate": 5.5604748555506706e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.193,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"epoch": 0.8064413061537479,
|
|
"grad_norm": 8.309075509709407,
|
|
"learning_rate": 5.490530138942806e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1863,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6310
|
|
},
|
|
{
|
|
"epoch": 0.8077193430890153,
|
|
"grad_norm": 10.253650621317618,
|
|
"learning_rate": 5.4209738192793146e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"epoch": 0.8089973800242827,
|
|
"grad_norm": 8.25575714700977,
|
|
"learning_rate": 5.3518072812960444e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6330
|
|
},
|
|
{
|
|
"epoch": 0.8102754169595501,
|
|
"grad_norm": 11.079645770003177,
|
|
"learning_rate": 5.283031901969001e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1864,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"epoch": 0.8115534538948176,
|
|
"grad_norm": 8.856148371837136,
|
|
"learning_rate": 5.214649050486961e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.184,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 6350
|
|
},
|
|
{
|
|
"epoch": 0.812831490830085,
|
|
"grad_norm": 11.181004151997012,
|
|
"learning_rate": 5.146660088224209e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2045,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"epoch": 0.8141095277653524,
|
|
"grad_norm": 9.153609232514398,
|
|
"learning_rate": 5.079066368713425e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1839,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6370
|
|
},
|
|
{
|
|
"epoch": 0.8153875647006198,
|
|
"grad_norm": 9.535738595895644,
|
|
"learning_rate": 5.011869237618768e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1878,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"epoch": 0.8166656016358873,
|
|
"grad_norm": 8.741327828462396,
|
|
"learning_rate": 4.9450700327090535e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2076,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6390
|
|
},
|
|
{
|
|
"epoch": 0.8179436385711547,
|
|
"grad_norm": 11.426080897454439,
|
|
"learning_rate": 4.8786700838311184e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2068,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"epoch": 0.8192216755064221,
|
|
"grad_norm": 11.034650603841335,
|
|
"learning_rate": 4.812670712883396e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2023,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6410
|
|
},
|
|
{
|
|
"epoch": 0.8204997124416895,
|
|
"grad_norm": 9.254914740173248,
|
|
"learning_rate": 4.7470732337895335e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"epoch": 0.821777749376957,
|
|
"grad_norm": 11.445609691444933,
|
|
"learning_rate": 4.681878952472276e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1843,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6430
|
|
},
|
|
{
|
|
"epoch": 0.8230557863122244,
|
|
"grad_norm": 10.758939904944432,
|
|
"learning_rate": 4.6170891668274616e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1844,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"epoch": 0.8243338232474918,
|
|
"grad_norm": 6.823878634242757,
|
|
"learning_rate": 4.552705166698165e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.198,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6450
|
|
},
|
|
{
|
|
"epoch": 0.8256118601827592,
|
|
"grad_norm": 11.462876385323876,
|
|
"learning_rate": 4.488728233849054e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1919,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"epoch": 0.8268898971180267,
|
|
"grad_norm": 11.511738897771243,
|
|
"learning_rate": 4.425159641940826e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6470
|
|
},
|
|
{
|
|
"epoch": 0.8281679340532941,
|
|
"grad_norm": 10.012274810790023,
|
|
"learning_rate": 4.362000656504902e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"epoch": 0.8294459709885615,
|
|
"grad_norm": 6.3696111029717715,
|
|
"learning_rate": 4.299252534918185e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6490
|
|
},
|
|
{
|
|
"epoch": 0.8307240079238289,
|
|
"grad_norm": 8.711459660714853,
|
|
"learning_rate": 4.2369165263780595e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1839,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"epoch": 0.8320020448590965,
|
|
"grad_norm": 8.586531255673448,
|
|
"learning_rate": 4.174993871877513e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6510
|
|
},
|
|
{
|
|
"epoch": 0.8332800817943639,
|
|
"grad_norm": 9.159130220138737,
|
|
"learning_rate": 4.1134858041804165e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2108,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"epoch": 0.8345581187296313,
|
|
"grad_norm": 14.110031544509095,
|
|
"learning_rate": 4.05239354779702e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1964,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6530
|
|
},
|
|
{
|
|
"epoch": 0.8358361556648987,
|
|
"grad_norm": 13.647332815864479,
|
|
"learning_rate": 3.991718318959528e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"epoch": 0.8371141926001662,
|
|
"grad_norm": 7.729306259968811,
|
|
"learning_rate": 3.931461325597924e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1847,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6550
|
|
},
|
|
{
|
|
"epoch": 0.8383922295354336,
|
|
"grad_norm": 12.245685775856511,
|
|
"learning_rate": 3.871623767315896e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2085,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"epoch": 0.839670266470701,
|
|
"grad_norm": 8.700079920104228,
|
|
"learning_rate": 3.812206835366977e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1891,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6570
|
|
},
|
|
{
|
|
"epoch": 0.8409483034059684,
|
|
"grad_norm": 9.331886204678842,
|
|
"learning_rate": 3.753211712630808e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"epoch": 0.8422263403412359,
|
|
"grad_norm": 9.558487053586102,
|
|
"learning_rate": 3.6946395735896e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6590
|
|
},
|
|
{
|
|
"epoch": 0.8435043772765033,
|
|
"grad_norm": 9.536711446586605,
|
|
"learning_rate": 3.636491584304774e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"epoch": 0.8447824142117707,
|
|
"grad_norm": 9.745832695689701,
|
|
"learning_rate": 3.578768902393697e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1645,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6610
|
|
},
|
|
{
|
|
"epoch": 0.8460604511470381,
|
|
"grad_norm": 6.944861472906955,
|
|
"learning_rate": 3.521472677006676e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.171,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"epoch": 0.8473384880823056,
|
|
"grad_norm": 6.74838826324146,
|
|
"learning_rate": 3.464604048804079e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6630
|
|
},
|
|
{
|
|
"epoch": 0.848616525017573,
|
|
"grad_norm": 11.60280140004919,
|
|
"learning_rate": 3.408164149933587e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"epoch": 0.8498945619528404,
|
|
"grad_norm": 8.055840883849848,
|
|
"learning_rate": 3.352154104007721e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.187,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6650
|
|
},
|
|
{
|
|
"epoch": 0.8511725988881078,
|
|
"grad_norm": 10.181553904665757,
|
|
"learning_rate": 3.296575026081408e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"epoch": 0.8524506358233753,
|
|
"grad_norm": 10.249516867924106,
|
|
"learning_rate": 3.241428022629822e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2086,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6670
|
|
},
|
|
{
|
|
"epoch": 0.8537286727586427,
|
|
"grad_norm": 12.054480356165014,
|
|
"learning_rate": 3.186714191526346e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1844,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"epoch": 0.8550067096939101,
|
|
"grad_norm": 6.844044080742228,
|
|
"learning_rate": 3.132434622020716e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 6690
|
|
},
|
|
{
|
|
"epoch": 0.8562847466291775,
|
|
"grad_norm": 11.715109071014988,
|
|
"learning_rate": 3.078590394717323e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1969,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"epoch": 0.857562783564445,
|
|
"grad_norm": 9.302220076847602,
|
|
"learning_rate": 3.0251825815537204e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1986,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6710
|
|
},
|
|
{
|
|
"epoch": 0.8588408204997124,
|
|
"grad_norm": 10.93280433427468,
|
|
"learning_rate": 2.972212245779271e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"epoch": 0.8601188574349798,
|
|
"grad_norm": 8.427937780048167,
|
|
"learning_rate": 2.9196804419339917e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1962,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6730
|
|
},
|
|
{
|
|
"epoch": 0.8613968943702472,
|
|
"grad_norm": 8.597293479711446,
|
|
"learning_rate": 2.867588215827538e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2005,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"epoch": 0.8626749313055148,
|
|
"grad_norm": 9.995814473290233,
|
|
"learning_rate": 2.8159366045184124e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1853,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6750
|
|
},
|
|
{
|
|
"epoch": 0.8639529682407822,
|
|
"grad_norm": 9.928836544697068,
|
|
"learning_rate": 2.7647266362932936e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"epoch": 0.8652310051760496,
|
|
"grad_norm": 10.113163869171162,
|
|
"learning_rate": 2.7139593306465758e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1788,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6770
|
|
},
|
|
{
|
|
"epoch": 0.866509042111317,
|
|
"grad_norm": 10.346262544298897,
|
|
"learning_rate": 2.6636356982600695e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"epoch": 0.8677870790465845,
|
|
"grad_norm": 8.842070789217034,
|
|
"learning_rate": 2.6137567409828864e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6790
|
|
},
|
|
{
|
|
"epoch": 0.8690651159818519,
|
|
"grad_norm": 10.348672126261382,
|
|
"learning_rate": 2.5643234518114955e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1875,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"epoch": 0.8703431529171193,
|
|
"grad_norm": 14.107914181886756,
|
|
"learning_rate": 2.515336814869942e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1811,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6810
|
|
},
|
|
{
|
|
"epoch": 0.8716211898523867,
|
|
"grad_norm": 10.208150894155457,
|
|
"learning_rate": 2.466797805390272e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1868,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"epoch": 0.8728992267876542,
|
|
"grad_norm": 9.143576947306267,
|
|
"learning_rate": 2.4187073896930977e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1748,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6830
|
|
},
|
|
{
|
|
"epoch": 0.8741772637229216,
|
|
"grad_norm": 9.619393237764903,
|
|
"learning_rate": 2.3710665251683775e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1968,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"epoch": 0.875455300658189,
|
|
"grad_norm": 12.665374011457072,
|
|
"learning_rate": 2.3238761602563496e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1823,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6850
|
|
},
|
|
{
|
|
"epoch": 0.8767333375934564,
|
|
"grad_norm": 8.602114064258634,
|
|
"learning_rate": 2.2771372344286417e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"epoch": 0.8780113745287239,
|
|
"grad_norm": 6.8825535391482715,
|
|
"learning_rate": 2.2308506781695906e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2029,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6870
|
|
},
|
|
{
|
|
"epoch": 0.8792894114639913,
|
|
"grad_norm": 11.099108660135526,
|
|
"learning_rate": 2.1850174129576933e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"epoch": 0.8805674483992587,
|
|
"grad_norm": 12.681665287086265,
|
|
"learning_rate": 2.1396383512472722e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6890
|
|
},
|
|
{
|
|
"epoch": 0.8818454853345261,
|
|
"grad_norm": 10.890987789058547,
|
|
"learning_rate": 2.09471439645032e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2022,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"epoch": 0.8831235222697936,
|
|
"grad_norm": 9.926453846449595,
|
|
"learning_rate": 2.0502464429184933e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6910
|
|
},
|
|
{
|
|
"epoch": 0.884401559205061,
|
|
"grad_norm": 11.278826880908559,
|
|
"learning_rate": 2.006235375925322e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1749,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"epoch": 0.8856795961403284,
|
|
"grad_norm": 11.038211397843158,
|
|
"learning_rate": 1.962682071648586e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2006,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6930
|
|
},
|
|
{
|
|
"epoch": 0.8869576330755958,
|
|
"grad_norm": 10.785169033376835,
|
|
"learning_rate": 1.919587397152861e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"epoch": 0.8882356700108633,
|
|
"grad_norm": 9.925891527391922,
|
|
"learning_rate": 1.8769522103722763e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6950
|
|
},
|
|
{
|
|
"epoch": 0.8895137069461307,
|
|
"grad_norm": 8.771182460846884,
|
|
"learning_rate": 1.8347773600934063e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"epoch": 0.8907917438813981,
|
|
"grad_norm": 9.61805627828746,
|
|
"learning_rate": 1.7930636859384064e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2153,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6970
|
|
},
|
|
{
|
|
"epoch": 0.8920697808166655,
|
|
"grad_norm": 10.059738064893434,
|
|
"learning_rate": 1.7518120183482755e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"epoch": 0.8933478177519331,
|
|
"grad_norm": 13.222656919567447,
|
|
"learning_rate": 1.7110231785663094e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6990
|
|
},
|
|
{
|
|
"epoch": 0.8946258546872005,
|
|
"grad_norm": 11.481242847946271,
|
|
"learning_rate": 1.670697978621802e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1726,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 0.8959038916224679,
|
|
"grad_norm": 12.332320196979492,
|
|
"learning_rate": 1.6308372213138244e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7010
|
|
},
|
|
{
|
|
"epoch": 0.8971819285577353,
|
|
"grad_norm": 12.808012844296925,
|
|
"learning_rate": 1.5914417001952785e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2049,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"epoch": 0.8984599654930028,
|
|
"grad_norm": 7.820818021151389,
|
|
"learning_rate": 1.5525121995570823e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7030
|
|
},
|
|
{
|
|
"epoch": 0.8997380024282702,
|
|
"grad_norm": 9.370835431954198,
|
|
"learning_rate": 1.5140494944125588e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"epoch": 0.9010160393635376,
|
|
"grad_norm": 11.067566635775739,
|
|
"learning_rate": 1.4760543504820088e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1862,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7050
|
|
},
|
|
{
|
|
"epoch": 0.902294076298805,
|
|
"grad_norm": 9.623901803980623,
|
|
"learning_rate": 1.438527524177463e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1872,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"epoch": 0.9035721132340725,
|
|
"grad_norm": 11.5687168737442,
|
|
"learning_rate": 1.4014697625876426e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1964,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7070
|
|
},
|
|
{
|
|
"epoch": 0.9048501501693399,
|
|
"grad_norm": 8.21021164784677,
|
|
"learning_rate": 1.3648818034630499e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2012,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"epoch": 0.9061281871046073,
|
|
"grad_norm": 11.17367745513911,
|
|
"learning_rate": 1.3287643752013162e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2041,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7090
|
|
},
|
|
{
|
|
"epoch": 0.9074062240398747,
|
|
"grad_norm": 9.879895150119761,
|
|
"learning_rate": 1.2931181968326765e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"epoch": 0.9086842609751422,
|
|
"grad_norm": 8.750603926836277,
|
|
"learning_rate": 1.257943978005674e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2044,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7110
|
|
},
|
|
{
|
|
"epoch": 0.9099622979104096,
|
|
"grad_norm": 9.732574600229979,
|
|
"learning_rate": 1.2232424189730084e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1951,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"epoch": 0.911240334845677,
|
|
"grad_norm": 10.799063558489175,
|
|
"learning_rate": 1.189014210577624e-08,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1805,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 7130
|
|
},
|
|
{
|
|
"epoch": 0.9125183717809444,
|
|
"grad_norm": 8.342418997195931,
|
|
"learning_rate": 1.1552600342389363e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.172,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"epoch": 0.9137964087162119,
|
|
"grad_norm": 10.71141399194098,
|
|
"learning_rate": 1.121980561939273e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1729,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7150
|
|
},
|
|
{
|
|
"epoch": 0.9150744456514793,
|
|
"grad_norm": 9.405664113940032,
|
|
"learning_rate": 1.0891764562104888e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1956,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"epoch": 0.9163524825867467,
|
|
"grad_norm": 11.026789747690092,
|
|
"learning_rate": 1.0568483701207941e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7170
|
|
},
|
|
{
|
|
"epoch": 0.9176305195220141,
|
|
"grad_norm": 9.18756797117969,
|
|
"learning_rate": 1.0249969472617304e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1915,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"epoch": 0.9189085564572816,
|
|
"grad_norm": 10.972057226223303,
|
|
"learning_rate": 9.936228217353709e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1814,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 7190
|
|
},
|
|
{
|
|
"epoch": 0.920186593392549,
|
|
"grad_norm": 7.87780338280655,
|
|
"learning_rate": 9.627266181416987e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"epoch": 0.9214646303278164,
|
|
"grad_norm": 7.847069398644123,
|
|
"learning_rate": 9.323089515661603e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1631,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7210
|
|
},
|
|
{
|
|
"epoch": 0.9227426672630838,
|
|
"grad_norm": 9.819571950781643,
|
|
"learning_rate": 9.023704275674394e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.194,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"epoch": 0.9240207041983514,
|
|
"grad_norm": 11.758652729017486,
|
|
"learning_rate": 8.729116421653748e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1746,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7230
|
|
},
|
|
{
|
|
"epoch": 0.9252987411336188,
|
|
"grad_norm": 9.074528747089998,
|
|
"learning_rate": 8.439331818291261e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"epoch": 0.9265767780688862,
|
|
"grad_norm": 10.439935658287324,
|
|
"learning_rate": 8.154356234654713e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.203,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7250
|
|
},
|
|
{
|
|
"epoch": 0.9278548150041536,
|
|
"grad_norm": 13.20542442648142,
|
|
"learning_rate": 7.874195344073297e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"epoch": 0.9291328519394211,
|
|
"grad_norm": 11.62024702139443,
|
|
"learning_rate": 7.598854724024828e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.185,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7270
|
|
},
|
|
{
|
|
"epoch": 0.9304108888746885,
|
|
"grad_norm": 10.099788710806347,
|
|
"learning_rate": 7.328339856024407e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"epoch": 0.9316889258099559,
|
|
"grad_norm": 9.897081640900788,
|
|
"learning_rate": 7.062656125515654e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1759,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7290
|
|
},
|
|
{
|
|
"epoch": 0.9329669627452233,
|
|
"grad_norm": 10.589111836344392,
|
|
"learning_rate": 6.801808821763178e-09,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2106,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"epoch": 0.9342449996804908,
|
|
"grad_norm": 10.724801842725016,
|
|
"learning_rate": 6.545803137747385e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7310
|
|
},
|
|
{
|
|
"epoch": 0.9355230366157582,
|
|
"grad_norm": 9.611442651274807,
|
|
"learning_rate": 6.294644170061258e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1943,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"epoch": 0.9368010735510256,
|
|
"grad_norm": 11.751486176205045,
|
|
"learning_rate": 6.0483369188085995e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1801,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7330
|
|
},
|
|
{
|
|
"epoch": 0.938079110486293,
|
|
"grad_norm": 11.512912312322182,
|
|
"learning_rate": 5.806886287504725e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1873,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"epoch": 0.9393571474215605,
|
|
"grad_norm": 7.192083175792538,
|
|
"learning_rate": 5.570297082978709e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1647,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7350
|
|
},
|
|
{
|
|
"epoch": 0.9406351843568279,
|
|
"grad_norm": 10.843092841652517,
|
|
"learning_rate": 5.3385740152777634e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2038,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.0,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"epoch": 0.9419132212920953,
|
|
"grad_norm": 8.467409445214516,
|
|
"learning_rate": 5.111721697573457e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7370
|
|
},
|
|
{
|
|
"epoch": 0.9431912582273627,
|
|
"grad_norm": 6.715800157562806,
|
|
"learning_rate": 4.889744646069837e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1889,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"epoch": 0.9444692951626302,
|
|
"grad_norm": 8.400806549616826,
|
|
"learning_rate": 4.672647279913622e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1971,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7390
|
|
},
|
|
{
|
|
"epoch": 0.9457473320978976,
|
|
"grad_norm": 13.972001950715883,
|
|
"learning_rate": 4.4604339211059824e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1976,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"epoch": 0.947025369033165,
|
|
"grad_norm": 9.677769647797021,
|
|
"learning_rate": 4.253108794416871e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 7410
|
|
},
|
|
{
|
|
"epoch": 0.9483034059684324,
|
|
"grad_norm": 9.345026975785517,
|
|
"learning_rate": 4.050676027300554e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2038,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"epoch": 0.9495814429037,
|
|
"grad_norm": 9.695675333701805,
|
|
"learning_rate": 3.853139649813625e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7430
|
|
},
|
|
{
|
|
"epoch": 0.9508594798389673,
|
|
"grad_norm": 7.770057515915393,
|
|
"learning_rate": 3.660503594534875e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1867,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"epoch": 0.9521375167742347,
|
|
"grad_norm": 12.504038666390727,
|
|
"learning_rate": 3.4727716964866595e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1886,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.375,
|
|
"step": 7450
|
|
},
|
|
{
|
|
"epoch": 0.9534155537095022,
|
|
"grad_norm": 9.181970318686727,
|
|
"learning_rate": 3.2899476930589863e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1931,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"epoch": 0.9546935906447697,
|
|
"grad_norm": 11.183563173203545,
|
|
"learning_rate": 3.112035223934689e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1982,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7470
|
|
},
|
|
{
|
|
"epoch": 0.9559716275800371,
|
|
"grad_norm": 10.636414438455274,
|
|
"learning_rate": 2.939037831017288e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"epoch": 0.9572496645153045,
|
|
"grad_norm": 11.633662618889964,
|
|
"learning_rate": 2.7709589583603254e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7490
|
|
},
|
|
{
|
|
"epoch": 0.9585277014505719,
|
|
"grad_norm": 12.38372710586855,
|
|
"learning_rate": 2.6078019520988093e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.196,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"epoch": 0.9598057383858394,
|
|
"grad_norm": 12.644481438195353,
|
|
"learning_rate": 2.449570060382711e-09,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7510
|
|
},
|
|
{
|
|
"epoch": 0.9610837753211068,
|
|
"grad_norm": 9.432942475563783,
|
|
"learning_rate": 2.2962664333120994e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1963,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"epoch": 0.9623618122563742,
|
|
"grad_norm": 11.928069118877714,
|
|
"learning_rate": 2.1478941228745828e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2113,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7530
|
|
},
|
|
{
|
|
"epoch": 0.9636398491916416,
|
|
"grad_norm": 7.886668745442734,
|
|
"learning_rate": 2.0044560828845205e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"epoch": 0.9649178861269091,
|
|
"grad_norm": 9.34734142715555,
|
|
"learning_rate": 1.8659551689241547e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2107,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7550
|
|
},
|
|
{
|
|
"epoch": 0.9661959230621765,
|
|
"grad_norm": 9.611208535032741,
|
|
"learning_rate": 1.7323941382869078e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"epoch": 0.9674739599974439,
|
|
"grad_norm": 9.527127269024511,
|
|
"learning_rate": 1.603775649922312e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1666,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7570
|
|
},
|
|
{
|
|
"epoch": 0.9687519969327113,
|
|
"grad_norm": 9.291770961378111,
|
|
"learning_rate": 1.4801022643831652e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"epoch": 0.9700300338679788,
|
|
"grad_norm": 12.925961238581618,
|
|
"learning_rate": 1.3613764437745978e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1738,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7590
|
|
},
|
|
{
|
|
"epoch": 0.9713080708032462,
|
|
"grad_norm": 10.408031687484408,
|
|
"learning_rate": 1.2476005517049192e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.185,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"epoch": 0.9725861077385136,
|
|
"grad_norm": 9.808605265046817,
|
|
"learning_rate": 1.138776853238682e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7610
|
|
},
|
|
{
|
|
"epoch": 0.973864144673781,
|
|
"grad_norm": 7.618616317950994,
|
|
"learning_rate": 1.0349075148516074e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1606,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"epoch": 0.9751421816090485,
|
|
"grad_norm": 11.794438615303235,
|
|
"learning_rate": 9.359946043873412e-10,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1954,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7630
|
|
},
|
|
{
|
|
"epoch": 0.9764202185443159,
|
|
"grad_norm": 12.563234469246394,
|
|
"learning_rate": 8.420400910163494e-10,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1973,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"epoch": 0.9776982554795833,
|
|
"grad_norm": 16.042841291187038,
|
|
"learning_rate": 7.530458451967814e-10,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1815,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7650
|
|
},
|
|
{
|
|
"epoch": 0.9789762924148507,
|
|
"grad_norm": 13.296730357123543,
|
|
"learning_rate": 6.69013638637056e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1816,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"epoch": 0.9802543293501182,
|
|
"grad_norm": 12.112711580204818,
|
|
"learning_rate": 5.899451442607784e-10,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1893,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7670
|
|
},
|
|
{
|
|
"epoch": 0.9815323662853856,
|
|
"grad_norm": 14.646352722458984,
|
|
"learning_rate": 5.158419361733501e-10,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1982,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"epoch": 0.982810403220653,
|
|
"grad_norm": 13.260198930878687,
|
|
"learning_rate": 4.4670548963063327e-10,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7690
|
|
},
|
|
{
|
|
"epoch": 0.9840884401559205,
|
|
"grad_norm": 8.788325394613999,
|
|
"learning_rate": 3.82537181009529e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"epoch": 0.985366477091188,
|
|
"grad_norm": 12.794711894897244,
|
|
"learning_rate": 3.233382877806945e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7710
|
|
},
|
|
{
|
|
"epoch": 0.9866445140264554,
|
|
"grad_norm": 11.753555535364036,
|
|
"learning_rate": 2.6910998848306276e-10,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"epoch": 0.9879225509617228,
|
|
"grad_norm": 14.594250427636789,
|
|
"learning_rate": 2.1985336270033405e-10,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7730
|
|
},
|
|
{
|
|
"epoch": 0.9892005878969902,
|
|
"grad_norm": 12.77554394664042,
|
|
"learning_rate": 1.7556939103952063e-10,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"epoch": 0.9904786248322577,
|
|
"grad_norm": 10.519067298735793,
|
|
"learning_rate": 1.3625895511137907e-10,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7750
|
|
},
|
|
{
|
|
"epoch": 0.9917566617675251,
|
|
"grad_norm": 9.931366042203358,
|
|
"learning_rate": 1.0192283751297992e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1821,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"epoch": 0.9930346987027925,
|
|
"grad_norm": 9.246116106532282,
|
|
"learning_rate": 7.256172181199783e-11,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1969,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7770
|
|
},
|
|
{
|
|
"epoch": 0.9943127356380599,
|
|
"grad_norm": 7.26117544128573,
|
|
"learning_rate": 4.8176192533139206e-11,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.194,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"epoch": 0.9955907725733274,
|
|
"grad_norm": 12.332218978053229,
|
|
"learning_rate": 2.876673514659589e-11,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1812,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7790
|
|
},
|
|
{
|
|
"epoch": 0.9968688095085948,
|
|
"grad_norm": 9.768254436897418,
|
|
"learning_rate": 1.4333736058219636e-11,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"epoch": 0.9981468464438622,
|
|
"grad_norm": 12.753093983260724,
|
|
"learning_rate": 4.8774826019448715e-12,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1805,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7810
|
|
},
|
|
{
|
|
"epoch": 0.9994248833791296,
|
|
"grad_norm": 9.444618426186262,
|
|
"learning_rate": 3.981630340710307e-13,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7820
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 7824,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 10000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|