23534 lines
692 KiB
JSON
23534 lines
692 KiB
JSON
{
|
|
"best_metric": 0.23926551640033722,
|
|
"best_model_checkpoint": "models/qwen2.5-3b-dpo-mini/checkpoint-10000",
|
|
"epoch": 1.9999360981532366,
|
|
"eval_steps": 10000,
|
|
"global_step": 15648,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.00012780369352674293,
|
|
"grad_norm": 1.583091404664238,
|
|
"learning_rate": 6.385696040868454e-10,
|
|
"logits/chosen": -2.140625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -141.0,
|
|
"logps/rejected": -147.0,
|
|
"loss": 0.6914,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.0012780369352674292,
|
|
"grad_norm": 1.831193607031671,
|
|
"learning_rate": 6.3856960408684546e-09,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -1.9375,
|
|
"logps/chosen": -156.0,
|
|
"logps/rejected": -136.0,
|
|
"loss": 0.6924,
|
|
"rewards/accuracies": 0.2361111044883728,
|
|
"rewards/chosen": 8.869171142578125e-05,
|
|
"rewards/margins": 8.726119995117188e-05,
|
|
"rewards/rejected": -4.246830940246582e-07,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0025560738705348585,
|
|
"grad_norm": 1.9051082450552785,
|
|
"learning_rate": 1.2771392081736909e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -167.0,
|
|
"logps/rejected": -134.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.33125001192092896,
|
|
"rewards/chosen": -0.00012493133544921875,
|
|
"rewards/margins": -0.000560760498046875,
|
|
"rewards/rejected": 0.000438690185546875,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.0038341108058022877,
|
|
"grad_norm": 2.0951178941868793,
|
|
"learning_rate": 1.915708812260536e-08,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -172.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.3812499940395355,
|
|
"rewards/chosen": -0.00020313262939453125,
|
|
"rewards/margins": 0.000377655029296875,
|
|
"rewards/rejected": -0.000579833984375,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.005112147741069717,
|
|
"grad_norm": 1.8582175367673581,
|
|
"learning_rate": 2.5542784163473818e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.0625,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -133.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.34375,
|
|
"rewards/chosen": -0.00213623046875,
|
|
"rewards/margins": -0.0004711151123046875,
|
|
"rewards/rejected": -0.00165557861328125,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.006390184676337146,
|
|
"grad_norm": 1.8191369002557825,
|
|
"learning_rate": 3.192848020434227e-08,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -135.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.3499999940395355,
|
|
"rewards/chosen": 6.29425048828125e-05,
|
|
"rewards/margins": -0.0002498626708984375,
|
|
"rewards/rejected": 0.0003147125244140625,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.007668221611604575,
|
|
"grad_norm": 1.8104389933900382,
|
|
"learning_rate": 3.831417624521072e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -126.0,
|
|
"loss": 0.6929,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": -7.82012939453125e-05,
|
|
"rewards/margins": -0.00019168853759765625,
|
|
"rewards/rejected": 0.00010967254638671875,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.008946258546872005,
|
|
"grad_norm": 1.9080659587821094,
|
|
"learning_rate": 4.469987228607918e-08,
|
|
"logits/chosen": -2.390625,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -164.0,
|
|
"logps/rejected": -141.0,
|
|
"loss": 0.6928,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": 0.000797271728515625,
|
|
"rewards/margins": 0.0012969970703125,
|
|
"rewards/rejected": -0.000499725341796875,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.010224295482139434,
|
|
"grad_norm": 1.7284420631380606,
|
|
"learning_rate": 5.1085568326947637e-08,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -163.0,
|
|
"logps/rejected": -135.0,
|
|
"loss": 0.6926,
|
|
"rewards/accuracies": 0.42500001192092896,
|
|
"rewards/chosen": 0.0013580322265625,
|
|
"rewards/margins": 0.0010833740234375,
|
|
"rewards/rejected": 0.00028228759765625,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.011502332417406863,
|
|
"grad_norm": 1.9985804873605706,
|
|
"learning_rate": 5.747126436781609e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -2.03125,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -142.0,
|
|
"loss": 0.6925,
|
|
"rewards/accuracies": 0.36250001192092896,
|
|
"rewards/chosen": 7.867813110351562e-05,
|
|
"rewards/margins": 0.0002193450927734375,
|
|
"rewards/rejected": -0.00014019012451171875,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.012780369352674292,
|
|
"grad_norm": 1.7673732788906307,
|
|
"learning_rate": 6.385696040868454e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -1.9296875,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -138.0,
|
|
"loss": 0.6923,
|
|
"rewards/accuracies": 0.3687500059604645,
|
|
"rewards/chosen": -0.000423431396484375,
|
|
"rewards/margins": -0.0003070831298828125,
|
|
"rewards/rejected": -0.00011539459228515625,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.014058406287941722,
|
|
"grad_norm": 1.8348538098786262,
|
|
"learning_rate": 7.024265644955301e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -1.96875,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -132.0,
|
|
"loss": 0.692,
|
|
"rewards/accuracies": 0.4124999940395355,
|
|
"rewards/chosen": 0.001129150390625,
|
|
"rewards/margins": 0.000797271728515625,
|
|
"rewards/rejected": 0.00032806396484375,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.01533644322320915,
|
|
"grad_norm": 2.0218021664474373,
|
|
"learning_rate": 7.662835249042144e-08,
|
|
"logits/chosen": -2.40625,
|
|
"logits/rejected": -1.859375,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -137.0,
|
|
"loss": 0.6918,
|
|
"rewards/accuracies": 0.48124998807907104,
|
|
"rewards/chosen": 0.002960205078125,
|
|
"rewards/margins": 0.0024261474609375,
|
|
"rewards/rejected": 0.000530242919921875,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.01661448015847658,
|
|
"grad_norm": 1.82922758998825,
|
|
"learning_rate": 8.301404853128991e-08,
|
|
"logits/chosen": -2.421875,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -144.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6915,
|
|
"rewards/accuracies": 0.543749988079071,
|
|
"rewards/chosen": 0.006317138671875,
|
|
"rewards/margins": 0.00396728515625,
|
|
"rewards/rejected": 0.002349853515625,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.01789251709374401,
|
|
"grad_norm": 1.7861618345539088,
|
|
"learning_rate": 8.939974457215836e-08,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -1.984375,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -141.0,
|
|
"loss": 0.6909,
|
|
"rewards/accuracies": 0.53125,
|
|
"rewards/chosen": 0.0064697265625,
|
|
"rewards/margins": 0.0040283203125,
|
|
"rewards/rejected": 0.0024261474609375,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.01917055402901144,
|
|
"grad_norm": 1.8958781010879984,
|
|
"learning_rate": 9.578544061302682e-08,
|
|
"logits/chosen": -2.46875,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -132.0,
|
|
"loss": 0.6903,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 0.0076904296875,
|
|
"rewards/margins": 0.0042724609375,
|
|
"rewards/rejected": 0.00341796875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.020448590964278868,
|
|
"grad_norm": 1.689453237922255,
|
|
"learning_rate": 1.0217113665389527e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -1.9140625,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6903,
|
|
"rewards/accuracies": 0.4937500059604645,
|
|
"rewards/chosen": 0.005950927734375,
|
|
"rewards/margins": 0.0031585693359375,
|
|
"rewards/rejected": 0.0027923583984375,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.021726627899546297,
|
|
"grad_norm": 1.6908724447283379,
|
|
"learning_rate": 1.0855683269476372e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -1.890625,
|
|
"logps/chosen": -153.0,
|
|
"logps/rejected": -151.0,
|
|
"loss": 0.6894,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": 0.01123046875,
|
|
"rewards/margins": 0.00665283203125,
|
|
"rewards/rejected": 0.004608154296875,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.023004664834813726,
|
|
"grad_norm": 1.9217126730360887,
|
|
"learning_rate": 1.1494252873563217e-07,
|
|
"logits/chosen": -2.453125,
|
|
"logits/rejected": -1.8828125,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -146.0,
|
|
"loss": 0.6875,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.01275634765625,
|
|
"rewards/margins": 0.00927734375,
|
|
"rewards/rejected": 0.003509521484375,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.024282701770081155,
|
|
"grad_norm": 1.7796346205464415,
|
|
"learning_rate": 1.2132822477650062e-07,
|
|
"logits/chosen": -2.5,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6877,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": 0.016845703125,
|
|
"rewards/margins": 0.00970458984375,
|
|
"rewards/rejected": 0.007080078125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.025560738705348585,
|
|
"grad_norm": 1.8062803830724468,
|
|
"learning_rate": 1.277139208173691e-07,
|
|
"logits/chosen": -2.484375,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -128.0,
|
|
"logps/rejected": -117.0,
|
|
"loss": 0.6868,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": 0.0159912109375,
|
|
"rewards/margins": 0.010986328125,
|
|
"rewards/rejected": 0.0050048828125,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.026838775640616014,
|
|
"grad_norm": 1.8109623854322845,
|
|
"learning_rate": 1.3409961685823753e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -152.0,
|
|
"logps/rejected": -135.0,
|
|
"loss": 0.6859,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": 0.0263671875,
|
|
"rewards/margins": 0.01507568359375,
|
|
"rewards/rejected": 0.01129150390625,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.028116812575883443,
|
|
"grad_norm": 1.7555569281404872,
|
|
"learning_rate": 1.4048531289910602e-07,
|
|
"logits/chosen": -2.4375,
|
|
"logits/rejected": -1.9609375,
|
|
"logps/chosen": -168.0,
|
|
"logps/rejected": -162.0,
|
|
"loss": 0.6846,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": 0.0260009765625,
|
|
"rewards/margins": 0.0228271484375,
|
|
"rewards/rejected": 0.003173828125,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.029394849511150872,
|
|
"grad_norm": 1.6784346166574386,
|
|
"learning_rate": 1.4687100893997445e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.046875,
|
|
"logps/chosen": -156.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6833,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": 0.024658203125,
|
|
"rewards/margins": 0.0225830078125,
|
|
"rewards/rejected": 0.002105712890625,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.0306728864464183,
|
|
"grad_norm": 2.017021630939051,
|
|
"learning_rate": 1.532567049808429e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.125,
|
|
"logps/chosen": -169.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6819,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": 0.0257568359375,
|
|
"rewards/margins": 0.0235595703125,
|
|
"rewards/rejected": 0.0020751953125,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.03195092338168573,
|
|
"grad_norm": 1.7084660802314153,
|
|
"learning_rate": 1.5964240102171135e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -154.0,
|
|
"logps/rejected": -145.0,
|
|
"loss": 0.6796,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 0.0361328125,
|
|
"rewards/margins": 0.0294189453125,
|
|
"rewards/rejected": 0.006744384765625,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.03322896031695316,
|
|
"grad_norm": 1.8365627743934905,
|
|
"learning_rate": 1.6602809706257982e-07,
|
|
"logits/chosen": -2.515625,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -157.0,
|
|
"logps/rejected": -152.0,
|
|
"loss": 0.6764,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": 0.032470703125,
|
|
"rewards/margins": 0.036865234375,
|
|
"rewards/rejected": -0.004364013671875,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.03450699725222059,
|
|
"grad_norm": 1.7335796846774567,
|
|
"learning_rate": 1.7241379310344828e-07,
|
|
"logits/chosen": -2.546875,
|
|
"logits/rejected": -1.9765625,
|
|
"logps/chosen": -158.0,
|
|
"logps/rejected": -139.0,
|
|
"loss": 0.6742,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": 0.028564453125,
|
|
"rewards/margins": 0.046630859375,
|
|
"rewards/rejected": -0.0179443359375,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.03578503418748802,
|
|
"grad_norm": 1.6763039372574249,
|
|
"learning_rate": 1.7879948914431672e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.0,
|
|
"logps/chosen": -146.0,
|
|
"logps/rejected": -144.0,
|
|
"loss": 0.6716,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": 0.03759765625,
|
|
"rewards/margins": 0.05615234375,
|
|
"rewards/rejected": -0.0184326171875,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.03706307112275545,
|
|
"grad_norm": 1.7476768718236924,
|
|
"learning_rate": 1.8518518518518516e-07,
|
|
"logits/chosen": -2.53125,
|
|
"logits/rejected": -2.015625,
|
|
"logps/chosen": -155.0,
|
|
"logps/rejected": -131.0,
|
|
"loss": 0.6693,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": 0.0230712890625,
|
|
"rewards/margins": 0.03955078125,
|
|
"rewards/rejected": -0.016357421875,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.03834110805802288,
|
|
"grad_norm": 1.9904155949768438,
|
|
"learning_rate": 1.9157088122605365e-07,
|
|
"logits/chosen": -2.625,
|
|
"logits/rejected": -2.15625,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -154.0,
|
|
"loss": 0.6654,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": 0.01300048828125,
|
|
"rewards/margins": 0.05810546875,
|
|
"rewards/rejected": -0.045166015625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.039619144993290306,
|
|
"grad_norm": 2.1919340946635275,
|
|
"learning_rate": 1.9795657726692208e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.09375,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6579,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": 0.00860595703125,
|
|
"rewards/margins": 0.08154296875,
|
|
"rewards/rejected": -0.07275390625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.040897181928557735,
|
|
"grad_norm": 2.050188028710248,
|
|
"learning_rate": 2.0434227330779055e-07,
|
|
"logits/chosen": -2.578125,
|
|
"logits/rejected": -2.25,
|
|
"logps/chosen": -151.0,
|
|
"logps/rejected": -155.0,
|
|
"loss": 0.654,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": 0.0015411376953125,
|
|
"rewards/margins": 0.08837890625,
|
|
"rewards/rejected": -0.0869140625,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.042175218863825165,
|
|
"grad_norm": 2.016463993421717,
|
|
"learning_rate": 2.1072796934865898e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.265625,
|
|
"logps/chosen": -164.0,
|
|
"logps/rejected": -165.0,
|
|
"loss": 0.6475,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.03125,
|
|
"rewards/margins": 0.10693359375,
|
|
"rewards/rejected": -0.1376953125,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.043453255799092594,
|
|
"grad_norm": 2.4323566534644714,
|
|
"learning_rate": 2.1711366538952745e-07,
|
|
"logits/chosen": -2.6875,
|
|
"logits/rejected": -2.171875,
|
|
"logps/chosen": -171.0,
|
|
"logps/rejected": -153.0,
|
|
"loss": 0.6386,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -0.05078125,
|
|
"rewards/margins": 0.1279296875,
|
|
"rewards/rejected": -0.177734375,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.04473129273436002,
|
|
"grad_norm": 2.202903045169722,
|
|
"learning_rate": 2.234993614303959e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -150.0,
|
|
"logps/rejected": -169.0,
|
|
"loss": 0.6285,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -0.0888671875,
|
|
"rewards/margins": 0.1650390625,
|
|
"rewards/rejected": -0.25390625,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.04600932966962745,
|
|
"grad_norm": 2.206602901584152,
|
|
"learning_rate": 2.2988505747126435e-07,
|
|
"logits/chosen": -2.703125,
|
|
"logits/rejected": -2.296875,
|
|
"logps/chosen": -181.0,
|
|
"logps/rejected": -158.0,
|
|
"loss": 0.6227,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.1357421875,
|
|
"rewards/margins": 0.1533203125,
|
|
"rewards/rejected": -0.2890625,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.04728736660489488,
|
|
"grad_norm": 2.077125523450064,
|
|
"learning_rate": 2.3627075351213284e-07,
|
|
"logits/chosen": -2.65625,
|
|
"logits/rejected": -2.203125,
|
|
"logps/chosen": -181.0,
|
|
"logps/rejected": -185.0,
|
|
"loss": 0.6089,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.1953125,
|
|
"rewards/margins": 0.2177734375,
|
|
"rewards/rejected": -0.4140625,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.04856540354016231,
|
|
"grad_norm": 2.286616738197532,
|
|
"learning_rate": 2.4265644955300125e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.21875,
|
|
"logps/chosen": -165.0,
|
|
"logps/rejected": -192.0,
|
|
"loss": 0.5919,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -0.2578125,
|
|
"rewards/margins": 0.2734375,
|
|
"rewards/rejected": -0.53125,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.04984344047542974,
|
|
"grad_norm": 2.3378312139893755,
|
|
"learning_rate": 2.490421455938697e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.34375,
|
|
"logps/chosen": -205.0,
|
|
"logps/rejected": -184.0,
|
|
"loss": 0.5987,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -0.38671875,
|
|
"rewards/margins": 0.19921875,
|
|
"rewards/rejected": -0.5859375,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.05112147741069717,
|
|
"grad_norm": 2.0376352660492505,
|
|
"learning_rate": 2.554278416347382e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.4375,
|
|
"logps/chosen": -205.0,
|
|
"logps/rejected": -225.0,
|
|
"loss": 0.5894,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.423828125,
|
|
"rewards/margins": 0.25,
|
|
"rewards/rejected": -0.671875,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.0523995143459646,
|
|
"grad_norm": 2.198540810606561,
|
|
"learning_rate": 2.6181353767560667e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.375,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -228.0,
|
|
"loss": 0.5725,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -0.423828125,
|
|
"rewards/margins": 0.341796875,
|
|
"rewards/rejected": -0.765625,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.05367755128123203,
|
|
"grad_norm": 2.5064207498697306,
|
|
"learning_rate": 2.6819923371647505e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.453125,
|
|
"logps/chosen": -203.0,
|
|
"logps/rejected": -231.0,
|
|
"loss": 0.5589,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -0.45703125,
|
|
"rewards/margins": 0.408203125,
|
|
"rewards/rejected": -0.86328125,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.05495558821649946,
|
|
"grad_norm": 2.7332223174524604,
|
|
"learning_rate": 2.7458492975734354e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -207.0,
|
|
"logps/rejected": -217.0,
|
|
"loss": 0.5532,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.53515625,
|
|
"rewards/margins": 0.384765625,
|
|
"rewards/rejected": -0.91796875,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.056233625151766886,
|
|
"grad_norm": 3.160820809244754,
|
|
"learning_rate": 2.8097062579821203e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -224.0,
|
|
"logps/rejected": -276.0,
|
|
"loss": 0.5351,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.63671875,
|
|
"rewards/margins": 0.609375,
|
|
"rewards/rejected": -1.25,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.057511662087034315,
|
|
"grad_norm": 3.1305847824157005,
|
|
"learning_rate": 2.873563218390804e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -225.0,
|
|
"logps/rejected": -266.0,
|
|
"loss": 0.5226,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.69921875,
|
|
"rewards/margins": 0.5859375,
|
|
"rewards/rejected": -1.2890625,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.058789699022301745,
|
|
"grad_norm": 3.2458132009733975,
|
|
"learning_rate": 2.937420178799489e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -212.0,
|
|
"logps/rejected": -284.0,
|
|
"loss": 0.5138,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.734375,
|
|
"rewards/margins": 0.7421875,
|
|
"rewards/rejected": -1.4765625,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.060067735957569174,
|
|
"grad_norm": 3.590934206396528,
|
|
"learning_rate": 3.001277139208174e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -227.0,
|
|
"logps/rejected": -286.0,
|
|
"loss": 0.5261,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.78515625,
|
|
"rewards/margins": 0.65234375,
|
|
"rewards/rejected": -1.4375,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.0613457728928366,
|
|
"grad_norm": 4.090558879807313,
|
|
"learning_rate": 3.065134099616858e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -241.0,
|
|
"logps/rejected": -296.0,
|
|
"loss": 0.5063,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.71484375,
|
|
"rewards/margins": 0.80078125,
|
|
"rewards/rejected": -1.515625,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.06262380982810403,
|
|
"grad_norm": 3.738095666639755,
|
|
"learning_rate": 3.1289910600255427e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -232.0,
|
|
"logps/rejected": -306.0,
|
|
"loss": 0.5051,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -0.78125,
|
|
"rewards/margins": 0.87890625,
|
|
"rewards/rejected": -1.6640625,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.06390184676337146,
|
|
"grad_norm": 4.03887506227773,
|
|
"learning_rate": 3.192848020434227e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -227.0,
|
|
"logps/rejected": -286.0,
|
|
"loss": 0.5208,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -0.8046875,
|
|
"rewards/margins": 0.63671875,
|
|
"rewards/rejected": -1.4453125,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.06517988369863889,
|
|
"grad_norm": 3.7784530781629444,
|
|
"learning_rate": 3.2567049808429114e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -240.0,
|
|
"logps/rejected": -296.0,
|
|
"loss": 0.4996,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -0.8515625,
|
|
"rewards/margins": 0.80078125,
|
|
"rewards/rejected": -1.6484375,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.06645792063390632,
|
|
"grad_norm": 4.907395725764965,
|
|
"learning_rate": 3.3205619412515963e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -236.0,
|
|
"logps/rejected": -312.0,
|
|
"loss": 0.4853,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -0.9765625,
|
|
"rewards/margins": 0.734375,
|
|
"rewards/rejected": -1.7109375,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.06773595756917375,
|
|
"grad_norm": 6.328166419969473,
|
|
"learning_rate": 3.3844189016602807e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -344.0,
|
|
"loss": 0.4789,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -1.1015625,
|
|
"rewards/margins": 0.8984375,
|
|
"rewards/rejected": -2.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.06901399450444118,
|
|
"grad_norm": 7.41169890134317,
|
|
"learning_rate": 3.4482758620689656e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -264.0,
|
|
"logps/rejected": -350.0,
|
|
"loss": 0.4664,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -0.9921875,
|
|
"rewards/margins": 0.97265625,
|
|
"rewards/rejected": -1.96875,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.07029203143970861,
|
|
"grad_norm": 6.329307259348903,
|
|
"learning_rate": 3.51213282247765e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -242.0,
|
|
"logps/rejected": -334.0,
|
|
"loss": 0.4756,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -0.97265625,
|
|
"rewards/margins": 0.92578125,
|
|
"rewards/rejected": -1.8984375,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.07157006837497604,
|
|
"grad_norm": 5.5382587319010605,
|
|
"learning_rate": 3.5759897828863344e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.4816,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.21875,
|
|
"rewards/margins": 0.94140625,
|
|
"rewards/rejected": -2.15625,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.07284810531024347,
|
|
"grad_norm": 5.59071348439989,
|
|
"learning_rate": 3.639846743295019e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -336.0,
|
|
"loss": 0.4814,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.1953125,
|
|
"rewards/margins": 0.87109375,
|
|
"rewards/rejected": -2.0625,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.0741261422455109,
|
|
"grad_norm": 7.500668748088703,
|
|
"learning_rate": 3.703703703703703e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -278.0,
|
|
"logps/rejected": -346.0,
|
|
"loss": 0.4581,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 0.87890625,
|
|
"rewards/rejected": -2.125,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.07540417918077832,
|
|
"grad_norm": 7.815381329656392,
|
|
"learning_rate": 3.767560664112388e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -366.0,
|
|
"loss": 0.4516,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.15625,
|
|
"rewards/margins": 0.953125,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.07668221611604575,
|
|
"grad_norm": 7.560002445698683,
|
|
"learning_rate": 3.831417624521073e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -414.0,
|
|
"loss": 0.4415,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.1796875,
|
|
"rewards/margins": 1.328125,
|
|
"rewards/rejected": -2.5,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.07796025305131318,
|
|
"grad_norm": 7.597044745493579,
|
|
"learning_rate": 3.895274584929757e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -404.0,
|
|
"loss": 0.4573,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.28125,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.453125,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.07923828998658061,
|
|
"grad_norm": 6.0294238622536085,
|
|
"learning_rate": 3.9591315453384417e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -352.0,
|
|
"loss": 0.433,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.171875,
|
|
"rewards/margins": 0.93359375,
|
|
"rewards/rejected": -2.109375,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.08051632692184804,
|
|
"grad_norm": 6.57580363708218,
|
|
"learning_rate": 4.0229885057471266e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -288.0,
|
|
"logps/rejected": -370.0,
|
|
"loss": 0.4397,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.34375,
|
|
"rewards/margins": 1.0546875,
|
|
"rewards/rejected": -2.40625,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.08179436385711547,
|
|
"grad_norm": 8.126812055550822,
|
|
"learning_rate": 4.086845466155811e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -396.0,
|
|
"loss": 0.4337,
|
|
"rewards/accuracies": 0.793749988079071,
|
|
"rewards/chosen": -1.2421875,
|
|
"rewards/margins": 1.1484375,
|
|
"rewards/rejected": -2.390625,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.0830724007923829,
|
|
"grad_norm": 7.662460966992449,
|
|
"learning_rate": 4.1507024265644953e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -436.0,
|
|
"loss": 0.4231,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.4140625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.08435043772765033,
|
|
"grad_norm": 6.26765621154057,
|
|
"learning_rate": 4.2145593869731797e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -268.0,
|
|
"logps/rejected": -368.0,
|
|
"loss": 0.4428,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.1796875,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.34375,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.08562847466291776,
|
|
"grad_norm": 10.966780568835159,
|
|
"learning_rate": 4.2784163473818646e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -272.0,
|
|
"logps/rejected": -384.0,
|
|
"loss": 0.4744,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -1.1484375,
|
|
"rewards/margins": 1.171875,
|
|
"rewards/rejected": -2.328125,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.08690651159818519,
|
|
"grad_norm": 7.995403962637887,
|
|
"learning_rate": 4.342273307790549e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -406.0,
|
|
"loss": 0.446,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.328125,
|
|
"rewards/margins": 1.1328125,
|
|
"rewards/rejected": -2.46875,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.08818454853345262,
|
|
"grad_norm": 7.767114033697263,
|
|
"learning_rate": 4.4061302681992333e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -386.0,
|
|
"loss": 0.4252,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": -1.359375,
|
|
"rewards/margins": 1.09375,
|
|
"rewards/rejected": -2.453125,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.08946258546872005,
|
|
"grad_norm": 10.782647668236367,
|
|
"learning_rate": 4.469987228607918e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -266.0,
|
|
"logps/rejected": -372.0,
|
|
"loss": 0.4396,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.1484375,
|
|
"rewards/margins": 1.1796875,
|
|
"rewards/rejected": -2.328125,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.09074062240398748,
|
|
"grad_norm": 13.262314383907762,
|
|
"learning_rate": 4.5338441890166026e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -296.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.412,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.4140625,
|
|
"rewards/margins": 1.2734375,
|
|
"rewards/rejected": -2.6875,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.0920186593392549,
|
|
"grad_norm": 8.452988029757332,
|
|
"learning_rate": 4.597701149425287e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -428.0,
|
|
"loss": 0.4053,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.390625,
|
|
"rewards/margins": 1.4296875,
|
|
"rewards/rejected": -2.8125,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.09329669627452233,
|
|
"grad_norm": 10.451528356808828,
|
|
"learning_rate": 4.661558109833972e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -286.0,
|
|
"logps/rejected": -418.0,
|
|
"loss": 0.4013,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -1.3671875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.09457473320978976,
|
|
"grad_norm": 11.00358677164108,
|
|
"learning_rate": 4.725415070242657e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -432.0,
|
|
"loss": 0.387,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.515625,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.09585277014505719,
|
|
"grad_norm": 13.575988451351572,
|
|
"learning_rate": 4.789272030651341e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -290.0,
|
|
"logps/rejected": -408.0,
|
|
"loss": 0.4079,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.4609375,
|
|
"rewards/margins": 1.3203125,
|
|
"rewards/rejected": -2.78125,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.09713080708032462,
|
|
"grad_norm": 9.325954891766974,
|
|
"learning_rate": 4.853128991060025e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -302.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.4176,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.34375,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -2.90625,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.09840884401559205,
|
|
"grad_norm": 7.52371689034163,
|
|
"learning_rate": 4.91698595146871e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -330.0,
|
|
"logps/rejected": -448.0,
|
|
"loss": 0.4036,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -1.7265625,
|
|
"rewards/margins": 1.21875,
|
|
"rewards/rejected": -2.953125,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.09968688095085948,
|
|
"grad_norm": 7.847386893700331,
|
|
"learning_rate": 4.980842911877394e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -450.0,
|
|
"loss": 0.374,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.5,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -3.03125,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.10096491788612691,
|
|
"grad_norm": 8.270767140969047,
|
|
"learning_rate": 4.999987806263758e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -298.0,
|
|
"logps/rejected": -418.0,
|
|
"loss": 0.4074,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.390625,
|
|
"rewards/margins": 1.34375,
|
|
"rewards/rejected": -2.734375,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.10224295482139434,
|
|
"grad_norm": 9.313330271911113,
|
|
"learning_rate": 4.999928082127696e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -446.0,
|
|
"loss": 0.4058,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.484375,
|
|
"rewards/margins": 1.5234375,
|
|
"rewards/rejected": -3.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.10352099175666177,
|
|
"grad_norm": 10.98658746507854,
|
|
"learning_rate": 4.999818589113488e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -282.0,
|
|
"logps/rejected": -398.0,
|
|
"loss": 0.3954,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.25,
|
|
"rewards/margins": 1.40625,
|
|
"rewards/rejected": -2.65625,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.1047990286919292,
|
|
"grad_norm": 10.59961166689991,
|
|
"learning_rate": 4.99965932940093e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.3963,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.10607706562719663,
|
|
"grad_norm": 13.077749744872387,
|
|
"learning_rate": 4.999450306160585e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.5546875,
|
|
"rewards/margins": 1.8359375,
|
|
"rewards/rejected": -3.390625,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.10735510256246406,
|
|
"grad_norm": 10.551892493562612,
|
|
"learning_rate": 4.999191523553715e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -304.0,
|
|
"logps/rejected": -468.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.3984375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -3.125,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.10863313949773148,
|
|
"grad_norm": 12.356141382859514,
|
|
"learning_rate": 4.998882986732195e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -482.0,
|
|
"loss": 0.3878,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.5390625,
|
|
"rewards/margins": 1.578125,
|
|
"rewards/rejected": -3.109375,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.10991117643299891,
|
|
"grad_norm": 15.20620984625719,
|
|
"learning_rate": 4.998524701838414e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -498.0,
|
|
"loss": 0.3599,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.515625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.11118921336826634,
|
|
"grad_norm": 14.444326022124622,
|
|
"learning_rate": 4.998116676005154e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.4075,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.6015625,
|
|
"rewards/rejected": -3.171875,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.11246725030353377,
|
|
"grad_norm": 7.828143338550613,
|
|
"learning_rate": 4.997658917355441e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -500.0,
|
|
"loss": 0.3833,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 1.6796875,
|
|
"rewards/rejected": -3.4375,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.1137452872388012,
|
|
"grad_norm": 18.788227330963323,
|
|
"learning_rate": 4.997151435002394e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.3749,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -1.5234375,
|
|
"rewards/margins": 1.7109375,
|
|
"rewards/rejected": -3.234375,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.11502332417406863,
|
|
"grad_norm": 11.597065629717148,
|
|
"learning_rate": 4.996594239049032e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -300.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.4025,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.4765625,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.11630136110933606,
|
|
"grad_norm": 10.476293056540518,
|
|
"learning_rate": 4.995987340588082e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -486.0,
|
|
"loss": 0.3728,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.53125,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -3.3125,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.11757939804460349,
|
|
"grad_norm": 9.355309673884802,
|
|
"learning_rate": 4.995330751701755e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -406.0,
|
|
"loss": 0.4006,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.359375,
|
|
"rewards/margins": 1.46875,
|
|
"rewards/rejected": -2.828125,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.11885743497987092,
|
|
"grad_norm": 10.550972596004549,
|
|
"learning_rate": 4.994624485461503e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.3842,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.5859375,
|
|
"rewards/margins": 1.796875,
|
|
"rewards/rejected": -3.390625,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.12013547191513835,
|
|
"grad_norm": 7.584312388304975,
|
|
"learning_rate": 4.99386855592776e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -306.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3669,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.46875,
|
|
"rewards/margins": 1.59375,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.12141350885040578,
|
|
"grad_norm": 14.923871887916482,
|
|
"learning_rate": 4.993062978149668e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -308.0,
|
|
"logps/rejected": -508.0,
|
|
"loss": 0.3733,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.4921875,
|
|
"rewards/margins": 1.984375,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.1226915457856732,
|
|
"grad_norm": 12.368485690167017,
|
|
"learning_rate": 4.992207768164769e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -274.0,
|
|
"logps/rejected": -396.0,
|
|
"loss": 0.4143,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.3125,
|
|
"rewards/margins": 1.296875,
|
|
"rewards/rejected": -2.609375,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.12396958272094064,
|
|
"grad_norm": 12.872066642334858,
|
|
"learning_rate": 4.991302942998686e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.3478,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.71875,
|
|
"rewards/margins": 1.75,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.12524761965620806,
|
|
"grad_norm": 8.76843030472801,
|
|
"learning_rate": 4.990348520664794e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -310.0,
|
|
"logps/rejected": -438.0,
|
|
"loss": 0.3752,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.421875,
|
|
"rewards/margins": 1.5546875,
|
|
"rewards/rejected": -2.96875,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.1265256565914755,
|
|
"grad_norm": 10.846462610025974,
|
|
"learning_rate": 4.989344520163849e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -294.0,
|
|
"logps/rejected": -452.0,
|
|
"loss": 0.3621,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 1.53125,
|
|
"rewards/rejected": -3.171875,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.12780369352674292,
|
|
"grad_norm": 24.815238960975694,
|
|
"learning_rate": 4.98829096148362e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -316.0,
|
|
"logps/rejected": -494.0,
|
|
"loss": 0.3789,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.8515625,
|
|
"rewards/rejected": -3.421875,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.12908173046201035,
|
|
"grad_norm": 12.469684746555005,
|
|
"learning_rate": 4.987187865598482e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -312.0,
|
|
"logps/rejected": -480.0,
|
|
"loss": 0.354,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.578125,
|
|
"rewards/margins": 1.96875,
|
|
"rewards/rejected": -3.546875,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.13035976739727778,
|
|
"grad_norm": 10.651703453569814,
|
|
"learning_rate": 4.986035254469005e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -280.0,
|
|
"logps/rejected": -460.0,
|
|
"loss": 0.3777,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.3671875,
|
|
"rewards/margins": 1.84375,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.1316378043325452,
|
|
"grad_norm": 10.169153192152455,
|
|
"learning_rate": 4.984833151041512e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -322.0,
|
|
"logps/rejected": -524.0,
|
|
"loss": 0.3529,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -1.734375,
|
|
"rewards/margins": 2.0625,
|
|
"rewards/rejected": -3.796875,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.13291584126781264,
|
|
"grad_norm": 17.70616690114393,
|
|
"learning_rate": 4.98358157924763e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.344,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.9921875,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -4.3125,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.13419387820308007,
|
|
"grad_norm": 14.910681564668302,
|
|
"learning_rate": 4.982280564003806e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -342.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3676,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.640625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -3.0625,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.1354719151383475,
|
|
"grad_norm": 9.401234339612973,
|
|
"learning_rate": 4.98093013121081e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -336.0,
|
|
"logps/rejected": -488.0,
|
|
"loss": 0.3497,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.734375,
|
|
"rewards/rejected": -3.453125,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.13674995207361493,
|
|
"grad_norm": 11.021445735908344,
|
|
"learning_rate": 4.979530307753227e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -472.0,
|
|
"loss": 0.3518,
|
|
"rewards/accuracies": 0.8062499761581421,
|
|
"rewards/chosen": -1.6328125,
|
|
"rewards/margins": 1.6875,
|
|
"rewards/rejected": -3.3125,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.13802798900888236,
|
|
"grad_norm": 13.943751456778854,
|
|
"learning_rate": 4.978081121498916e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -344.0,
|
|
"logps/rejected": -516.0,
|
|
"loss": 0.3489,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.9609375,
|
|
"rewards/margins": 1.7578125,
|
|
"rewards/rejected": -3.71875,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.13930602594414979,
|
|
"grad_norm": 10.338244132399032,
|
|
"learning_rate": 4.976582601298456e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -332.0,
|
|
"logps/rejected": -510.0,
|
|
"loss": 0.3353,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -1.7109375,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.14058406287941722,
|
|
"grad_norm": 8.71888191275971,
|
|
"learning_rate": 4.975034776984573e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -338.0,
|
|
"logps/rejected": -486.0,
|
|
"loss": 0.3553,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.8203125,
|
|
"rewards/margins": 1.65625,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.14186209981468464,
|
|
"grad_norm": 12.216205581090218,
|
|
"learning_rate": 4.973437679371546e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -292.0,
|
|
"logps/rejected": -456.0,
|
|
"loss": 0.3496,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.5,
|
|
"rewards/margins": 1.7890625,
|
|
"rewards/rejected": -3.296875,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.14314013674995207,
|
|
"grad_norm": 12.423722309973517,
|
|
"learning_rate": 4.971791340254592e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -320.0,
|
|
"logps/rejected": -462.0,
|
|
"loss": 0.3644,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -1.6171875,
|
|
"rewards/margins": 1.640625,
|
|
"rewards/rejected": -3.25,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.1444181736852195,
|
|
"grad_norm": 11.14194864610794,
|
|
"learning_rate": 4.970095792409233e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -454.0,
|
|
"loss": 0.3456,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.5859375,
|
|
"rewards/margins": 1.625,
|
|
"rewards/rejected": -3.21875,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.14569621062048693,
|
|
"grad_norm": 11.464691538377055,
|
|
"learning_rate": 4.968351069590646e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -506.0,
|
|
"loss": 0.3693,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -1.9453125,
|
|
"rewards/margins": 1.703125,
|
|
"rewards/rejected": -3.65625,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.14697424755575436,
|
|
"grad_norm": 9.505788032958687,
|
|
"learning_rate": 4.966557206532992e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -352.0,
|
|
"logps/rejected": -474.0,
|
|
"loss": 0.3805,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.8515625,
|
|
"rewards/margins": 1.4140625,
|
|
"rewards/rejected": -3.265625,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.1482522844910218,
|
|
"grad_norm": 9.689853960676805,
|
|
"learning_rate": 4.964714238948715e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -328.0,
|
|
"logps/rejected": -520.0,
|
|
"loss": 0.3429,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.7265625,
|
|
"rewards/margins": 1.9609375,
|
|
"rewards/rejected": -3.6875,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.14953032142628922,
|
|
"grad_norm": 8.325164199298303,
|
|
"learning_rate": 4.962822203527845e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -326.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3277,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -1.7578125,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -4.25,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.15080835836155665,
|
|
"grad_norm": 15.52096274687861,
|
|
"learning_rate": 4.960881137937256e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -312.0,
|
|
"logps/rejected": -492.0,
|
|
"loss": 0.3235,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.546875,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -3.46875,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.15208639529682408,
|
|
"grad_norm": 12.639275691252218,
|
|
"learning_rate": 4.958891080819923e-07,
|
|
"logits/chosen": -3.34375,
|
|
"logits/rejected": -3.34375,
|
|
"logps/chosen": -356.0,
|
|
"logps/rejected": -552.0,
|
|
"loss": 0.3541,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.125,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -4.125,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.1533644322320915,
|
|
"grad_norm": 8.869328760760212,
|
|
"learning_rate": 4.956852071794151e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -324.0,
|
|
"logps/rejected": -470.0,
|
|
"loss": 0.3544,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -1.5703125,
|
|
"rewards/margins": 1.71875,
|
|
"rewards/rejected": -3.28125,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.15464246916735894,
|
|
"grad_norm": 10.63346925167416,
|
|
"learning_rate": 4.954764151452782e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -332.0,
|
|
"logps/rejected": -512.0,
|
|
"loss": 0.3265,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -1.828125,
|
|
"rewards/margins": 1.8125,
|
|
"rewards/rejected": -3.640625,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.15592050610262637,
|
|
"grad_norm": 11.334624897642897,
|
|
"learning_rate": 4.952627361362395e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -340.0,
|
|
"logps/rejected": -540.0,
|
|
"loss": 0.3435,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -1.9765625,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -3.90625,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.1571985430378938,
|
|
"grad_norm": 11.849129116829852,
|
|
"learning_rate": 4.950441744062471e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.28125,
|
|
"logps/chosen": -364.0,
|
|
"logps/rejected": -564.0,
|
|
"loss": 0.3435,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.046875,
|
|
"rewards/margins": 1.9453125,
|
|
"rewards/rejected": -3.984375,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.15847657997316122,
|
|
"grad_norm": 9.577666300992249,
|
|
"learning_rate": 4.948207343064551e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -374.0,
|
|
"logps/rejected": -596.0,
|
|
"loss": 0.3173,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.203125,
|
|
"rewards/margins": 2.34375,
|
|
"rewards/rejected": -4.5625,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.15975461690842865,
|
|
"grad_norm": 10.910196783813214,
|
|
"learning_rate": 4.945924202851366e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -398.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.3307,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.328125,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -4.6875,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.16103265384369608,
|
|
"grad_norm": 10.57466378686154,
|
|
"learning_rate": 4.943592368875955e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -414.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 0.3491,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -2.59375,
|
|
"rewards/margins": 1.921875,
|
|
"rewards/rejected": -4.53125,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.1623106907789635,
|
|
"grad_norm": 11.230589436335967,
|
|
"learning_rate": 4.941211887560757e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -458.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.3023,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.859375,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.16358872771423094,
|
|
"grad_norm": 11.961477038376033,
|
|
"learning_rate": 4.938782806296691e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3028,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -2.875,
|
|
"rewards/margins": 2.078125,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.16486676464949837,
|
|
"grad_norm": 9.972580283679875,
|
|
"learning_rate": 4.936305173442206e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -608.0,
|
|
"loss": 0.3068,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.5,
|
|
"rewards/margins": 2.203125,
|
|
"rewards/rejected": -4.71875,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.1661448015847658,
|
|
"grad_norm": 10.211364589134622,
|
|
"learning_rate": 4.933779038322324e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -636.0,
|
|
"loss": 0.3251,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.59375,
|
|
"rewards/margins": 2.234375,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.16742283852003323,
|
|
"grad_norm": 8.909898711500084,
|
|
"learning_rate": 4.931204451227658e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -386.0,
|
|
"logps/rejected": -572.0,
|
|
"loss": 0.3113,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -2.234375,
|
|
"rewards/margins": 2.0,
|
|
"rewards/rejected": -4.25,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.16870087545530066,
|
|
"grad_norm": 13.917027005009777,
|
|
"learning_rate": 4.928581463413405e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -354.0,
|
|
"logps/rejected": -584.0,
|
|
"loss": 0.3264,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.140625,
|
|
"rewards/margins": 2.265625,
|
|
"rewards/rejected": -4.40625,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.1699789123905681,
|
|
"grad_norm": 9.075174861427325,
|
|
"learning_rate": 4.925910127098333e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -342.0,
|
|
"logps/rejected": -560.0,
|
|
"loss": 0.3096,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -1.90625,
|
|
"rewards/margins": 2.390625,
|
|
"rewards/rejected": -4.28125,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.17125694932583552,
|
|
"grad_norm": 10.492320216709347,
|
|
"learning_rate": 4.923190495463736e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -390.0,
|
|
"logps/rejected": -624.0,
|
|
"loss": 0.2988,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -2.296875,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -4.75,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.17253498626110295,
|
|
"grad_norm": 11.104892250574451,
|
|
"learning_rate": 4.920422622652377e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -376.0,
|
|
"logps/rejected": -576.0,
|
|
"loss": 0.3202,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.078125,
|
|
"rewards/margins": 2.140625,
|
|
"rewards/rejected": -4.21875,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.17381302319637038,
|
|
"grad_norm": 8.174183970674996,
|
|
"learning_rate": 4.917606563767411e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.3258,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -2.359375,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -4.9375,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.1750910601316378,
|
|
"grad_norm": 9.822967964793463,
|
|
"learning_rate": 4.914742374871289e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.2794,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.90625,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -5.125,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.17636909706690523,
|
|
"grad_norm": 13.046396762006776,
|
|
"learning_rate": 4.911830112984638e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.28125,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 0.3158,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.515625,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -4.75,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.17764713400217266,
|
|
"grad_norm": 9.634796199540869,
|
|
"learning_rate": 4.908869836085127e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.109375,
|
|
"logps/chosen": -400.0,
|
|
"logps/rejected": -620.0,
|
|
"loss": 0.2978,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.453125,
|
|
"rewards/margins": 2.375,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.1789251709374401,
|
|
"grad_norm": 10.036265985004444,
|
|
"learning_rate": 4.905861603106318e-07,
|
|
"logits/chosen": -3.40625,
|
|
"logits/rejected": -3.328125,
|
|
"logps/chosen": -404.0,
|
|
"logps/rejected": -644.0,
|
|
"loss": 0.3018,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.296875,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -4.90625,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.18020320787270752,
|
|
"grad_norm": 18.756659281595155,
|
|
"learning_rate": 4.902805473936483e-07,
|
|
"logits/chosen": -3.4375,
|
|
"logits/rejected": -3.296875,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -2.5625,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.18148124480797495,
|
|
"grad_norm": 11.083399006007372,
|
|
"learning_rate": 4.899701509417423e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -664.0,
|
|
"loss": 0.2897,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -5.15625,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.18275928174324238,
|
|
"grad_norm": 9.072585092087222,
|
|
"learning_rate": 4.896549771343247e-07,
|
|
"logits/chosen": -3.390625,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.2857,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.109375,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.1840373186785098,
|
|
"grad_norm": 7.519453357476137,
|
|
"learning_rate": 4.893350322459149e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -676.0,
|
|
"loss": 0.2972,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -2.71875,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.18531535561377724,
|
|
"grad_norm": 10.926070436917199,
|
|
"learning_rate": 4.890103226460152e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3027,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -2.828125,
|
|
"rewards/margins": 2.21875,
|
|
"rewards/rejected": -5.0625,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.18659339254904467,
|
|
"grad_norm": 9.889333734788899,
|
|
"learning_rate": 4.886808547989846e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.3162,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.1878714294843121,
|
|
"grad_norm": 11.48343624861157,
|
|
"learning_rate": 4.883466352639099e-07,
|
|
"logits/chosen": -3.296875,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.3164,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.625,
|
|
"rewards/margins": 2.359375,
|
|
"rewards/rejected": -4.96875,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.18914946641957953,
|
|
"grad_norm": 11.028607347761977,
|
|
"learning_rate": 4.88007670694475e-07,
|
|
"logits/chosen": -3.421875,
|
|
"logits/rejected": -3.21875,
|
|
"logps/chosen": -440.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.292,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -2.734375,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.25,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.19042750335484696,
|
|
"grad_norm": 9.137715141189384,
|
|
"learning_rate": 4.876639678388285e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -420.0,
|
|
"logps/rejected": -628.0,
|
|
"loss": 0.3172,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -2.515625,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -4.84375,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.19170554029011438,
|
|
"grad_norm": 8.392316890933923,
|
|
"learning_rate": 4.873155335394497e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -656.0,
|
|
"loss": 0.2895,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.21875,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.1929835772253818,
|
|
"grad_norm": 9.776765970586137,
|
|
"learning_rate": 4.869623747330116e-07,
|
|
"logits/chosen": -3.265625,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -474.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.3101,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.19426161416064924,
|
|
"grad_norm": 10.113772712494775,
|
|
"learning_rate": 4.866044984502436e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2819,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.765625,
|
|
"rewards/margins": 2.5625,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.19553965109591667,
|
|
"grad_norm": 12.373484208276247,
|
|
"learning_rate": 4.862419118157909e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -648.0,
|
|
"loss": 0.2809,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.75,
|
|
"rewards/margins": 2.328125,
|
|
"rewards/rejected": -5.09375,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.1968176880311841,
|
|
"grad_norm": 11.78683291379729,
|
|
"learning_rate": 4.858746220480734e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -412.0,
|
|
"logps/rejected": -632.0,
|
|
"loss": 0.3016,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -2.453125,
|
|
"rewards/margins": 2.4375,
|
|
"rewards/rejected": -4.875,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.19809572496645153,
|
|
"grad_norm": 12.160856594312754,
|
|
"learning_rate": 4.855026364591413e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -482.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2759,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.625,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.19937376190171896,
|
|
"grad_norm": 7.635186752817528,
|
|
"learning_rate": 4.851259624545297e-07,
|
|
"logits/chosen": -3.296875,
|
|
"logits/rejected": -3.203125,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -700.0,
|
|
"loss": 0.2999,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.46875,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.2006517988369864,
|
|
"grad_norm": 10.580854781397509,
|
|
"learning_rate": 4.847446075331114e-07,
|
|
"logits/chosen": -3.28125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -672.0,
|
|
"loss": 0.298,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -2.8125,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.28125,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.20192983577225382,
|
|
"grad_norm": 15.508397767776305,
|
|
"learning_rate": 4.843585792869476e-07,
|
|
"logits/chosen": -3.375,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.3035,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.625,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.20320787270752125,
|
|
"grad_norm": 12.29502011282903,
|
|
"learning_rate": 4.839678854011362e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -680.0,
|
|
"loss": 0.3106,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.109375,
|
|
"rewards/rejected": -5.34375,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.20448590964278868,
|
|
"grad_norm": 9.221266332248716,
|
|
"learning_rate": 4.835725336536598e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.25,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2795,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -5.875,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.2057639465780561,
|
|
"grad_norm": 21.949201663234852,
|
|
"learning_rate": 4.831725319152298e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.359375,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2822,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 2.796875,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.20704198351332354,
|
|
"grad_norm": 8.390379977679709,
|
|
"learning_rate": 4.827678881491305e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2692,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.359375,
|
|
"rewards/margins": 2.65625,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.20832002044859096,
|
|
"grad_norm": 9.085937257418248,
|
|
"learning_rate": 4.8235861041106e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2757,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.3125,
|
|
"rewards/rejected": -5.78125,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.2095980573838584,
|
|
"grad_norm": 10.915204034593046,
|
|
"learning_rate": 4.819447068489705e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.2838,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.96875,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.21087609431912582,
|
|
"grad_norm": 10.623276883304097,
|
|
"learning_rate": 4.815261857029052e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -462.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2661,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.21215413125439325,
|
|
"grad_norm": 15.529572832655484,
|
|
"learning_rate": 4.811030553048351e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2895,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.4375,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.21343216818966068,
|
|
"grad_norm": 8.986111656832815,
|
|
"learning_rate": 4.806753240784924e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -452.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2822,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -2.796875,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -5.59375,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.2147102051249281,
|
|
"grad_norm": 10.662265185336627,
|
|
"learning_rate": 4.802430005392037e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -668.0,
|
|
"loss": 0.3085,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -2.96875,
|
|
"rewards/margins": 2.53125,
|
|
"rewards/rejected": -5.5,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.21598824206019554,
|
|
"grad_norm": 18.32782419922393,
|
|
"learning_rate": 4.798060932937194e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -464.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2825,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.0,
|
|
"rewards/margins": 2.453125,
|
|
"rewards/rejected": -5.46875,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.21726627899546297,
|
|
"grad_norm": 8.37402052085016,
|
|
"learning_rate": 4.79364611040043e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -456.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2776,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.0,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.15625,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.2185443159307304,
|
|
"grad_norm": 7.388938797100764,
|
|
"learning_rate": 4.789185625672583e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2688,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 2.421875,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.21982235286599783,
|
|
"grad_norm": 11.18964791369844,
|
|
"learning_rate": 4.784679567553531e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2807,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -6.375,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.22110038980126526,
|
|
"grad_norm": 12.788195666651463,
|
|
"learning_rate": 4.780128025750442e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -466.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2667,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.15625,
|
|
"rewards/margins": 2.578125,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.22237842673653269,
|
|
"grad_norm": 13.452879203275968,
|
|
"learning_rate": 4.775531090875971e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2936,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.34375,
|
|
"rewards/rejected": -5.75,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.22365646367180012,
|
|
"grad_norm": 13.978917804241819,
|
|
"learning_rate": 4.770888854446471e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -462.0,
|
|
"logps/rejected": -696.0,
|
|
"loss": 0.2665,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.09375,
|
|
"rewards/margins": 2.625,
|
|
"rewards/rejected": -5.71875,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.22493450060706754,
|
|
"grad_norm": 14.827300418751848,
|
|
"learning_rate": 4.766201408880156e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -752.0,
|
|
"loss": 0.3005,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.22621253754233497,
|
|
"grad_norm": 8.780665034348093,
|
|
"learning_rate": 4.761468847495277e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -748.0,
|
|
"loss": 0.268,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.46875,
|
|
"rewards/rejected": -5.9375,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.2274905744776024,
|
|
"grad_norm": 8.913295529168444,
|
|
"learning_rate": 4.756691264508251e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2597,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.140625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.22876861141286983,
|
|
"grad_norm": 9.149720146692632,
|
|
"learning_rate": 4.751868755031793e-07,
|
|
"logits/chosen": -3.3125,
|
|
"logits/rejected": -3.234375,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2978,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.23004664834813726,
|
|
"grad_norm": 9.864161436507205,
|
|
"learning_rate": 4.747001415073018e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -684.0,
|
|
"loss": 0.2801,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.171875,
|
|
"rewards/margins": 2.25,
|
|
"rewards/rejected": -5.40625,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.2313246852834047,
|
|
"grad_norm": 12.049630916813516,
|
|
"learning_rate": 4.742089341531535e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.109375,
|
|
"logps/chosen": -488.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2725,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.23260272221867212,
|
|
"grad_norm": 9.245650817902728,
|
|
"learning_rate": 4.7371326321975127e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2838,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.23388075915393955,
|
|
"grad_norm": 10.565374823511835,
|
|
"learning_rate": 4.7321313857497336e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.3001,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.23515879608920698,
|
|
"grad_norm": 11.728663046788858,
|
|
"learning_rate": 4.7270857017536335e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2755,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.2364368330244744,
|
|
"grad_norm": 13.400694062249938,
|
|
"learning_rate": 4.7219956806593143e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2901,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.671875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.23771486995974184,
|
|
"grad_norm": 8.737541772532301,
|
|
"learning_rate": 4.716861423799546e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2871,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.9375,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.23899290689500927,
|
|
"grad_norm": 11.454317171770168,
|
|
"learning_rate": 4.711683033387752e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2612,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.2402709438302767,
|
|
"grad_norm": 8.019796544220595,
|
|
"learning_rate": 4.70646061251597e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2603,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -7.0,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.24154898076554412,
|
|
"grad_norm": 9.374363546593047,
|
|
"learning_rate": 4.701194265152802e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2726,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.125,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.24282701770081155,
|
|
"grad_norm": 13.860826584264853,
|
|
"learning_rate": 4.6958840961413447e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -724.0,
|
|
"loss": 0.2811,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.546875,
|
|
"rewards/margins": 2.4375,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.24410505463607898,
|
|
"grad_norm": 10.57554715572998,
|
|
"learning_rate": 4.6905302111971004e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -428.0,
|
|
"logps/rejected": -712.0,
|
|
"loss": 0.2829,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -2.609375,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -5.59375,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.2453830915713464,
|
|
"grad_norm": 14.259221382970665,
|
|
"learning_rate": 4.6851327169058735e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.2912,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 2.484375,
|
|
"rewards/rejected": -5.875,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.24666112850661384,
|
|
"grad_norm": 7.366111663404775,
|
|
"learning_rate": 4.679691720721651e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2693,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.28125,
|
|
"rewards/rejected": -6.0,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.24793916544188127,
|
|
"grad_norm": 11.096803393586505,
|
|
"learning_rate": 4.674207330964458e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2789,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.2492172023771487,
|
|
"grad_norm": 8.956111080301984,
|
|
"learning_rate": 4.668679656818207e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2746,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.25049523931241613,
|
|
"grad_norm": 9.513508988130903,
|
|
"learning_rate": 4.663108808328519e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2819,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 2.296875,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.25177327624768353,
|
|
"grad_norm": 10.234008316257968,
|
|
"learning_rate": 4.6574948964005367e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2871,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.546875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.253051313182951,
|
|
"grad_norm": 8.01274333751014,
|
|
"learning_rate": 4.6518380327967145e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.1875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.247,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.515625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.2543293501182184,
|
|
"grad_norm": 12.33132352378727,
|
|
"learning_rate": 4.6461383301345947e-07,
|
|
"logits/chosen": -3.328125,
|
|
"logits/rejected": -3.265625,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2821,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.25560738705348585,
|
|
"grad_norm": 10.395762625615612,
|
|
"learning_rate": 4.6403959018845637e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -3.09375,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2645,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.703125,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.25688542398875325,
|
|
"grad_norm": 15.793711134680974,
|
|
"learning_rate": 4.6346108623675954e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.2914,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.2581634609240207,
|
|
"grad_norm": 10.102379635568527,
|
|
"learning_rate": 4.628783326752974e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.261,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.2594414978592881,
|
|
"grad_norm": 11.19917017437885,
|
|
"learning_rate": 4.622913411056e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2563,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.26071953479455556,
|
|
"grad_norm": 9.591252819586105,
|
|
"learning_rate": 4.617001232135684e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.046875,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.2804,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.5,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -6.3125,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.26199757172982296,
|
|
"grad_norm": 10.927105525570921,
|
|
"learning_rate": 4.6110469076924153e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2661,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.5625,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.2632756086650904,
|
|
"grad_norm": 8.081281860104301,
|
|
"learning_rate": 4.605050556265624e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2664,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.2645536456003578,
|
|
"grad_norm": 8.719157157796769,
|
|
"learning_rate": 4.599012297231417e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2605,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.328125,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -6.34375,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.2658316825356253,
|
|
"grad_norm": 9.889601643083903,
|
|
"learning_rate": 4.5929322508002045e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -776.0,
|
|
"loss": 0.264,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.2671097194708927,
|
|
"grad_norm": 9.01425592242334,
|
|
"learning_rate": 4.586810538014304e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.109375,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -740.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.28125,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.09375,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.26838775640616014,
|
|
"grad_norm": 9.917816992815437,
|
|
"learning_rate": 4.580647280745532e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -490.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.2651,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.0625,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.26966579334142754,
|
|
"grad_norm": 9.08237633986445,
|
|
"learning_rate": 4.5744426016927783e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2618,
|
|
"rewards/accuracies": 0.8187500238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.6875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.270943830276695,
|
|
"grad_norm": 10.374914225435086,
|
|
"learning_rate": 4.5681966243795645e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2563,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -3.65625,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 0.2722218672119624,
|
|
"grad_norm": 9.39552251607615,
|
|
"learning_rate": 4.5619094731515783e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2488,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 0.27349990414722986,
|
|
"grad_norm": 9.255807709157049,
|
|
"learning_rate": 4.5555812731742085e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2436,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 2.71875,
|
|
"rewards/rejected": -6.4375,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 0.27477794108249726,
|
|
"grad_norm": 13.023248862747112,
|
|
"learning_rate": 4.549212150430042e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2483,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.453125,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -6.375,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"epoch": 0.2760559780177647,
|
|
"grad_norm": 10.069695705444243,
|
|
"learning_rate": 4.5428022317163654e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -504.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2538,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.390625,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.53125,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 0.2773340149530321,
|
|
"grad_norm": 7.905785619069278,
|
|
"learning_rate": 4.5363516446426353e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -476.0,
|
|
"logps/rejected": -716.0,
|
|
"loss": 0.2553,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.21875,
|
|
"rewards/margins": 2.59375,
|
|
"rewards/rejected": -5.8125,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"epoch": 0.27861205188829957,
|
|
"grad_norm": 14.110926580452388,
|
|
"learning_rate": 4.5298605176279383e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2546,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 0.279890088823567,
|
|
"grad_norm": 11.437656708234213,
|
|
"learning_rate": 4.5233289798984355e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -506.0,
|
|
"logps/rejected": -772.0,
|
|
"loss": 0.2515,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.28125,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"epoch": 0.28116812575883443,
|
|
"grad_norm": 10.92037760467791,
|
|
"learning_rate": 4.51675716148479e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -468.0,
|
|
"logps/rejected": -744.0,
|
|
"loss": 0.27,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.0625,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.03125,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 0.28244616269410183,
|
|
"grad_norm": 9.082318419309797,
|
|
"learning_rate": 4.510145193219577e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2611,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.03125,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"epoch": 0.2837241996293693,
|
|
"grad_norm": 9.696968012792881,
|
|
"learning_rate": 4.503493206734681e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.171875,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -732.0,
|
|
"loss": 0.2665,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.078125,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -5.875,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 0.2850022365646367,
|
|
"grad_norm": 8.32597970165281,
|
|
"learning_rate": 4.4968013344586723e-07,
|
|
"logits/chosen": -3.234375,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2618,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"epoch": 0.28628027349990415,
|
|
"grad_norm": 9.205123244163184,
|
|
"learning_rate": 4.4900697096141754e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2936,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 2.40625,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 0.28755831043517155,
|
|
"grad_norm": 8.173644269044605,
|
|
"learning_rate": 4.483298466215211e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2481,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.484375,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.1875,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"epoch": 0.288836347370439,
|
|
"grad_norm": 13.842959897880784,
|
|
"learning_rate": 4.4764877390645317e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2398,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 0.2901143843057064,
|
|
"grad_norm": 10.298507286958628,
|
|
"learning_rate": 4.469637663750939e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2618,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"epoch": 0.29139242124097386,
|
|
"grad_norm": 11.781378177678372,
|
|
"learning_rate": 4.4627483766465813e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2419,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 0.29267045817624127,
|
|
"grad_norm": 9.975143928231438,
|
|
"learning_rate": 4.4558200149042393e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -848.0,
|
|
"loss": 0.2498,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"epoch": 0.2939484951115087,
|
|
"grad_norm": 9.81920229020163,
|
|
"learning_rate": 4.4488527164545976e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -3.0625,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2174,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 0.2952265320467761,
|
|
"grad_norm": 8.085865328061837,
|
|
"learning_rate": 4.4418466200034974e-07,
|
|
"logits/chosen": -3.265625,
|
|
"logits/rejected": -3.140625,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2624,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.625,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"epoch": 0.2965045689820436,
|
|
"grad_norm": 10.263278935222155,
|
|
"learning_rate": 4.4348018650291764e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -496.0,
|
|
"logps/rejected": -756.0,
|
|
"loss": 0.2542,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.265625,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -6.21875,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"epoch": 0.297782605917311,
|
|
"grad_norm": 8.200991204991244,
|
|
"learning_rate": 4.427718591779489e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -484.0,
|
|
"logps/rejected": -780.0,
|
|
"loss": 0.2513,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.25,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"epoch": 0.29906064285257844,
|
|
"grad_norm": 9.045676093918619,
|
|
"learning_rate": 4.4205969412691167e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2433,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.765625,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"epoch": 0.30033867978784584,
|
|
"grad_norm": 9.307362421836325,
|
|
"learning_rate": 4.4134370552767617e-07,
|
|
"logits/chosen": -3.25,
|
|
"logits/rejected": -3.125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"epoch": 0.3016167167231133,
|
|
"grad_norm": 10.633510285123654,
|
|
"learning_rate": 4.406239076342322e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -800.0,
|
|
"loss": 0.2616,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 2.890625,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"epoch": 0.3028947536583807,
|
|
"grad_norm": 7.178556035609511,
|
|
"learning_rate": 4.399003147764053e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -508.0,
|
|
"logps/rejected": -760.0,
|
|
"loss": 0.2394,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.40625,
|
|
"rewards/margins": 2.5,
|
|
"rewards/rejected": -5.90625,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"epoch": 0.30417279059364816,
|
|
"grad_norm": 8.025880286415548,
|
|
"learning_rate": 4.3917294135957185e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -736.0,
|
|
"loss": 0.2697,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.515625,
|
|
"rewards/margins": 2.609375,
|
|
"rewards/rejected": -6.125,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"epoch": 0.30545082752891556,
|
|
"grad_norm": 9.150701110291598,
|
|
"learning_rate": 4.3844180186437205e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.252,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.640625,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"epoch": 0.306728864464183,
|
|
"grad_norm": 9.06253052565922,
|
|
"learning_rate": 4.3770691084642153e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2478,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.828125,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.0625,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"epoch": 0.3080069013994504,
|
|
"grad_norm": 8.361504252843975,
|
|
"learning_rate": 4.3696828293602185e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -3.15625,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2583,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.71875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"epoch": 0.3092849383347179,
|
|
"grad_norm": 9.674493266614986,
|
|
"learning_rate": 4.3622593283786895e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.235,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.84375,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"epoch": 0.3105629752699853,
|
|
"grad_norm": 8.99066458935145,
|
|
"learning_rate": 4.354798753307606e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -832.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -3.796875,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"epoch": 0.31184101220525273,
|
|
"grad_norm": 10.245096178761992,
|
|
"learning_rate": 4.3473012526730216e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.258,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -3.953125,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2440
|
|
},
|
|
{
|
|
"epoch": 0.31311904914052013,
|
|
"grad_norm": 8.095300746315656,
|
|
"learning_rate": 4.339766975736109e-07,
|
|
"logits/chosen": -2.71875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2535,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.78125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 2450
|
|
},
|
|
{
|
|
"epoch": 0.3143970860757876,
|
|
"grad_norm": 9.33334356795753,
|
|
"learning_rate": 4.332196072490185e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.247,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 2460
|
|
},
|
|
{
|
|
"epoch": 0.315675123011055,
|
|
"grad_norm": 9.68096887344856,
|
|
"learning_rate": 4.324588693657733e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2589,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.625,
|
|
"step": 2470
|
|
},
|
|
{
|
|
"epoch": 0.31695315994632245,
|
|
"grad_norm": 8.74974145611664,
|
|
"learning_rate": 4.3169449906873925e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2480
|
|
},
|
|
{
|
|
"epoch": 0.31823119688158985,
|
|
"grad_norm": 6.971622374452891,
|
|
"learning_rate": 4.309265115750949e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2366,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -7.53125,
|
|
"step": 2490
|
|
},
|
|
{
|
|
"epoch": 0.3195092338168573,
|
|
"grad_norm": 11.17024488529161,
|
|
"learning_rate": 4.301549221740305e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2874,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2500
|
|
},
|
|
{
|
|
"epoch": 0.3207872707521247,
|
|
"grad_norm": 10.477310622785978,
|
|
"learning_rate": 4.293797462264436e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2387,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.859375,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2510
|
|
},
|
|
{
|
|
"epoch": 0.32206530768739217,
|
|
"grad_norm": 10.024656692443616,
|
|
"learning_rate": 4.286009991646329e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.253,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.25,
|
|
"step": 2520
|
|
},
|
|
{
|
|
"epoch": 0.32334334462265957,
|
|
"grad_norm": 8.009802554248187,
|
|
"learning_rate": 4.2781869649199153e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -864.0,
|
|
"loss": 0.2492,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2530
|
|
},
|
|
{
|
|
"epoch": 0.324621381557927,
|
|
"grad_norm": 8.945237247295104,
|
|
"learning_rate": 4.2703285378269815e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 2.9375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2540
|
|
},
|
|
{
|
|
"epoch": 0.3258994184931944,
|
|
"grad_norm": 10.551216216391607,
|
|
"learning_rate": 4.262434866814067e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2183,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.75,
|
|
"step": 2550
|
|
},
|
|
{
|
|
"epoch": 0.3271774554284619,
|
|
"grad_norm": 8.688942019994915,
|
|
"learning_rate": 4.254506109029353e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2568,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2560
|
|
},
|
|
{
|
|
"epoch": 0.3284554923637293,
|
|
"grad_norm": 6.564420476695321,
|
|
"learning_rate": 4.2465424223195327e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2343,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 2570
|
|
},
|
|
{
|
|
"epoch": 0.32973352929899674,
|
|
"grad_norm": 9.43421095366614,
|
|
"learning_rate": 4.238543965226668e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2473,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.015625,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 2580
|
|
},
|
|
{
|
|
"epoch": 0.33101156623426414,
|
|
"grad_norm": 13.150142319397695,
|
|
"learning_rate": 4.230510896985035e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2324,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 2590
|
|
},
|
|
{
|
|
"epoch": 0.3322896031695316,
|
|
"grad_norm": 9.050124476173716,
|
|
"learning_rate": 4.2224433775179506e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.268,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 2600
|
|
},
|
|
{
|
|
"epoch": 0.333567640104799,
|
|
"grad_norm": 8.614894031818137,
|
|
"learning_rate": 4.2143415674345937e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2273,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 2610
|
|
},
|
|
{
|
|
"epoch": 0.33484567704006646,
|
|
"grad_norm": 8.111143191701691,
|
|
"learning_rate": 4.2062056280268033e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2433,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 2620
|
|
},
|
|
{
|
|
"epoch": 0.33612371397533386,
|
|
"grad_norm": 11.117676038308154,
|
|
"learning_rate": 4.198035721265869e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2518,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2630
|
|
},
|
|
{
|
|
"epoch": 0.3374017509106013,
|
|
"grad_norm": 9.258130845121494,
|
|
"learning_rate": 4.1898320097993085e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2846,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 2640
|
|
},
|
|
{
|
|
"epoch": 0.3386797878458687,
|
|
"grad_norm": 9.194807847235614,
|
|
"learning_rate": 4.181594656947625e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -796.0,
|
|
"loss": 0.2202,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.75,
|
|
"rewards/margins": 2.75,
|
|
"rewards/rejected": -6.5,
|
|
"step": 2650
|
|
},
|
|
{
|
|
"epoch": 0.3399578247811362,
|
|
"grad_norm": 9.12609259279053,
|
|
"learning_rate": 4.173323826701062e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.235,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 2660
|
|
},
|
|
{
|
|
"epoch": 0.3412358617164036,
|
|
"grad_norm": 8.558030417957948,
|
|
"learning_rate": 4.165019683716332e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2471,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 2.734375,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2670
|
|
},
|
|
{
|
|
"epoch": 0.34251389865167103,
|
|
"grad_norm": 7.570195174697268,
|
|
"learning_rate": 4.1566823933133444e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2322,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.8125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2680
|
|
},
|
|
{
|
|
"epoch": 0.34379193558693844,
|
|
"grad_norm": 12.163376095462333,
|
|
"learning_rate": 4.148312121471908e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2238,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 2690
|
|
},
|
|
{
|
|
"epoch": 0.3450699725222059,
|
|
"grad_norm": 8.261895776179635,
|
|
"learning_rate": 4.1399090348284336e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -520.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.249,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.65625,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -6.71875,
|
|
"step": 2700
|
|
},
|
|
{
|
|
"epoch": 0.3463480094574733,
|
|
"grad_norm": 7.401313646780075,
|
|
"learning_rate": 4.1314733006726116e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -784.0,
|
|
"loss": 0.2496,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.59375,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -6.5625,
|
|
"step": 2710
|
|
},
|
|
{
|
|
"epoch": 0.34762604639274075,
|
|
"grad_norm": 8.802407402306804,
|
|
"learning_rate": 4.1230050869440825e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2272,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 2.8125,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2720
|
|
},
|
|
{
|
|
"epoch": 0.34890408332800815,
|
|
"grad_norm": 8.551492444117791,
|
|
"learning_rate": 4.114504562229096e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -820.0,
|
|
"loss": 0.2462,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.875,
|
|
"step": 2730
|
|
},
|
|
{
|
|
"epoch": 0.3501821202632756,
|
|
"grad_norm": 13.237697656186098,
|
|
"learning_rate": 4.105971895757151e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -528.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.262,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -3.90625,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2740
|
|
},
|
|
{
|
|
"epoch": 0.351460157198543,
|
|
"grad_norm": 9.794093445613948,
|
|
"learning_rate": 4.0974072573976295e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -510.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2488,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 3.046875,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2750
|
|
},
|
|
{
|
|
"epoch": 0.35273819413381047,
|
|
"grad_norm": 9.723773756216174,
|
|
"learning_rate": 4.088810817656414e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.671875,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2760
|
|
},
|
|
{
|
|
"epoch": 0.35401623106907787,
|
|
"grad_norm": 8.723909251690763,
|
|
"learning_rate": 4.0801827476724923e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -816.0,
|
|
"loss": 0.2428,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 2.859375,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2770
|
|
},
|
|
{
|
|
"epoch": 0.3552942680043453,
|
|
"grad_norm": 8.462907156480545,
|
|
"learning_rate": 4.071523219214551e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2365,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.03125,
|
|
"rewards/rejected": -7.25,
|
|
"step": 2780
|
|
},
|
|
{
|
|
"epoch": 0.3565723049396127,
|
|
"grad_norm": 9.665834891128702,
|
|
"learning_rate": 4.062832404677556e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -824.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -6.8125,
|
|
"step": 2790
|
|
},
|
|
{
|
|
"epoch": 0.3578503418748802,
|
|
"grad_norm": 8.989459813679218,
|
|
"learning_rate": 4.054110477079321e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.2434,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -6.75,
|
|
"step": 2800
|
|
},
|
|
{
|
|
"epoch": 0.3591283788101476,
|
|
"grad_norm": 8.431501557282072,
|
|
"learning_rate": 4.0453576100570606e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -460.0,
|
|
"logps/rejected": -728.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -3.125,
|
|
"rewards/margins": 2.703125,
|
|
"rewards/rejected": -5.84375,
|
|
"step": 2810
|
|
},
|
|
{
|
|
"epoch": 0.36040641574541504,
|
|
"grad_norm": 15.875947653262912,
|
|
"learning_rate": 4.036573977863938e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.96875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2820
|
|
},
|
|
{
|
|
"epoch": 0.36168445268068244,
|
|
"grad_norm": 11.089723641989362,
|
|
"learning_rate": 4.027759755365591e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2521,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 2830
|
|
},
|
|
{
|
|
"epoch": 0.3629624896159499,
|
|
"grad_norm": 7.979671614832609,
|
|
"learning_rate": 4.018915118036653e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -512.0,
|
|
"logps/rejected": -808.0,
|
|
"loss": 0.2409,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.625,
|
|
"step": 2840
|
|
},
|
|
{
|
|
"epoch": 0.3642405265512173,
|
|
"grad_norm": 7.47703355045046,
|
|
"learning_rate": 4.0100402419572617e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -3.46875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -6.65625,
|
|
"step": 2850
|
|
},
|
|
{
|
|
"epoch": 0.36551856348648476,
|
|
"grad_norm": 6.881027994969947,
|
|
"learning_rate": 4.0011353038095497e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2217,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.375,
|
|
"step": 2860
|
|
},
|
|
{
|
|
"epoch": 0.36679660042175216,
|
|
"grad_norm": 8.387929456445715,
|
|
"learning_rate": 3.992200480874128e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2132,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.0,
|
|
"step": 2870
|
|
},
|
|
{
|
|
"epoch": 0.3680746373570196,
|
|
"grad_norm": 11.868719055234116,
|
|
"learning_rate": 3.983235951026562e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -852.0,
|
|
"loss": 0.2397,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -7.125,
|
|
"step": 2880
|
|
},
|
|
{
|
|
"epoch": 0.369352674292287,
|
|
"grad_norm": 6.6165920564533565,
|
|
"learning_rate": 3.974241892733821e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -556.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.125,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 2890
|
|
},
|
|
{
|
|
"epoch": 0.3706307112275545,
|
|
"grad_norm": 9.666254997669043,
|
|
"learning_rate": 3.965218485050733e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2361,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.09375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2900
|
|
},
|
|
{
|
|
"epoch": 0.3719087481628219,
|
|
"grad_norm": 8.197217719753612,
|
|
"learning_rate": 3.9561659076164166e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -516.0,
|
|
"logps/rejected": -868.0,
|
|
"loss": 0.2378,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -7.21875,
|
|
"step": 2910
|
|
},
|
|
{
|
|
"epoch": 0.37318678509808934,
|
|
"grad_norm": 7.426775845977119,
|
|
"learning_rate": 3.947084340650706e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2383,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 2920
|
|
},
|
|
{
|
|
"epoch": 0.37446482203335674,
|
|
"grad_norm": 6.536302581696631,
|
|
"learning_rate": 3.937973964950561e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -540.0,
|
|
"logps/rejected": -836.0,
|
|
"loss": 0.2321,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.84375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -6.96875,
|
|
"step": 2930
|
|
},
|
|
{
|
|
"epoch": 0.3757428589686242,
|
|
"grad_norm": 12.669430664625045,
|
|
"learning_rate": 3.928834961886472e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -502.0,
|
|
"logps/rejected": -788.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.3125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -6.46875,
|
|
"step": 2940
|
|
},
|
|
{
|
|
"epoch": 0.3770208959038916,
|
|
"grad_norm": 10.03412970439382,
|
|
"learning_rate": 3.919667513398843e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -500.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2145,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -3.421875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -6.90625,
|
|
"step": 2950
|
|
},
|
|
{
|
|
"epoch": 0.37829893283915905,
|
|
"grad_norm": 7.385896415249684,
|
|
"learning_rate": 3.910471801994377e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2349,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -7.75,
|
|
"step": 2960
|
|
},
|
|
{
|
|
"epoch": 0.37957696977442645,
|
|
"grad_norm": 9.839499943864618,
|
|
"learning_rate": 3.901248010742435e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.40625,
|
|
"step": 2970
|
|
},
|
|
{
|
|
"epoch": 0.3808550067096939,
|
|
"grad_norm": 10.643804579819335,
|
|
"learning_rate": 3.891996323271396e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -532.0,
|
|
"logps/rejected": -844.0,
|
|
"loss": 0.2548,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -3.734375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.0,
|
|
"step": 2980
|
|
},
|
|
{
|
|
"epoch": 0.3821330436449613,
|
|
"grad_norm": 6.260351646142021,
|
|
"learning_rate": 3.8827169237650023e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -812.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.6875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -6.6875,
|
|
"step": 2990
|
|
},
|
|
{
|
|
"epoch": 0.38341108058022877,
|
|
"grad_norm": 9.26864693273647,
|
|
"learning_rate": 3.87340999695869e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -536.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2277,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -3.921875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3000
|
|
},
|
|
{
|
|
"epoch": 0.38468911751549617,
|
|
"grad_norm": 11.798346785124874,
|
|
"learning_rate": 3.8640757281359104e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -524.0,
|
|
"logps/rejected": -804.0,
|
|
"loss": 0.2409,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -3.609375,
|
|
"rewards/margins": 2.984375,
|
|
"rewards/rejected": -6.59375,
|
|
"step": 3010
|
|
},
|
|
{
|
|
"epoch": 0.3859671544507636,
|
|
"grad_norm": 9.13276020421161,
|
|
"learning_rate": 3.8547143031244454e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.2363,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -7.25,
|
|
"step": 3020
|
|
},
|
|
{
|
|
"epoch": 0.38724519138603103,
|
|
"grad_norm": 11.67031797954755,
|
|
"learning_rate": 3.845325908292704e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2292,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3030
|
|
},
|
|
{
|
|
"epoch": 0.3885232283212985,
|
|
"grad_norm": 10.639147098006879,
|
|
"learning_rate": 3.835910730546013e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -828.0,
|
|
"loss": 0.245,
|
|
"rewards/accuracies": 0.831250011920929,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -6.875,
|
|
"step": 3040
|
|
},
|
|
{
|
|
"epoch": 0.3898012652565659,
|
|
"grad_norm": 8.44223529175865,
|
|
"learning_rate": 3.8264689573229e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -552.0,
|
|
"logps/rejected": -880.0,
|
|
"loss": 0.236,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -3.875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3050
|
|
},
|
|
{
|
|
"epoch": 0.39107930219183334,
|
|
"grad_norm": 8.488987566619226,
|
|
"learning_rate": 3.8170007765913563e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2073,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3060
|
|
},
|
|
{
|
|
"epoch": 0.39235733912710075,
|
|
"grad_norm": 8.533803292750349,
|
|
"learning_rate": 3.8075063768450977e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2315,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3070
|
|
},
|
|
{
|
|
"epoch": 0.3936353760623682,
|
|
"grad_norm": 13.47605711618702,
|
|
"learning_rate": 3.79798594709981e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2244,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -7.90625,
|
|
"step": 3080
|
|
},
|
|
{
|
|
"epoch": 0.3949134129976356,
|
|
"grad_norm": 9.088959049871905,
|
|
"learning_rate": 3.78843967688939e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2113,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3090
|
|
},
|
|
{
|
|
"epoch": 0.39619144993290306,
|
|
"grad_norm": 6.990268151404954,
|
|
"learning_rate": 3.7788677562621676e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2411,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 2.96875,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3100
|
|
},
|
|
{
|
|
"epoch": 0.39746948686817046,
|
|
"grad_norm": 10.21876475077512,
|
|
"learning_rate": 3.769270375777126e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2438,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.625,
|
|
"step": 3110
|
|
},
|
|
{
|
|
"epoch": 0.3987475238034379,
|
|
"grad_norm": 8.94493406660772,
|
|
"learning_rate": 3.7596477265001053e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -548.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2207,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -3.984375,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -7.15625,
|
|
"step": 3120
|
|
},
|
|
{
|
|
"epoch": 0.4000255607387053,
|
|
"grad_norm": 8.81703733566685,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2356,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3130
|
|
},
|
|
{
|
|
"epoch": 0.4013035976739728,
|
|
"grad_norm": 9.976485922500268,
|
|
"learning_rate": 3.740327388344945e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2087,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.15625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3140
|
|
},
|
|
{
|
|
"epoch": 0.4025816346092402,
|
|
"grad_norm": 6.8661146681276835,
|
|
"learning_rate": 3.7306300840984927e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2409,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3150
|
|
},
|
|
{
|
|
"epoch": 0.40385967154450764,
|
|
"grad_norm": 10.705188172801805,
|
|
"learning_rate": 3.720908280315777e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -584.0,
|
|
"logps/rejected": -892.0,
|
|
"loss": 0.2195,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3160
|
|
},
|
|
{
|
|
"epoch": 0.40513770847977504,
|
|
"grad_norm": 7.260979308776354,
|
|
"learning_rate": 3.711162170539673e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2157,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3170
|
|
},
|
|
{
|
|
"epoch": 0.4064157454150425,
|
|
"grad_norm": 13.386154459966885,
|
|
"learning_rate": 3.701391948796945e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 2.90625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3180
|
|
},
|
|
{
|
|
"epoch": 0.4076937823503099,
|
|
"grad_norm": 10.71858268901349,
|
|
"learning_rate": 3.6915978095943745e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2193,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.15625,
|
|
"rewards/rejected": -7.46875,
|
|
"step": 3190
|
|
},
|
|
{
|
|
"epoch": 0.40897181928557735,
|
|
"grad_norm": 7.112292506270725,
|
|
"learning_rate": 3.6817799479149007e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2479,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3200
|
|
},
|
|
{
|
|
"epoch": 0.41024985622084476,
|
|
"grad_norm": 12.88518280843065,
|
|
"learning_rate": 3.6719385592137306e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -872.0,
|
|
"loss": 0.2387,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 3210
|
|
},
|
|
{
|
|
"epoch": 0.4115278931561122,
|
|
"grad_norm": 10.14123838967259,
|
|
"learning_rate": 3.662073839414452e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -876.0,
|
|
"loss": 0.2255,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.28125,
|
|
"step": 3220
|
|
},
|
|
{
|
|
"epoch": 0.4128059300913796,
|
|
"grad_norm": 9.792291066078825,
|
|
"learning_rate": 3.6521859849051274e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2431,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3230
|
|
},
|
|
{
|
|
"epoch": 0.41408396702664707,
|
|
"grad_norm": 8.93660540981452,
|
|
"learning_rate": 3.6422751925343906e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3240
|
|
},
|
|
{
|
|
"epoch": 0.41536200396191447,
|
|
"grad_norm": 9.207018470233994,
|
|
"learning_rate": 3.6323416596075244e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2373,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.34375,
|
|
"step": 3250
|
|
},
|
|
{
|
|
"epoch": 0.41664004089718193,
|
|
"grad_norm": 8.798998468661477,
|
|
"learning_rate": 3.622385583882535e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -544.0,
|
|
"logps/rejected": -840.0,
|
|
"loss": 0.2528,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.0,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.125,
|
|
"step": 3260
|
|
},
|
|
{
|
|
"epoch": 0.41791807783244933,
|
|
"grad_norm": 7.39070684420073,
|
|
"learning_rate": 3.6124071635662107e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -856.0,
|
|
"loss": 0.2159,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.109375,
|
|
"rewards/rejected": -7.1875,
|
|
"step": 3270
|
|
},
|
|
{
|
|
"epoch": 0.4191961147677168,
|
|
"grad_norm": 10.572409208940522,
|
|
"learning_rate": 3.6024065973101803e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -560.0,
|
|
"logps/rejected": -860.0,
|
|
"loss": 0.2188,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.0625,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.3125,
|
|
"step": 3280
|
|
},
|
|
{
|
|
"epoch": 0.4204741517029842,
|
|
"grad_norm": 11.489752500259677,
|
|
"learning_rate": 3.5923840842069583e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2227,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3290
|
|
},
|
|
{
|
|
"epoch": 0.42175218863825165,
|
|
"grad_norm": 8.808785722348407,
|
|
"learning_rate": 3.5823398237859746e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -916.0,
|
|
"loss": 0.2303,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 2.953125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3300
|
|
},
|
|
{
|
|
"epoch": 0.42303022557351905,
|
|
"grad_norm": 8.69574601913517,
|
|
"learning_rate": 3.572274016009613e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2233,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3310
|
|
},
|
|
{
|
|
"epoch": 0.4243082625087865,
|
|
"grad_norm": 9.178583055571378,
|
|
"learning_rate": 3.562186861269223e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2118,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3320
|
|
},
|
|
{
|
|
"epoch": 0.4255862994440539,
|
|
"grad_norm": 11.47875876274727,
|
|
"learning_rate": 3.5520785603811296e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2562,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3330
|
|
},
|
|
{
|
|
"epoch": 0.42686433637932136,
|
|
"grad_norm": 9.725339945230063,
|
|
"learning_rate": 3.541949314582641e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 3340
|
|
},
|
|
{
|
|
"epoch": 0.42814237331458876,
|
|
"grad_norm": 8.970902307435473,
|
|
"learning_rate": 3.5317993255280383e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2017,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 3350
|
|
},
|
|
{
|
|
"epoch": 0.4294204102498562,
|
|
"grad_norm": 8.382183001221023,
|
|
"learning_rate": 3.5216287952845614e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1958,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 3360
|
|
},
|
|
{
|
|
"epoch": 0.4306984471851236,
|
|
"grad_norm": 9.189979249063406,
|
|
"learning_rate": 3.511437926328387e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.246,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3370
|
|
},
|
|
{
|
|
"epoch": 0.4319764841203911,
|
|
"grad_norm": 8.228711642134167,
|
|
"learning_rate": 3.501226921540598e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.2335,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 3380
|
|
},
|
|
{
|
|
"epoch": 0.4332545210556585,
|
|
"grad_norm": 9.21386830259457,
|
|
"learning_rate": 3.4909959842031436e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -884.0,
|
|
"loss": 0.2189,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 2.84375,
|
|
"rewards/rejected": -7.5,
|
|
"step": 3390
|
|
},
|
|
{
|
|
"epoch": 0.43453255799092594,
|
|
"grad_norm": 6.248955823376214,
|
|
"learning_rate": 3.480745317994793e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2178,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3400
|
|
},
|
|
{
|
|
"epoch": 0.43581059492619334,
|
|
"grad_norm": 9.020550855639287,
|
|
"learning_rate": 3.4704751269870774e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.1958,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3410
|
|
},
|
|
{
|
|
"epoch": 0.4370886318614608,
|
|
"grad_norm": 10.664082807354959,
|
|
"learning_rate": 3.460185615640234e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2194,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3420
|
|
},
|
|
{
|
|
"epoch": 0.4383666687967282,
|
|
"grad_norm": 10.56454935423742,
|
|
"learning_rate": 3.4498769887991316e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2377,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3430
|
|
},
|
|
{
|
|
"epoch": 0.43964470573199566,
|
|
"grad_norm": 8.491123471406155,
|
|
"learning_rate": 3.4395494516891885e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2178,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 2.875,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3440
|
|
},
|
|
{
|
|
"epoch": 0.44092274266726306,
|
|
"grad_norm": 9.201408062848294,
|
|
"learning_rate": 3.4292032099122956e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2088,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3450
|
|
},
|
|
{
|
|
"epoch": 0.4422007796025305,
|
|
"grad_norm": 7.860836704868255,
|
|
"learning_rate": 3.4188384694427166e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -908.0,
|
|
"loss": 0.2384,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 2.921875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 3460
|
|
},
|
|
{
|
|
"epoch": 0.4434788165377979,
|
|
"grad_norm": 13.747547646724954,
|
|
"learning_rate": 3.4084554366229884e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.2205,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.078125,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 3470
|
|
},
|
|
{
|
|
"epoch": 0.44475685347306537,
|
|
"grad_norm": 8.848263859657068,
|
|
"learning_rate": 3.398054318159816e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2116,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3480
|
|
},
|
|
{
|
|
"epoch": 0.4460348904083328,
|
|
"grad_norm": 10.597332854509538,
|
|
"learning_rate": 3.387635321119955e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.228,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3490
|
|
},
|
|
{
|
|
"epoch": 0.44731292734360023,
|
|
"grad_norm": 9.674325942945405,
|
|
"learning_rate": 3.3771986529260914e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2123,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3500
|
|
},
|
|
{
|
|
"epoch": 0.44859096427886763,
|
|
"grad_norm": 12.193772222585018,
|
|
"learning_rate": 3.3667445213527075e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2273,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3510
|
|
},
|
|
{
|
|
"epoch": 0.4498690012141351,
|
|
"grad_norm": 11.935477705606276,
|
|
"learning_rate": 3.356273134521951e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -3.03125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2136,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3520
|
|
},
|
|
{
|
|
"epoch": 0.4511470381494025,
|
|
"grad_norm": 10.16354356392599,
|
|
"learning_rate": 3.3457847008994895e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2291,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3530
|
|
},
|
|
{
|
|
"epoch": 0.45242507508466995,
|
|
"grad_norm": 9.401934005328973,
|
|
"learning_rate": 3.3352794292903584e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2383,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.0625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 3540
|
|
},
|
|
{
|
|
"epoch": 0.45370311201993735,
|
|
"grad_norm": 12.304466333106431,
|
|
"learning_rate": 3.324757528834809e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2339,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3550
|
|
},
|
|
{
|
|
"epoch": 0.4549811489552048,
|
|
"grad_norm": 10.335666794847032,
|
|
"learning_rate": 3.314219209004139e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2134,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3560
|
|
},
|
|
{
|
|
"epoch": 0.4562591858904722,
|
|
"grad_norm": 8.135726139180283,
|
|
"learning_rate": 3.303664679596526e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2045,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3570
|
|
},
|
|
{
|
|
"epoch": 0.45753722282573966,
|
|
"grad_norm": 10.36951976978052,
|
|
"learning_rate": 3.293094150732849e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1959,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3580
|
|
},
|
|
{
|
|
"epoch": 0.45881525976100707,
|
|
"grad_norm": 12.805385346829228,
|
|
"learning_rate": 3.2825078328525076e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2227,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3590
|
|
},
|
|
{
|
|
"epoch": 0.4600932966962745,
|
|
"grad_norm": 8.737872062348318,
|
|
"learning_rate": 3.271905936709231e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2004,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3600
|
|
},
|
|
{
|
|
"epoch": 0.4613713336315419,
|
|
"grad_norm": 5.90450642703934,
|
|
"learning_rate": 3.261288673366881e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2222,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3610
|
|
},
|
|
{
|
|
"epoch": 0.4626493705668094,
|
|
"grad_norm": 10.917396256677707,
|
|
"learning_rate": 3.250656254195252e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -912.0,
|
|
"loss": 0.2324,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3620
|
|
},
|
|
{
|
|
"epoch": 0.4639274075020768,
|
|
"grad_norm": 8.521570669595315,
|
|
"learning_rate": 3.240008890865864e-07,
|
|
"logits/chosen": -3.203125,
|
|
"logits/rejected": -3.078125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2087,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3630
|
|
},
|
|
{
|
|
"epoch": 0.46520544443734424,
|
|
"grad_norm": 8.202418931720207,
|
|
"learning_rate": 3.2293467953477454e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2127,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 2.828125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3640
|
|
},
|
|
{
|
|
"epoch": 0.46648348137261164,
|
|
"grad_norm": 6.5224761862396585,
|
|
"learning_rate": 3.218670179903216e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 3650
|
|
},
|
|
{
|
|
"epoch": 0.4677615183078791,
|
|
"grad_norm": 11.98921345415937,
|
|
"learning_rate": 3.207979257083658e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2271,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3660
|
|
},
|
|
{
|
|
"epoch": 0.4690395552431465,
|
|
"grad_norm": 9.018316199665886,
|
|
"learning_rate": 3.19727423972529e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2005,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3670
|
|
},
|
|
{
|
|
"epoch": 0.47031759217841396,
|
|
"grad_norm": 7.973706582819973,
|
|
"learning_rate": 3.186555340944923e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2111,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3680
|
|
},
|
|
{
|
|
"epoch": 0.47159562911368136,
|
|
"grad_norm": 8.551400923945243,
|
|
"learning_rate": 3.175822774135724e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2167,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3690
|
|
},
|
|
{
|
|
"epoch": 0.4728736660489488,
|
|
"grad_norm": 9.50674868300808,
|
|
"learning_rate": 3.165076752962962e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3700
|
|
},
|
|
{
|
|
"epoch": 0.4741517029842162,
|
|
"grad_norm": 8.941779392618068,
|
|
"learning_rate": 3.15431749135976e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -576.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2065,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -7.5625,
|
|
"step": 3710
|
|
},
|
|
{
|
|
"epoch": 0.4754297399194837,
|
|
"grad_norm": 13.024991648002365,
|
|
"learning_rate": 3.1435452035228306e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2345,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 3720
|
|
},
|
|
{
|
|
"epoch": 0.4767077768547511,
|
|
"grad_norm": 9.638702132762546,
|
|
"learning_rate": 3.132760103908216e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.221,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3730
|
|
},
|
|
{
|
|
"epoch": 0.47798581379001853,
|
|
"grad_norm": 10.2285096933021,
|
|
"learning_rate": 3.1219624072270166e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2144,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3740
|
|
},
|
|
{
|
|
"epoch": 0.47926385072528593,
|
|
"grad_norm": 9.481196208270266,
|
|
"learning_rate": 3.111152328441115e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2314,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3750
|
|
},
|
|
{
|
|
"epoch": 0.4805418876605534,
|
|
"grad_norm": 11.176541898064906,
|
|
"learning_rate": 3.100330082758901e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.193,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 3760
|
|
},
|
|
{
|
|
"epoch": 0.4818199245958208,
|
|
"grad_norm": 10.520661709895942,
|
|
"learning_rate": 3.089495885630983e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2282,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.75,
|
|
"step": 3770
|
|
},
|
|
{
|
|
"epoch": 0.48309796153108825,
|
|
"grad_norm": 12.274708641803729,
|
|
"learning_rate": 3.0786499527459005e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3780
|
|
},
|
|
{
|
|
"epoch": 0.48437599846635565,
|
|
"grad_norm": 12.872720663919305,
|
|
"learning_rate": 3.0677925000258285e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2301,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3790
|
|
},
|
|
{
|
|
"epoch": 0.4856540354016231,
|
|
"grad_norm": 9.708450732405078,
|
|
"learning_rate": 3.056923743622283e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -564.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3800
|
|
},
|
|
{
|
|
"epoch": 0.4869320723368905,
|
|
"grad_norm": 8.19563346716996,
|
|
"learning_rate": 3.0460438999118144e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.215,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 3810
|
|
},
|
|
{
|
|
"epoch": 0.48821010927215797,
|
|
"grad_norm": 8.106052628533215,
|
|
"learning_rate": 3.035153185491698e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3820
|
|
},
|
|
{
|
|
"epoch": 0.48948814620742537,
|
|
"grad_norm": 13.685997047539583,
|
|
"learning_rate": 3.024251817175628e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2047,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3830
|
|
},
|
|
{
|
|
"epoch": 0.4907661831426928,
|
|
"grad_norm": 8.488651222369613,
|
|
"learning_rate": 3.0133400119893947e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2136,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3840
|
|
},
|
|
{
|
|
"epoch": 0.4920442200779602,
|
|
"grad_norm": 12.500560067856066,
|
|
"learning_rate": 3.0024179871665706e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2082,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3850
|
|
},
|
|
{
|
|
"epoch": 0.4933222570132277,
|
|
"grad_norm": 9.56821539066649,
|
|
"learning_rate": 2.9914859601441774e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3860
|
|
},
|
|
{
|
|
"epoch": 0.4946002939484951,
|
|
"grad_norm": 11.642104475986267,
|
|
"learning_rate": 2.980544148558365e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2271,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 3870
|
|
},
|
|
{
|
|
"epoch": 0.49587833088376254,
|
|
"grad_norm": 11.975382621959218,
|
|
"learning_rate": 2.9695927702400767e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.216,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3880
|
|
},
|
|
{
|
|
"epoch": 0.49715636781902994,
|
|
"grad_norm": 8.674175640451184,
|
|
"learning_rate": 2.958632043210706e-07,
|
|
"logits/chosen": -2.734375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2073,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 3890
|
|
},
|
|
{
|
|
"epoch": 0.4984344047542974,
|
|
"grad_norm": 13.531335502879033,
|
|
"learning_rate": 2.9476621856777686e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2201,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 3900
|
|
},
|
|
{
|
|
"epoch": 0.4997124416895648,
|
|
"grad_norm": 9.753557812476503,
|
|
"learning_rate": 2.9366834160305466e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1996,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 3910
|
|
},
|
|
{
|
|
"epoch": 0.5009904786248323,
|
|
"grad_norm": 8.623771874932421,
|
|
"learning_rate": 2.925695952835746e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 3920
|
|
},
|
|
{
|
|
"epoch": 0.5022685155600997,
|
|
"grad_norm": 8.619272047417937,
|
|
"learning_rate": 2.9147000148331483e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1965,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 3930
|
|
},
|
|
{
|
|
"epoch": 0.5035465524953671,
|
|
"grad_norm": 9.340092455147365,
|
|
"learning_rate": 2.903695820931249e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2062,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 3940
|
|
},
|
|
{
|
|
"epoch": 0.5048245894306346,
|
|
"grad_norm": 10.45509856513219,
|
|
"learning_rate": 2.8926835902029043e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.25,
|
|
"step": 3950
|
|
},
|
|
{
|
|
"epoch": 0.506102626365902,
|
|
"grad_norm": 10.115987074829942,
|
|
"learning_rate": 2.8816635418809693e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2186,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 3960
|
|
},
|
|
{
|
|
"epoch": 0.5073806633011694,
|
|
"grad_norm": 9.859559908143657,
|
|
"learning_rate": 2.870635895353931e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -900.0,
|
|
"loss": 0.1997,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.65625,
|
|
"step": 3970
|
|
},
|
|
{
|
|
"epoch": 0.5086587002364368,
|
|
"grad_norm": 10.87319879727593,
|
|
"learning_rate": 2.8596008701615456e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2066,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 3980
|
|
},
|
|
{
|
|
"epoch": 0.5099367371717043,
|
|
"grad_norm": 7.8658320285041405,
|
|
"learning_rate": 2.8485586859904596e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2377,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 3990
|
|
},
|
|
{
|
|
"epoch": 0.5112147741069717,
|
|
"grad_norm": 8.505547152770252,
|
|
"learning_rate": 2.8375095626698464e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -896.0,
|
|
"loss": 0.2167,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.71875,
|
|
"step": 4000
|
|
},
|
|
{
|
|
"epoch": 0.5124928110422391,
|
|
"grad_norm": 11.91791149654105,
|
|
"learning_rate": 2.826453720167021e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2182,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4010
|
|
},
|
|
{
|
|
"epoch": 0.5137708479775065,
|
|
"grad_norm": 9.968567086380052,
|
|
"learning_rate": 2.815391378583069e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2166,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4020
|
|
},
|
|
{
|
|
"epoch": 0.515048884912774,
|
|
"grad_norm": 9.35733243554509,
|
|
"learning_rate": 2.8043227581484556e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4030
|
|
},
|
|
{
|
|
"epoch": 0.5163269218480414,
|
|
"grad_norm": 12.215635212530254,
|
|
"learning_rate": 2.79324807921865e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.227,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4040
|
|
},
|
|
{
|
|
"epoch": 0.5176049587833088,
|
|
"grad_norm": 13.845726328872498,
|
|
"learning_rate": 2.7821675622697317e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.2227,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4050
|
|
},
|
|
{
|
|
"epoch": 0.5188829957185762,
|
|
"grad_norm": 11.50962600886563,
|
|
"learning_rate": 2.7710814278940057e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2251,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4060
|
|
},
|
|
{
|
|
"epoch": 0.5201610326538437,
|
|
"grad_norm": 9.136976034548045,
|
|
"learning_rate": 2.759989896795611e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2297,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4070
|
|
},
|
|
{
|
|
"epoch": 0.5214390695891111,
|
|
"grad_norm": 8.603249208409274,
|
|
"learning_rate": 2.748893189786122e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4080
|
|
},
|
|
{
|
|
"epoch": 0.5227171065243785,
|
|
"grad_norm": 8.84947914185045,
|
|
"learning_rate": 2.7377915277801586e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1985,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4090
|
|
},
|
|
{
|
|
"epoch": 0.5239951434596459,
|
|
"grad_norm": 10.135619925874042,
|
|
"learning_rate": 2.726685131790983e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2223,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4100
|
|
},
|
|
{
|
|
"epoch": 0.5252731803949134,
|
|
"grad_norm": 8.893574056072689,
|
|
"learning_rate": 2.715574222926105e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2135,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4110
|
|
},
|
|
{
|
|
"epoch": 0.5265512173301808,
|
|
"grad_norm": 5.695805913492098,
|
|
"learning_rate": 2.704459022382874e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4120
|
|
},
|
|
{
|
|
"epoch": 0.5278292542654482,
|
|
"grad_norm": 6.965426261729773,
|
|
"learning_rate": 2.6933397514440796e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4130
|
|
},
|
|
{
|
|
"epoch": 0.5291072912007156,
|
|
"grad_norm": 8.527321271713554,
|
|
"learning_rate": 2.6822166314735483e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2099,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4140
|
|
},
|
|
{
|
|
"epoch": 0.5303853281359832,
|
|
"grad_norm": 11.360590075652906,
|
|
"learning_rate": 2.67108988391173e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2048,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4150
|
|
},
|
|
{
|
|
"epoch": 0.5316633650712506,
|
|
"grad_norm": 8.516403048337656,
|
|
"learning_rate": 2.659959730271295e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2067,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4160
|
|
},
|
|
{
|
|
"epoch": 0.532941402006518,
|
|
"grad_norm": 8.679204620956797,
|
|
"learning_rate": 2.64882639213272e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4170
|
|
},
|
|
{
|
|
"epoch": 0.5342194389417854,
|
|
"grad_norm": 8.214012990646285,
|
|
"learning_rate": 2.6376900911398823e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.208,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4180
|
|
},
|
|
{
|
|
"epoch": 0.5354974758770529,
|
|
"grad_norm": 6.735540115827085,
|
|
"learning_rate": 2.626551048995643e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4190
|
|
},
|
|
{
|
|
"epoch": 0.5367755128123203,
|
|
"grad_norm": 13.73261695569995,
|
|
"learning_rate": 2.6154094874574326e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2123,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4200
|
|
},
|
|
{
|
|
"epoch": 0.5380535497475877,
|
|
"grad_norm": 8.6629838322733,
|
|
"learning_rate": 2.60426562833284e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2017,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4210
|
|
},
|
|
{
|
|
"epoch": 0.5393315866828551,
|
|
"grad_norm": 8.083638442585006,
|
|
"learning_rate": 2.593119693475193e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4220
|
|
},
|
|
{
|
|
"epoch": 0.5406096236181226,
|
|
"grad_norm": 9.85692839498437,
|
|
"learning_rate": 2.5819719047791467e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2145,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4230
|
|
},
|
|
{
|
|
"epoch": 0.54188766055339,
|
|
"grad_norm": 8.33356041058765,
|
|
"learning_rate": 2.570822484176257e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.1869,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 4240
|
|
},
|
|
{
|
|
"epoch": 0.5431656974886574,
|
|
"grad_norm": 10.85907589382582,
|
|
"learning_rate": 2.559671653630574e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1961,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4250
|
|
},
|
|
{
|
|
"epoch": 0.5444437344239248,
|
|
"grad_norm": 12.760498280651152,
|
|
"learning_rate": 2.5485196351342146e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2171,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4260
|
|
},
|
|
{
|
|
"epoch": 0.5457217713591923,
|
|
"grad_norm": 11.083493467932884,
|
|
"learning_rate": 2.537366650702944e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4270
|
|
},
|
|
{
|
|
"epoch": 0.5469998082944597,
|
|
"grad_norm": 9.755652576792773,
|
|
"learning_rate": 2.526212922371758e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2014,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4280
|
|
},
|
|
{
|
|
"epoch": 0.5482778452297271,
|
|
"grad_norm": 8.621966933478847,
|
|
"learning_rate": 2.5150586721904653e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1909,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4290
|
|
},
|
|
{
|
|
"epoch": 0.5495558821649945,
|
|
"grad_norm": 9.93825493293039,
|
|
"learning_rate": 2.50390412221926e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2184,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4300
|
|
},
|
|
{
|
|
"epoch": 0.550833919100262,
|
|
"grad_norm": 6.4926059090968735,
|
|
"learning_rate": 2.492749494524305e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2104,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -7.875,
|
|
"step": 4310
|
|
},
|
|
{
|
|
"epoch": 0.5521119560355294,
|
|
"grad_norm": 8.178900908424412,
|
|
"learning_rate": 2.481595011173312e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.75,
|
|
"step": 4320
|
|
},
|
|
{
|
|
"epoch": 0.5533899929707968,
|
|
"grad_norm": 8.902539400477867,
|
|
"learning_rate": 2.470440894231118e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4330
|
|
},
|
|
{
|
|
"epoch": 0.5546680299060642,
|
|
"grad_norm": 10.017993004725309,
|
|
"learning_rate": 2.4592873657552665e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.2036,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.0,
|
|
"step": 4340
|
|
},
|
|
{
|
|
"epoch": 0.5559460668413317,
|
|
"grad_norm": 9.086190763346574,
|
|
"learning_rate": 2.4481346477915834e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.1972,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.21875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 4350
|
|
},
|
|
{
|
|
"epoch": 0.5572241037765991,
|
|
"grad_norm": 8.170239096649071,
|
|
"learning_rate": 2.436982962369761e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4360
|
|
},
|
|
{
|
|
"epoch": 0.5585021407118665,
|
|
"grad_norm": 9.91347550249988,
|
|
"learning_rate": 2.4258325314989364e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1968,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4370
|
|
},
|
|
{
|
|
"epoch": 0.559780177647134,
|
|
"grad_norm": 12.649043344414686,
|
|
"learning_rate": 2.4146835771632674e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2218,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4380
|
|
},
|
|
{
|
|
"epoch": 0.5610582145824015,
|
|
"grad_norm": 10.059811667819446,
|
|
"learning_rate": 2.4035363213175206e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2293,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4390
|
|
},
|
|
{
|
|
"epoch": 0.5623362515176689,
|
|
"grad_norm": 7.752963384126441,
|
|
"learning_rate": 2.3923909858826473e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -888.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -7.59375,
|
|
"step": 4400
|
|
},
|
|
{
|
|
"epoch": 0.5636142884529363,
|
|
"grad_norm": 8.121775627856461,
|
|
"learning_rate": 2.3812477927413648e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4410
|
|
},
|
|
{
|
|
"epoch": 0.5648923253882037,
|
|
"grad_norm": 10.889500920755431,
|
|
"learning_rate": 2.370106963733746e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2137,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4420
|
|
},
|
|
{
|
|
"epoch": 0.5661703623234712,
|
|
"grad_norm": 7.787520343491865,
|
|
"learning_rate": 2.358968720652794e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4430
|
|
},
|
|
{
|
|
"epoch": 0.5674483992587386,
|
|
"grad_norm": 7.334229409175379,
|
|
"learning_rate": 2.3478332852400337e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1986,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4440
|
|
},
|
|
{
|
|
"epoch": 0.568726436194006,
|
|
"grad_norm": 9.306378482430759,
|
|
"learning_rate": 2.3367008791810936e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2612,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.3125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4450
|
|
},
|
|
{
|
|
"epoch": 0.5700044731292734,
|
|
"grad_norm": 7.851220791242363,
|
|
"learning_rate": 2.3255717241012922e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1944,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 4460
|
|
},
|
|
{
|
|
"epoch": 0.5712825100645409,
|
|
"grad_norm": 8.633601777057988,
|
|
"learning_rate": 2.314446041561231e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1993,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 4470
|
|
},
|
|
{
|
|
"epoch": 0.5725605469998083,
|
|
"grad_norm": 10.040452596074198,
|
|
"learning_rate": 2.303324053052374e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2164,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4480
|
|
},
|
|
{
|
|
"epoch": 0.5738385839350757,
|
|
"grad_norm": 9.442768793474356,
|
|
"learning_rate": 2.29220597999265e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4490
|
|
},
|
|
{
|
|
"epoch": 0.5751166208703431,
|
|
"grad_norm": 10.471888355465538,
|
|
"learning_rate": 2.281092043722034e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1983,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4500
|
|
},
|
|
{
|
|
"epoch": 0.5763946578056106,
|
|
"grad_norm": 10.315495473189406,
|
|
"learning_rate": 2.269982465498147e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.191,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4510
|
|
},
|
|
{
|
|
"epoch": 0.577672694740878,
|
|
"grad_norm": 9.421774060820772,
|
|
"learning_rate": 2.2588774664918503e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2269,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4520
|
|
},
|
|
{
|
|
"epoch": 0.5789507316761454,
|
|
"grad_norm": 10.653247108656883,
|
|
"learning_rate": 2.2477772677828396e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 4530
|
|
},
|
|
{
|
|
"epoch": 0.5802287686114128,
|
|
"grad_norm": 12.414812092039039,
|
|
"learning_rate": 2.2366820903552478e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4540
|
|
},
|
|
{
|
|
"epoch": 0.5815068055466803,
|
|
"grad_norm": 15.625418407059193,
|
|
"learning_rate": 2.2255921550932418e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1943,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4550
|
|
},
|
|
{
|
|
"epoch": 0.5827848424819477,
|
|
"grad_norm": 9.748298215017456,
|
|
"learning_rate": 2.214507682776627e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4560
|
|
},
|
|
{
|
|
"epoch": 0.5840628794172151,
|
|
"grad_norm": 9.206249587748667,
|
|
"learning_rate": 2.203428894076454e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.2063,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.25,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4570
|
|
},
|
|
{
|
|
"epoch": 0.5853409163524825,
|
|
"grad_norm": 10.742870208744552,
|
|
"learning_rate": 2.1923560095506196e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2239,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4580
|
|
},
|
|
{
|
|
"epoch": 0.58661895328775,
|
|
"grad_norm": 9.987652609751306,
|
|
"learning_rate": 2.1812892496394835e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2105,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 4590
|
|
},
|
|
{
|
|
"epoch": 0.5878969902230174,
|
|
"grad_norm": 10.530610611497671,
|
|
"learning_rate": 2.170228834661472e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 4600
|
|
},
|
|
{
|
|
"epoch": 0.5891750271582848,
|
|
"grad_norm": 9.50246727866108,
|
|
"learning_rate": 2.1591749848086992e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2007,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4610
|
|
},
|
|
{
|
|
"epoch": 0.5904530640935522,
|
|
"grad_norm": 8.38828507992591,
|
|
"learning_rate": 2.148127920142577e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4620
|
|
},
|
|
{
|
|
"epoch": 0.5917311010288198,
|
|
"grad_norm": 11.01986987749802,
|
|
"learning_rate": 2.1370878605894377e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4630
|
|
},
|
|
{
|
|
"epoch": 0.5930091379640872,
|
|
"grad_norm": 10.543139498864132,
|
|
"learning_rate": 2.1260550259361576e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.09375,
|
|
"rewards/rejected": -7.84375,
|
|
"step": 4640
|
|
},
|
|
{
|
|
"epoch": 0.5942871748993546,
|
|
"grad_norm": 9.129152697037581,
|
|
"learning_rate": 2.115029635825774e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2175,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.359375,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4650
|
|
},
|
|
{
|
|
"epoch": 0.595565211834622,
|
|
"grad_norm": 8.38626724144314,
|
|
"learning_rate": 2.104011909753122e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2239,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4660
|
|
},
|
|
{
|
|
"epoch": 0.5968432487698895,
|
|
"grad_norm": 8.949485168979885,
|
|
"learning_rate": 2.093002067060457e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2056,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4670
|
|
},
|
|
{
|
|
"epoch": 0.5981212857051569,
|
|
"grad_norm": 10.070633848403004,
|
|
"learning_rate": 2.082000326933092e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.2003,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 4680
|
|
},
|
|
{
|
|
"epoch": 0.5993993226404243,
|
|
"grad_norm": 6.916035441529172,
|
|
"learning_rate": 2.0710069083950342e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2108,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4690
|
|
},
|
|
{
|
|
"epoch": 0.6006773595756917,
|
|
"grad_norm": 9.474816951227877,
|
|
"learning_rate": 2.060022030304621e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2323,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4700
|
|
},
|
|
{
|
|
"epoch": 0.6019553965109592,
|
|
"grad_norm": 6.119960790414473,
|
|
"learning_rate": 2.0490459113501686e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4710
|
|
},
|
|
{
|
|
"epoch": 0.6032334334462266,
|
|
"grad_norm": 8.27937601473009,
|
|
"learning_rate": 2.0380787700456132e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4720
|
|
},
|
|
{
|
|
"epoch": 0.604511470381494,
|
|
"grad_norm": 8.980086860385386,
|
|
"learning_rate": 2.027120824726162e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2223,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4730
|
|
},
|
|
{
|
|
"epoch": 0.6057895073167614,
|
|
"grad_norm": 9.660168225938056,
|
|
"learning_rate": 2.0161722935439513e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 4740
|
|
},
|
|
{
|
|
"epoch": 0.6070675442520289,
|
|
"grad_norm": 9.627236009510261,
|
|
"learning_rate": 2.0052333944636954e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.2116,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.203125,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 4750
|
|
},
|
|
{
|
|
"epoch": 0.6083455811872963,
|
|
"grad_norm": 11.942856212349632,
|
|
"learning_rate": 1.994304345258354e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1951,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4760
|
|
},
|
|
{
|
|
"epoch": 0.6096236181225637,
|
|
"grad_norm": 8.668637933993713,
|
|
"learning_rate": 1.983385363504793e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4770
|
|
},
|
|
{
|
|
"epoch": 0.6109016550578311,
|
|
"grad_norm": 8.513667560083928,
|
|
"learning_rate": 1.9724766665794541e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1825,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4780
|
|
},
|
|
{
|
|
"epoch": 0.6121796919930986,
|
|
"grad_norm": 12.364360217821515,
|
|
"learning_rate": 1.9615784716540293e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 4790
|
|
},
|
|
{
|
|
"epoch": 0.613457728928366,
|
|
"grad_norm": 13.291905969721395,
|
|
"learning_rate": 1.950690995691132e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1938,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 4800
|
|
},
|
|
{
|
|
"epoch": 0.6147357658636334,
|
|
"grad_norm": 12.195901066043673,
|
|
"learning_rate": 1.939814455439986e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2181,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4810
|
|
},
|
|
{
|
|
"epoch": 0.6160138027989008,
|
|
"grad_norm": 10.210694409079894,
|
|
"learning_rate": 1.9289490674321017e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 4820
|
|
},
|
|
{
|
|
"epoch": 0.6172918397341683,
|
|
"grad_norm": 11.462328801893005,
|
|
"learning_rate": 1.9180950479769687e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2051,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4830
|
|
},
|
|
{
|
|
"epoch": 0.6185698766694357,
|
|
"grad_norm": 11.618759475704962,
|
|
"learning_rate": 1.9072526131577542e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1983,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 4840
|
|
},
|
|
{
|
|
"epoch": 0.6198479136047031,
|
|
"grad_norm": 8.771287614014419,
|
|
"learning_rate": 1.8964219788269936e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2049,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 4850
|
|
},
|
|
{
|
|
"epoch": 0.6211259505399706,
|
|
"grad_norm": 8.382633721719383,
|
|
"learning_rate": 1.885603360602298e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4860
|
|
},
|
|
{
|
|
"epoch": 0.6224039874752381,
|
|
"grad_norm": 11.874740935105166,
|
|
"learning_rate": 1.8747969738620588e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4870
|
|
},
|
|
{
|
|
"epoch": 0.6236820244105055,
|
|
"grad_norm": 10.358959890573601,
|
|
"learning_rate": 1.8640030337411655e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2078,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4880
|
|
},
|
|
{
|
|
"epoch": 0.6249600613457729,
|
|
"grad_norm": 6.897393388027196,
|
|
"learning_rate": 1.8532217551267143e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.211,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4890
|
|
},
|
|
{
|
|
"epoch": 0.6262380982810403,
|
|
"grad_norm": 8.014943203049432,
|
|
"learning_rate": 1.842453352653735e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1946,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 4900
|
|
},
|
|
{
|
|
"epoch": 0.6275161352163078,
|
|
"grad_norm": 12.135630997047638,
|
|
"learning_rate": 1.8316980407009213e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2056,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 4910
|
|
},
|
|
{
|
|
"epoch": 0.6287941721515752,
|
|
"grad_norm": 8.388293855177318,
|
|
"learning_rate": 1.820956033386355e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2067,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.625,
|
|
"step": 4920
|
|
},
|
|
{
|
|
"epoch": 0.6300722090868426,
|
|
"grad_norm": 8.19919332061552,
|
|
"learning_rate": 1.8102275445632508e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2086,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 4930
|
|
},
|
|
{
|
|
"epoch": 0.63135024602211,
|
|
"grad_norm": 11.524825983155498,
|
|
"learning_rate": 1.7995127878156934e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4940
|
|
},
|
|
{
|
|
"epoch": 0.6326282829573775,
|
|
"grad_norm": 11.935553547051724,
|
|
"learning_rate": 1.788811976454388e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2033,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 4950
|
|
},
|
|
{
|
|
"epoch": 0.6339063198926449,
|
|
"grad_norm": 8.72541909787161,
|
|
"learning_rate": 1.7781253235124143e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.217,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4960
|
|
},
|
|
{
|
|
"epoch": 0.6351843568279123,
|
|
"grad_norm": 9.174733860455342,
|
|
"learning_rate": 1.7674530417409823e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 4970
|
|
},
|
|
{
|
|
"epoch": 0.6364623937631797,
|
|
"grad_norm": 8.931555059599443,
|
|
"learning_rate": 1.7567953436052025e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.219,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 4980
|
|
},
|
|
{
|
|
"epoch": 0.6377404306984472,
|
|
"grad_norm": 14.476630420450562,
|
|
"learning_rate": 1.7461524412798503e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2224,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 4990
|
|
},
|
|
{
|
|
"epoch": 0.6390184676337146,
|
|
"grad_norm": 9.976279367626784,
|
|
"learning_rate": 1.735524546645142e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1926,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5000
|
|
},
|
|
{
|
|
"epoch": 0.640296504568982,
|
|
"grad_norm": 10.403381451664748,
|
|
"learning_rate": 1.7249118712825246e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -924.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.1875,
|
|
"rewards/rejected": -7.78125,
|
|
"step": 5010
|
|
},
|
|
{
|
|
"epoch": 0.6415745415042494,
|
|
"grad_norm": 10.392003155571185,
|
|
"learning_rate": 1.7143146264704522e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.21875,
|
|
"rewards/rejected": -7.8125,
|
|
"step": 5020
|
|
},
|
|
{
|
|
"epoch": 0.6428525784395169,
|
|
"grad_norm": 9.045623892838288,
|
|
"learning_rate": 1.70373302318019e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5030
|
|
},
|
|
{
|
|
"epoch": 0.6441306153747843,
|
|
"grad_norm": 10.532156784041835,
|
|
"learning_rate": 1.6931672720716072e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2078,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5040
|
|
},
|
|
{
|
|
"epoch": 0.6454086523100517,
|
|
"grad_norm": 13.016020098769182,
|
|
"learning_rate": 1.6826175834889866e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5050
|
|
},
|
|
{
|
|
"epoch": 0.6466866892453191,
|
|
"grad_norm": 8.620907567456209,
|
|
"learning_rate": 1.6720841674568365e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2049,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5060
|
|
},
|
|
{
|
|
"epoch": 0.6479647261805866,
|
|
"grad_norm": 12.006083637867917,
|
|
"learning_rate": 1.6615672336757074e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1859,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5070
|
|
},
|
|
{
|
|
"epoch": 0.649242763115854,
|
|
"grad_norm": 8.58425824054044,
|
|
"learning_rate": 1.6510669915180238e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5080
|
|
},
|
|
{
|
|
"epoch": 0.6505208000511215,
|
|
"grad_norm": 10.81312666340287,
|
|
"learning_rate": 1.6405836500239062e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5090
|
|
},
|
|
{
|
|
"epoch": 0.6517988369863889,
|
|
"grad_norm": 10.6110280495848,
|
|
"learning_rate": 1.6301174178970162e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1746,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5100
|
|
},
|
|
{
|
|
"epoch": 0.6530768739216564,
|
|
"grad_norm": 9.639168092530126,
|
|
"learning_rate": 1.6196685035004015e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.188,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5110
|
|
},
|
|
{
|
|
"epoch": 0.6543549108569238,
|
|
"grad_norm": 9.11174679070626,
|
|
"learning_rate": 1.6092371148523442e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5120
|
|
},
|
|
{
|
|
"epoch": 0.6556329477921912,
|
|
"grad_norm": 13.377523718184259,
|
|
"learning_rate": 1.5988234596222234e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5130
|
|
},
|
|
{
|
|
"epoch": 0.6569109847274586,
|
|
"grad_norm": 7.383701674519595,
|
|
"learning_rate": 1.5884277451263772e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5140
|
|
},
|
|
{
|
|
"epoch": 0.6581890216627261,
|
|
"grad_norm": 8.953980108485803,
|
|
"learning_rate": 1.57805017832398e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.1813,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5150
|
|
},
|
|
{
|
|
"epoch": 0.6594670585979935,
|
|
"grad_norm": 9.211324366300165,
|
|
"learning_rate": 1.567690965812918e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5160
|
|
},
|
|
{
|
|
"epoch": 0.6607450955332609,
|
|
"grad_norm": 10.95169892297747,
|
|
"learning_rate": 1.5573503138256756e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.2315,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5170
|
|
},
|
|
{
|
|
"epoch": 0.6620231324685283,
|
|
"grad_norm": 7.565071174200895,
|
|
"learning_rate": 1.5470284282252367e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2028,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5180
|
|
},
|
|
{
|
|
"epoch": 0.6633011694037958,
|
|
"grad_norm": 8.40586608624689,
|
|
"learning_rate": 1.5367255145009784e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1959,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5190
|
|
},
|
|
{
|
|
"epoch": 0.6645792063390632,
|
|
"grad_norm": 8.935789239896653,
|
|
"learning_rate": 1.5264417777645849e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1954,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5200
|
|
},
|
|
{
|
|
"epoch": 0.6658572432743306,
|
|
"grad_norm": 10.318660688920453,
|
|
"learning_rate": 1.516177422745962e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2219,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5210
|
|
},
|
|
{
|
|
"epoch": 0.667135280209598,
|
|
"grad_norm": 10.599513311823376,
|
|
"learning_rate": 1.5059326537891602e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2089,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5220
|
|
},
|
|
{
|
|
"epoch": 0.6684133171448655,
|
|
"grad_norm": 7.096941650990519,
|
|
"learning_rate": 1.4957076748483121e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5230
|
|
},
|
|
{
|
|
"epoch": 0.6696913540801329,
|
|
"grad_norm": 9.134873030915466,
|
|
"learning_rate": 1.4855026894835636e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2057,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5240
|
|
},
|
|
{
|
|
"epoch": 0.6709693910154003,
|
|
"grad_norm": 9.184838305556584,
|
|
"learning_rate": 1.4753179008570304e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5250
|
|
},
|
|
{
|
|
"epoch": 0.6722474279506677,
|
|
"grad_norm": 10.491068837200642,
|
|
"learning_rate": 1.4651535117287462e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5260
|
|
},
|
|
{
|
|
"epoch": 0.6735254648859352,
|
|
"grad_norm": 13.654106335755474,
|
|
"learning_rate": 1.4550097244526294e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5270
|
|
},
|
|
{
|
|
"epoch": 0.6748035018212026,
|
|
"grad_norm": 7.2043046993448305,
|
|
"learning_rate": 1.444886740972455e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2154,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5280
|
|
},
|
|
{
|
|
"epoch": 0.67608153875647,
|
|
"grad_norm": 10.381553344000324,
|
|
"learning_rate": 1.4347847628178315e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -932.0,
|
|
"loss": 0.1997,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 5290
|
|
},
|
|
{
|
|
"epoch": 0.6773595756917374,
|
|
"grad_norm": 8.206259981152979,
|
|
"learning_rate": 1.4247039911001952e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2015,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.234375,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 5300
|
|
},
|
|
{
|
|
"epoch": 0.678637612627005,
|
|
"grad_norm": 9.733010610608309,
|
|
"learning_rate": 1.4146446265087966e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5310
|
|
},
|
|
{
|
|
"epoch": 0.6799156495622724,
|
|
"grad_norm": 7.818977341913373,
|
|
"learning_rate": 1.4046068693067142e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5320
|
|
},
|
|
{
|
|
"epoch": 0.6811936864975398,
|
|
"grad_norm": 11.569940395290942,
|
|
"learning_rate": 1.3945909193268618e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5330
|
|
},
|
|
{
|
|
"epoch": 0.6824717234328072,
|
|
"grad_norm": 7.915367886076844,
|
|
"learning_rate": 1.3845969759680125e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5340
|
|
},
|
|
{
|
|
"epoch": 0.6837497603680747,
|
|
"grad_norm": 10.40013069384589,
|
|
"learning_rate": 1.374625238190832e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5350
|
|
},
|
|
{
|
|
"epoch": 0.6850277973033421,
|
|
"grad_norm": 8.748911256197829,
|
|
"learning_rate": 1.36467590451391e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5360
|
|
},
|
|
{
|
|
"epoch": 0.6863058342386095,
|
|
"grad_norm": 10.775546531691665,
|
|
"learning_rate": 1.354749173009815e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2074,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.4375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5370
|
|
},
|
|
{
|
|
"epoch": 0.6875838711738769,
|
|
"grad_norm": 10.000688484948581,
|
|
"learning_rate": 1.3448452413011487e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 5380
|
|
},
|
|
{
|
|
"epoch": 0.6888619081091444,
|
|
"grad_norm": 8.846111947475448,
|
|
"learning_rate": 1.3349643065566117e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2055,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.28125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5390
|
|
},
|
|
{
|
|
"epoch": 0.6901399450444118,
|
|
"grad_norm": 6.691267912708803,
|
|
"learning_rate": 1.325106565487077e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.169,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5400
|
|
},
|
|
{
|
|
"epoch": 0.6914179819796792,
|
|
"grad_norm": 10.58421750198331,
|
|
"learning_rate": 1.3152722143416748e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2012,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5410
|
|
},
|
|
{
|
|
"epoch": 0.6926960189149466,
|
|
"grad_norm": 10.823577363873508,
|
|
"learning_rate": 1.305461448903889e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1942,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.328125,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5420
|
|
},
|
|
{
|
|
"epoch": 0.6939740558502141,
|
|
"grad_norm": 10.703964915891877,
|
|
"learning_rate": 1.2956744644876527e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1891,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5430
|
|
},
|
|
{
|
|
"epoch": 0.6952520927854815,
|
|
"grad_norm": 11.074215783268322,
|
|
"learning_rate": 1.2859114559334643e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2077,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5440
|
|
},
|
|
{
|
|
"epoch": 0.6965301297207489,
|
|
"grad_norm": 9.906928540767801,
|
|
"learning_rate": 1.27617261760451e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.421875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5450
|
|
},
|
|
{
|
|
"epoch": 0.6978081666560163,
|
|
"grad_norm": 7.989791957700388,
|
|
"learning_rate": 1.26645814338279e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -944.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5460
|
|
},
|
|
{
|
|
"epoch": 0.6990862035912838,
|
|
"grad_norm": 9.439843209419791,
|
|
"learning_rate": 1.2567682266652603e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1897,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5470
|
|
},
|
|
{
|
|
"epoch": 0.7003642405265512,
|
|
"grad_norm": 12.246737461104036,
|
|
"learning_rate": 1.2471030603599848e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5480
|
|
},
|
|
{
|
|
"epoch": 0.7016422774618186,
|
|
"grad_norm": 10.904656119606363,
|
|
"learning_rate": 1.2374628368822914e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -948.0,
|
|
"loss": 0.1955,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5490
|
|
},
|
|
{
|
|
"epoch": 0.702920314397086,
|
|
"grad_norm": 10.223805884896036,
|
|
"learning_rate": 1.2278477481509462e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -7.9375,
|
|
"step": 5500
|
|
},
|
|
{
|
|
"epoch": 0.7041983513323535,
|
|
"grad_norm": 10.18405106135057,
|
|
"learning_rate": 1.2182579855843247e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.1946,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5510
|
|
},
|
|
{
|
|
"epoch": 0.7054763882676209,
|
|
"grad_norm": 9.516374791611842,
|
|
"learning_rate": 1.2086937400966129e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1833,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5520
|
|
},
|
|
{
|
|
"epoch": 0.7067544252028883,
|
|
"grad_norm": 9.178417627197618,
|
|
"learning_rate": 1.1991552020939918e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5530
|
|
},
|
|
{
|
|
"epoch": 0.7080324621381557,
|
|
"grad_norm": 7.880000574893539,
|
|
"learning_rate": 1.1896425614708589e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5540
|
|
},
|
|
{
|
|
"epoch": 0.7093104990734233,
|
|
"grad_norm": 8.442036728841282,
|
|
"learning_rate": 1.1801560076060444e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5550
|
|
},
|
|
{
|
|
"epoch": 0.7105885360086907,
|
|
"grad_norm": 9.96646161178662,
|
|
"learning_rate": 1.170695729359037e-07,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5560
|
|
},
|
|
{
|
|
"epoch": 0.711866572943958,
|
|
"grad_norm": 9.055717430232225,
|
|
"learning_rate": 1.1612619150662303e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1913,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5570
|
|
},
|
|
{
|
|
"epoch": 0.7131446098792255,
|
|
"grad_norm": 10.28494479666354,
|
|
"learning_rate": 1.151854752537168e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5580
|
|
},
|
|
{
|
|
"epoch": 0.714422646814493,
|
|
"grad_norm": 32.22289570587893,
|
|
"learning_rate": 1.1424744290508078e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.189,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5590
|
|
},
|
|
{
|
|
"epoch": 0.7157006837497604,
|
|
"grad_norm": 12.269299820154668,
|
|
"learning_rate": 1.1331211313517925e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5600
|
|
},
|
|
{
|
|
"epoch": 0.7169787206850278,
|
|
"grad_norm": 9.260295940193947,
|
|
"learning_rate": 1.1237950456467308e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.19,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5610
|
|
},
|
|
{
|
|
"epoch": 0.7182567576202952,
|
|
"grad_norm": 10.398934473010499,
|
|
"learning_rate": 1.1144963576004954e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2061,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5620
|
|
},
|
|
{
|
|
"epoch": 0.7195347945555627,
|
|
"grad_norm": 12.87874977657243,
|
|
"learning_rate": 1.105225252332519e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5630
|
|
},
|
|
{
|
|
"epoch": 0.7208128314908301,
|
|
"grad_norm": 10.678499925600923,
|
|
"learning_rate": 1.0959819144131144e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.188,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 5640
|
|
},
|
|
{
|
|
"epoch": 0.7220908684260975,
|
|
"grad_norm": 10.020846983046951,
|
|
"learning_rate": 1.0867665278597999e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1866,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5650
|
|
},
|
|
{
|
|
"epoch": 0.7233689053613649,
|
|
"grad_norm": 11.621911848428802,
|
|
"learning_rate": 1.0775792761336324e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1806,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5660
|
|
},
|
|
{
|
|
"epoch": 0.7246469422966324,
|
|
"grad_norm": 10.137810011465284,
|
|
"learning_rate": 1.0684203421355587e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1977,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 5670
|
|
},
|
|
{
|
|
"epoch": 0.7259249792318998,
|
|
"grad_norm": 7.480549183495816,
|
|
"learning_rate": 1.0592899082027715e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1947,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5680
|
|
},
|
|
{
|
|
"epoch": 0.7272030161671672,
|
|
"grad_norm": 8.95863895797164,
|
|
"learning_rate": 1.050188156105081e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2002,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5690
|
|
},
|
|
{
|
|
"epoch": 0.7284810531024346,
|
|
"grad_norm": 10.004556249685614,
|
|
"learning_rate": 1.0411152670412975e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1985,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 5700
|
|
},
|
|
{
|
|
"epoch": 0.7297590900377021,
|
|
"grad_norm": 9.942783279630579,
|
|
"learning_rate": 1.03207142163562e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1943,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5710
|
|
},
|
|
{
|
|
"epoch": 0.7310371269729695,
|
|
"grad_norm": 11.150110790338827,
|
|
"learning_rate": 1.023056799934045e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.2104,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5720
|
|
},
|
|
{
|
|
"epoch": 0.7323151639082369,
|
|
"grad_norm": 11.60741920901857,
|
|
"learning_rate": 1.0140715814007783e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2133,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5730
|
|
},
|
|
{
|
|
"epoch": 0.7335932008435043,
|
|
"grad_norm": 11.162758346658364,
|
|
"learning_rate": 1.0051159449146643e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.40625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1933,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.265625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 5740
|
|
},
|
|
{
|
|
"epoch": 0.7348712377787718,
|
|
"grad_norm": 8.587527124699529,
|
|
"learning_rate": 9.961900687656242e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1919,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5750
|
|
},
|
|
{
|
|
"epoch": 0.7361492747140392,
|
|
"grad_norm": 13.109963319894257,
|
|
"learning_rate": 9.872941306511056e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5760
|
|
},
|
|
{
|
|
"epoch": 0.7374273116493066,
|
|
"grad_norm": 9.579077500243383,
|
|
"learning_rate": 9.784283076725494e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 5770
|
|
},
|
|
{
|
|
"epoch": 0.738705348584574,
|
|
"grad_norm": 10.231998859747685,
|
|
"learning_rate": 9.695927763318568e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5780
|
|
},
|
|
{
|
|
"epoch": 0.7399833855198416,
|
|
"grad_norm": 9.12737106467952,
|
|
"learning_rate": 9.607877125278823e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 5790
|
|
},
|
|
{
|
|
"epoch": 0.741261422455109,
|
|
"grad_norm": 10.593757644763386,
|
|
"learning_rate": 9.520132915529269e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5800
|
|
},
|
|
{
|
|
"epoch": 0.7425394593903764,
|
|
"grad_norm": 9.210720279818025,
|
|
"learning_rate": 9.432696880892513e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 5810
|
|
},
|
|
{
|
|
"epoch": 0.7438174963256438,
|
|
"grad_norm": 7.802302327948419,
|
|
"learning_rate": 9.345570762055968e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2068,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 5820
|
|
},
|
|
{
|
|
"epoch": 0.7450955332609113,
|
|
"grad_norm": 10.07799824095782,
|
|
"learning_rate": 9.2587562935372e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2027,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.375,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 5830
|
|
},
|
|
{
|
|
"epoch": 0.7463735701961787,
|
|
"grad_norm": 10.091705801781881,
|
|
"learning_rate": 9.172255203649429e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1918,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5840
|
|
},
|
|
{
|
|
"epoch": 0.7476516071314461,
|
|
"grad_norm": 8.580309136599698,
|
|
"learning_rate": 9.086069214467062e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2003,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 5850
|
|
},
|
|
{
|
|
"epoch": 0.7489296440667135,
|
|
"grad_norm": 8.566545143121003,
|
|
"learning_rate": 9.00020004179145e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1871,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5860
|
|
},
|
|
{
|
|
"epoch": 0.750207681001981,
|
|
"grad_norm": 12.927074227474392,
|
|
"learning_rate": 8.914649395116752e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2015,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5870
|
|
},
|
|
{
|
|
"epoch": 0.7514857179372484,
|
|
"grad_norm": 10.661616736265797,
|
|
"learning_rate": 8.829418977595815e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2289,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5880
|
|
},
|
|
{
|
|
"epoch": 0.7527637548725158,
|
|
"grad_norm": 10.401588278458307,
|
|
"learning_rate": 8.744510486006376e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1992,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5890
|
|
},
|
|
{
|
|
"epoch": 0.7540417918077832,
|
|
"grad_norm": 8.981678643474089,
|
|
"learning_rate": 8.659925610717208e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2206,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 5900
|
|
},
|
|
{
|
|
"epoch": 0.7553198287430507,
|
|
"grad_norm": 9.88446374371275,
|
|
"learning_rate": 8.575666035654482e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 5910
|
|
},
|
|
{
|
|
"epoch": 0.7565978656783181,
|
|
"grad_norm": 8.026599552996204,
|
|
"learning_rate": 8.491733438268281e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 5920
|
|
},
|
|
{
|
|
"epoch": 0.7578759026135855,
|
|
"grad_norm": 11.0399148587071,
|
|
"learning_rate": 8.408129489499147e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2214,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 5930
|
|
},
|
|
{
|
|
"epoch": 0.7591539395488529,
|
|
"grad_norm": 9.850294858942783,
|
|
"learning_rate": 8.324855853744858e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1793,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 5940
|
|
},
|
|
{
|
|
"epoch": 0.7604319764841204,
|
|
"grad_norm": 8.702468926444062,
|
|
"learning_rate": 8.241914188827278e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1857,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5950
|
|
},
|
|
{
|
|
"epoch": 0.7617100134193878,
|
|
"grad_norm": 13.512408481784156,
|
|
"learning_rate": 8.159306145959346e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1995,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 5960
|
|
},
|
|
{
|
|
"epoch": 0.7629880503546552,
|
|
"grad_norm": 13.572590071392261,
|
|
"learning_rate": 8.077033369712233e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 5970
|
|
},
|
|
{
|
|
"epoch": 0.7642660872899226,
|
|
"grad_norm": 10.395036087264575,
|
|
"learning_rate": 7.99509749798255e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.1916,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.0,
|
|
"rewards/rejected": -7.875,
|
|
"step": 5980
|
|
},
|
|
{
|
|
"epoch": 0.7655441242251901,
|
|
"grad_norm": 11.431047760742594,
|
|
"learning_rate": 7.913500161959808e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1859,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 5990
|
|
},
|
|
{
|
|
"epoch": 0.7668221611604575,
|
|
"grad_norm": 12.12350658078121,
|
|
"learning_rate": 7.832242986093877e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6000
|
|
},
|
|
{
|
|
"epoch": 0.7681001980957249,
|
|
"grad_norm": 12.154386070056951,
|
|
"learning_rate": 7.751327588062686e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1784,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6010
|
|
},
|
|
{
|
|
"epoch": 0.7693782350309923,
|
|
"grad_norm": 8.9004131803929,
|
|
"learning_rate": 7.670755578740012e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1882,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6020
|
|
},
|
|
{
|
|
"epoch": 0.7706562719662599,
|
|
"grad_norm": 13.277525759657642,
|
|
"learning_rate": 7.590528562163393e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.53125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6030
|
|
},
|
|
{
|
|
"epoch": 0.7719343089015273,
|
|
"grad_norm": 10.410706613714614,
|
|
"learning_rate": 7.51064813550224e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -940.0,
|
|
"loss": 0.1864,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6040
|
|
},
|
|
{
|
|
"epoch": 0.7732123458367947,
|
|
"grad_norm": 9.515020477677249,
|
|
"learning_rate": 7.431115889025969e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1924,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6050
|
|
},
|
|
{
|
|
"epoch": 0.7744903827720621,
|
|
"grad_norm": 13.072675219028618,
|
|
"learning_rate": 7.35193340607242e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6060
|
|
},
|
|
{
|
|
"epoch": 0.7757684197073296,
|
|
"grad_norm": 12.782269185266843,
|
|
"learning_rate": 7.27310226301627e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2286,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6070
|
|
},
|
|
{
|
|
"epoch": 0.777046456642597,
|
|
"grad_norm": 9.418209612849605,
|
|
"learning_rate": 7.194624029237686e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6080
|
|
},
|
|
{
|
|
"epoch": 0.7783244935778644,
|
|
"grad_norm": 12.973911791232169,
|
|
"learning_rate": 7.116500267091077e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1974,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6090
|
|
},
|
|
{
|
|
"epoch": 0.7796025305131318,
|
|
"grad_norm": 10.172661744116416,
|
|
"learning_rate": 7.038732531873972e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1876,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6100
|
|
},
|
|
{
|
|
"epoch": 0.7808805674483993,
|
|
"grad_norm": 11.421418477500511,
|
|
"learning_rate": 6.961322371796105e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.181,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6110
|
|
},
|
|
{
|
|
"epoch": 0.7821586043836667,
|
|
"grad_norm": 11.428398005666034,
|
|
"learning_rate": 6.884271327948524e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6120
|
|
},
|
|
{
|
|
"epoch": 0.7834366413189341,
|
|
"grad_norm": 8.869925031073004,
|
|
"learning_rate": 6.807580934272961e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2008,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6130
|
|
},
|
|
{
|
|
"epoch": 0.7847146782542015,
|
|
"grad_norm": 10.839154644975004,
|
|
"learning_rate": 6.731252717531288e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1885,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6140
|
|
},
|
|
{
|
|
"epoch": 0.785992715189469,
|
|
"grad_norm": 8.44382605953575,
|
|
"learning_rate": 6.655288197275103e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6150
|
|
},
|
|
{
|
|
"epoch": 0.7872707521247364,
|
|
"grad_norm": 11.179183535261652,
|
|
"learning_rate": 6.579688885815488e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1932,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6160
|
|
},
|
|
{
|
|
"epoch": 0.7885487890600038,
|
|
"grad_norm": 11.052997512838385,
|
|
"learning_rate": 6.504456288192911e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6170
|
|
},
|
|
{
|
|
"epoch": 0.7898268259952712,
|
|
"grad_norm": 11.267748953593356,
|
|
"learning_rate": 6.429591902147238e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1826,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6180
|
|
},
|
|
{
|
|
"epoch": 0.7911048629305387,
|
|
"grad_norm": 8.88100812958174,
|
|
"learning_rate": 6.355097218087968e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1853,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6190
|
|
},
|
|
{
|
|
"epoch": 0.7923828998658061,
|
|
"grad_norm": 13.932434101787049,
|
|
"learning_rate": 6.280973719064486e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.209,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6200
|
|
},
|
|
{
|
|
"epoch": 0.7936609368010735,
|
|
"grad_norm": 12.849144357607967,
|
|
"learning_rate": 6.207222880736618e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.2326,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6210
|
|
},
|
|
{
|
|
"epoch": 0.7949389737363409,
|
|
"grad_norm": 13.733842838610288,
|
|
"learning_rate": 6.133846171345189e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2189,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6220
|
|
},
|
|
{
|
|
"epoch": 0.7962170106716084,
|
|
"grad_norm": 8.758429764508215,
|
|
"learning_rate": 6.060845051682809e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.1997,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.40625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -7.875,
|
|
"step": 6230
|
|
},
|
|
{
|
|
"epoch": 0.7974950476068758,
|
|
"grad_norm": 10.236369223553389,
|
|
"learning_rate": 5.988220975064839e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6240
|
|
},
|
|
{
|
|
"epoch": 0.7987730845421432,
|
|
"grad_norm": 8.991486561608422,
|
|
"learning_rate": 5.915975387300382e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6250
|
|
},
|
|
{
|
|
"epoch": 0.8000511214774106,
|
|
"grad_norm": 10.298101814631558,
|
|
"learning_rate": 5.844109726663568e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1978,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6260
|
|
},
|
|
{
|
|
"epoch": 0.8013291584126782,
|
|
"grad_norm": 11.9546027222749,
|
|
"learning_rate": 5.772625423864866e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -936.0,
|
|
"loss": 0.2097,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -7.96875,
|
|
"step": 6270
|
|
},
|
|
{
|
|
"epoch": 0.8026071953479456,
|
|
"grad_norm": 10.119304805105461,
|
|
"learning_rate": 5.701523902022659e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6280
|
|
},
|
|
{
|
|
"epoch": 0.803885232283213,
|
|
"grad_norm": 11.201139785327099,
|
|
"learning_rate": 5.6308065766348335e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6290
|
|
},
|
|
{
|
|
"epoch": 0.8051632692184804,
|
|
"grad_norm": 9.377393381822943,
|
|
"learning_rate": 5.5604748555506706e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.193,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6300
|
|
},
|
|
{
|
|
"epoch": 0.8064413061537479,
|
|
"grad_norm": 8.309075509709407,
|
|
"learning_rate": 5.490530138942806e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1863,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6310
|
|
},
|
|
{
|
|
"epoch": 0.8077193430890153,
|
|
"grad_norm": 10.253650621317618,
|
|
"learning_rate": 5.4209738192793146e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1884,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6320
|
|
},
|
|
{
|
|
"epoch": 0.8089973800242827,
|
|
"grad_norm": 8.25575714700977,
|
|
"learning_rate": 5.3518072812960444e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6330
|
|
},
|
|
{
|
|
"epoch": 0.8102754169595501,
|
|
"grad_norm": 11.079645770003177,
|
|
"learning_rate": 5.283031901969001e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1864,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.3125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6340
|
|
},
|
|
{
|
|
"epoch": 0.8115534538948176,
|
|
"grad_norm": 8.856148371837136,
|
|
"learning_rate": 5.214649050486961e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.184,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.0625,
|
|
"step": 6350
|
|
},
|
|
{
|
|
"epoch": 0.812831490830085,
|
|
"grad_norm": 11.181004151997012,
|
|
"learning_rate": 5.146660088224209e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.2045,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6360
|
|
},
|
|
{
|
|
"epoch": 0.8141095277653524,
|
|
"grad_norm": 9.153609232514398,
|
|
"learning_rate": 5.079066368713425e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1839,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6370
|
|
},
|
|
{
|
|
"epoch": 0.8153875647006198,
|
|
"grad_norm": 9.535738595895644,
|
|
"learning_rate": 5.011869237618768e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1878,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6380
|
|
},
|
|
{
|
|
"epoch": 0.8166656016358873,
|
|
"grad_norm": 8.741327828462396,
|
|
"learning_rate": 4.9450700327090535e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2076,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6390
|
|
},
|
|
{
|
|
"epoch": 0.8179436385711547,
|
|
"grad_norm": 11.426080897454439,
|
|
"learning_rate": 4.8786700838311184e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.2068,
|
|
"rewards/accuracies": 0.856249988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6400
|
|
},
|
|
{
|
|
"epoch": 0.8192216755064221,
|
|
"grad_norm": 11.034650603841335,
|
|
"learning_rate": 4.812670712883396e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2023,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6410
|
|
},
|
|
{
|
|
"epoch": 0.8204997124416895,
|
|
"grad_norm": 9.254914740173248,
|
|
"learning_rate": 4.7470732337895335e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6420
|
|
},
|
|
{
|
|
"epoch": 0.821777749376957,
|
|
"grad_norm": 11.445609691444933,
|
|
"learning_rate": 4.681878952472276e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1843,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6430
|
|
},
|
|
{
|
|
"epoch": 0.8230557863122244,
|
|
"grad_norm": 10.758939904944432,
|
|
"learning_rate": 4.6170891668274616e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1844,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6440
|
|
},
|
|
{
|
|
"epoch": 0.8243338232474918,
|
|
"grad_norm": 6.823878634242757,
|
|
"learning_rate": 4.552705166698165e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.198,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.140625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 6450
|
|
},
|
|
{
|
|
"epoch": 0.8256118601827592,
|
|
"grad_norm": 11.462876385323876,
|
|
"learning_rate": 4.488728233849054e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1919,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6460
|
|
},
|
|
{
|
|
"epoch": 0.8268898971180267,
|
|
"grad_norm": 11.511738897771243,
|
|
"learning_rate": 4.425159641940826e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1908,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6470
|
|
},
|
|
{
|
|
"epoch": 0.8281679340532941,
|
|
"grad_norm": 10.012274810790023,
|
|
"learning_rate": 4.362000656504902e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6480
|
|
},
|
|
{
|
|
"epoch": 0.8294459709885615,
|
|
"grad_norm": 6.3696111029717715,
|
|
"learning_rate": 4.299252534918185e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.125,
|
|
"step": 6490
|
|
},
|
|
{
|
|
"epoch": 0.8307240079238289,
|
|
"grad_norm": 8.711459660714853,
|
|
"learning_rate": 4.2369165263780595e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1839,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6500
|
|
},
|
|
{
|
|
"epoch": 0.8320020448590965,
|
|
"grad_norm": 8.586531255673448,
|
|
"learning_rate": 4.174993871877513e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6510
|
|
},
|
|
{
|
|
"epoch": 0.8332800817943639,
|
|
"grad_norm": 9.159130220138737,
|
|
"learning_rate": 4.1134858041804165e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2108,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6520
|
|
},
|
|
{
|
|
"epoch": 0.8345581187296313,
|
|
"grad_norm": 14.110031544509095,
|
|
"learning_rate": 4.05239354779702e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1964,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 6530
|
|
},
|
|
{
|
|
"epoch": 0.8358361556648987,
|
|
"grad_norm": 13.647332815864479,
|
|
"learning_rate": 3.991718318959528e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1952,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6540
|
|
},
|
|
{
|
|
"epoch": 0.8371141926001662,
|
|
"grad_norm": 7.729306259968811,
|
|
"learning_rate": 3.931461325597924e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1847,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6550
|
|
},
|
|
{
|
|
"epoch": 0.8383922295354336,
|
|
"grad_norm": 12.245685775856511,
|
|
"learning_rate": 3.871623767315896e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.2085,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6560
|
|
},
|
|
{
|
|
"epoch": 0.839670266470701,
|
|
"grad_norm": 8.700079920104228,
|
|
"learning_rate": 3.812206835366977e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1891,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6570
|
|
},
|
|
{
|
|
"epoch": 0.8409483034059684,
|
|
"grad_norm": 9.331886204678842,
|
|
"learning_rate": 3.753211712630808e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 6580
|
|
},
|
|
{
|
|
"epoch": 0.8422263403412359,
|
|
"grad_norm": 9.558487053586102,
|
|
"learning_rate": 3.6946395735896e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6590
|
|
},
|
|
{
|
|
"epoch": 0.8435043772765033,
|
|
"grad_norm": 9.536711446586605,
|
|
"learning_rate": 3.636491584304774e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.2018,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6600
|
|
},
|
|
{
|
|
"epoch": 0.8447824142117707,
|
|
"grad_norm": 9.745832695689701,
|
|
"learning_rate": 3.578768902393697e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1645,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 6610
|
|
},
|
|
{
|
|
"epoch": 0.8460604511470381,
|
|
"grad_norm": 6.944861472906955,
|
|
"learning_rate": 3.521472677006676e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.171,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 6620
|
|
},
|
|
{
|
|
"epoch": 0.8473384880823056,
|
|
"grad_norm": 6.74838826324146,
|
|
"learning_rate": 3.464604048804079e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6630
|
|
},
|
|
{
|
|
"epoch": 0.848616525017573,
|
|
"grad_norm": 11.60280140004919,
|
|
"learning_rate": 3.408164149933587e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1966,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6640
|
|
},
|
|
{
|
|
"epoch": 0.8498945619528404,
|
|
"grad_norm": 8.055840883849848,
|
|
"learning_rate": 3.352154104007721e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.187,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6650
|
|
},
|
|
{
|
|
"epoch": 0.8511725988881078,
|
|
"grad_norm": 10.181553904665757,
|
|
"learning_rate": 3.296575026081408e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.390625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6660
|
|
},
|
|
{
|
|
"epoch": 0.8524506358233753,
|
|
"grad_norm": 10.249516867924106,
|
|
"learning_rate": 3.241428022629822e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.2086,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.40625,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 6670
|
|
},
|
|
{
|
|
"epoch": 0.8537286727586427,
|
|
"grad_norm": 12.054480356165014,
|
|
"learning_rate": 3.186714191526346e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1844,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6680
|
|
},
|
|
{
|
|
"epoch": 0.8550067096939101,
|
|
"grad_norm": 6.844044080742228,
|
|
"learning_rate": 3.132434622020716e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -928.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.34375,
|
|
"rewards/rejected": -8.0,
|
|
"step": 6690
|
|
},
|
|
{
|
|
"epoch": 0.8562847466291775,
|
|
"grad_norm": 11.715109071014988,
|
|
"learning_rate": 3.078590394717323e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1969,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6700
|
|
},
|
|
{
|
|
"epoch": 0.857562783564445,
|
|
"grad_norm": 9.302220076847602,
|
|
"learning_rate": 3.0251825815537204e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1986,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6710
|
|
},
|
|
{
|
|
"epoch": 0.8588408204997124,
|
|
"grad_norm": 10.93280433427468,
|
|
"learning_rate": 2.972212245779271e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.5625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6720
|
|
},
|
|
{
|
|
"epoch": 0.8601188574349798,
|
|
"grad_norm": 8.427937780048167,
|
|
"learning_rate": 2.9196804419339917e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1962,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6730
|
|
},
|
|
{
|
|
"epoch": 0.8613968943702472,
|
|
"grad_norm": 8.597293479711446,
|
|
"learning_rate": 2.867588215827538e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.2005,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 6740
|
|
},
|
|
{
|
|
"epoch": 0.8626749313055148,
|
|
"grad_norm": 9.995814473290233,
|
|
"learning_rate": 2.8159366045184124e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1853,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6750
|
|
},
|
|
{
|
|
"epoch": 0.8639529682407822,
|
|
"grad_norm": 9.928836544697068,
|
|
"learning_rate": 2.7647266362932936e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6760
|
|
},
|
|
{
|
|
"epoch": 0.8652310051760496,
|
|
"grad_norm": 10.113163869171162,
|
|
"learning_rate": 2.7139593306465758e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1788,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6770
|
|
},
|
|
{
|
|
"epoch": 0.866509042111317,
|
|
"grad_norm": 10.346262544298897,
|
|
"learning_rate": 2.6636356982600695e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6780
|
|
},
|
|
{
|
|
"epoch": 0.8677870790465845,
|
|
"grad_norm": 8.842070789217034,
|
|
"learning_rate": 2.6137567409828864e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6790
|
|
},
|
|
{
|
|
"epoch": 0.8690651159818519,
|
|
"grad_norm": 10.348672126261382,
|
|
"learning_rate": 2.5643234518114955e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1875,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6800
|
|
},
|
|
{
|
|
"epoch": 0.8703431529171193,
|
|
"grad_norm": 14.107914181886756,
|
|
"learning_rate": 2.515336814869942e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1811,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6810
|
|
},
|
|
{
|
|
"epoch": 0.8716211898523867,
|
|
"grad_norm": 10.208150894155457,
|
|
"learning_rate": 2.466797805390272e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1868,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.53125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6820
|
|
},
|
|
{
|
|
"epoch": 0.8728992267876542,
|
|
"grad_norm": 9.143576947306267,
|
|
"learning_rate": 2.4187073896930977e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1748,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6830
|
|
},
|
|
{
|
|
"epoch": 0.8741772637229216,
|
|
"grad_norm": 9.619393237764903,
|
|
"learning_rate": 2.3710665251683775e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1968,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6840
|
|
},
|
|
{
|
|
"epoch": 0.875455300658189,
|
|
"grad_norm": 12.665374011457072,
|
|
"learning_rate": 2.3238761602563496e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1823,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 6850
|
|
},
|
|
{
|
|
"epoch": 0.8767333375934564,
|
|
"grad_norm": 8.602114064258634,
|
|
"learning_rate": 2.2771372344286417e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2042,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6860
|
|
},
|
|
{
|
|
"epoch": 0.8780113745287239,
|
|
"grad_norm": 6.8825535391482715,
|
|
"learning_rate": 2.2308506781695906e-08,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.2029,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6870
|
|
},
|
|
{
|
|
"epoch": 0.8792894114639913,
|
|
"grad_norm": 11.099108660135526,
|
|
"learning_rate": 2.1850174129576933e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.2016,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.375,
|
|
"step": 6880
|
|
},
|
|
{
|
|
"epoch": 0.8805674483992587,
|
|
"grad_norm": 12.681665287086265,
|
|
"learning_rate": 2.1396383512472722e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6890
|
|
},
|
|
{
|
|
"epoch": 0.8818454853345261,
|
|
"grad_norm": 10.890987789058547,
|
|
"learning_rate": 2.09471439645032e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2022,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 6900
|
|
},
|
|
{
|
|
"epoch": 0.8831235222697936,
|
|
"grad_norm": 9.926453846449595,
|
|
"learning_rate": 2.0502464429184933e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 6910
|
|
},
|
|
{
|
|
"epoch": 0.884401559205061,
|
|
"grad_norm": 11.278826880908559,
|
|
"learning_rate": 2.006235375925322e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1749,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6920
|
|
},
|
|
{
|
|
"epoch": 0.8856795961403284,
|
|
"grad_norm": 11.038211397843158,
|
|
"learning_rate": 1.962682071648586e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2006,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6930
|
|
},
|
|
{
|
|
"epoch": 0.8869576330755958,
|
|
"grad_norm": 10.785169033376835,
|
|
"learning_rate": 1.919587397152861e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1896,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 6940
|
|
},
|
|
{
|
|
"epoch": 0.8882356700108633,
|
|
"grad_norm": 9.925891527391922,
|
|
"learning_rate": 1.8769522103722763e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1796,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 6950
|
|
},
|
|
{
|
|
"epoch": 0.8895137069461307,
|
|
"grad_norm": 8.771182460846884,
|
|
"learning_rate": 1.8347773600934063e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 6960
|
|
},
|
|
{
|
|
"epoch": 0.8907917438813981,
|
|
"grad_norm": 9.61805627828746,
|
|
"learning_rate": 1.7930636859384064e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2153,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.25,
|
|
"step": 6970
|
|
},
|
|
{
|
|
"epoch": 0.8920697808166655,
|
|
"grad_norm": 10.059738064893434,
|
|
"learning_rate": 1.7518120183482755e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 6980
|
|
},
|
|
{
|
|
"epoch": 0.8933478177519331,
|
|
"grad_norm": 13.222656919567447,
|
|
"learning_rate": 1.7110231785663094e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.2009,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 6990
|
|
},
|
|
{
|
|
"epoch": 0.8946258546872005,
|
|
"grad_norm": 11.481242847946271,
|
|
"learning_rate": 1.670697978621802e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1726,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7000
|
|
},
|
|
{
|
|
"epoch": 0.8959038916224679,
|
|
"grad_norm": 12.332320196979492,
|
|
"learning_rate": 1.6308372213138244e-08,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7010
|
|
},
|
|
{
|
|
"epoch": 0.8971819285577353,
|
|
"grad_norm": 12.808012844296925,
|
|
"learning_rate": 1.5914417001952785e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2049,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7020
|
|
},
|
|
{
|
|
"epoch": 0.8984599654930028,
|
|
"grad_norm": 7.820818021151389,
|
|
"learning_rate": 1.5525121995570823e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7030
|
|
},
|
|
{
|
|
"epoch": 0.8997380024282702,
|
|
"grad_norm": 9.370835431954198,
|
|
"learning_rate": 1.5140494944125588e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7040
|
|
},
|
|
{
|
|
"epoch": 0.9010160393635376,
|
|
"grad_norm": 11.067566635775739,
|
|
"learning_rate": 1.4760543504820088e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1862,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7050
|
|
},
|
|
{
|
|
"epoch": 0.902294076298805,
|
|
"grad_norm": 9.623901803980623,
|
|
"learning_rate": 1.438527524177463e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1872,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7060
|
|
},
|
|
{
|
|
"epoch": 0.9035721132340725,
|
|
"grad_norm": 11.5687168737442,
|
|
"learning_rate": 1.4014697625876426e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1964,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7070
|
|
},
|
|
{
|
|
"epoch": 0.9048501501693399,
|
|
"grad_norm": 8.21021164784677,
|
|
"learning_rate": 1.3648818034630499e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2012,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7080
|
|
},
|
|
{
|
|
"epoch": 0.9061281871046073,
|
|
"grad_norm": 11.17367745513911,
|
|
"learning_rate": 1.3287643752013162e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.2041,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7090
|
|
},
|
|
{
|
|
"epoch": 0.9074062240398747,
|
|
"grad_norm": 9.879895150119761,
|
|
"learning_rate": 1.2931181968326765e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.2031,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7100
|
|
},
|
|
{
|
|
"epoch": 0.9086842609751422,
|
|
"grad_norm": 8.750603926836277,
|
|
"learning_rate": 1.257943978005674e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2044,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7110
|
|
},
|
|
{
|
|
"epoch": 0.9099622979104096,
|
|
"grad_norm": 9.732574600229979,
|
|
"learning_rate": 1.2232424189730084e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1951,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.609375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7120
|
|
},
|
|
{
|
|
"epoch": 0.911240334845677,
|
|
"grad_norm": 10.799063558489175,
|
|
"learning_rate": 1.189014210577624e-08,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1805,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.421875,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 7130
|
|
},
|
|
{
|
|
"epoch": 0.9125183717809444,
|
|
"grad_norm": 8.342418997195931,
|
|
"learning_rate": 1.1552600342389363e-08,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.172,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7140
|
|
},
|
|
{
|
|
"epoch": 0.9137964087162119,
|
|
"grad_norm": 10.71141399194098,
|
|
"learning_rate": 1.121980561939273e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1729,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7150
|
|
},
|
|
{
|
|
"epoch": 0.9150744456514793,
|
|
"grad_norm": 9.405664113940032,
|
|
"learning_rate": 1.0891764562104888e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1956,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7160
|
|
},
|
|
{
|
|
"epoch": 0.9163524825867467,
|
|
"grad_norm": 11.026789747690092,
|
|
"learning_rate": 1.0568483701207941e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.515625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7170
|
|
},
|
|
{
|
|
"epoch": 0.9176305195220141,
|
|
"grad_norm": 9.18756797117969,
|
|
"learning_rate": 1.0249969472617304e-08,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1915,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7180
|
|
},
|
|
{
|
|
"epoch": 0.9189085564572816,
|
|
"grad_norm": 10.972057226223303,
|
|
"learning_rate": 9.936228217353709e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -952.0,
|
|
"loss": 0.1814,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.453125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 7190
|
|
},
|
|
{
|
|
"epoch": 0.920186593392549,
|
|
"grad_norm": 7.87780338280655,
|
|
"learning_rate": 9.627266181416987e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7200
|
|
},
|
|
{
|
|
"epoch": 0.9214646303278164,
|
|
"grad_norm": 7.847069398644123,
|
|
"learning_rate": 9.323089515661603e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1631,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7210
|
|
},
|
|
{
|
|
"epoch": 0.9227426672630838,
|
|
"grad_norm": 9.819571950781643,
|
|
"learning_rate": 9.023704275674394e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.194,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7220
|
|
},
|
|
{
|
|
"epoch": 0.9240207041983514,
|
|
"grad_norm": 11.758652729017486,
|
|
"learning_rate": 8.729116421653748e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1746,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7230
|
|
},
|
|
{
|
|
"epoch": 0.9252987411336188,
|
|
"grad_norm": 9.074528747089998,
|
|
"learning_rate": 8.439331818291261e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.2084,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7240
|
|
},
|
|
{
|
|
"epoch": 0.9265767780688862,
|
|
"grad_norm": 10.439935658287324,
|
|
"learning_rate": 8.154356234654713e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.203,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7250
|
|
},
|
|
{
|
|
"epoch": 0.9278548150041536,
|
|
"grad_norm": 13.20542442648142,
|
|
"learning_rate": 7.874195344073297e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7260
|
|
},
|
|
{
|
|
"epoch": 0.9291328519394211,
|
|
"grad_norm": 11.62024702139443,
|
|
"learning_rate": 7.598854724024828e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.185,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7270
|
|
},
|
|
{
|
|
"epoch": 0.9304108888746885,
|
|
"grad_norm": 10.099788710806347,
|
|
"learning_rate": 7.328339856024407e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7280
|
|
},
|
|
{
|
|
"epoch": 0.9316889258099559,
|
|
"grad_norm": 9.897081640900788,
|
|
"learning_rate": 7.062656125515654e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1759,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7290
|
|
},
|
|
{
|
|
"epoch": 0.9329669627452233,
|
|
"grad_norm": 10.589111836344392,
|
|
"learning_rate": 6.801808821763178e-09,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.2106,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7300
|
|
},
|
|
{
|
|
"epoch": 0.9342449996804908,
|
|
"grad_norm": 10.724801842725016,
|
|
"learning_rate": 6.545803137747385e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7310
|
|
},
|
|
{
|
|
"epoch": 0.9355230366157582,
|
|
"grad_norm": 9.611442651274807,
|
|
"learning_rate": 6.294644170061258e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1943,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7320
|
|
},
|
|
{
|
|
"epoch": 0.9368010735510256,
|
|
"grad_norm": 11.751486176205045,
|
|
"learning_rate": 6.0483369188085995e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1801,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7330
|
|
},
|
|
{
|
|
"epoch": 0.938079110486293,
|
|
"grad_norm": 11.512912312322182,
|
|
"learning_rate": 5.806886287504725e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1873,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7340
|
|
},
|
|
{
|
|
"epoch": 0.9393571474215605,
|
|
"grad_norm": 7.192083175792538,
|
|
"learning_rate": 5.570297082978709e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1647,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7350
|
|
},
|
|
{
|
|
"epoch": 0.9406351843568279,
|
|
"grad_norm": 10.843092841652517,
|
|
"learning_rate": 5.3385740152777634e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2038,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.0,
|
|
"step": 7360
|
|
},
|
|
{
|
|
"epoch": 0.9419132212920953,
|
|
"grad_norm": 8.467409445214516,
|
|
"learning_rate": 5.111721697573457e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7370
|
|
},
|
|
{
|
|
"epoch": 0.9431912582273627,
|
|
"grad_norm": 6.715800157562806,
|
|
"learning_rate": 4.889744646069837e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1889,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 7380
|
|
},
|
|
{
|
|
"epoch": 0.9444692951626302,
|
|
"grad_norm": 8.400806549616826,
|
|
"learning_rate": 4.672647279913622e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1971,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7390
|
|
},
|
|
{
|
|
"epoch": 0.9457473320978976,
|
|
"grad_norm": 13.972001950715883,
|
|
"learning_rate": 4.4604339211059824e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1976,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7400
|
|
},
|
|
{
|
|
"epoch": 0.947025369033165,
|
|
"grad_norm": 9.677769647797021,
|
|
"learning_rate": 4.253108794416871e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 7410
|
|
},
|
|
{
|
|
"epoch": 0.9483034059684324,
|
|
"grad_norm": 9.345026975785517,
|
|
"learning_rate": 4.050676027300554e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2038,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7420
|
|
},
|
|
{
|
|
"epoch": 0.9495814429037,
|
|
"grad_norm": 9.695675333701805,
|
|
"learning_rate": 3.853139649813625e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1957,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7430
|
|
},
|
|
{
|
|
"epoch": 0.9508594798389673,
|
|
"grad_norm": 7.770057515915393,
|
|
"learning_rate": 3.660503594534875e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1867,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7440
|
|
},
|
|
{
|
|
"epoch": 0.9521375167742347,
|
|
"grad_norm": 12.504038666390727,
|
|
"learning_rate": 3.4727716964866595e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1886,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.5,
|
|
"rewards/rejected": -8.375,
|
|
"step": 7450
|
|
},
|
|
{
|
|
"epoch": 0.9534155537095022,
|
|
"grad_norm": 9.181970318686727,
|
|
"learning_rate": 3.2899476930589863e-09,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1931,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7460
|
|
},
|
|
{
|
|
"epoch": 0.9546935906447697,
|
|
"grad_norm": 11.183563173203545,
|
|
"learning_rate": 3.112035223934689e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1982,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7470
|
|
},
|
|
{
|
|
"epoch": 0.9559716275800371,
|
|
"grad_norm": 10.636414438455274,
|
|
"learning_rate": 2.939037831017288e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7480
|
|
},
|
|
{
|
|
"epoch": 0.9572496645153045,
|
|
"grad_norm": 11.633662618889964,
|
|
"learning_rate": 2.7709589583603254e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1904,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7490
|
|
},
|
|
{
|
|
"epoch": 0.9585277014505719,
|
|
"grad_norm": 12.38372710586855,
|
|
"learning_rate": 2.6078019520988093e-09,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.196,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.484375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7500
|
|
},
|
|
{
|
|
"epoch": 0.9598057383858394,
|
|
"grad_norm": 12.644481438195353,
|
|
"learning_rate": 2.449570060382711e-09,
|
|
"logits/chosen": -2.765625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7510
|
|
},
|
|
{
|
|
"epoch": 0.9610837753211068,
|
|
"grad_norm": 9.432942475563783,
|
|
"learning_rate": 2.2962664333120994e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1963,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7520
|
|
},
|
|
{
|
|
"epoch": 0.9623618122563742,
|
|
"grad_norm": 11.928069118877714,
|
|
"learning_rate": 2.1478941228745828e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2113,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7530
|
|
},
|
|
{
|
|
"epoch": 0.9636398491916416,
|
|
"grad_norm": 7.886668745442734,
|
|
"learning_rate": 2.0044560828845205e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 7540
|
|
},
|
|
{
|
|
"epoch": 0.9649178861269091,
|
|
"grad_norm": 9.34734142715555,
|
|
"learning_rate": 1.8659551689241547e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.2107,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7550
|
|
},
|
|
{
|
|
"epoch": 0.9661959230621765,
|
|
"grad_norm": 9.611208535032741,
|
|
"learning_rate": 1.7323941382869078e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1923,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7560
|
|
},
|
|
{
|
|
"epoch": 0.9674739599974439,
|
|
"grad_norm": 9.527127269024511,
|
|
"learning_rate": 1.603775649922312e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1666,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7570
|
|
},
|
|
{
|
|
"epoch": 0.9687519969327113,
|
|
"grad_norm": 9.291770961378111,
|
|
"learning_rate": 1.4801022643831652e-09,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.2,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7580
|
|
},
|
|
{
|
|
"epoch": 0.9700300338679788,
|
|
"grad_norm": 12.925961238581618,
|
|
"learning_rate": 1.3613764437745978e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1738,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7590
|
|
},
|
|
{
|
|
"epoch": 0.9713080708032462,
|
|
"grad_norm": 10.408031687484408,
|
|
"learning_rate": 1.2476005517049192e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.185,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7600
|
|
},
|
|
{
|
|
"epoch": 0.9725861077385136,
|
|
"grad_norm": 9.808605265046817,
|
|
"learning_rate": 1.138776853238682e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1914,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 7610
|
|
},
|
|
{
|
|
"epoch": 0.973864144673781,
|
|
"grad_norm": 7.618616317950994,
|
|
"learning_rate": 1.0349075148516074e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1606,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7620
|
|
},
|
|
{
|
|
"epoch": 0.9751421816090485,
|
|
"grad_norm": 11.794438615303235,
|
|
"learning_rate": 9.359946043873412e-10,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1954,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7630
|
|
},
|
|
{
|
|
"epoch": 0.9764202185443159,
|
|
"grad_norm": 12.563234469246394,
|
|
"learning_rate": 8.420400910163494e-10,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1973,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7640
|
|
},
|
|
{
|
|
"epoch": 0.9776982554795833,
|
|
"grad_norm": 16.042841291187038,
|
|
"learning_rate": 7.530458451967814e-10,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1815,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7650
|
|
},
|
|
{
|
|
"epoch": 0.9789762924148507,
|
|
"grad_norm": 13.296730357123543,
|
|
"learning_rate": 6.69013638637056e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1816,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7660
|
|
},
|
|
{
|
|
"epoch": 0.9802543293501182,
|
|
"grad_norm": 12.112711580204818,
|
|
"learning_rate": 5.899451442607784e-10,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1893,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7670
|
|
},
|
|
{
|
|
"epoch": 0.9815323662853856,
|
|
"grad_norm": 14.646352722458984,
|
|
"learning_rate": 5.158419361733501e-10,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1982,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7680
|
|
},
|
|
{
|
|
"epoch": 0.982810403220653,
|
|
"grad_norm": 13.260198930878687,
|
|
"learning_rate": 4.4670548963063327e-10,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1835,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.546875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7690
|
|
},
|
|
{
|
|
"epoch": 0.9840884401559205,
|
|
"grad_norm": 8.788325394613999,
|
|
"learning_rate": 3.82537181009529e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1928,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 7700
|
|
},
|
|
{
|
|
"epoch": 0.985366477091188,
|
|
"grad_norm": 12.794711894897244,
|
|
"learning_rate": 3.233382877806945e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.2011,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.765625,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7710
|
|
},
|
|
{
|
|
"epoch": 0.9866445140264554,
|
|
"grad_norm": 11.753555535364036,
|
|
"learning_rate": 2.6910998848306276e-10,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7720
|
|
},
|
|
{
|
|
"epoch": 0.9879225509617228,
|
|
"grad_norm": 14.594250427636789,
|
|
"learning_rate": 2.1985336270033405e-10,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1975,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7730
|
|
},
|
|
{
|
|
"epoch": 0.9892005878969902,
|
|
"grad_norm": 12.77554394664042,
|
|
"learning_rate": 1.7556939103952063e-10,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -956.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.59375,
|
|
"rewards/rejected": -8.25,
|
|
"step": 7740
|
|
},
|
|
{
|
|
"epoch": 0.9904786248322577,
|
|
"grad_norm": 10.519067298735793,
|
|
"learning_rate": 1.3625895511137907e-10,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 7750
|
|
},
|
|
{
|
|
"epoch": 0.9917566617675251,
|
|
"grad_norm": 9.931366042203358,
|
|
"learning_rate": 1.0192283751297992e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1821,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7760
|
|
},
|
|
{
|
|
"epoch": 0.9930346987027925,
|
|
"grad_norm": 9.246116106532282,
|
|
"learning_rate": 7.256172181199783e-11,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1969,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7770
|
|
},
|
|
{
|
|
"epoch": 0.9943127356380599,
|
|
"grad_norm": 7.26117544128573,
|
|
"learning_rate": 4.8176192533139206e-11,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.194,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7780
|
|
},
|
|
{
|
|
"epoch": 0.9955907725733274,
|
|
"grad_norm": 12.332218978053229,
|
|
"learning_rate": 2.876673514659589e-11,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1812,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7790
|
|
},
|
|
{
|
|
"epoch": 0.9968688095085948,
|
|
"grad_norm": 9.768254436897418,
|
|
"learning_rate": 1.4333736058219636e-11,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7800
|
|
},
|
|
{
|
|
"epoch": 0.9981468464438622,
|
|
"grad_norm": 12.753093983260724,
|
|
"learning_rate": 4.8774826019448715e-12,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1805,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 7810
|
|
},
|
|
{
|
|
"epoch": 0.9994248833791296,
|
|
"grad_norm": 9.444618426186262,
|
|
"learning_rate": 3.981630340710307e-13,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7820
|
|
},
|
|
{
|
|
"epoch": 1.0007668221611605,
|
|
"grad_norm": 9.793739897002116,
|
|
"learning_rate": 2.93088577631297e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1666,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 7830
|
|
},
|
|
{
|
|
"epoch": 1.002044859096428,
|
|
"grad_norm": 9.002669484334344,
|
|
"learning_rate": 2.925391243611856e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1877,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 7840
|
|
},
|
|
{
|
|
"epoch": 1.0033228960316953,
|
|
"grad_norm": 7.3436812439478585,
|
|
"learning_rate": 2.9198945940233674e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -992.0,
|
|
"loss": 0.1676,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7850
|
|
},
|
|
{
|
|
"epoch": 1.0046009329669627,
|
|
"grad_norm": 10.110088255433219,
|
|
"learning_rate": 2.914395854900648e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7860
|
|
},
|
|
{
|
|
"epoch": 1.0058789699022301,
|
|
"grad_norm": 10.536172769097313,
|
|
"learning_rate": 2.908895053607241e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 7870
|
|
},
|
|
{
|
|
"epoch": 1.0071570068374975,
|
|
"grad_norm": 11.272134541218382,
|
|
"learning_rate": 2.903392217516952e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1912,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7880
|
|
},
|
|
{
|
|
"epoch": 1.008435043772765,
|
|
"grad_norm": 10.96979479582042,
|
|
"learning_rate": 2.89788737401371e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1673,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 7890
|
|
},
|
|
{
|
|
"epoch": 1.0097130807080326,
|
|
"grad_norm": 9.981606177601087,
|
|
"learning_rate": 2.8923805504914363e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1789,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.5,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.5,
|
|
"step": 7900
|
|
},
|
|
{
|
|
"epoch": 1.0109911176433,
|
|
"grad_norm": 11.993598319058572,
|
|
"learning_rate": 2.886871774353903e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.182,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 7910
|
|
},
|
|
{
|
|
"epoch": 1.0122691545785674,
|
|
"grad_norm": 13.605038367767165,
|
|
"learning_rate": 2.8813610730146003e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.2072,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 7920
|
|
},
|
|
{
|
|
"epoch": 1.0135471915138348,
|
|
"grad_norm": 11.172416814175007,
|
|
"learning_rate": 2.8758484738966e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -568.0,
|
|
"logps/rejected": -904.0,
|
|
"loss": 0.1721,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.03125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -7.6875,
|
|
"step": 7930
|
|
},
|
|
{
|
|
"epoch": 1.0148252284491022,
|
|
"grad_norm": 9.814062210000673,
|
|
"learning_rate": 2.870334004432416e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1868,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 7940
|
|
},
|
|
{
|
|
"epoch": 1.0161032653843696,
|
|
"grad_norm": 13.3322442728074,
|
|
"learning_rate": 2.86481769206387e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -592.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1819,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.25,
|
|
"step": 7950
|
|
},
|
|
{
|
|
"epoch": 1.017381302319637,
|
|
"grad_norm": 8.583508636588496,
|
|
"learning_rate": 2.859299564241955e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -920.0,
|
|
"loss": 0.1972,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -4.4375,
|
|
"rewards/margins": 3.296875,
|
|
"rewards/rejected": -7.75,
|
|
"step": 7960
|
|
},
|
|
{
|
|
"epoch": 1.0186593392549044,
|
|
"grad_norm": 9.09390192789269,
|
|
"learning_rate": 2.853779648426697e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1918,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.1875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 7970
|
|
},
|
|
{
|
|
"epoch": 1.019937376190172,
|
|
"grad_norm": 9.317681243705513,
|
|
"learning_rate": 2.8482579720870234e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1859,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 7980
|
|
},
|
|
{
|
|
"epoch": 1.0212154131254394,
|
|
"grad_norm": 9.655269321006157,
|
|
"learning_rate": 2.842734562700618e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.184,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 7990
|
|
},
|
|
{
|
|
"epoch": 1.0224934500607068,
|
|
"grad_norm": 9.513634678774741,
|
|
"learning_rate": 2.837209447753793e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -572.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1792,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -3.9375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -8.125,
|
|
"step": 8000
|
|
},
|
|
{
|
|
"epoch": 1.0237714869959742,
|
|
"grad_norm": 10.067010169627945,
|
|
"learning_rate": 2.8316826547413423e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1777,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.59375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 8010
|
|
},
|
|
{
|
|
"epoch": 1.0250495239312416,
|
|
"grad_norm": 10.700118461764552,
|
|
"learning_rate": 2.826154211166417e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -596.0,
|
|
"logps/rejected": -972.0,
|
|
"loss": 0.1872,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8020
|
|
},
|
|
{
|
|
"epoch": 1.026327560866509,
|
|
"grad_norm": 11.209752195570033,
|
|
"learning_rate": 2.820624144540377e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 8030
|
|
},
|
|
{
|
|
"epoch": 1.0276055978017764,
|
|
"grad_norm": 6.347408079079281,
|
|
"learning_rate": 2.8150924823826625e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1655,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8040
|
|
},
|
|
{
|
|
"epoch": 1.0288836347370438,
|
|
"grad_norm": 12.357991396104282,
|
|
"learning_rate": 2.809559252220652e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1649,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.90625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8050
|
|
},
|
|
{
|
|
"epoch": 1.0301616716723114,
|
|
"grad_norm": 9.8048668569966,
|
|
"learning_rate": 2.8040244815895254e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1731,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 8060
|
|
},
|
|
{
|
|
"epoch": 1.0314397086075788,
|
|
"grad_norm": 7.393877631897585,
|
|
"learning_rate": 2.798488198032131e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1781,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8070
|
|
},
|
|
{
|
|
"epoch": 1.0327177455428462,
|
|
"grad_norm": 10.729316226388566,
|
|
"learning_rate": 2.792950429098844e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8080
|
|
},
|
|
{
|
|
"epoch": 1.0339957824781136,
|
|
"grad_norm": 7.516989134163046,
|
|
"learning_rate": 2.7874112023474333e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.17,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8090
|
|
},
|
|
{
|
|
"epoch": 1.035273819413381,
|
|
"grad_norm": 6.442338927759854,
|
|
"learning_rate": 2.781870545342921e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8100
|
|
},
|
|
{
|
|
"epoch": 1.0365518563486484,
|
|
"grad_norm": 11.18028552973371,
|
|
"learning_rate": 2.7763284856574483e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1788,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 8110
|
|
},
|
|
{
|
|
"epoch": 1.0378298932839158,
|
|
"grad_norm": 10.878882803166348,
|
|
"learning_rate": 2.7707850508701323e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 8120
|
|
},
|
|
{
|
|
"epoch": 1.0391079302191832,
|
|
"grad_norm": 9.574475689760854,
|
|
"learning_rate": 2.7652402685669387e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1775,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8130
|
|
},
|
|
{
|
|
"epoch": 1.0403859671544509,
|
|
"grad_norm": 11.199987026614055,
|
|
"learning_rate": 2.7596941663405345e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1726,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8140
|
|
},
|
|
{
|
|
"epoch": 1.0416640040897183,
|
|
"grad_norm": 8.36200035927947,
|
|
"learning_rate": 2.7541467717901573e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1811,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8150
|
|
},
|
|
{
|
|
"epoch": 1.0429420410249857,
|
|
"grad_norm": 8.854793907312779,
|
|
"learning_rate": 2.7485981125214764e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1955,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 8160
|
|
},
|
|
{
|
|
"epoch": 1.044220077960253,
|
|
"grad_norm": 7.9267747325867095,
|
|
"learning_rate": 2.743048216146452e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1542,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8170
|
|
},
|
|
{
|
|
"epoch": 1.0454981148955205,
|
|
"grad_norm": 10.39920919594746,
|
|
"learning_rate": 2.737497110283202e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1638,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 8180
|
|
},
|
|
{
|
|
"epoch": 1.0467761518307879,
|
|
"grad_norm": 9.61248623593858,
|
|
"learning_rate": 2.731944822555864e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1757,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 8190
|
|
},
|
|
{
|
|
"epoch": 1.0480541887660553,
|
|
"grad_norm": 12.611978808856993,
|
|
"learning_rate": 2.726391380594456e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1872,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8200
|
|
},
|
|
{
|
|
"epoch": 1.0493322257013227,
|
|
"grad_norm": 5.8579043608799015,
|
|
"learning_rate": 2.720836812034741e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1775,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8210
|
|
},
|
|
{
|
|
"epoch": 1.0506102626365903,
|
|
"grad_norm": 11.77064222897185,
|
|
"learning_rate": 2.715281144518085e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1777,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8220
|
|
},
|
|
{
|
|
"epoch": 1.0518882995718577,
|
|
"grad_norm": 9.77725061181982,
|
|
"learning_rate": 2.709724405691327e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 8230
|
|
},
|
|
{
|
|
"epoch": 1.053166336507125,
|
|
"grad_norm": 8.648896874296353,
|
|
"learning_rate": 2.7041666232066346e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.65625,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 8240
|
|
},
|
|
{
|
|
"epoch": 1.0544443734423925,
|
|
"grad_norm": 9.555378197262757,
|
|
"learning_rate": 2.69860782472137e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1941,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8250
|
|
},
|
|
{
|
|
"epoch": 1.05572241037766,
|
|
"grad_norm": 8.749517199368952,
|
|
"learning_rate": 2.6930480378979513e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -624.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 8260
|
|
},
|
|
{
|
|
"epoch": 1.0570004473129273,
|
|
"grad_norm": 9.905567957439871,
|
|
"learning_rate": 2.687487290403715e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1736,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8270
|
|
},
|
|
{
|
|
"epoch": 1.0582784842481947,
|
|
"grad_norm": 10.75370904435311,
|
|
"learning_rate": 2.6819256099107756e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1674,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 8280
|
|
},
|
|
{
|
|
"epoch": 1.059556521183462,
|
|
"grad_norm": 10.255362189541565,
|
|
"learning_rate": 2.6763630240958946e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1728,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8290
|
|
},
|
|
{
|
|
"epoch": 1.0608345581187297,
|
|
"grad_norm": 10.246388181727587,
|
|
"learning_rate": 2.6707995606403366e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1617,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 8300
|
|
},
|
|
{
|
|
"epoch": 1.0621125950539971,
|
|
"grad_norm": 12.929394573191676,
|
|
"learning_rate": 2.665235247229734e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1687,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8310
|
|
},
|
|
{
|
|
"epoch": 1.0633906319892645,
|
|
"grad_norm": 10.050061531106172,
|
|
"learning_rate": 2.659670111553949e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1782,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8320
|
|
},
|
|
{
|
|
"epoch": 1.064668668924532,
|
|
"grad_norm": 11.590725397037096,
|
|
"learning_rate": 2.654104181306935e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1947,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -8.75,
|
|
"step": 8330
|
|
},
|
|
{
|
|
"epoch": 1.0659467058597993,
|
|
"grad_norm": 10.884747764045423,
|
|
"learning_rate": 2.6485374841865994e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8340
|
|
},
|
|
{
|
|
"epoch": 1.0672247427950667,
|
|
"grad_norm": 9.412234086973548,
|
|
"learning_rate": 2.6429700478946667e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8350
|
|
},
|
|
{
|
|
"epoch": 1.0685027797303341,
|
|
"grad_norm": 9.866005395581741,
|
|
"learning_rate": 2.6374019001365396e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1699,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 8360
|
|
},
|
|
{
|
|
"epoch": 1.0697808166656015,
|
|
"grad_norm": 12.839280711409044,
|
|
"learning_rate": 2.631833068621161e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.185,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8370
|
|
},
|
|
{
|
|
"epoch": 1.0710588536008692,
|
|
"grad_norm": 9.543783056734359,
|
|
"learning_rate": 2.6262635810608763e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8380
|
|
},
|
|
{
|
|
"epoch": 1.0723368905361366,
|
|
"grad_norm": 8.913154707551678,
|
|
"learning_rate": 2.620693465171295e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1592,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 8390
|
|
},
|
|
{
|
|
"epoch": 1.073614927471404,
|
|
"grad_norm": 12.493619267004417,
|
|
"learning_rate": 2.6151227486711547e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1827,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8400
|
|
},
|
|
{
|
|
"epoch": 1.0748929644066714,
|
|
"grad_norm": 11.245710975859144,
|
|
"learning_rate": 2.6095514592821814e-07,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1703,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8410
|
|
},
|
|
{
|
|
"epoch": 1.0761710013419388,
|
|
"grad_norm": 10.870414471136572,
|
|
"learning_rate": 2.603979624728952e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1000.0,
|
|
"loss": 0.1635,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 3.6875,
|
|
"rewards/rejected": -8.375,
|
|
"step": 8420
|
|
},
|
|
{
|
|
"epoch": 1.0774490382772062,
|
|
"grad_norm": 11.81784499074986,
|
|
"learning_rate": 2.598407272738756e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1618,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 8430
|
|
},
|
|
{
|
|
"epoch": 1.0787270752124736,
|
|
"grad_norm": 8.920813120865647,
|
|
"learning_rate": 2.592834431041457e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1677,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8440
|
|
},
|
|
{
|
|
"epoch": 1.080005112147741,
|
|
"grad_norm": 9.651821827804635,
|
|
"learning_rate": 2.5872611273693573e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8450
|
|
},
|
|
{
|
|
"epoch": 1.0812831490830086,
|
|
"grad_norm": 7.782821307879692,
|
|
"learning_rate": 2.581687389457058e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.5,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1506,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 8460
|
|
},
|
|
{
|
|
"epoch": 1.082561186018276,
|
|
"grad_norm": 11.993079437671195,
|
|
"learning_rate": 2.576113245041319e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 8470
|
|
},
|
|
{
|
|
"epoch": 1.0838392229535434,
|
|
"grad_norm": 9.027091864147854,
|
|
"learning_rate": 2.5705387218609267e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1875,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8480
|
|
},
|
|
{
|
|
"epoch": 1.0851172598888108,
|
|
"grad_norm": 11.172202529858502,
|
|
"learning_rate": 2.5649638476565485e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1774,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 8490
|
|
},
|
|
{
|
|
"epoch": 1.0863952968240782,
|
|
"grad_norm": 8.366326928331228,
|
|
"learning_rate": 2.5593886501705994e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1814,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.125,
|
|
"step": 8500
|
|
},
|
|
{
|
|
"epoch": 1.0876733337593456,
|
|
"grad_norm": 15.49608599918426,
|
|
"learning_rate": 2.553813157147107e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8510
|
|
},
|
|
{
|
|
"epoch": 1.088951370694613,
|
|
"grad_norm": 13.194232623449643,
|
|
"learning_rate": 2.548237396331564e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1812,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 8520
|
|
},
|
|
{
|
|
"epoch": 1.0902294076298804,
|
|
"grad_norm": 10.570147031293036,
|
|
"learning_rate": 2.5426613954708e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1892,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.78125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8530
|
|
},
|
|
{
|
|
"epoch": 1.091507444565148,
|
|
"grad_norm": 12.792969719649745,
|
|
"learning_rate": 2.5370851823128376e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1781,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 8540
|
|
},
|
|
{
|
|
"epoch": 1.0927854815004154,
|
|
"grad_norm": 8.155866731584963,
|
|
"learning_rate": 2.5315087846067546e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1661,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8550
|
|
},
|
|
{
|
|
"epoch": 1.0940635184356828,
|
|
"grad_norm": 11.43299208228307,
|
|
"learning_rate": 2.5259322301025493e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.125,
|
|
"step": 8560
|
|
},
|
|
{
|
|
"epoch": 1.0953415553709502,
|
|
"grad_norm": 11.581246848208384,
|
|
"learning_rate": 2.520355546550999e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1762,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 8570
|
|
},
|
|
{
|
|
"epoch": 1.0966195923062176,
|
|
"grad_norm": 7.5531191014349375,
|
|
"learning_rate": 2.5147787617035235e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 8580
|
|
},
|
|
{
|
|
"epoch": 1.097897629241485,
|
|
"grad_norm": 12.22511911705573,
|
|
"learning_rate": 2.5092019033120484e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1901,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 8590
|
|
},
|
|
{
|
|
"epoch": 1.0991756661767524,
|
|
"grad_norm": 10.172246205013995,
|
|
"learning_rate": 2.5036249991288614e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1892,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 8600
|
|
},
|
|
{
|
|
"epoch": 1.1004537031120198,
|
|
"grad_norm": 8.523153345006731,
|
|
"learning_rate": 2.4980480769064805e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -604.0,
|
|
"logps/rejected": -964.0,
|
|
"loss": 0.1713,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.5625,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -8.3125,
|
|
"step": 8610
|
|
},
|
|
{
|
|
"epoch": 1.1017317400472875,
|
|
"grad_norm": 13.46349720976093,
|
|
"learning_rate": 2.492471164397514e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -988.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.46875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.4375,
|
|
"step": 8620
|
|
},
|
|
{
|
|
"epoch": 1.1030097769825549,
|
|
"grad_norm": 12.692605192581905,
|
|
"learning_rate": 2.486894289354521e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -580.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1991,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.34375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8630
|
|
},
|
|
{
|
|
"epoch": 1.1042878139178223,
|
|
"grad_norm": 11.684460877635107,
|
|
"learning_rate": 2.4813174795298753e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -588.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1707,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.28125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8640
|
|
},
|
|
{
|
|
"epoch": 1.1055658508530897,
|
|
"grad_norm": 11.138998731809462,
|
|
"learning_rate": 2.475740762675623e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -968.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.25,
|
|
"rewards/margins": 3.625,
|
|
"rewards/rejected": -7.875,
|
|
"step": 8650
|
|
},
|
|
{
|
|
"epoch": 1.106843887788357,
|
|
"grad_norm": 10.182232860207785,
|
|
"learning_rate": 2.4701641665433514e-07,
|
|
"logits/chosen": -2.78125,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -980.0,
|
|
"loss": 0.1732,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8660
|
|
},
|
|
{
|
|
"epoch": 1.1081219247236245,
|
|
"grad_norm": 11.533017525853683,
|
|
"learning_rate": 2.464587718884044e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -996.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8670
|
|
},
|
|
{
|
|
"epoch": 1.1093999616588919,
|
|
"grad_norm": 10.19906423287545,
|
|
"learning_rate": 2.459011447447947e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -600.0,
|
|
"logps/rejected": -960.0,
|
|
"loss": 0.1667,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.1875,
|
|
"step": 8680
|
|
},
|
|
{
|
|
"epoch": 1.1106779985941593,
|
|
"grad_norm": 12.307725593269353,
|
|
"learning_rate": 2.45343537998443e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1712,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.53125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8690
|
|
},
|
|
{
|
|
"epoch": 1.111956035529427,
|
|
"grad_norm": 13.901330347133428,
|
|
"learning_rate": 2.4478595442418473e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 8700
|
|
},
|
|
{
|
|
"epoch": 1.1132340724646943,
|
|
"grad_norm": 8.802904680915939,
|
|
"learning_rate": 2.4422839679673996e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -984.0,
|
|
"loss": 0.1607,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 8710
|
|
},
|
|
{
|
|
"epoch": 1.1145121093999617,
|
|
"grad_norm": 10.39181707302118,
|
|
"learning_rate": 2.4367086789069954e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1968,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 8720
|
|
},
|
|
{
|
|
"epoch": 1.115790146335229,
|
|
"grad_norm": 8.971322923592636,
|
|
"learning_rate": 2.4311337048051163e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1911,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 8730
|
|
},
|
|
{
|
|
"epoch": 1.1170681832704965,
|
|
"grad_norm": 8.71816008693693,
|
|
"learning_rate": 2.425559073404675e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -612.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8740
|
|
},
|
|
{
|
|
"epoch": 1.118346220205764,
|
|
"grad_norm": 11.110149770588588,
|
|
"learning_rate": 2.41998481244688e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1735,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 8750
|
|
},
|
|
{
|
|
"epoch": 1.1196242571410313,
|
|
"grad_norm": 8.80837913365687,
|
|
"learning_rate": 2.4144109496710936e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1754,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 8760
|
|
},
|
|
{
|
|
"epoch": 1.1209022940762987,
|
|
"grad_norm": 8.1589054813181,
|
|
"learning_rate": 2.4088375128146997e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.161,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8770
|
|
},
|
|
{
|
|
"epoch": 1.1221803310115663,
|
|
"grad_norm": 11.945639642235644,
|
|
"learning_rate": 2.403264529612962e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1504,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.78125,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 8780
|
|
},
|
|
{
|
|
"epoch": 1.1234583679468337,
|
|
"grad_norm": 11.569953335116981,
|
|
"learning_rate": 2.3976920277988854e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 8790
|
|
},
|
|
{
|
|
"epoch": 1.1247364048821011,
|
|
"grad_norm": 13.598732599331584,
|
|
"learning_rate": 2.3921200351030807e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1702,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 8800
|
|
},
|
|
{
|
|
"epoch": 1.1260144418173685,
|
|
"grad_norm": 7.749172338773482,
|
|
"learning_rate": 2.386548579253624e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1687,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 8810
|
|
},
|
|
{
|
|
"epoch": 1.127292478752636,
|
|
"grad_norm": 9.869203051606881,
|
|
"learning_rate": 2.3809776879759203e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.179,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8820
|
|
},
|
|
{
|
|
"epoch": 1.1285705156879033,
|
|
"grad_norm": 9.212740949988463,
|
|
"learning_rate": 2.3754073889925655e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8830
|
|
},
|
|
{
|
|
"epoch": 1.1298485526231707,
|
|
"grad_norm": 8.355055868788307,
|
|
"learning_rate": 2.3698377100232073e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1797,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 8840
|
|
},
|
|
{
|
|
"epoch": 1.1311265895584381,
|
|
"grad_norm": 8.253615583958148,
|
|
"learning_rate": 2.3642686787844095e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1772,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8850
|
|
},
|
|
{
|
|
"epoch": 1.1324046264937055,
|
|
"grad_norm": 11.200655287502787,
|
|
"learning_rate": 2.3587003229895108e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1020.0,
|
|
"loss": 0.1658,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -8.875,
|
|
"step": 8860
|
|
},
|
|
{
|
|
"epoch": 1.1336826634289732,
|
|
"grad_norm": 9.215990244514574,
|
|
"learning_rate": 2.3531326703484895e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -620.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1758,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 8870
|
|
},
|
|
{
|
|
"epoch": 1.1349607003642406,
|
|
"grad_norm": 14.804600873057607,
|
|
"learning_rate": 2.3475657485678235e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -1008.0,
|
|
"loss": 0.1901,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -4.625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.625,
|
|
"step": 8880
|
|
},
|
|
{
|
|
"epoch": 1.136238737299508,
|
|
"grad_norm": 9.707687199553083,
|
|
"learning_rate": 2.341999585350356e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -616.0,
|
|
"logps/rejected": -976.0,
|
|
"loss": 0.1887,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.6875,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -8.5,
|
|
"step": 8890
|
|
},
|
|
{
|
|
"epoch": 1.1375167742347754,
|
|
"grad_norm": 12.930001235443862,
|
|
"learning_rate": 2.336434208395154e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.18,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 8900
|
|
},
|
|
{
|
|
"epoch": 1.1387948111700428,
|
|
"grad_norm": 10.257419846335008,
|
|
"learning_rate": 2.3308696453973726e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1751,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8910
|
|
},
|
|
{
|
|
"epoch": 1.1400728481053102,
|
|
"grad_norm": 8.426972164720423,
|
|
"learning_rate": 2.325305924048115e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1769,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 8920
|
|
},
|
|
{
|
|
"epoch": 1.1413508850405776,
|
|
"grad_norm": 9.51570501986898,
|
|
"learning_rate": 2.319743072034297e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1778,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8930
|
|
},
|
|
{
|
|
"epoch": 1.1426289219758452,
|
|
"grad_norm": 13.154249035166893,
|
|
"learning_rate": 2.3141811170385078e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -632.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1927,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 8940
|
|
},
|
|
{
|
|
"epoch": 1.1439069589111126,
|
|
"grad_norm": 8.837785292901133,
|
|
"learning_rate": 2.308620086738874e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1012.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 8950
|
|
},
|
|
{
|
|
"epoch": 1.14518499584638,
|
|
"grad_norm": 10.641521132461545,
|
|
"learning_rate": 2.303060008808919e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1879,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 8960
|
|
},
|
|
{
|
|
"epoch": 1.1464630327816474,
|
|
"grad_norm": 7.139401203739757,
|
|
"learning_rate": 2.2975009109174282e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1647,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 8970
|
|
},
|
|
{
|
|
"epoch": 1.1477410697169148,
|
|
"grad_norm": 9.674884483133546,
|
|
"learning_rate": 2.291942820728308e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.183,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 8980
|
|
},
|
|
{
|
|
"epoch": 1.1490191066521822,
|
|
"grad_norm": 7.8995664380627435,
|
|
"learning_rate": 2.286385765900453e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1661,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.65625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 8990
|
|
},
|
|
{
|
|
"epoch": 1.1502971435874496,
|
|
"grad_norm": 10.201099604073594,
|
|
"learning_rate": 2.2808297740876038e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 9000
|
|
},
|
|
{
|
|
"epoch": 1.151575180522717,
|
|
"grad_norm": 13.045559871799567,
|
|
"learning_rate": 2.2752748729382108e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.162,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9010
|
|
},
|
|
{
|
|
"epoch": 1.1528532174579844,
|
|
"grad_norm": 10.931551568686894,
|
|
"learning_rate": 2.269721090095298e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1973,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9020
|
|
},
|
|
{
|
|
"epoch": 1.154131254393252,
|
|
"grad_norm": 9.588251761779542,
|
|
"learning_rate": 2.2641684531963233e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1832,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 9030
|
|
},
|
|
{
|
|
"epoch": 1.1554092913285194,
|
|
"grad_norm": 10.087702912942275,
|
|
"learning_rate": 2.2586169898730425e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1692,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 9040
|
|
},
|
|
{
|
|
"epoch": 1.1566873282637868,
|
|
"grad_norm": 12.558033292582566,
|
|
"learning_rate": 2.2530667277513713e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 9050
|
|
},
|
|
{
|
|
"epoch": 1.1579653651990542,
|
|
"grad_norm": 9.862234243673335,
|
|
"learning_rate": 2.247517694451248e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9060
|
|
},
|
|
{
|
|
"epoch": 1.1592434021343216,
|
|
"grad_norm": 8.483102506930782,
|
|
"learning_rate": 2.2419699175864964e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1684,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.0,
|
|
"step": 9070
|
|
},
|
|
{
|
|
"epoch": 1.160521439069589,
|
|
"grad_norm": 10.365093327270756,
|
|
"learning_rate": 2.2364234247646853e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1785,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 9080
|
|
},
|
|
{
|
|
"epoch": 1.1617994760048564,
|
|
"grad_norm": 11.078748531859485,
|
|
"learning_rate": 2.230878243586997e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1895,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 9090
|
|
},
|
|
{
|
|
"epoch": 1.163077512940124,
|
|
"grad_norm": 11.756344722412415,
|
|
"learning_rate": 2.225334401648084e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9100
|
|
},
|
|
{
|
|
"epoch": 1.1643555498753915,
|
|
"grad_norm": 12.728237864635561,
|
|
"learning_rate": 2.219791926535936e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9110
|
|
},
|
|
{
|
|
"epoch": 1.1656335868106589,
|
|
"grad_norm": 8.251473794642155,
|
|
"learning_rate": 2.2142508458317406e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.172,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9120
|
|
},
|
|
{
|
|
"epoch": 1.1669116237459263,
|
|
"grad_norm": 13.422178539672343,
|
|
"learning_rate": 2.2087111871097455e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1675,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9130
|
|
},
|
|
{
|
|
"epoch": 1.1681896606811937,
|
|
"grad_norm": 15.75211685357834,
|
|
"learning_rate": 2.2031729779371237e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1719,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9140
|
|
},
|
|
{
|
|
"epoch": 1.169467697616461,
|
|
"grad_norm": 9.128192258566758,
|
|
"learning_rate": 2.1976362458738333e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9150
|
|
},
|
|
{
|
|
"epoch": 1.1707457345517285,
|
|
"grad_norm": 9.468398215689838,
|
|
"learning_rate": 2.1921010184724816e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9160
|
|
},
|
|
{
|
|
"epoch": 1.1720237714869959,
|
|
"grad_norm": 13.013947158173366,
|
|
"learning_rate": 2.1865673232781901e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1881,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9170
|
|
},
|
|
{
|
|
"epoch": 1.1733018084222633,
|
|
"grad_norm": 10.06906757226758,
|
|
"learning_rate": 2.1810351878284538e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1936,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9180
|
|
},
|
|
{
|
|
"epoch": 1.174579845357531,
|
|
"grad_norm": 13.95054752379249,
|
|
"learning_rate": 2.1755046396530074e-07,
|
|
"logits/chosen": -2.75,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1618,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 9190
|
|
},
|
|
{
|
|
"epoch": 1.1758578822927983,
|
|
"grad_norm": 12.364774430012691,
|
|
"learning_rate": 2.1699757062736845e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.155,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9200
|
|
},
|
|
{
|
|
"epoch": 1.1771359192280657,
|
|
"grad_norm": 8.761179985697268,
|
|
"learning_rate": 2.164448415204285e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1498,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9210
|
|
},
|
|
{
|
|
"epoch": 1.178413956163333,
|
|
"grad_norm": 13.107311109960998,
|
|
"learning_rate": 2.158922793950435e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 9220
|
|
},
|
|
{
|
|
"epoch": 1.1796919930986005,
|
|
"grad_norm": 8.684428498052904,
|
|
"learning_rate": 2.153398870009453e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1783,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9230
|
|
},
|
|
{
|
|
"epoch": 1.180970030033868,
|
|
"grad_norm": 8.989886993845838,
|
|
"learning_rate": 2.1478766708702074e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.484375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.157,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.796875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 9240
|
|
},
|
|
{
|
|
"epoch": 1.1822480669691353,
|
|
"grad_norm": 9.158778773808871,
|
|
"learning_rate": 2.1423562240129869e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9250
|
|
},
|
|
{
|
|
"epoch": 1.183526103904403,
|
|
"grad_norm": 12.167043112847624,
|
|
"learning_rate": 2.1368375569093587e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1717,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9260
|
|
},
|
|
{
|
|
"epoch": 1.1848041408396703,
|
|
"grad_norm": 9.956515597502806,
|
|
"learning_rate": 2.1313206970220338e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1736,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9270
|
|
},
|
|
{
|
|
"epoch": 1.1860821777749377,
|
|
"grad_norm": 8.82080127275406,
|
|
"learning_rate": 2.1258056718047298e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1758,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 9280
|
|
},
|
|
{
|
|
"epoch": 1.1873602147102051,
|
|
"grad_norm": 10.48275029797209,
|
|
"learning_rate": 2.1202925087020346e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1759,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9290
|
|
},
|
|
{
|
|
"epoch": 1.1886382516454725,
|
|
"grad_norm": 10.416324291817167,
|
|
"learning_rate": 2.1147812351492687e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1829,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 9300
|
|
},
|
|
{
|
|
"epoch": 1.18991628858074,
|
|
"grad_norm": 12.359961817177224,
|
|
"learning_rate": 2.1092718785723497e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9310
|
|
},
|
|
{
|
|
"epoch": 1.1911943255160073,
|
|
"grad_norm": 9.133749012355143,
|
|
"learning_rate": 2.1037644663876566e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1815,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9320
|
|
},
|
|
{
|
|
"epoch": 1.1924723624512747,
|
|
"grad_norm": 8.607099471081787,
|
|
"learning_rate": 2.0982590260018923e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1873,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9330
|
|
},
|
|
{
|
|
"epoch": 1.1937503993865421,
|
|
"grad_norm": 7.027959067289001,
|
|
"learning_rate": 2.0927555848119472e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1758,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.859375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9340
|
|
},
|
|
{
|
|
"epoch": 1.1950284363218098,
|
|
"grad_norm": 9.677106712552908,
|
|
"learning_rate": 2.0872541702047635e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9350
|
|
},
|
|
{
|
|
"epoch": 1.1963064732570772,
|
|
"grad_norm": 9.172920122127982,
|
|
"learning_rate": 2.0817548095571965e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1778,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9360
|
|
},
|
|
{
|
|
"epoch": 1.1975845101923446,
|
|
"grad_norm": 14.141110110747428,
|
|
"learning_rate": 2.0762575302358827e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1882,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.15625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 9370
|
|
},
|
|
{
|
|
"epoch": 1.198862547127612,
|
|
"grad_norm": 9.339406721803096,
|
|
"learning_rate": 2.0707623595970995e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1756,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 9380
|
|
},
|
|
{
|
|
"epoch": 1.2001405840628794,
|
|
"grad_norm": 10.981534315380854,
|
|
"learning_rate": 2.0652693249866328e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9390
|
|
},
|
|
{
|
|
"epoch": 1.2014186209981468,
|
|
"grad_norm": 8.495947332387841,
|
|
"learning_rate": 2.059778453739637e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.71875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 9400
|
|
},
|
|
{
|
|
"epoch": 1.2026966579334142,
|
|
"grad_norm": 10.158284185990194,
|
|
"learning_rate": 2.0542897731805013e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9410
|
|
},
|
|
{
|
|
"epoch": 1.2039746948686818,
|
|
"grad_norm": 9.832753006390973,
|
|
"learning_rate": 2.0488033106227138e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1718,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9420
|
|
},
|
|
{
|
|
"epoch": 1.2052527318039492,
|
|
"grad_norm": 16.372998096961464,
|
|
"learning_rate": 2.0433190933687246e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.181,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 9430
|
|
},
|
|
{
|
|
"epoch": 1.2065307687392166,
|
|
"grad_norm": 11.30856271193926,
|
|
"learning_rate": 2.0378371487098115e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1747,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 9440
|
|
},
|
|
{
|
|
"epoch": 1.207808805674484,
|
|
"grad_norm": 9.448324520200229,
|
|
"learning_rate": 2.0323575039259419e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1657,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 9450
|
|
},
|
|
{
|
|
"epoch": 1.2090868426097514,
|
|
"grad_norm": 8.391376110479527,
|
|
"learning_rate": 2.0268801862856383e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9460
|
|
},
|
|
{
|
|
"epoch": 1.2103648795450188,
|
|
"grad_norm": 11.48008422005276,
|
|
"learning_rate": 2.0214052230458434e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1579,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9470
|
|
},
|
|
{
|
|
"epoch": 1.2116429164802862,
|
|
"grad_norm": 9.857979206562652,
|
|
"learning_rate": 2.015932641451783e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 9480
|
|
},
|
|
{
|
|
"epoch": 1.2129209534155536,
|
|
"grad_norm": 13.596627334327412,
|
|
"learning_rate": 2.010462468736831e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1004.0,
|
|
"loss": 0.1696,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -8.75,
|
|
"step": 9490
|
|
},
|
|
{
|
|
"epoch": 1.214198990350821,
|
|
"grad_norm": 8.00173857521025,
|
|
"learning_rate": 2.004994732122376e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 9500
|
|
},
|
|
{
|
|
"epoch": 1.2154770272860886,
|
|
"grad_norm": 13.184302458769404,
|
|
"learning_rate": 1.9995294588176787e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1756,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.25,
|
|
"step": 9510
|
|
},
|
|
{
|
|
"epoch": 1.216755064221356,
|
|
"grad_norm": 7.550889466931553,
|
|
"learning_rate": 1.9940666760197469e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9520
|
|
},
|
|
{
|
|
"epoch": 1.2180331011566234,
|
|
"grad_norm": 9.583242395878697,
|
|
"learning_rate": 1.9886064109131917e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1665,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -8.75,
|
|
"step": 9530
|
|
},
|
|
{
|
|
"epoch": 1.2193111380918908,
|
|
"grad_norm": 8.254257411573414,
|
|
"learning_rate": 1.9831486906700966e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -636.0,
|
|
"logps/rejected": -1024.0,
|
|
"loss": 0.1581,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -8.875,
|
|
"step": 9540
|
|
},
|
|
{
|
|
"epoch": 1.2205891750271582,
|
|
"grad_norm": 10.525671293076426,
|
|
"learning_rate": 1.9776935424498812e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1593,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9550
|
|
},
|
|
{
|
|
"epoch": 1.2218672119624256,
|
|
"grad_norm": 12.569317958096246,
|
|
"learning_rate": 1.9722409933991647e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.16,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 9560
|
|
},
|
|
{
|
|
"epoch": 1.223145248897693,
|
|
"grad_norm": 8.083359903906649,
|
|
"learning_rate": 1.9667910706516336e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1593,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9570
|
|
},
|
|
{
|
|
"epoch": 1.2244232858329607,
|
|
"grad_norm": 9.861583608780453,
|
|
"learning_rate": 1.9613438013279028e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1597,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9580
|
|
},
|
|
{
|
|
"epoch": 1.225701322768228,
|
|
"grad_norm": 8.484208509238131,
|
|
"learning_rate": 1.955899212535385e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1682,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9590
|
|
},
|
|
{
|
|
"epoch": 1.2269793597034955,
|
|
"grad_norm": 8.632396255358831,
|
|
"learning_rate": 1.950457331368153e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1537,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 9600
|
|
},
|
|
{
|
|
"epoch": 1.2282573966387629,
|
|
"grad_norm": 12.19240726948117,
|
|
"learning_rate": 1.945018184906805e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.0,
|
|
"step": 9610
|
|
},
|
|
{
|
|
"epoch": 1.2295354335740303,
|
|
"grad_norm": 9.047734921539412,
|
|
"learning_rate": 1.9395818002183316e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 9620
|
|
},
|
|
{
|
|
"epoch": 1.2308134705092977,
|
|
"grad_norm": 11.425663980736191,
|
|
"learning_rate": 1.934148204355978e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.515625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9630
|
|
},
|
|
{
|
|
"epoch": 1.232091507444565,
|
|
"grad_norm": 8.965728172000757,
|
|
"learning_rate": 1.9287174243591134e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9640
|
|
},
|
|
{
|
|
"epoch": 1.2333695443798325,
|
|
"grad_norm": 8.57673320822646,
|
|
"learning_rate": 1.9232894872530928e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.188,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9650
|
|
},
|
|
{
|
|
"epoch": 1.2346475813150999,
|
|
"grad_norm": 11.365561069929564,
|
|
"learning_rate": 1.9178644200491248e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1525,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9660
|
|
},
|
|
{
|
|
"epoch": 1.2359256182503675,
|
|
"grad_norm": 12.077874161518164,
|
|
"learning_rate": 1.9124422497441366e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1489,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9670
|
|
},
|
|
{
|
|
"epoch": 1.237203655185635,
|
|
"grad_norm": 11.7683730356947,
|
|
"learning_rate": 1.9070230033206385e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1536,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9680
|
|
},
|
|
{
|
|
"epoch": 1.2384816921209023,
|
|
"grad_norm": 9.769742084109476,
|
|
"learning_rate": 1.9016067077465912e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1748,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 9690
|
|
},
|
|
{
|
|
"epoch": 1.2397597290561697,
|
|
"grad_norm": 10.711727609047516,
|
|
"learning_rate": 1.896193389975271e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9700
|
|
},
|
|
{
|
|
"epoch": 1.2410377659914371,
|
|
"grad_norm": 8.709962769552265,
|
|
"learning_rate": 1.890783076945136e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1737,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 9710
|
|
},
|
|
{
|
|
"epoch": 1.2423158029267045,
|
|
"grad_norm": 15.964742767418263,
|
|
"learning_rate": 1.8853757955796924e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1666,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9720
|
|
},
|
|
{
|
|
"epoch": 1.243593839861972,
|
|
"grad_norm": 11.204694589329842,
|
|
"learning_rate": 1.8799715727873565e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1752,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9730
|
|
},
|
|
{
|
|
"epoch": 1.2448718767972395,
|
|
"grad_norm": 13.582403295228488,
|
|
"learning_rate": 1.8745704354613278e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1652,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9740
|
|
},
|
|
{
|
|
"epoch": 1.246149913732507,
|
|
"grad_norm": 9.514114838209917,
|
|
"learning_rate": 1.8691724104794493e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1769,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9750
|
|
},
|
|
{
|
|
"epoch": 1.2474279506677743,
|
|
"grad_norm": 10.536194154771247,
|
|
"learning_rate": 1.8637775247040777e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -776.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1832,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9760
|
|
},
|
|
{
|
|
"epoch": 1.2487059876030417,
|
|
"grad_norm": 13.45511967286933,
|
|
"learning_rate": 1.8583858049819464e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1734,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 3.671875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 9770
|
|
},
|
|
{
|
|
"epoch": 1.2499840245383091,
|
|
"grad_norm": 13.317071101199458,
|
|
"learning_rate": 1.8529972781440345e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.17,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9780
|
|
},
|
|
{
|
|
"epoch": 1.2512620614735765,
|
|
"grad_norm": 10.23423676486201,
|
|
"learning_rate": 1.8476119710054323e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.46875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 9790
|
|
},
|
|
{
|
|
"epoch": 1.252540098408844,
|
|
"grad_norm": 10.124955326944248,
|
|
"learning_rate": 1.8422299103652063e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1892,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 9800
|
|
},
|
|
{
|
|
"epoch": 1.2538181353441114,
|
|
"grad_norm": 8.042244637589551,
|
|
"learning_rate": 1.8368511230062695e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1708,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9810
|
|
},
|
|
{
|
|
"epoch": 1.2550961722793788,
|
|
"grad_norm": 9.73056988456358,
|
|
"learning_rate": 1.8314756356952438e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 9820
|
|
},
|
|
{
|
|
"epoch": 1.2563742092146464,
|
|
"grad_norm": 9.299217725085608,
|
|
"learning_rate": 1.8261034751823316e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.161,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9830
|
|
},
|
|
{
|
|
"epoch": 1.2576522461499138,
|
|
"grad_norm": 12.508446663068236,
|
|
"learning_rate": 1.820734668201177e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1526,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.625,
|
|
"step": 9840
|
|
},
|
|
{
|
|
"epoch": 1.2589302830851812,
|
|
"grad_norm": 12.023850516348142,
|
|
"learning_rate": 1.815369241468738e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1774,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 9850
|
|
},
|
|
{
|
|
"epoch": 1.2602083200204486,
|
|
"grad_norm": 14.490328800702788,
|
|
"learning_rate": 1.8100072216851508e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -772.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 3.734375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 9860
|
|
},
|
|
{
|
|
"epoch": 1.261486356955716,
|
|
"grad_norm": 10.975165721589365,
|
|
"learning_rate": 1.8046486355335972e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1568,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.125,
|
|
"step": 9870
|
|
},
|
|
{
|
|
"epoch": 1.2627643938909834,
|
|
"grad_norm": 9.49015858824327,
|
|
"learning_rate": 1.799293509680173e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.96875,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9880
|
|
},
|
|
{
|
|
"epoch": 1.264042430826251,
|
|
"grad_norm": 9.126750240492946,
|
|
"learning_rate": 1.793941870773753e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1575,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9890
|
|
},
|
|
{
|
|
"epoch": 1.2653204677615184,
|
|
"grad_norm": 8.488616696424357,
|
|
"learning_rate": 1.7885937454458605e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1381,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9900
|
|
},
|
|
{
|
|
"epoch": 1.2665985046967858,
|
|
"grad_norm": 9.060288256850551,
|
|
"learning_rate": 1.7832491603105346e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1625,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0,
|
|
"step": 9910
|
|
},
|
|
{
|
|
"epoch": 1.2678765416320532,
|
|
"grad_norm": 13.196118121665412,
|
|
"learning_rate": 1.7779081419641963e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1741,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 9920
|
|
},
|
|
{
|
|
"epoch": 1.2691545785673206,
|
|
"grad_norm": 16.944706503313817,
|
|
"learning_rate": 1.772570716985519e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.176,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 3.875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 9930
|
|
},
|
|
{
|
|
"epoch": 1.270432615502588,
|
|
"grad_norm": 14.012718602248546,
|
|
"learning_rate": 1.7672369119352903e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 9940
|
|
},
|
|
{
|
|
"epoch": 1.2717106524378554,
|
|
"grad_norm": 13.799438014368707,
|
|
"learning_rate": 1.7619067533562887e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1762,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 9950
|
|
},
|
|
{
|
|
"epoch": 1.2729886893731228,
|
|
"grad_norm": 9.789102158125887,
|
|
"learning_rate": 1.7565802677731423e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9960
|
|
},
|
|
{
|
|
"epoch": 1.2742667263083902,
|
|
"grad_norm": 11.670905648113973,
|
|
"learning_rate": 1.7512574816922044e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 9970
|
|
},
|
|
{
|
|
"epoch": 1.2755447632436576,
|
|
"grad_norm": 10.907827426264625,
|
|
"learning_rate": 1.745938421601417e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1774,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 9980
|
|
},
|
|
{
|
|
"epoch": 1.2768228001789252,
|
|
"grad_norm": 9.678391040150258,
|
|
"learning_rate": 1.7406231139701806e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.173,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 9990
|
|
},
|
|
{
|
|
"epoch": 1.2781008371141926,
|
|
"grad_norm": 13.97763085395111,
|
|
"learning_rate": 1.735311585249222e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1801,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 1.2781008371141926,
|
|
"eval_logits/chosen": -2.984375,
|
|
"eval_logits/rejected": -2.765625,
|
|
"eval_logps/chosen": -720.0,
|
|
"eval_logps/rejected": -1080.0,
|
|
"eval_loss": 0.23926551640033722,
|
|
"eval_rewards/accuracies": 0.8974654674530029,
|
|
"eval_rewards/chosen": -5.625,
|
|
"eval_rewards/margins": 3.78125,
|
|
"eval_rewards/rejected": -9.375,
|
|
"eval_runtime": 1884.4699,
|
|
"eval_samples_per_second": 29.454,
|
|
"eval_steps_per_second": 0.461,
|
|
"step": 10000
|
|
},
|
|
{
|
|
"epoch": 1.27937887404946,
|
|
"grad_norm": 11.61678325539143,
|
|
"learning_rate": 1.7300038618704626e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1605,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10010
|
|
},
|
|
{
|
|
"epoch": 1.2806569109847274,
|
|
"grad_norm": 7.438166564405569,
|
|
"learning_rate": 1.7246999702468884e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1613,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10020
|
|
},
|
|
{
|
|
"epoch": 1.2819349479199948,
|
|
"grad_norm": 7.976608253366539,
|
|
"learning_rate": 1.7193999367724156e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1434,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10030
|
|
},
|
|
{
|
|
"epoch": 1.2832129848552623,
|
|
"grad_norm": 12.29189907278104,
|
|
"learning_rate": 1.7141037878217624e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1792,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10040
|
|
},
|
|
{
|
|
"epoch": 1.2844910217905299,
|
|
"grad_norm": 8.263784179594328,
|
|
"learning_rate": 1.7088115497503163e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 10050
|
|
},
|
|
{
|
|
"epoch": 1.2857690587257973,
|
|
"grad_norm": 9.788017604997307,
|
|
"learning_rate": 1.7035232488940013e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1743,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 10060
|
|
},
|
|
{
|
|
"epoch": 1.2870470956610647,
|
|
"grad_norm": 12.141864032176112,
|
|
"learning_rate": 1.6982389115691502e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1547,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10070
|
|
},
|
|
{
|
|
"epoch": 1.288325132596332,
|
|
"grad_norm": 8.15304150250457,
|
|
"learning_rate": 1.6929585640723715e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.141,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10080
|
|
},
|
|
{
|
|
"epoch": 1.2896031695315995,
|
|
"grad_norm": 12.365972275709149,
|
|
"learning_rate": 1.687682232680419e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10090
|
|
},
|
|
{
|
|
"epoch": 1.2908812064668669,
|
|
"grad_norm": 11.148985181311243,
|
|
"learning_rate": 1.682409943650061e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.147,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10100
|
|
},
|
|
{
|
|
"epoch": 1.2921592434021343,
|
|
"grad_norm": 9.910706559244055,
|
|
"learning_rate": 1.6771417232179491e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -644.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.125,
|
|
"step": 10110
|
|
},
|
|
{
|
|
"epoch": 1.2934372803374017,
|
|
"grad_norm": 14.49757489823456,
|
|
"learning_rate": 1.6718775976004896e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.84375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.125,
|
|
"step": 10120
|
|
},
|
|
{
|
|
"epoch": 1.294715317272669,
|
|
"grad_norm": 8.542572240275735,
|
|
"learning_rate": 1.66661759299371e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1454,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 10130
|
|
},
|
|
{
|
|
"epoch": 1.2959933542079365,
|
|
"grad_norm": 8.145781278602318,
|
|
"learning_rate": 1.6613617355731323e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 3.9375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 10140
|
|
},
|
|
{
|
|
"epoch": 1.2972713911432041,
|
|
"grad_norm": 13.633869435963755,
|
|
"learning_rate": 1.6561100514936384e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1618,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10150
|
|
},
|
|
{
|
|
"epoch": 1.2985494280784715,
|
|
"grad_norm": 12.090786431572226,
|
|
"learning_rate": 1.650862566889343e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1743,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 5.03125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10160
|
|
},
|
|
{
|
|
"epoch": 1.299827465013739,
|
|
"grad_norm": 10.97838475586465,
|
|
"learning_rate": 1.6456193078734648e-07,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10170
|
|
},
|
|
{
|
|
"epoch": 1.3011055019490063,
|
|
"grad_norm": 9.673857765098305,
|
|
"learning_rate": 1.6403803005381913e-07,
|
|
"logits/chosen": -2.796875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10180
|
|
},
|
|
{
|
|
"epoch": 1.3023835388842737,
|
|
"grad_norm": 12.851507523488168,
|
|
"learning_rate": 1.635145570954555e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1687,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10190
|
|
},
|
|
{
|
|
"epoch": 1.3036615758195411,
|
|
"grad_norm": 11.353888250383354,
|
|
"learning_rate": 1.6299151451722991e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 10200
|
|
},
|
|
{
|
|
"epoch": 1.3049396127548087,
|
|
"grad_norm": 9.29453192994112,
|
|
"learning_rate": 1.6246890492197516e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10210
|
|
},
|
|
{
|
|
"epoch": 1.3062176496900761,
|
|
"grad_norm": 11.056729555742567,
|
|
"learning_rate": 1.619467309103692e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.148,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10220
|
|
},
|
|
{
|
|
"epoch": 1.3074956866253435,
|
|
"grad_norm": 9.619998922145582,
|
|
"learning_rate": 1.614249950809224e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10230
|
|
},
|
|
{
|
|
"epoch": 1.308773723560611,
|
|
"grad_norm": 13.306882835676573,
|
|
"learning_rate": 1.6090370002996458e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1372,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10240
|
|
},
|
|
{
|
|
"epoch": 1.3100517604958783,
|
|
"grad_norm": 13.535705683821893,
|
|
"learning_rate": 1.6038284835163227e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.152,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10250
|
|
},
|
|
{
|
|
"epoch": 1.3113297974311457,
|
|
"grad_norm": 9.303734802663968,
|
|
"learning_rate": 1.5986244263785543e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.154,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10260
|
|
},
|
|
{
|
|
"epoch": 1.3126078343664132,
|
|
"grad_norm": 12.80378526578538,
|
|
"learning_rate": 1.5934248547834493e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10270
|
|
},
|
|
{
|
|
"epoch": 1.3138858713016806,
|
|
"grad_norm": 9.8353883410435,
|
|
"learning_rate": 1.5882297946057922e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1494,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10280
|
|
},
|
|
{
|
|
"epoch": 1.315163908236948,
|
|
"grad_norm": 10.334995200129333,
|
|
"learning_rate": 1.58303927169792e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10290
|
|
},
|
|
{
|
|
"epoch": 1.3164419451722154,
|
|
"grad_norm": 15.548721968057658,
|
|
"learning_rate": 1.5778533118895893e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1048.0,
|
|
"loss": 0.1984,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.0,
|
|
"step": 10300
|
|
},
|
|
{
|
|
"epoch": 1.317719982107483,
|
|
"grad_norm": 9.965301691514663,
|
|
"learning_rate": 1.5726719409878497e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -628.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1522,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -4.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -8.9375,
|
|
"step": 10310
|
|
},
|
|
{
|
|
"epoch": 1.3189980190427504,
|
|
"grad_norm": 11.883410302000634,
|
|
"learning_rate": 1.5674951847769143e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1708,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10320
|
|
},
|
|
{
|
|
"epoch": 1.3202760559780178,
|
|
"grad_norm": 10.437698687232169,
|
|
"learning_rate": 1.5623230690180328e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.578125,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 10330
|
|
},
|
|
{
|
|
"epoch": 1.3215540929132852,
|
|
"grad_norm": 11.893821802995003,
|
|
"learning_rate": 1.5571556194493615e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1591,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 10340
|
|
},
|
|
{
|
|
"epoch": 1.3228321298485526,
|
|
"grad_norm": 10.899103930979841,
|
|
"learning_rate": 1.5519928617858373e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.162,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10350
|
|
},
|
|
{
|
|
"epoch": 1.32411016678382,
|
|
"grad_norm": 11.345899735465693,
|
|
"learning_rate": 1.5468348217190492e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10360
|
|
},
|
|
{
|
|
"epoch": 1.3253882037190876,
|
|
"grad_norm": 12.30558614485336,
|
|
"learning_rate": 1.541681524917106e-07,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1729,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 3.84375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10370
|
|
},
|
|
{
|
|
"epoch": 1.326666240654355,
|
|
"grad_norm": 10.070533445591067,
|
|
"learning_rate": 1.5365329970245173e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10380
|
|
},
|
|
{
|
|
"epoch": 1.3279442775896224,
|
|
"grad_norm": 9.555068680623833,
|
|
"learning_rate": 1.5313892636620572e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1376,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 3.90625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10390
|
|
},
|
|
{
|
|
"epoch": 1.3292223145248898,
|
|
"grad_norm": 12.8211655716973,
|
|
"learning_rate": 1.5262503504266441e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10400
|
|
},
|
|
{
|
|
"epoch": 1.3305003514601572,
|
|
"grad_norm": 7.33947423183935,
|
|
"learning_rate": 1.5211162828912073e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.146,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10410
|
|
},
|
|
{
|
|
"epoch": 1.3317783883954246,
|
|
"grad_norm": 16.553187562669738,
|
|
"learning_rate": 1.5159870866045632e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1598,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10420
|
|
},
|
|
{
|
|
"epoch": 1.333056425330692,
|
|
"grad_norm": 8.249018554748698,
|
|
"learning_rate": 1.5108627870912877e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1599,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10430
|
|
},
|
|
{
|
|
"epoch": 1.3343344622659594,
|
|
"grad_norm": 11.885385205335053,
|
|
"learning_rate": 1.5057434098515871e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10440
|
|
},
|
|
{
|
|
"epoch": 1.3356124992012268,
|
|
"grad_norm": 12.606879170463433,
|
|
"learning_rate": 1.500628980361175e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10450
|
|
},
|
|
{
|
|
"epoch": 1.3368905361364942,
|
|
"grad_norm": 8.510890461580383,
|
|
"learning_rate": 1.4955195240711416e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 10460
|
|
},
|
|
{
|
|
"epoch": 1.3381685730717618,
|
|
"grad_norm": 13.307022774845727,
|
|
"learning_rate": 1.49041506640783e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 10470
|
|
},
|
|
{
|
|
"epoch": 1.3394466100070292,
|
|
"grad_norm": 10.194898408199766,
|
|
"learning_rate": 1.4853156327727083e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1436,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.125,
|
|
"step": 10480
|
|
},
|
|
{
|
|
"epoch": 1.3407246469422966,
|
|
"grad_norm": 9.840893728306792,
|
|
"learning_rate": 1.4802212485422423e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1665,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10490
|
|
},
|
|
{
|
|
"epoch": 1.342002683877564,
|
|
"grad_norm": 16.030708438907578,
|
|
"learning_rate": 1.475131939067771e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10500
|
|
},
|
|
{
|
|
"epoch": 1.3432807208128315,
|
|
"grad_norm": 14.6898022036467,
|
|
"learning_rate": 1.4700477296753805e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.8125,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10510
|
|
},
|
|
{
|
|
"epoch": 1.3445587577480989,
|
|
"grad_norm": 7.640691836956104,
|
|
"learning_rate": 1.4649686456657764e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1655,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 10520
|
|
},
|
|
{
|
|
"epoch": 1.3458367946833665,
|
|
"grad_norm": 10.471801000652848,
|
|
"learning_rate": 1.4598947123141593e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1324,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -4.90625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 10530
|
|
},
|
|
{
|
|
"epoch": 1.3471148316186339,
|
|
"grad_norm": 14.327591713140697,
|
|
"learning_rate": 1.4548259548700958e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10540
|
|
},
|
|
{
|
|
"epoch": 1.3483928685539013,
|
|
"grad_norm": 10.818716941448443,
|
|
"learning_rate": 1.4497623985574003e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1545,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 10550
|
|
},
|
|
{
|
|
"epoch": 1.3496709054891687,
|
|
"grad_norm": 10.701973869348382,
|
|
"learning_rate": 1.4447040685740011e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1774,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.5,
|
|
"step": 10560
|
|
},
|
|
{
|
|
"epoch": 1.350948942424436,
|
|
"grad_norm": 10.557227140466773,
|
|
"learning_rate": 1.439650990091819e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1641,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10570
|
|
},
|
|
{
|
|
"epoch": 1.3522269793597035,
|
|
"grad_norm": 11.232062901600258,
|
|
"learning_rate": 1.4346031882566422e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 10580
|
|
},
|
|
{
|
|
"epoch": 1.3535050162949709,
|
|
"grad_norm": 8.121003160184413,
|
|
"learning_rate": 1.4295606881879994e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1371,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10590
|
|
},
|
|
{
|
|
"epoch": 1.3547830532302383,
|
|
"grad_norm": 11.168731072310887,
|
|
"learning_rate": 1.4245235149790393e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1594,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -4.96875,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10600
|
|
},
|
|
{
|
|
"epoch": 1.3560610901655057,
|
|
"grad_norm": 10.801917621179639,
|
|
"learning_rate": 1.4194916936963975e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10610
|
|
},
|
|
{
|
|
"epoch": 1.357339127100773,
|
|
"grad_norm": 12.255284522095959,
|
|
"learning_rate": 1.4144652493800813e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -4.875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -8.875,
|
|
"step": 10620
|
|
},
|
|
{
|
|
"epoch": 1.3586171640360407,
|
|
"grad_norm": 9.63680109894397,
|
|
"learning_rate": 1.409444207043337e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.25,
|
|
"step": 10630
|
|
},
|
|
{
|
|
"epoch": 1.3598952009713081,
|
|
"grad_norm": 11.406361911753756,
|
|
"learning_rate": 1.40442859167253e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10640
|
|
},
|
|
{
|
|
"epoch": 1.3611732379065755,
|
|
"grad_norm": 8.724674918938067,
|
|
"learning_rate": 1.3994184282270212e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 10650
|
|
},
|
|
{
|
|
"epoch": 1.362451274841843,
|
|
"grad_norm": 8.713189202090533,
|
|
"learning_rate": 1.3944137416390367e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1326,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10660
|
|
},
|
|
{
|
|
"epoch": 1.3637293117771103,
|
|
"grad_norm": 11.092642328430658,
|
|
"learning_rate": 1.3894145568135532e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.640625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 10670
|
|
},
|
|
{
|
|
"epoch": 1.3650073487123777,
|
|
"grad_norm": 15.316488295385675,
|
|
"learning_rate": 1.3844208986281647e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1637,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10680
|
|
},
|
|
{
|
|
"epoch": 1.3662853856476453,
|
|
"grad_norm": 11.215655709405532,
|
|
"learning_rate": 1.379432791932963e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10690
|
|
},
|
|
{
|
|
"epoch": 1.3675634225829127,
|
|
"grad_norm": 8.734870719545054,
|
|
"learning_rate": 1.3744502615504177e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1546,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10700
|
|
},
|
|
{
|
|
"epoch": 1.3688414595181801,
|
|
"grad_norm": 10.872876644870878,
|
|
"learning_rate": 1.3694733322752433e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1501,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 10710
|
|
},
|
|
{
|
|
"epoch": 1.3701194964534475,
|
|
"grad_norm": 11.29460016079079,
|
|
"learning_rate": 1.3645020288742872e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1671,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10720
|
|
},
|
|
{
|
|
"epoch": 1.371397533388715,
|
|
"grad_norm": 13.2399522838317,
|
|
"learning_rate": 1.3595363760863958e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1558,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 10730
|
|
},
|
|
{
|
|
"epoch": 1.3726755703239824,
|
|
"grad_norm": 9.725638941764556,
|
|
"learning_rate": 1.3545763986222973e-07,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1628,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10740
|
|
},
|
|
{
|
|
"epoch": 1.3739536072592498,
|
|
"grad_norm": 16.159984692053747,
|
|
"learning_rate": 1.349622121164481e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1739,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 10750
|
|
},
|
|
{
|
|
"epoch": 1.3752316441945172,
|
|
"grad_norm": 10.45347181336293,
|
|
"learning_rate": 1.344673568367065e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.151,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10760
|
|
},
|
|
{
|
|
"epoch": 1.3765096811297846,
|
|
"grad_norm": 12.043752161415977,
|
|
"learning_rate": 1.3397307648556861e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1639,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10770
|
|
},
|
|
{
|
|
"epoch": 1.377787718065052,
|
|
"grad_norm": 8.3067657319515,
|
|
"learning_rate": 1.334793735227366e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1579,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10780
|
|
},
|
|
{
|
|
"epoch": 1.3790657550003196,
|
|
"grad_norm": 10.000745025884786,
|
|
"learning_rate": 1.329862504050395e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.145,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10790
|
|
},
|
|
{
|
|
"epoch": 1.380343791935587,
|
|
"grad_norm": 9.27281728403169,
|
|
"learning_rate": 1.32493709586421e-07,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1402,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.25,
|
|
"step": 10800
|
|
},
|
|
{
|
|
"epoch": 1.3816218288708544,
|
|
"grad_norm": 11.177095928758252,
|
|
"learning_rate": 1.3200175351792676e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.159,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10810
|
|
},
|
|
{
|
|
"epoch": 1.3828998658061218,
|
|
"grad_norm": 13.81811898136344,
|
|
"learning_rate": 1.3151038464769288e-07,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.167,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 10820
|
|
},
|
|
{
|
|
"epoch": 1.3841779027413892,
|
|
"grad_norm": 12.668241973578079,
|
|
"learning_rate": 1.3101960542093306e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1615,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 10830
|
|
},
|
|
{
|
|
"epoch": 1.3854559396766566,
|
|
"grad_norm": 14.666040788124336,
|
|
"learning_rate": 1.3052941827992708e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1744,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 3.578125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 10840
|
|
},
|
|
{
|
|
"epoch": 1.3867339766119242,
|
|
"grad_norm": 8.587958353569526,
|
|
"learning_rate": 1.3003982566400802e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1293,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10850
|
|
},
|
|
{
|
|
"epoch": 1.3880120135471916,
|
|
"grad_norm": 10.462369727921462,
|
|
"learning_rate": 1.295508300095503e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10860
|
|
},
|
|
{
|
|
"epoch": 1.389290050482459,
|
|
"grad_norm": 7.445306803662397,
|
|
"learning_rate": 1.2906243374995818e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10870
|
|
},
|
|
{
|
|
"epoch": 1.3905680874177264,
|
|
"grad_norm": 10.953212359779702,
|
|
"learning_rate": 1.2857463931565256e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 10880
|
|
},
|
|
{
|
|
"epoch": 1.3918461243529938,
|
|
"grad_norm": 6.7514582868915225,
|
|
"learning_rate": 1.280874491340599e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1522,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 10890
|
|
},
|
|
{
|
|
"epoch": 1.3931241612882612,
|
|
"grad_norm": 14.749439493951504,
|
|
"learning_rate": 1.2760086562959937e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1664,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 10900
|
|
},
|
|
{
|
|
"epoch": 1.3944021982235286,
|
|
"grad_norm": 10.625720834052649,
|
|
"learning_rate": 1.2711489122367124e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 10910
|
|
},
|
|
{
|
|
"epoch": 1.395680235158796,
|
|
"grad_norm": 12.248756314421437,
|
|
"learning_rate": 1.2662952833464485e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1634,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10920
|
|
},
|
|
{
|
|
"epoch": 1.3969582720940634,
|
|
"grad_norm": 9.829887044914619,
|
|
"learning_rate": 1.2614477937784617e-07,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1473,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 10930
|
|
},
|
|
{
|
|
"epoch": 1.3982363090293308,
|
|
"grad_norm": 11.549751875940906,
|
|
"learning_rate": 1.2566064676554628e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.375,
|
|
"step": 10940
|
|
},
|
|
{
|
|
"epoch": 1.3995143459645984,
|
|
"grad_norm": 10.503560647884871,
|
|
"learning_rate": 1.2517713290694892e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -3.0,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1522,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 10950
|
|
},
|
|
{
|
|
"epoch": 1.4007923828998659,
|
|
"grad_norm": 14.679390504503392,
|
|
"learning_rate": 1.246942402081788e-07,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -3.015625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1611,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 10960
|
|
},
|
|
{
|
|
"epoch": 1.4020704198351333,
|
|
"grad_norm": 8.841059671351234,
|
|
"learning_rate": 1.242119710722696e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1721,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 10970
|
|
},
|
|
{
|
|
"epoch": 1.4033484567704007,
|
|
"grad_norm": 11.857415726709224,
|
|
"learning_rate": 1.2373032789915176e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -608.0,
|
|
"logps/rejected": -1016.0,
|
|
"loss": 0.1661,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -4.71875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -8.8125,
|
|
"step": 10980
|
|
},
|
|
{
|
|
"epoch": 1.404626493705668,
|
|
"grad_norm": 9.710749089708392,
|
|
"learning_rate": 1.2324931308564103e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1040.0,
|
|
"loss": 0.1659,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.03125,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.0,
|
|
"step": 10990
|
|
},
|
|
{
|
|
"epoch": 1.4059045306409355,
|
|
"grad_norm": 11.438152431601049,
|
|
"learning_rate": 1.2276892902542596e-07,
|
|
"logits/chosen": -2.84375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1669,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11000
|
|
},
|
|
{
|
|
"epoch": 1.407182567576203,
|
|
"grad_norm": 9.709697271547052,
|
|
"learning_rate": 1.2228917810905631e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -652.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1474,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 11010
|
|
},
|
|
{
|
|
"epoch": 1.4084606045114705,
|
|
"grad_norm": 10.24605596244792,
|
|
"learning_rate": 1.2181006272393115e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1763,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 11020
|
|
},
|
|
{
|
|
"epoch": 1.4097386414467379,
|
|
"grad_norm": 12.25722659236921,
|
|
"learning_rate": 1.2133158525428686e-07,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1032.0,
|
|
"loss": 0.1666,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.0625,
|
|
"rewards/margins": 3.828125,
|
|
"rewards/rejected": -8.875,
|
|
"step": 11030
|
|
},
|
|
{
|
|
"epoch": 1.4110166783820053,
|
|
"grad_norm": 12.072026666485739,
|
|
"learning_rate": 1.2085374808118558e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -640.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1611,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -4.9375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 11040
|
|
},
|
|
{
|
|
"epoch": 1.4122947153172727,
|
|
"grad_norm": 7.932169471982789,
|
|
"learning_rate": 1.2037655358250282e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1501,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 11050
|
|
},
|
|
{
|
|
"epoch": 1.41357275225254,
|
|
"grad_norm": 7.953752156821214,
|
|
"learning_rate": 1.199000041329162e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.375,
|
|
"step": 11060
|
|
},
|
|
{
|
|
"epoch": 1.4148507891878075,
|
|
"grad_norm": 13.426621520621833,
|
|
"learning_rate": 1.1942410210389317e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1732,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11070
|
|
},
|
|
{
|
|
"epoch": 1.416128826123075,
|
|
"grad_norm": 9.374559807282663,
|
|
"learning_rate": 1.1894884986367934e-07,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11080
|
|
},
|
|
{
|
|
"epoch": 1.4174068630583423,
|
|
"grad_norm": 9.973310613947785,
|
|
"learning_rate": 1.184742497772871e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1436,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11090
|
|
},
|
|
{
|
|
"epoch": 1.4186848999936097,
|
|
"grad_norm": 13.250219412695905,
|
|
"learning_rate": 1.1800030420648313e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11100
|
|
},
|
|
{
|
|
"epoch": 1.4199629369288773,
|
|
"grad_norm": 13.773396859073182,
|
|
"learning_rate": 1.1752701550977715e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1240.0,
|
|
"loss": 0.1541,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 5.0625,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 11110
|
|
},
|
|
{
|
|
"epoch": 1.4212409738641447,
|
|
"grad_norm": 8.993057070559015,
|
|
"learning_rate": 1.1705438604241025e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1575,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11120
|
|
},
|
|
{
|
|
"epoch": 1.4225190107994121,
|
|
"grad_norm": 12.61693065803415,
|
|
"learning_rate": 1.1658241815634259e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1807,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.25,
|
|
"step": 11130
|
|
},
|
|
{
|
|
"epoch": 1.4237970477346795,
|
|
"grad_norm": 11.057436063689746,
|
|
"learning_rate": 1.1611111420024255e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.145,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11140
|
|
},
|
|
{
|
|
"epoch": 1.425075084669947,
|
|
"grad_norm": 12.071787660113134,
|
|
"learning_rate": 1.1564047651947423e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1734,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11150
|
|
},
|
|
{
|
|
"epoch": 1.4263531216052143,
|
|
"grad_norm": 8.663142571001057,
|
|
"learning_rate": 1.1517050745608611e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 3.8125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 11160
|
|
},
|
|
{
|
|
"epoch": 1.427631158540482,
|
|
"grad_norm": 10.17682640212931,
|
|
"learning_rate": 1.147012093487997e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1556,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 11170
|
|
},
|
|
{
|
|
"epoch": 1.4289091954757493,
|
|
"grad_norm": 9.483691786180756,
|
|
"learning_rate": 1.1423258453299728e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.145,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 11180
|
|
},
|
|
{
|
|
"epoch": 1.4301872324110168,
|
|
"grad_norm": 9.211669175543367,
|
|
"learning_rate": 1.1376463534071093e-07,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1559,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 11190
|
|
},
|
|
{
|
|
"epoch": 1.4314652693462842,
|
|
"grad_norm": 12.093964271729028,
|
|
"learning_rate": 1.1329736410061033e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1554,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11200
|
|
},
|
|
{
|
|
"epoch": 1.4327433062815516,
|
|
"grad_norm": 9.29114146915148,
|
|
"learning_rate": 1.128307731379915e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.5,
|
|
"step": 11210
|
|
},
|
|
{
|
|
"epoch": 1.434021343216819,
|
|
"grad_norm": 12.657395691854575,
|
|
"learning_rate": 1.1236486477476551e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 11220
|
|
},
|
|
{
|
|
"epoch": 1.4352993801520864,
|
|
"grad_norm": 11.348547431972344,
|
|
"learning_rate": 1.1189964132944588e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 11230
|
|
},
|
|
{
|
|
"epoch": 1.4365774170873538,
|
|
"grad_norm": 8.516011381449768,
|
|
"learning_rate": 1.1143510511713849e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1064.0,
|
|
"loss": 0.1633,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.3125,
|
|
"step": 11240
|
|
},
|
|
{
|
|
"epoch": 1.4378554540226212,
|
|
"grad_norm": 12.710258105717188,
|
|
"learning_rate": 1.1097125844952885e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1456,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 11250
|
|
},
|
|
{
|
|
"epoch": 1.4391334909578886,
|
|
"grad_norm": 9.096742033397492,
|
|
"learning_rate": 1.1050810363487139e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1644,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11260
|
|
},
|
|
{
|
|
"epoch": 1.4404115278931562,
|
|
"grad_norm": 9.27229207508295,
|
|
"learning_rate": 1.1004564297797741e-07,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1396,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11270
|
|
},
|
|
{
|
|
"epoch": 1.4416895648284236,
|
|
"grad_norm": 14.009678507550513,
|
|
"learning_rate": 1.0958387878020381e-07,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1682,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11280
|
|
},
|
|
{
|
|
"epoch": 1.442967601763691,
|
|
"grad_norm": 12.862568319486723,
|
|
"learning_rate": 1.0912281333944204e-07,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.133,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11290
|
|
},
|
|
{
|
|
"epoch": 1.4442456386989584,
|
|
"grad_norm": 10.33161006641078,
|
|
"learning_rate": 1.0866244895010584e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1567,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11300
|
|
},
|
|
{
|
|
"epoch": 1.4455236756342258,
|
|
"grad_norm": 14.720700918357485,
|
|
"learning_rate": 1.082027879031207e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11310
|
|
},
|
|
{
|
|
"epoch": 1.4468017125694932,
|
|
"grad_norm": 9.199043302635625,
|
|
"learning_rate": 1.077438324859118e-07,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -656.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.181,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.0,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.5,
|
|
"step": 11320
|
|
},
|
|
{
|
|
"epoch": 1.4480797495047608,
|
|
"grad_norm": 9.957779437846934,
|
|
"learning_rate": 1.072855849823928e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1457,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 11330
|
|
},
|
|
{
|
|
"epoch": 1.4493577864400282,
|
|
"grad_norm": 10.62616037496165,
|
|
"learning_rate": 1.0682804767295486e-07,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 11340
|
|
},
|
|
{
|
|
"epoch": 1.4506358233752956,
|
|
"grad_norm": 13.753631734488085,
|
|
"learning_rate": 1.0637122283445466e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 11350
|
|
},
|
|
{
|
|
"epoch": 1.451913860310563,
|
|
"grad_norm": 10.84099368492692,
|
|
"learning_rate": 1.0591511274020364e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1693,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11360
|
|
},
|
|
{
|
|
"epoch": 1.4531918972458304,
|
|
"grad_norm": 10.784148259297346,
|
|
"learning_rate": 1.0545971965995634e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.141,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11370
|
|
},
|
|
{
|
|
"epoch": 1.4544699341810978,
|
|
"grad_norm": 8.135502688800099,
|
|
"learning_rate": 1.05005045859899e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11380
|
|
},
|
|
{
|
|
"epoch": 1.4557479711163652,
|
|
"grad_norm": 8.00246592998164,
|
|
"learning_rate": 1.0455109360263889e-07,
|
|
"logits/chosen": -2.875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1657,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5,
|
|
"step": 11390
|
|
},
|
|
{
|
|
"epoch": 1.4570260080516326,
|
|
"grad_norm": 9.797097532732542,
|
|
"learning_rate": 1.0409786514719223e-07,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1604,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11400
|
|
},
|
|
{
|
|
"epoch": 1.4583040449869,
|
|
"grad_norm": 9.694349670220472,
|
|
"learning_rate": 1.0364536274897376e-07,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1455,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11410
|
|
},
|
|
{
|
|
"epoch": 1.4595820819221674,
|
|
"grad_norm": 12.38283245894702,
|
|
"learning_rate": 1.031935886597848e-07,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1535,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 5.1875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11420
|
|
},
|
|
{
|
|
"epoch": 1.460860118857435,
|
|
"grad_norm": 8.936517008363177,
|
|
"learning_rate": 1.027425451278025e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 5.15625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 11430
|
|
},
|
|
{
|
|
"epoch": 1.4621381557927025,
|
|
"grad_norm": 10.831220985154136,
|
|
"learning_rate": 1.0229223439756851e-07,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1656,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11440
|
|
},
|
|
{
|
|
"epoch": 1.4634161927279699,
|
|
"grad_norm": 8.556358700103702,
|
|
"learning_rate": 1.0184265870997768e-07,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1367,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11450
|
|
},
|
|
{
|
|
"epoch": 1.4646942296632373,
|
|
"grad_norm": 12.077581357401632,
|
|
"learning_rate": 1.0139382030226737e-07,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.96875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 11460
|
|
},
|
|
{
|
|
"epoch": 1.4659722665985047,
|
|
"grad_norm": 10.787672273909923,
|
|
"learning_rate": 1.0094572140800558e-07,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1739,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11470
|
|
},
|
|
{
|
|
"epoch": 1.467250303533772,
|
|
"grad_norm": 9.948886157796718,
|
|
"learning_rate": 1.004983642570807e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11480
|
|
},
|
|
{
|
|
"epoch": 1.4685283404690397,
|
|
"grad_norm": 15.864211760797932,
|
|
"learning_rate": 1.0005175107568953e-07,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1541,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11490
|
|
},
|
|
{
|
|
"epoch": 1.469806377404307,
|
|
"grad_norm": 12.462596811906824,
|
|
"learning_rate": 9.960588408632683e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1577,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 11500
|
|
},
|
|
{
|
|
"epoch": 1.4710844143395745,
|
|
"grad_norm": 9.807942803597852,
|
|
"learning_rate": 9.916076550777417e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1628,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 11510
|
|
},
|
|
{
|
|
"epoch": 1.4723624512748419,
|
|
"grad_norm": 12.380375604436118,
|
|
"learning_rate": 9.871639755508852e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.178,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11520
|
|
},
|
|
{
|
|
"epoch": 1.4736404882101093,
|
|
"grad_norm": 12.77801489441428,
|
|
"learning_rate": 9.827278243959176e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1753,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11530
|
|
},
|
|
{
|
|
"epoch": 1.4749185251453767,
|
|
"grad_norm": 10.35489180421786,
|
|
"learning_rate": 9.78299223688592e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1516,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11540
|
|
},
|
|
{
|
|
"epoch": 1.476196562080644,
|
|
"grad_norm": 14.764091673164994,
|
|
"learning_rate": 9.73878195467088e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11550
|
|
},
|
|
{
|
|
"epoch": 1.4774745990159115,
|
|
"grad_norm": 8.91578021211425,
|
|
"learning_rate": 9.694647617319044e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11560
|
|
},
|
|
{
|
|
"epoch": 1.478752635951179,
|
|
"grad_norm": 10.9841675460549,
|
|
"learning_rate": 9.650589444457435e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1397,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11570
|
|
},
|
|
{
|
|
"epoch": 1.4800306728864463,
|
|
"grad_norm": 9.89509879622352,
|
|
"learning_rate": 9.606607655334103e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1384,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11580
|
|
},
|
|
{
|
|
"epoch": 1.481308709821714,
|
|
"grad_norm": 8.810197963037124,
|
|
"learning_rate": 9.562702468816946e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11590
|
|
},
|
|
{
|
|
"epoch": 1.4825867467569813,
|
|
"grad_norm": 11.187235524321606,
|
|
"learning_rate": 9.518874103392673e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1545,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11600
|
|
},
|
|
{
|
|
"epoch": 1.4838647836922487,
|
|
"grad_norm": 8.308168361331765,
|
|
"learning_rate": 9.475122777165729e-08,
|
|
"logits/chosen": -3.21875,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1392,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11610
|
|
},
|
|
{
|
|
"epoch": 1.4851428206275161,
|
|
"grad_norm": 12.759456416581942,
|
|
"learning_rate": 9.431448707857148e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1407,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11620
|
|
},
|
|
{
|
|
"epoch": 1.4864208575627835,
|
|
"grad_norm": 9.265918162123217,
|
|
"learning_rate": 9.387852112803546e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.146,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11630
|
|
},
|
|
{
|
|
"epoch": 1.487698894498051,
|
|
"grad_norm": 8.332858163699392,
|
|
"learning_rate": 9.34433320895598e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1364,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 11640
|
|
},
|
|
{
|
|
"epoch": 1.4889769314333186,
|
|
"grad_norm": 12.235024940851668,
|
|
"learning_rate": 9.300892212878886e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 11650
|
|
},
|
|
{
|
|
"epoch": 1.490254968368586,
|
|
"grad_norm": 13.052203756824563,
|
|
"learning_rate": 9.257529340749021e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1563,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11660
|
|
},
|
|
{
|
|
"epoch": 1.4915330053038534,
|
|
"grad_norm": 10.747014663632283,
|
|
"learning_rate": 9.214244808354351e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.178,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11670
|
|
},
|
|
{
|
|
"epoch": 1.4928110422391208,
|
|
"grad_norm": 10.663336523174248,
|
|
"learning_rate": 9.171038831093028e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11680
|
|
},
|
|
{
|
|
"epoch": 1.4940890791743882,
|
|
"grad_norm": 9.739080278989183,
|
|
"learning_rate": 9.127911623972253e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1298,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11690
|
|
},
|
|
{
|
|
"epoch": 1.4953671161096556,
|
|
"grad_norm": 10.515221950290618,
|
|
"learning_rate": 9.084863401607273e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -648.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1541,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.96875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11700
|
|
},
|
|
{
|
|
"epoch": 1.496645153044923,
|
|
"grad_norm": 10.246625757921912,
|
|
"learning_rate": 9.041894378220258e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1573,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11710
|
|
},
|
|
{
|
|
"epoch": 1.4979231899801904,
|
|
"grad_norm": 12.99095492752731,
|
|
"learning_rate": 8.999004767639256e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 11720
|
|
},
|
|
{
|
|
"epoch": 1.4992012269154578,
|
|
"grad_norm": 10.437109592299482,
|
|
"learning_rate": 8.95619478329716e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 11730
|
|
},
|
|
{
|
|
"epoch": 1.5004792638507252,
|
|
"grad_norm": 10.487452986000255,
|
|
"learning_rate": 8.913464638230575e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1591,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11740
|
|
},
|
|
{
|
|
"epoch": 1.5017573007859926,
|
|
"grad_norm": 9.032769583796481,
|
|
"learning_rate": 8.870814545078839e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1459,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11750
|
|
},
|
|
{
|
|
"epoch": 1.5030353377212602,
|
|
"grad_norm": 12.474367808376874,
|
|
"learning_rate": 8.828244716082897e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11760
|
|
},
|
|
{
|
|
"epoch": 1.5043133746565276,
|
|
"grad_norm": 13.831164839214665,
|
|
"learning_rate": 8.785755363084277e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1741,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11770
|
|
},
|
|
{
|
|
"epoch": 1.505591411591795,
|
|
"grad_norm": 11.80319684012857,
|
|
"learning_rate": 8.743346697524054e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11780
|
|
},
|
|
{
|
|
"epoch": 1.5068694485270624,
|
|
"grad_norm": 11.340568045210375,
|
|
"learning_rate": 8.701018930441739e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1596,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 11790
|
|
},
|
|
{
|
|
"epoch": 1.50814748546233,
|
|
"grad_norm": 10.031009539482778,
|
|
"learning_rate": 8.658772272474307e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11800
|
|
},
|
|
{
|
|
"epoch": 1.5094255223975974,
|
|
"grad_norm": 10.222246813978815,
|
|
"learning_rate": 8.616606933855077e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1406,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 11810
|
|
},
|
|
{
|
|
"epoch": 1.5107035593328648,
|
|
"grad_norm": 10.522623597133993,
|
|
"learning_rate": 8.574523124412697e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11820
|
|
},
|
|
{
|
|
"epoch": 1.5119815962681322,
|
|
"grad_norm": 11.181079924622459,
|
|
"learning_rate": 8.532521053570121e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1502,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11830
|
|
},
|
|
{
|
|
"epoch": 1.5132596332033996,
|
|
"grad_norm": 7.667202990627593,
|
|
"learning_rate": 8.490600930343514e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1623,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11840
|
|
},
|
|
{
|
|
"epoch": 1.514537670138667,
|
|
"grad_norm": 9.646765032562787,
|
|
"learning_rate": 8.44876296334128e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1487,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 11850
|
|
},
|
|
{
|
|
"epoch": 1.5158157070739344,
|
|
"grad_norm": 10.012416218533366,
|
|
"learning_rate": 8.407007360762955e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.159,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11860
|
|
},
|
|
{
|
|
"epoch": 1.5170937440092018,
|
|
"grad_norm": 8.150935233146557,
|
|
"learning_rate": 8.36533433039821e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 11870
|
|
},
|
|
{
|
|
"epoch": 1.5183717809444692,
|
|
"grad_norm": 10.803015402481362,
|
|
"learning_rate": 8.323744079625819e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1701,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.96875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 11880
|
|
},
|
|
{
|
|
"epoch": 1.5196498178797366,
|
|
"grad_norm": 11.958688449421555,
|
|
"learning_rate": 8.282236815412599e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11890
|
|
},
|
|
{
|
|
"epoch": 1.520927854815004,
|
|
"grad_norm": 10.106095459532787,
|
|
"learning_rate": 8.240812744312428e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 11900
|
|
},
|
|
{
|
|
"epoch": 1.5222058917502714,
|
|
"grad_norm": 12.37592965783719,
|
|
"learning_rate": 8.199472072465153e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 11910
|
|
},
|
|
{
|
|
"epoch": 1.523483928685539,
|
|
"grad_norm": 10.377992192363955,
|
|
"learning_rate": 8.158215005595628e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 11920
|
|
},
|
|
{
|
|
"epoch": 1.5247619656208065,
|
|
"grad_norm": 10.792908757384343,
|
|
"learning_rate": 8.117041749012649e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1531,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 11930
|
|
},
|
|
{
|
|
"epoch": 1.5260400025560739,
|
|
"grad_norm": 11.001792418578807,
|
|
"learning_rate": 8.075952507607931e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1603,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 11940
|
|
},
|
|
{
|
|
"epoch": 1.5273180394913413,
|
|
"grad_norm": 10.242356822888798,
|
|
"learning_rate": 8.034947485855131e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 11950
|
|
},
|
|
{
|
|
"epoch": 1.5285960764266089,
|
|
"grad_norm": 9.251414278750838,
|
|
"learning_rate": 7.994026887808772e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.5,
|
|
"step": 11960
|
|
},
|
|
{
|
|
"epoch": 1.5298741133618763,
|
|
"grad_norm": 11.853751107996388,
|
|
"learning_rate": 7.95319091710328e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 11970
|
|
},
|
|
{
|
|
"epoch": 1.5311521502971437,
|
|
"grad_norm": 14.040158388167082,
|
|
"learning_rate": 7.912439776951935e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1735,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 11980
|
|
},
|
|
{
|
|
"epoch": 1.532430187232411,
|
|
"grad_norm": 9.573096797550356,
|
|
"learning_rate": 7.871773670145862e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 11990
|
|
},
|
|
{
|
|
"epoch": 1.5337082241676785,
|
|
"grad_norm": 9.396314193149403,
|
|
"learning_rate": 7.831192799053066e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1399,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12000
|
|
},
|
|
{
|
|
"epoch": 1.534986261102946,
|
|
"grad_norm": 16.69418623861175,
|
|
"learning_rate": 7.790697365617347e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1415,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12010
|
|
},
|
|
{
|
|
"epoch": 1.5362642980382133,
|
|
"grad_norm": 10.567016182091837,
|
|
"learning_rate": 7.750287571357377e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1586,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.875,
|
|
"step": 12020
|
|
},
|
|
{
|
|
"epoch": 1.5375423349734807,
|
|
"grad_norm": 11.50615338821914,
|
|
"learning_rate": 7.709963617365642e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1753,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12030
|
|
},
|
|
{
|
|
"epoch": 1.538820371908748,
|
|
"grad_norm": 10.65143104102225,
|
|
"learning_rate": 7.669725704307445e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1668,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 12040
|
|
},
|
|
{
|
|
"epoch": 1.5400984088440155,
|
|
"grad_norm": 10.025028948194606,
|
|
"learning_rate": 7.629574032419958e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1433,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.875,
|
|
"step": 12050
|
|
},
|
|
{
|
|
"epoch": 1.541376445779283,
|
|
"grad_norm": 9.686910173982989,
|
|
"learning_rate": 7.589508801511143e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.142,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12060
|
|
},
|
|
{
|
|
"epoch": 1.5426544827145503,
|
|
"grad_norm": 7.269425221448671,
|
|
"learning_rate": 7.549530210958854e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1409,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12070
|
|
},
|
|
{
|
|
"epoch": 1.543932519649818,
|
|
"grad_norm": 12.223446432641076,
|
|
"learning_rate": 7.509638459709739e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12080
|
|
},
|
|
{
|
|
"epoch": 1.5452105565850853,
|
|
"grad_norm": 8.641546278848681,
|
|
"learning_rate": 7.469833746278346e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12090
|
|
},
|
|
{
|
|
"epoch": 1.5464885935203527,
|
|
"grad_norm": 9.917101746029982,
|
|
"learning_rate": 7.430116268746078e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1714,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12100
|
|
},
|
|
{
|
|
"epoch": 1.5477666304556201,
|
|
"grad_norm": 13.761179512544745,
|
|
"learning_rate": 7.39048622476022e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12110
|
|
},
|
|
{
|
|
"epoch": 1.5490446673908878,
|
|
"grad_norm": 11.119246022232648,
|
|
"learning_rate": 7.350943811532976e-08,
|
|
"logits/chosen": -2.859375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1634,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 12120
|
|
},
|
|
{
|
|
"epoch": 1.5503227043261552,
|
|
"grad_norm": 9.249195090157448,
|
|
"learning_rate": 7.311489225840444e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1543,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12130
|
|
},
|
|
{
|
|
"epoch": 1.5516007412614226,
|
|
"grad_norm": 10.447651696391043,
|
|
"learning_rate": 7.272122664021695e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1677,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12140
|
|
},
|
|
{
|
|
"epoch": 1.55287877819669,
|
|
"grad_norm": 10.864502757673646,
|
|
"learning_rate": 7.232844321977738e-08,
|
|
"logits/chosen": -2.828125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1459,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12150
|
|
},
|
|
{
|
|
"epoch": 1.5541568151319574,
|
|
"grad_norm": 8.85828421121414,
|
|
"learning_rate": 7.193654395170573e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1511,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12160
|
|
},
|
|
{
|
|
"epoch": 1.5554348520672248,
|
|
"grad_norm": 16.50517282532781,
|
|
"learning_rate": 7.154553078622239e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12170
|
|
},
|
|
{
|
|
"epoch": 1.5567128890024922,
|
|
"grad_norm": 8.574202855048568,
|
|
"learning_rate": 7.115540566913797e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1436,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12180
|
|
},
|
|
{
|
|
"epoch": 1.5579909259377596,
|
|
"grad_norm": 10.810888515464885,
|
|
"learning_rate": 7.076617054184389e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1574,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12190
|
|
},
|
|
{
|
|
"epoch": 1.559268962873027,
|
|
"grad_norm": 7.457960346140366,
|
|
"learning_rate": 7.03778273413029e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12200
|
|
},
|
|
{
|
|
"epoch": 1.5605469998082944,
|
|
"grad_norm": 7.279709759728956,
|
|
"learning_rate": 6.99903780000389e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1338,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.625,
|
|
"step": 12210
|
|
},
|
|
{
|
|
"epoch": 1.5618250367435618,
|
|
"grad_norm": 9.062437716884702,
|
|
"learning_rate": 6.960382444612801e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.5625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1722,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 5.125,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 12220
|
|
},
|
|
{
|
|
"epoch": 1.5631030736788292,
|
|
"grad_norm": 13.459270325223686,
|
|
"learning_rate": 6.92181686031883e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -764.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1632,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -6.0625,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12230
|
|
},
|
|
{
|
|
"epoch": 1.5643811106140968,
|
|
"grad_norm": 11.420081863784505,
|
|
"learning_rate": 6.883341239037064e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1602,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12240
|
|
},
|
|
{
|
|
"epoch": 1.5656591475493642,
|
|
"grad_norm": 13.417134293230164,
|
|
"learning_rate": 6.844955772234917e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1536,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 12250
|
|
},
|
|
{
|
|
"epoch": 1.5669371844846316,
|
|
"grad_norm": 9.667616154823163,
|
|
"learning_rate": 6.806660650931142e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1428,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 12260
|
|
},
|
|
{
|
|
"epoch": 1.568215221419899,
|
|
"grad_norm": 8.997991857537537,
|
|
"learning_rate": 6.76845606569493e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1452,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12270
|
|
},
|
|
{
|
|
"epoch": 1.5694932583551666,
|
|
"grad_norm": 9.04008816490061,
|
|
"learning_rate": 6.73034220664491e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 12280
|
|
},
|
|
{
|
|
"epoch": 1.570771295290434,
|
|
"grad_norm": 9.432414905692434,
|
|
"learning_rate": 6.69231926344824e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1475,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12290
|
|
},
|
|
{
|
|
"epoch": 1.5720493322257014,
|
|
"grad_norm": 12.035295485184237,
|
|
"learning_rate": 6.654387425319646e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1629,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12300
|
|
},
|
|
{
|
|
"epoch": 1.5733273691609688,
|
|
"grad_norm": 14.534600801356092,
|
|
"learning_rate": 6.616546881020479e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1555,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12310
|
|
},
|
|
{
|
|
"epoch": 1.5746054060962362,
|
|
"grad_norm": 10.748006072313448,
|
|
"learning_rate": 6.578797818857806e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1497,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12320
|
|
},
|
|
{
|
|
"epoch": 1.5758834430315036,
|
|
"grad_norm": 9.366457097549711,
|
|
"learning_rate": 6.54114042668342e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1628,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 12330
|
|
},
|
|
{
|
|
"epoch": 1.577161479966771,
|
|
"grad_norm": 14.29220140565374,
|
|
"learning_rate": 6.503574891892954e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1496,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12340
|
|
},
|
|
{
|
|
"epoch": 1.5784395169020384,
|
|
"grad_norm": 14.439666635744588,
|
|
"learning_rate": 6.46610140142492e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1514,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -6.09375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12350
|
|
},
|
|
{
|
|
"epoch": 1.5797175538373058,
|
|
"grad_norm": 9.351590074519404,
|
|
"learning_rate": 6.428720141759778e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12360
|
|
},
|
|
{
|
|
"epoch": 1.5809955907725732,
|
|
"grad_norm": 13.470209400982355,
|
|
"learning_rate": 6.39143129891904e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1572,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12370
|
|
},
|
|
{
|
|
"epoch": 1.5822736277078406,
|
|
"grad_norm": 11.158096087311577,
|
|
"learning_rate": 6.354235058464295e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12380
|
|
},
|
|
{
|
|
"epoch": 1.583551664643108,
|
|
"grad_norm": 9.951528485560331,
|
|
"learning_rate": 6.317131605496334e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12390
|
|
},
|
|
{
|
|
"epoch": 1.5848297015783757,
|
|
"grad_norm": 10.644975027048147,
|
|
"learning_rate": 6.280121124654201e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.136,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12400
|
|
},
|
|
{
|
|
"epoch": 1.586107738513643,
|
|
"grad_norm": 11.807603959654157,
|
|
"learning_rate": 6.243203800114257e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1471,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12410
|
|
},
|
|
{
|
|
"epoch": 1.5873857754489105,
|
|
"grad_norm": 7.108280544801186,
|
|
"learning_rate": 6.206379815589333e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1553,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 5.21875,
|
|
"rewards/rejected": -10.6875,
|
|
"step": 12420
|
|
},
|
|
{
|
|
"epoch": 1.5886638123841779,
|
|
"grad_norm": 12.019450109413693,
|
|
"learning_rate": 6.169649354327719e-08,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1654,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12430
|
|
},
|
|
{
|
|
"epoch": 1.5899418493194455,
|
|
"grad_norm": 13.551368457160118,
|
|
"learning_rate": 6.133012599112341e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1552,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12440
|
|
},
|
|
{
|
|
"epoch": 1.591219886254713,
|
|
"grad_norm": 17.599148853785525,
|
|
"learning_rate": 6.096469732259795e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1427,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12450
|
|
},
|
|
{
|
|
"epoch": 1.5924979231899803,
|
|
"grad_norm": 9.710638561537065,
|
|
"learning_rate": 6.06002093561945e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1316,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12460
|
|
},
|
|
{
|
|
"epoch": 1.5937759601252477,
|
|
"grad_norm": 11.967214456719551,
|
|
"learning_rate": 6.023666390572576e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1272.0,
|
|
"loss": 0.1695,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 5.21875,
|
|
"rewards/rejected": -11.0,
|
|
"step": 12470
|
|
},
|
|
{
|
|
"epoch": 1.595053997060515,
|
|
"grad_norm": 9.109161920815124,
|
|
"learning_rate": 5.987406278031384e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12480
|
|
},
|
|
{
|
|
"epoch": 1.5963320339957825,
|
|
"grad_norm": 9.579879903189044,
|
|
"learning_rate": 5.951240778438188e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1656,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12490
|
|
},
|
|
{
|
|
"epoch": 1.59761007093105,
|
|
"grad_norm": 9.228594375064981,
|
|
"learning_rate": 5.915170071764461e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1375,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12500
|
|
},
|
|
{
|
|
"epoch": 1.5988881078663173,
|
|
"grad_norm": 12.547914385348141,
|
|
"learning_rate": 5.879194337509949e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1435,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12510
|
|
},
|
|
{
|
|
"epoch": 1.6001661448015847,
|
|
"grad_norm": 13.744472128530129,
|
|
"learning_rate": 5.843313754701798e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1396,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12520
|
|
},
|
|
{
|
|
"epoch": 1.601444181736852,
|
|
"grad_norm": 9.517400972891124,
|
|
"learning_rate": 5.8075285018936325e-08,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1186,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12530
|
|
},
|
|
{
|
|
"epoch": 1.6027222186721195,
|
|
"grad_norm": 10.271480402842696,
|
|
"learning_rate": 5.77183875716471e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12540
|
|
},
|
|
{
|
|
"epoch": 1.604000255607387,
|
|
"grad_norm": 12.276563592099997,
|
|
"learning_rate": 5.7362446981189775e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1523,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12550
|
|
},
|
|
{
|
|
"epoch": 1.6052782925426545,
|
|
"grad_norm": 10.450582546903629,
|
|
"learning_rate": 5.7007465018842435e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12560
|
|
},
|
|
{
|
|
"epoch": 1.606556329477922,
|
|
"grad_norm": 12.859229216456761,
|
|
"learning_rate": 5.6653443451112585e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12570
|
|
},
|
|
{
|
|
"epoch": 1.6078343664131893,
|
|
"grad_norm": 10.183791967823463,
|
|
"learning_rate": 5.6300384039728424e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1614,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12580
|
|
},
|
|
{
|
|
"epoch": 1.6091124033484567,
|
|
"grad_norm": 12.161865277394519,
|
|
"learning_rate": 5.594828854163036e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12590
|
|
},
|
|
{
|
|
"epoch": 1.6103904402837244,
|
|
"grad_norm": 8.506154794757979,
|
|
"learning_rate": 5.559715870896178e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1548,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 12600
|
|
},
|
|
{
|
|
"epoch": 1.6116684772189918,
|
|
"grad_norm": 10.724959243801147,
|
|
"learning_rate": 5.52469962890609e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1491,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.5,
|
|
"step": 12610
|
|
},
|
|
{
|
|
"epoch": 1.6129465141542592,
|
|
"grad_norm": 11.613970832785595,
|
|
"learning_rate": 5.4897803024451545e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1596,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 12620
|
|
},
|
|
{
|
|
"epoch": 1.6142245510895266,
|
|
"grad_norm": 15.729825565373556,
|
|
"learning_rate": 5.454958065283466e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1657,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12630
|
|
},
|
|
{
|
|
"epoch": 1.615502588024794,
|
|
"grad_norm": 12.184563504498001,
|
|
"learning_rate": 5.420233090708001e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1444,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 12640
|
|
},
|
|
{
|
|
"epoch": 1.6167806249600614,
|
|
"grad_norm": 8.8771990213809,
|
|
"learning_rate": 5.3856055515216906e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1232.0,
|
|
"loss": 0.1403,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 5.0625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 12650
|
|
},
|
|
{
|
|
"epoch": 1.6180586618953288,
|
|
"grad_norm": 12.427781605965897,
|
|
"learning_rate": 5.351075620042622e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12660
|
|
},
|
|
{
|
|
"epoch": 1.6193366988305962,
|
|
"grad_norm": 8.807307663163346,
|
|
"learning_rate": 5.316643468103138e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1491,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12670
|
|
},
|
|
{
|
|
"epoch": 1.6206147357658636,
|
|
"grad_norm": 9.757673892289993,
|
|
"learning_rate": 5.282309267048995e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1662,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.75,
|
|
"step": 12680
|
|
},
|
|
{
|
|
"epoch": 1.621892772701131,
|
|
"grad_norm": 11.35318485146273,
|
|
"learning_rate": 5.2480731877385356e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1416,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12690
|
|
},
|
|
{
|
|
"epoch": 1.6231708096363984,
|
|
"grad_norm": 9.466002182166104,
|
|
"learning_rate": 5.2139354005417856e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1443,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12700
|
|
},
|
|
{
|
|
"epoch": 1.6244488465716658,
|
|
"grad_norm": 9.89133474990578,
|
|
"learning_rate": 5.179896075339671e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1601,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12710
|
|
},
|
|
{
|
|
"epoch": 1.6257268835069334,
|
|
"grad_norm": 9.510344550642348,
|
|
"learning_rate": 5.1459553815231094e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1591,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.75,
|
|
"rewards/rejected": -9.5,
|
|
"step": 12720
|
|
},
|
|
{
|
|
"epoch": 1.6270049204422008,
|
|
"grad_norm": 14.356022509608808,
|
|
"learning_rate": 5.112113487992209e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1766,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12730
|
|
},
|
|
{
|
|
"epoch": 1.6282829573774682,
|
|
"grad_norm": 10.447214245640378,
|
|
"learning_rate": 5.078370563155413e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1386,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 12740
|
|
},
|
|
{
|
|
"epoch": 1.6295609943127356,
|
|
"grad_norm": 11.237649051877147,
|
|
"learning_rate": 5.044726774928657e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.163,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.9375,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12750
|
|
},
|
|
{
|
|
"epoch": 1.6308390312480032,
|
|
"grad_norm": 10.054893207508396,
|
|
"learning_rate": 5.011182290734561e-08,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1602,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12760
|
|
},
|
|
{
|
|
"epoch": 1.6321170681832706,
|
|
"grad_norm": 10.084195714615372,
|
|
"learning_rate": 4.9777372775015456e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1447,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.375,
|
|
"step": 12770
|
|
},
|
|
{
|
|
"epoch": 1.633395105118538,
|
|
"grad_norm": 13.263484573524702,
|
|
"learning_rate": 4.9443919016630645e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1703,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12780
|
|
},
|
|
{
|
|
"epoch": 1.6346731420538054,
|
|
"grad_norm": 8.383512346411408,
|
|
"learning_rate": 4.9111463291567125e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1256.0,
|
|
"loss": 0.1559,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 5.5625,
|
|
"rewards/rejected": -11.125,
|
|
"step": 12790
|
|
},
|
|
{
|
|
"epoch": 1.6359511789890728,
|
|
"grad_norm": 10.580332639012939,
|
|
"learning_rate": 4.8780007254234394e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12800
|
|
},
|
|
{
|
|
"epoch": 1.6372292159243402,
|
|
"grad_norm": 19.51848775776392,
|
|
"learning_rate": 4.844955255406732e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1828,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12810
|
|
},
|
|
{
|
|
"epoch": 1.6385072528596076,
|
|
"grad_norm": 11.931564428067116,
|
|
"learning_rate": 4.812010083551757e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.16,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 12820
|
|
},
|
|
{
|
|
"epoch": 1.639785289794875,
|
|
"grad_norm": 11.081839896001414,
|
|
"learning_rate": 4.779165373804586e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1284,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12830
|
|
},
|
|
{
|
|
"epoch": 1.6410633267301424,
|
|
"grad_norm": 8.912648287414607,
|
|
"learning_rate": 4.746421289611338e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -752.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1423,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12840
|
|
},
|
|
{
|
|
"epoch": 1.6423413636654098,
|
|
"grad_norm": 11.476184210415996,
|
|
"learning_rate": 4.7137779939173893e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1474,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 12850
|
|
},
|
|
{
|
|
"epoch": 1.6436194006006772,
|
|
"grad_norm": 12.492110875247722,
|
|
"learning_rate": 4.6812356491665764e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.96875,
|
|
"rewards/rejected": -10.5,
|
|
"step": 12860
|
|
},
|
|
{
|
|
"epoch": 1.6448974375359446,
|
|
"grad_norm": 10.879943565961865,
|
|
"learning_rate": 4.6487944173003476e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1442,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12870
|
|
},
|
|
{
|
|
"epoch": 1.6461754744712123,
|
|
"grad_norm": 15.928020637284542,
|
|
"learning_rate": 4.616454459757002e-08,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 12880
|
|
},
|
|
{
|
|
"epoch": 1.6474535114064797,
|
|
"grad_norm": 7.269730657997192,
|
|
"learning_rate": 4.584215937470848e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 12890
|
|
},
|
|
{
|
|
"epoch": 1.648731548341747,
|
|
"grad_norm": 10.28920428876983,
|
|
"learning_rate": 4.552079010871415e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1585,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 12900
|
|
},
|
|
{
|
|
"epoch": 1.6500095852770145,
|
|
"grad_norm": 11.316966507611001,
|
|
"learning_rate": 4.520043839882684e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12910
|
|
},
|
|
{
|
|
"epoch": 1.651287622212282,
|
|
"grad_norm": 9.069174759597042,
|
|
"learning_rate": 4.488110583922236e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1694,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 12920
|
|
},
|
|
{
|
|
"epoch": 1.6525656591475495,
|
|
"grad_norm": 12.417263699859777,
|
|
"learning_rate": 4.456279401900523e-08,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 12930
|
|
},
|
|
{
|
|
"epoch": 1.653843696082817,
|
|
"grad_norm": 17.65844858555798,
|
|
"learning_rate": 4.424550452219994e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1847,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 12940
|
|
},
|
|
{
|
|
"epoch": 1.6551217330180843,
|
|
"grad_norm": 9.332315629733964,
|
|
"learning_rate": 4.392923892774411e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1723,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12950
|
|
},
|
|
{
|
|
"epoch": 1.6563997699533517,
|
|
"grad_norm": 11.230195658475209,
|
|
"learning_rate": 4.361399880947972e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12960
|
|
},
|
|
{
|
|
"epoch": 1.657677806888619,
|
|
"grad_norm": 10.757072801678362,
|
|
"learning_rate": 4.329978573614568e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.167,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 12970
|
|
},
|
|
{
|
|
"epoch": 1.6589558438238865,
|
|
"grad_norm": 12.568097778396032,
|
|
"learning_rate": 4.298660127137024e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1395,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 12980
|
|
},
|
|
{
|
|
"epoch": 1.660233880759154,
|
|
"grad_norm": 8.681036794407342,
|
|
"learning_rate": 4.26744469736626e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1503,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 12990
|
|
},
|
|
{
|
|
"epoch": 1.6615119176944213,
|
|
"grad_norm": 10.748730385170303,
|
|
"learning_rate": 4.2363324396405805e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1422,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 13000
|
|
},
|
|
{
|
|
"epoch": 1.6627899546296887,
|
|
"grad_norm": 9.54558033538067,
|
|
"learning_rate": 4.205323508784855e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.5,
|
|
"step": 13010
|
|
},
|
|
{
|
|
"epoch": 1.664067991564956,
|
|
"grad_norm": 9.69882496731971,
|
|
"learning_rate": 4.1744180591097605e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1611,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 13020
|
|
},
|
|
{
|
|
"epoch": 1.6653460285002235,
|
|
"grad_norm": 9.404584846040253,
|
|
"learning_rate": 4.1436162444110386e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1633,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13030
|
|
},
|
|
{
|
|
"epoch": 1.6666240654354911,
|
|
"grad_norm": 9.751104808340164,
|
|
"learning_rate": 4.112918217968683e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1687,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13040
|
|
},
|
|
{
|
|
"epoch": 1.6679021023707585,
|
|
"grad_norm": 9.070501787724893,
|
|
"learning_rate": 4.082324132546231e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1596,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13050
|
|
},
|
|
{
|
|
"epoch": 1.669180139306026,
|
|
"grad_norm": 9.824126557878227,
|
|
"learning_rate": 4.051834140389951e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1308,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 13060
|
|
},
|
|
{
|
|
"epoch": 1.6704581762412933,
|
|
"grad_norm": 13.590904771054559,
|
|
"learning_rate": 4.0214483932281106e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1474,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13070
|
|
},
|
|
{
|
|
"epoch": 1.671736213176561,
|
|
"grad_norm": 10.099487968529644,
|
|
"learning_rate": 3.991167042270241e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1365,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13080
|
|
},
|
|
{
|
|
"epoch": 1.6730142501118284,
|
|
"grad_norm": 13.315177673304635,
|
|
"learning_rate": 3.9609902382063353e-08,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1655,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 13090
|
|
},
|
|
{
|
|
"epoch": 1.6742922870470958,
|
|
"grad_norm": 10.878570202665678,
|
|
"learning_rate": 3.930918131206157e-08,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1493,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13100
|
|
},
|
|
{
|
|
"epoch": 1.6755703239823632,
|
|
"grad_norm": 10.960054640663829,
|
|
"learning_rate": 3.900950870918435e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1381,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13110
|
|
},
|
|
{
|
|
"epoch": 1.6768483609176306,
|
|
"grad_norm": 11.730572519108025,
|
|
"learning_rate": 3.871088606470152e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13120
|
|
},
|
|
{
|
|
"epoch": 1.678126397852898,
|
|
"grad_norm": 16.351532699106595,
|
|
"learning_rate": 3.841331486465818e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1627,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13130
|
|
},
|
|
{
|
|
"epoch": 1.6794044347881654,
|
|
"grad_norm": 10.074693097060957,
|
|
"learning_rate": 3.81167965898668e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13140
|
|
},
|
|
{
|
|
"epoch": 1.6806824717234328,
|
|
"grad_norm": 10.356973881084844,
|
|
"learning_rate": 3.782133271590046e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.124,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13150
|
|
},
|
|
{
|
|
"epoch": 1.6819605086587002,
|
|
"grad_norm": 7.5109538996121135,
|
|
"learning_rate": 3.7526924713084776e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.146,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13160
|
|
},
|
|
{
|
|
"epoch": 1.6832385455939676,
|
|
"grad_norm": 7.43584171219965,
|
|
"learning_rate": 3.7233574046491386e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -744.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13170
|
|
},
|
|
{
|
|
"epoch": 1.684516582529235,
|
|
"grad_norm": 8.091461209680139,
|
|
"learning_rate": 3.694128217593012e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -760.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1472,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -6.0,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13180
|
|
},
|
|
{
|
|
"epoch": 1.6857946194645024,
|
|
"grad_norm": 11.278883290132143,
|
|
"learning_rate": 3.665005055594183e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1577,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13190
|
|
},
|
|
{
|
|
"epoch": 1.68707265639977,
|
|
"grad_norm": 7.690946330940168,
|
|
"learning_rate": 3.635988063579137e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1398,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 5.0625,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13200
|
|
},
|
|
{
|
|
"epoch": 1.6883506933350374,
|
|
"grad_norm": 14.1181961523169,
|
|
"learning_rate": 3.607077385946003e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 13210
|
|
},
|
|
{
|
|
"epoch": 1.6896287302703048,
|
|
"grad_norm": 12.185505585818044,
|
|
"learning_rate": 3.578273166563878e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.157,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13220
|
|
},
|
|
{
|
|
"epoch": 1.6909067672055722,
|
|
"grad_norm": 10.486951579951219,
|
|
"learning_rate": 3.549575548772066e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.984375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1339,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13230
|
|
},
|
|
{
|
|
"epoch": 1.6921848041408398,
|
|
"grad_norm": 10.874078113401561,
|
|
"learning_rate": 3.52098467537939e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1447,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13240
|
|
},
|
|
{
|
|
"epoch": 1.6934628410761072,
|
|
"grad_norm": 8.559857941272588,
|
|
"learning_rate": 3.492500688663494e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 13250
|
|
},
|
|
{
|
|
"epoch": 1.6947408780113746,
|
|
"grad_norm": 11.945439184291894,
|
|
"learning_rate": 3.4641237303701e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1382,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13260
|
|
},
|
|
{
|
|
"epoch": 1.696018914946642,
|
|
"grad_norm": 10.703184305003559,
|
|
"learning_rate": 3.435853941712322e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1502,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.96875,
|
|
"rewards/rejected": -10.4375,
|
|
"step": 13270
|
|
},
|
|
{
|
|
"epoch": 1.6972969518819094,
|
|
"grad_norm": 9.473424218245944,
|
|
"learning_rate": 3.4076914633699825e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.9375,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1615,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13280
|
|
},
|
|
{
|
|
"epoch": 1.6985749888171768,
|
|
"grad_norm": 12.309402631962879,
|
|
"learning_rate": 3.379636435488864e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1200.0,
|
|
"loss": 0.1562,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 5.09375,
|
|
"rewards/rejected": -10.5,
|
|
"step": 13290
|
|
},
|
|
{
|
|
"epoch": 1.6998530257524442,
|
|
"grad_norm": 12.714540982337835,
|
|
"learning_rate": 3.351688997680069e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13300
|
|
},
|
|
{
|
|
"epoch": 1.7011310626877116,
|
|
"grad_norm": 12.042285791699674,
|
|
"learning_rate": 3.323849289019273e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13310
|
|
},
|
|
{
|
|
"epoch": 1.702409099622979,
|
|
"grad_norm": 8.951963033610214,
|
|
"learning_rate": 3.296117448046062e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1688,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13320
|
|
},
|
|
{
|
|
"epoch": 1.7036871365582464,
|
|
"grad_norm": 8.891445504358778,
|
|
"learning_rate": 3.268493612763246e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1341,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.875,
|
|
"step": 13330
|
|
},
|
|
{
|
|
"epoch": 1.7049651734935138,
|
|
"grad_norm": 9.866638871693574,
|
|
"learning_rate": 3.240977920636148e-08,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1541,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 13340
|
|
},
|
|
{
|
|
"epoch": 1.7062432104287812,
|
|
"grad_norm": 15.709750146528357,
|
|
"learning_rate": 3.2135705085919534e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.158,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13350
|
|
},
|
|
{
|
|
"epoch": 1.7075212473640489,
|
|
"grad_norm": 14.42470319889697,
|
|
"learning_rate": 3.186271513018993e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1594,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13360
|
|
},
|
|
{
|
|
"epoch": 1.7087992842993163,
|
|
"grad_norm": 8.930022776253411,
|
|
"learning_rate": 3.15908106976609e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1438,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13370
|
|
},
|
|
{
|
|
"epoch": 1.7100773212345837,
|
|
"grad_norm": 10.266994731297885,
|
|
"learning_rate": 3.131999314141873e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1345,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13380
|
|
},
|
|
{
|
|
"epoch": 1.711355358169851,
|
|
"grad_norm": 7.991014015948914,
|
|
"learning_rate": 3.1050263809141e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13390
|
|
},
|
|
{
|
|
"epoch": 1.7126333951051187,
|
|
"grad_norm": 9.533028636727256,
|
|
"learning_rate": 3.078162404309009e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1437,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13400
|
|
},
|
|
{
|
|
"epoch": 1.713911432040386,
|
|
"grad_norm": 10.166254139871006,
|
|
"learning_rate": 3.051407518010612e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1374,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13410
|
|
},
|
|
{
|
|
"epoch": 1.7151894689756535,
|
|
"grad_norm": 20.501536359023945,
|
|
"learning_rate": 3.024761855160077e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1432,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13420
|
|
},
|
|
{
|
|
"epoch": 1.716467505910921,
|
|
"grad_norm": 12.16897294342861,
|
|
"learning_rate": 2.9982255483550154e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13430
|
|
},
|
|
{
|
|
"epoch": 1.7177455428461883,
|
|
"grad_norm": 7.095714092157576,
|
|
"learning_rate": 2.9717987296488555e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13440
|
|
},
|
|
{
|
|
"epoch": 1.7190235797814557,
|
|
"grad_norm": 10.085330881143296,
|
|
"learning_rate": 2.9454815305501867e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.15,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.90625,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13450
|
|
},
|
|
{
|
|
"epoch": 1.720301616716723,
|
|
"grad_norm": 10.080544930632923,
|
|
"learning_rate": 2.9192740820220773e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1782,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13460
|
|
},
|
|
{
|
|
"epoch": 1.7215796536519905,
|
|
"grad_norm": 11.512873439831605,
|
|
"learning_rate": 2.893176514481463e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1597,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13470
|
|
},
|
|
{
|
|
"epoch": 1.722857690587258,
|
|
"grad_norm": 16.09918469753171,
|
|
"learning_rate": 2.8671889577984527e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1817,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13480
|
|
},
|
|
{
|
|
"epoch": 1.7241357275225253,
|
|
"grad_norm": 8.307363455025255,
|
|
"learning_rate": 2.8413115412957127e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 13490
|
|
},
|
|
{
|
|
"epoch": 1.7254137644577927,
|
|
"grad_norm": 14.318859134640787,
|
|
"learning_rate": 2.8155443937478264e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.165,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13500
|
|
},
|
|
{
|
|
"epoch": 1.72669180139306,
|
|
"grad_norm": 8.121559167464675,
|
|
"learning_rate": 2.78988764338062e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1377,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13510
|
|
},
|
|
{
|
|
"epoch": 1.7279698383283277,
|
|
"grad_norm": 11.152849450848047,
|
|
"learning_rate": 2.7643414178705748e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1072.0,
|
|
"loss": 0.1507,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 13520
|
|
},
|
|
{
|
|
"epoch": 1.7292478752635951,
|
|
"grad_norm": 9.953380185633824,
|
|
"learning_rate": 2.738905844344144e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1679,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.375,
|
|
"step": 13530
|
|
},
|
|
{
|
|
"epoch": 1.7305259121988625,
|
|
"grad_norm": 9.098672192531483,
|
|
"learning_rate": 2.7135810493771366e-08,
|
|
"logits/chosen": -2.890625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1056.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 13540
|
|
},
|
|
{
|
|
"epoch": 1.73180394913413,
|
|
"grad_norm": 13.080306160681328,
|
|
"learning_rate": 2.688367158994115e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1576,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13550
|
|
},
|
|
{
|
|
"epoch": 1.7330819860693976,
|
|
"grad_norm": 11.270529218973435,
|
|
"learning_rate": 2.663264298667711e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 13560
|
|
},
|
|
{
|
|
"epoch": 1.734360023004665,
|
|
"grad_norm": 10.513696283143453,
|
|
"learning_rate": 2.638272593318072e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1435,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 13570
|
|
},
|
|
{
|
|
"epoch": 1.7356380599399324,
|
|
"grad_norm": 8.904662666522817,
|
|
"learning_rate": 2.613392167312159e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1451,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13580
|
|
},
|
|
{
|
|
"epoch": 1.7369160968751998,
|
|
"grad_norm": 6.71524883761725,
|
|
"learning_rate": 2.588623144463206e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1224.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.90625,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 13590
|
|
},
|
|
{
|
|
"epoch": 1.7381941338104672,
|
|
"grad_norm": 7.921988234050129,
|
|
"learning_rate": 2.563965648030053e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1424,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 13600
|
|
},
|
|
{
|
|
"epoch": 1.7394721707457346,
|
|
"grad_norm": 10.377797310654964,
|
|
"learning_rate": 2.539419800716541e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1698,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13610
|
|
},
|
|
{
|
|
"epoch": 1.740750207681002,
|
|
"grad_norm": 10.564238013114593,
|
|
"learning_rate": 2.5149857246709262e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1525,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13620
|
|
},
|
|
{
|
|
"epoch": 1.7420282446162694,
|
|
"grad_norm": 11.163471055282269,
|
|
"learning_rate": 2.4906635414852328e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13630
|
|
},
|
|
{
|
|
"epoch": 1.7433062815515368,
|
|
"grad_norm": 7.886940638043434,
|
|
"learning_rate": 2.4664533721946906e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1346,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13640
|
|
},
|
|
{
|
|
"epoch": 1.7445843184868042,
|
|
"grad_norm": 8.139565850536563,
|
|
"learning_rate": 2.4423553372770995e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1431,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 13650
|
|
},
|
|
{
|
|
"epoch": 1.7458623554220716,
|
|
"grad_norm": 10.93235959546461,
|
|
"learning_rate": 2.4183695566522332e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.146,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.953125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 13660
|
|
},
|
|
{
|
|
"epoch": 1.747140392357339,
|
|
"grad_norm": 12.129375239791715,
|
|
"learning_rate": 2.3944961496812744e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.159,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 13670
|
|
},
|
|
{
|
|
"epoch": 1.7484184292926066,
|
|
"grad_norm": 9.591038353306839,
|
|
"learning_rate": 2.370735235166174e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13680
|
|
},
|
|
{
|
|
"epoch": 1.749696466227874,
|
|
"grad_norm": 8.64814249707982,
|
|
"learning_rate": 2.3470869313490987e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1526,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 13690
|
|
},
|
|
{
|
|
"epoch": 1.7509745031631414,
|
|
"grad_norm": 12.638604359599785,
|
|
"learning_rate": 2.3235513559118207e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -660.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1536,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13700
|
|
},
|
|
{
|
|
"epoch": 1.7522525400984088,
|
|
"grad_norm": 9.80199416228049,
|
|
"learning_rate": 2.3001286259751313e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.143,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 13710
|
|
},
|
|
{
|
|
"epoch": 1.7535305770336764,
|
|
"grad_norm": 11.838193078041444,
|
|
"learning_rate": 2.2768188580982866e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1622,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13720
|
|
},
|
|
{
|
|
"epoch": 1.7548086139689438,
|
|
"grad_norm": 11.25848518919155,
|
|
"learning_rate": 2.2536221682783794e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1411,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13730
|
|
},
|
|
{
|
|
"epoch": 1.7560866509042112,
|
|
"grad_norm": 10.808883838407986,
|
|
"learning_rate": 2.2305386719498165e-08,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.155,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13740
|
|
},
|
|
{
|
|
"epoch": 1.7573646878394786,
|
|
"grad_norm": 12.822469636687895,
|
|
"learning_rate": 2.2075684839836933e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1472,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13750
|
|
},
|
|
{
|
|
"epoch": 1.758642724774746,
|
|
"grad_norm": 10.580264723553709,
|
|
"learning_rate": 2.1847117186872516e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1519,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 13760
|
|
},
|
|
{
|
|
"epoch": 1.7599207617100134,
|
|
"grad_norm": 13.29768195211083,
|
|
"learning_rate": 2.1619684898033174e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1479,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13770
|
|
},
|
|
{
|
|
"epoch": 1.7611987986452808,
|
|
"grad_norm": 9.244016852518765,
|
|
"learning_rate": 2.1393389105097026e-08,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 13780
|
|
},
|
|
{
|
|
"epoch": 1.7624768355805482,
|
|
"grad_norm": 8.150376865571264,
|
|
"learning_rate": 2.1168230934186793e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 13790
|
|
},
|
|
{
|
|
"epoch": 1.7637548725158156,
|
|
"grad_norm": 8.674885967333253,
|
|
"learning_rate": 2.094421150576381e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1353,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 13800
|
|
},
|
|
{
|
|
"epoch": 1.765032909451083,
|
|
"grad_norm": 10.827112291973835,
|
|
"learning_rate": 2.072133193462286e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1453,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13810
|
|
},
|
|
{
|
|
"epoch": 1.7663109463863504,
|
|
"grad_norm": 11.82887235352997,
|
|
"learning_rate": 2.0499593329886293e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1395,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 13820
|
|
},
|
|
{
|
|
"epoch": 1.7675889833216178,
|
|
"grad_norm": 12.534723631836071,
|
|
"learning_rate": 2.0278996794998616e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1487,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13830
|
|
},
|
|
{
|
|
"epoch": 1.7688670202568855,
|
|
"grad_norm": 17.467097863848547,
|
|
"learning_rate": 2.0059543427721177e-08,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1671,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.625,
|
|
"step": 13840
|
|
},
|
|
{
|
|
"epoch": 1.7701450571921529,
|
|
"grad_norm": 9.588420485753964,
|
|
"learning_rate": 1.9841234320126356e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1525,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13850
|
|
},
|
|
{
|
|
"epoch": 1.7714230941274203,
|
|
"grad_norm": 14.998857660958935,
|
|
"learning_rate": 1.9624070558592444e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1446,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 13860
|
|
},
|
|
{
|
|
"epoch": 1.7727011310626877,
|
|
"grad_norm": 11.498974088453098,
|
|
"learning_rate": 1.9408053223798044e-08,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1605,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13870
|
|
},
|
|
{
|
|
"epoch": 1.7739791679979553,
|
|
"grad_norm": 11.34559480835344,
|
|
"learning_rate": 1.9193183390716768e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1512,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13880
|
|
},
|
|
{
|
|
"epoch": 1.7752572049332227,
|
|
"grad_norm": 15.385336092658148,
|
|
"learning_rate": 1.8979462128611957e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1687,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13890
|
|
},
|
|
{
|
|
"epoch": 1.77653524186849,
|
|
"grad_norm": 9.563306457518955,
|
|
"learning_rate": 1.8766890501031142e-08,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1571,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 13900
|
|
},
|
|
{
|
|
"epoch": 1.7778132788037575,
|
|
"grad_norm": 11.95208502989548,
|
|
"learning_rate": 1.8555469565801062e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1475,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13910
|
|
},
|
|
{
|
|
"epoch": 1.779091315739025,
|
|
"grad_norm": 12.686984192311087,
|
|
"learning_rate": 1.834520037502213e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1642,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 13920
|
|
},
|
|
{
|
|
"epoch": 1.7803693526742923,
|
|
"grad_norm": 9.574801828792555,
|
|
"learning_rate": 1.8136083975063193e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 13930
|
|
},
|
|
{
|
|
"epoch": 1.7816473896095597,
|
|
"grad_norm": 11.876562802028962,
|
|
"learning_rate": 1.7928121406556696e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1597,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 13940
|
|
},
|
|
{
|
|
"epoch": 1.782925426544827,
|
|
"grad_norm": 10.726637400908194,
|
|
"learning_rate": 1.7721313704392977e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1529,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.25,
|
|
"step": 13950
|
|
},
|
|
{
|
|
"epoch": 1.7842034634800945,
|
|
"grad_norm": 13.423853508050184,
|
|
"learning_rate": 1.751566189771561e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.143,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 13960
|
|
},
|
|
{
|
|
"epoch": 1.785481500415362,
|
|
"grad_norm": 10.942400408166316,
|
|
"learning_rate": 1.7311167009915833e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1715,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 13970
|
|
},
|
|
{
|
|
"epoch": 1.7867595373506293,
|
|
"grad_norm": 12.61138052341031,
|
|
"learning_rate": 1.7107830058627758e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1483,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.625,
|
|
"step": 13980
|
|
},
|
|
{
|
|
"epoch": 1.7880375742858967,
|
|
"grad_norm": 9.296731970428597,
|
|
"learning_rate": 1.6905652055723334e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1677,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 13990
|
|
},
|
|
{
|
|
"epoch": 1.7893156112211643,
|
|
"grad_norm": 17.27131332653722,
|
|
"learning_rate": 1.6704634007307068e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1597,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14000
|
|
},
|
|
{
|
|
"epoch": 1.7905936481564317,
|
|
"grad_norm": 13.643812551024203,
|
|
"learning_rate": 1.6504776913711132e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 14010
|
|
},
|
|
{
|
|
"epoch": 1.7918716850916991,
|
|
"grad_norm": 9.285428705838198,
|
|
"learning_rate": 1.6306081769490443e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1317,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14020
|
|
},
|
|
{
|
|
"epoch": 1.7931497220269665,
|
|
"grad_norm": 11.831120919568061,
|
|
"learning_rate": 1.6108549563417756e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1536,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.15625,
|
|
"rewards/margins": 4.9375,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14030
|
|
},
|
|
{
|
|
"epoch": 1.7944277589622342,
|
|
"grad_norm": 11.774157531421034,
|
|
"learning_rate": 1.5912181278478597e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14040
|
|
},
|
|
{
|
|
"epoch": 1.7957057958975016,
|
|
"grad_norm": 12.111032807950124,
|
|
"learning_rate": 1.5716977891866352e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1532,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14050
|
|
},
|
|
{
|
|
"epoch": 1.796983832832769,
|
|
"grad_norm": 12.49771071125514,
|
|
"learning_rate": 1.5522940374977618e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1382,
|
|
"rewards/accuracies": 0.981249988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.90625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 14060
|
|
},
|
|
{
|
|
"epoch": 1.7982618697680364,
|
|
"grad_norm": 15.091897925618236,
|
|
"learning_rate": 1.5330069693407165e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14070
|
|
},
|
|
{
|
|
"epoch": 1.7995399067033038,
|
|
"grad_norm": 15.012264420287543,
|
|
"learning_rate": 1.5138366806943302e-08,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1699,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14080
|
|
},
|
|
{
|
|
"epoch": 1.8008179436385712,
|
|
"grad_norm": 12.879249742583966,
|
|
"learning_rate": 1.4947832669562844e-08,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1372,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14090
|
|
},
|
|
{
|
|
"epoch": 1.8020959805738386,
|
|
"grad_norm": 10.117385918315096,
|
|
"learning_rate": 1.4758468229426612e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 14100
|
|
},
|
|
{
|
|
"epoch": 1.803374017509106,
|
|
"grad_norm": 10.434715411629035,
|
|
"learning_rate": 1.4570274428874679e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1402,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14110
|
|
},
|
|
{
|
|
"epoch": 1.8046520544443734,
|
|
"grad_norm": 10.257712949256526,
|
|
"learning_rate": 1.438325220442152e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1672,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14120
|
|
},
|
|
{
|
|
"epoch": 1.8059300913796408,
|
|
"grad_norm": 8.785574136152228,
|
|
"learning_rate": 1.4197402486751581e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.162,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14130
|
|
},
|
|
{
|
|
"epoch": 1.8072081283149082,
|
|
"grad_norm": 12.447729841189695,
|
|
"learning_rate": 1.401272620071442e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1643,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14140
|
|
},
|
|
{
|
|
"epoch": 1.8084861652501756,
|
|
"grad_norm": 10.070325070925708,
|
|
"learning_rate": 1.3829224265320211e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1333,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14150
|
|
},
|
|
{
|
|
"epoch": 1.8097642021854432,
|
|
"grad_norm": 12.903234521283592,
|
|
"learning_rate": 1.3646897593735273e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14160
|
|
},
|
|
{
|
|
"epoch": 1.8110422391207106,
|
|
"grad_norm": 12.686883856670724,
|
|
"learning_rate": 1.346574709327733e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.25,
|
|
"step": 14170
|
|
},
|
|
{
|
|
"epoch": 1.812320276055978,
|
|
"grad_norm": 14.22861593842286,
|
|
"learning_rate": 1.3285773665411143e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 5.125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 14180
|
|
},
|
|
{
|
|
"epoch": 1.8135983129912454,
|
|
"grad_norm": 9.789271909311166,
|
|
"learning_rate": 1.3106978205743968e-08,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1674,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14190
|
|
},
|
|
{
|
|
"epoch": 1.814876349926513,
|
|
"grad_norm": 12.328876632604587,
|
|
"learning_rate": 1.2929361604021028e-08,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1405,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14200
|
|
},
|
|
{
|
|
"epoch": 1.8161543868617804,
|
|
"grad_norm": 7.992402341168219,
|
|
"learning_rate": 1.275292474412129e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1485,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14210
|
|
},
|
|
{
|
|
"epoch": 1.8174324237970478,
|
|
"grad_norm": 11.691821378605601,
|
|
"learning_rate": 1.2577668504052813e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1412,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.9375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 14220
|
|
},
|
|
{
|
|
"epoch": 1.8187104607323152,
|
|
"grad_norm": 12.155431721595809,
|
|
"learning_rate": 1.2403593755948572e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1481,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14230
|
|
},
|
|
{
|
|
"epoch": 1.8199884976675826,
|
|
"grad_norm": 9.91908769920145,
|
|
"learning_rate": 1.2230701366061969e-08,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1344,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14240
|
|
},
|
|
{
|
|
"epoch": 1.82126653460285,
|
|
"grad_norm": 9.339881834605144,
|
|
"learning_rate": 1.2058992194762724e-08,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1335,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14250
|
|
},
|
|
{
|
|
"epoch": 1.8225445715381174,
|
|
"grad_norm": 11.10352886253188,
|
|
"learning_rate": 1.1888467096532406e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.96875,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1716,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 3.703125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14260
|
|
},
|
|
{
|
|
"epoch": 1.8238226084733848,
|
|
"grad_norm": 14.93846998373106,
|
|
"learning_rate": 1.1719126919960187e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1538,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14270
|
|
},
|
|
{
|
|
"epoch": 1.8251006454086522,
|
|
"grad_norm": 10.202931139450149,
|
|
"learning_rate": 1.1550972507738815e-08,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1596,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14280
|
|
},
|
|
{
|
|
"epoch": 1.8263786823439196,
|
|
"grad_norm": 8.27863731117087,
|
|
"learning_rate": 1.1384004696660148e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14290
|
|
},
|
|
{
|
|
"epoch": 1.827656719279187,
|
|
"grad_norm": 12.30629947950159,
|
|
"learning_rate": 1.1218224317611159e-08,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1425,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14300
|
|
},
|
|
{
|
|
"epoch": 1.8289347562144544,
|
|
"grad_norm": 6.7009280288802735,
|
|
"learning_rate": 1.1053632195569845e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1359,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 14310
|
|
},
|
|
{
|
|
"epoch": 1.830212793149722,
|
|
"grad_norm": 14.503511364519225,
|
|
"learning_rate": 1.0890229149600943e-08,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -740.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1624,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.84375,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14320
|
|
},
|
|
{
|
|
"epoch": 1.8314908300849895,
|
|
"grad_norm": 13.359834100545635,
|
|
"learning_rate": 1.072801599285203e-08,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 5.125,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 14330
|
|
},
|
|
{
|
|
"epoch": 1.8327688670202569,
|
|
"grad_norm": 9.795144374083144,
|
|
"learning_rate": 1.0566993532549363e-08,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1464,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.125,
|
|
"step": 14340
|
|
},
|
|
{
|
|
"epoch": 1.8340469039555243,
|
|
"grad_norm": 11.966682565809306,
|
|
"learning_rate": 1.0407162569993855e-08,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1441,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14350
|
|
},
|
|
{
|
|
"epoch": 1.835324940890792,
|
|
"grad_norm": 11.976332678053666,
|
|
"learning_rate": 1.024852390055722e-08,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1437,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14360
|
|
},
|
|
{
|
|
"epoch": 1.8366029778260593,
|
|
"grad_norm": 11.073959106533652,
|
|
"learning_rate": 1.0091078313677892e-08,
|
|
"logits/chosen": -2.8125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1351,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14370
|
|
},
|
|
{
|
|
"epoch": 1.8378810147613267,
|
|
"grad_norm": 13.967643551885132,
|
|
"learning_rate": 9.93482659285716e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1459,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14380
|
|
},
|
|
{
|
|
"epoch": 1.839159051696594,
|
|
"grad_norm": 12.418189122707291,
|
|
"learning_rate": 9.779769515655179e-09,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1549,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.1875,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14390
|
|
},
|
|
{
|
|
"epoch": 1.8404370886318615,
|
|
"grad_norm": 10.408033015979928,
|
|
"learning_rate": 9.625907853687165e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14400
|
|
},
|
|
{
|
|
"epoch": 1.841715125567129,
|
|
"grad_norm": 9.263369906030954,
|
|
"learning_rate": 9.473242372619621e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1467,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14410
|
|
},
|
|
{
|
|
"epoch": 1.8429931625023963,
|
|
"grad_norm": 12.739122397711153,
|
|
"learning_rate": 9.321773832166397e-09,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1606,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14420
|
|
},
|
|
{
|
|
"epoch": 1.8442711994376637,
|
|
"grad_norm": 11.839371678365538,
|
|
"learning_rate": 9.171502986085072e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1584,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14430
|
|
},
|
|
{
|
|
"epoch": 1.8455492363729311,
|
|
"grad_norm": 12.856022816191095,
|
|
"learning_rate": 9.022430582172919e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14440
|
|
},
|
|
{
|
|
"epoch": 1.8468272733081985,
|
|
"grad_norm": 8.926987752379693,
|
|
"learning_rate": 8.874557362263529e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1439,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14450
|
|
},
|
|
{
|
|
"epoch": 1.848105310243466,
|
|
"grad_norm": 12.864364750595266,
|
|
"learning_rate": 8.727884062222885e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1666,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14460
|
|
},
|
|
{
|
|
"epoch": 1.8493833471787333,
|
|
"grad_norm": 12.647684825072547,
|
|
"learning_rate": 8.582411411945768e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1537,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.03125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14470
|
|
},
|
|
{
|
|
"epoch": 1.850661384114001,
|
|
"grad_norm": 11.822844846768222,
|
|
"learning_rate": 8.438140135352211e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.546875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1717,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14480
|
|
},
|
|
{
|
|
"epoch": 1.8519394210492683,
|
|
"grad_norm": 13.036845289997391,
|
|
"learning_rate": 8.295070950383647e-09,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1613,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14490
|
|
},
|
|
{
|
|
"epoch": 1.8532174579845357,
|
|
"grad_norm": 9.44512506950682,
|
|
"learning_rate": 8.153204568999733e-09,
|
|
"logits/chosen": -3.1875,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -664.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1337,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14500
|
|
},
|
|
{
|
|
"epoch": 1.8544954949198031,
|
|
"grad_norm": 10.391187249363702,
|
|
"learning_rate": 8.012541697174414e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.703125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1514,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14510
|
|
},
|
|
{
|
|
"epoch": 1.8557735318550708,
|
|
"grad_norm": 14.04567191399965,
|
|
"learning_rate": 7.873083034892537e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1443,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -10.1875,
|
|
"step": 14520
|
|
},
|
|
{
|
|
"epoch": 1.8570515687903382,
|
|
"grad_norm": 13.633018900248839,
|
|
"learning_rate": 7.734829276146576e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.609375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1552,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14530
|
|
},
|
|
{
|
|
"epoch": 1.8583296057256056,
|
|
"grad_norm": 14.81761718347149,
|
|
"learning_rate": 7.59778110893286e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1421,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.4375,
|
|
"step": 14540
|
|
},
|
|
{
|
|
"epoch": 1.859607642660873,
|
|
"grad_norm": 8.88382465270999,
|
|
"learning_rate": 7.461939215248403e-09,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1575,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14550
|
|
},
|
|
{
|
|
"epoch": 1.8608856795961404,
|
|
"grad_norm": 10.324327735645847,
|
|
"learning_rate": 7.327304271087298e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1674,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 3.890625,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14560
|
|
},
|
|
{
|
|
"epoch": 1.8621637165314078,
|
|
"grad_norm": 11.893562246279691,
|
|
"learning_rate": 7.1938769464375e-09,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1663,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 5.03125,
|
|
"rewards/rejected": -10.375,
|
|
"step": 14570
|
|
},
|
|
{
|
|
"epoch": 1.8634417534666752,
|
|
"grad_norm": 12.310902619295284,
|
|
"learning_rate": 7.061657905277518e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1505,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14580
|
|
},
|
|
{
|
|
"epoch": 1.8647197904019426,
|
|
"grad_norm": 12.099849619715219,
|
|
"learning_rate": 6.9306478055729254e-09,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1593,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 14590
|
|
},
|
|
{
|
|
"epoch": 1.86599782733721,
|
|
"grad_norm": 16.938882516568054,
|
|
"learning_rate": 6.800847299273327e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1653,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14600
|
|
},
|
|
{
|
|
"epoch": 1.8672758642724774,
|
|
"grad_norm": 11.587208367743546,
|
|
"learning_rate": 6.672257032308864e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1547,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.375,
|
|
"step": 14610
|
|
},
|
|
{
|
|
"epoch": 1.8685539012077448,
|
|
"grad_norm": 11.619646400256674,
|
|
"learning_rate": 6.544877644587193e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1465,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 3.984375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14620
|
|
},
|
|
{
|
|
"epoch": 1.8698319381430122,
|
|
"grad_norm": 12.217282596762695,
|
|
"learning_rate": 6.418709769990316e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1638,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.5,
|
|
"step": 14630
|
|
},
|
|
{
|
|
"epoch": 1.8711099750782798,
|
|
"grad_norm": 10.678100699980412,
|
|
"learning_rate": 6.293754036371141e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1413,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14640
|
|
},
|
|
{
|
|
"epoch": 1.8723880120135472,
|
|
"grad_norm": 16.210974014943545,
|
|
"learning_rate": 6.170011065550818e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1088.0,
|
|
"loss": 0.1387,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 14650
|
|
},
|
|
{
|
|
"epoch": 1.8736660489488146,
|
|
"grad_norm": 10.127056266258883,
|
|
"learning_rate": 6.047481473315102e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1491,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.5625,
|
|
"step": 14660
|
|
},
|
|
{
|
|
"epoch": 1.874944085884082,
|
|
"grad_norm": 11.71800474936891,
|
|
"learning_rate": 5.926165869411859e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1671,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.09375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.625,
|
|
"step": 14670
|
|
},
|
|
{
|
|
"epoch": 1.8762221228193496,
|
|
"grad_norm": 10.427549178650835,
|
|
"learning_rate": 5.8060648575475326e-09,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1589,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14680
|
|
},
|
|
{
|
|
"epoch": 1.877500159754617,
|
|
"grad_norm": 9.623257167478432,
|
|
"learning_rate": 5.68717903538446e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1331,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14690
|
|
},
|
|
{
|
|
"epoch": 1.8787781966898844,
|
|
"grad_norm": 10.184536050278489,
|
|
"learning_rate": 5.569508994537814e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.144,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14700
|
|
},
|
|
{
|
|
"epoch": 1.8800562336251518,
|
|
"grad_norm": 11.21262756503898,
|
|
"learning_rate": 5.453055320572525e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1438,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14710
|
|
},
|
|
{
|
|
"epoch": 1.8813342705604192,
|
|
"grad_norm": 8.276067538916859,
|
|
"learning_rate": 5.337818593000615e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.65625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14720
|
|
},
|
|
{
|
|
"epoch": 1.8826123074956866,
|
|
"grad_norm": 13.181049739902091,
|
|
"learning_rate": 5.22379938527806e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1453,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14730
|
|
},
|
|
{
|
|
"epoch": 1.883890344430954,
|
|
"grad_norm": 13.959442989381273,
|
|
"learning_rate": 5.1109982648021296e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.640625,
|
|
"logps/chosen": -724.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1455,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.6875,
|
|
"rewards/margins": 4.0625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14740
|
|
},
|
|
{
|
|
"epoch": 1.8851683813662214,
|
|
"grad_norm": 13.495500872308062,
|
|
"learning_rate": 4.999415792908495e-09,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1599,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14750
|
|
},
|
|
{
|
|
"epoch": 1.8864464183014888,
|
|
"grad_norm": 14.023588667374646,
|
|
"learning_rate": 4.889052524868348e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1312,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14760
|
|
},
|
|
{
|
|
"epoch": 1.8877244552367562,
|
|
"grad_norm": 7.947339683870802,
|
|
"learning_rate": 4.779909009885841e-09,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1537,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14770
|
|
},
|
|
{
|
|
"epoch": 1.8890024921720237,
|
|
"grad_norm": 10.536940544419787,
|
|
"learning_rate": 4.6719857910950955e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1208.0,
|
|
"loss": 0.1319,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 5.1875,
|
|
"rewards/rejected": -10.5625,
|
|
"step": 14780
|
|
},
|
|
{
|
|
"epoch": 1.890280529107291,
|
|
"grad_norm": 11.818957062933153,
|
|
"learning_rate": 4.565283405557757e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.139,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 14790
|
|
},
|
|
{
|
|
"epoch": 1.8915585660425587,
|
|
"grad_norm": 9.286164362041376,
|
|
"learning_rate": 4.4598023842601085e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1468,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14800
|
|
},
|
|
{
|
|
"epoch": 1.892836602977826,
|
|
"grad_norm": 11.296676357308892,
|
|
"learning_rate": 4.3555432521105996e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1618,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 14810
|
|
},
|
|
{
|
|
"epoch": 1.8941146399130935,
|
|
"grad_norm": 13.00937492775704,
|
|
"learning_rate": 4.2525065279370995e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.164,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14820
|
|
},
|
|
{
|
|
"epoch": 1.8953926768483609,
|
|
"grad_norm": 10.15410344253207,
|
|
"learning_rate": 4.1506927244844004e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1349,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14830
|
|
},
|
|
{
|
|
"epoch": 1.8966707137836285,
|
|
"grad_norm": 9.119767621505575,
|
|
"learning_rate": 4.050102348411605e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1548,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14840
|
|
},
|
|
{
|
|
"epoch": 1.897948750718896,
|
|
"grad_norm": 14.469398940553326,
|
|
"learning_rate": 3.9507359002897145e-09,
|
|
"logits/chosen": -3.171875,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1709,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.25,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14850
|
|
},
|
|
{
|
|
"epoch": 1.8992267876541633,
|
|
"grad_norm": 12.231871395300658,
|
|
"learning_rate": 3.852593874598936e-09,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -748.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1719,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.875,
|
|
"rewards/margins": 3.921875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14860
|
|
},
|
|
{
|
|
"epoch": 1.9005048245894307,
|
|
"grad_norm": 12.073682499643972,
|
|
"learning_rate": 3.755676759726434e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1477,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 14870
|
|
},
|
|
{
|
|
"epoch": 1.901782861524698,
|
|
"grad_norm": 12.282004370609691,
|
|
"learning_rate": 3.6599850379638032e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1394,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.75,
|
|
"step": 14880
|
|
},
|
|
{
|
|
"epoch": 1.9030608984599655,
|
|
"grad_norm": 10.207448871691478,
|
|
"learning_rate": 3.565519185504684e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1445,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14890
|
|
},
|
|
{
|
|
"epoch": 1.904338935395233,
|
|
"grad_norm": 11.856070963380358,
|
|
"learning_rate": 3.4722796724423466e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1632,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 14900
|
|
},
|
|
{
|
|
"epoch": 1.9056169723305003,
|
|
"grad_norm": 14.22172415536922,
|
|
"learning_rate": 3.380266962767386e-09,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1582,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.96875,
|
|
"rewards/rejected": -10.375,
|
|
"step": 14910
|
|
},
|
|
{
|
|
"epoch": 1.9068950092657677,
|
|
"grad_norm": 13.602757040527935,
|
|
"learning_rate": 3.289481514365533e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1192.0,
|
|
"loss": 0.1476,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 5.09375,
|
|
"rewards/rejected": -10.375,
|
|
"step": 14920
|
|
},
|
|
{
|
|
"epoch": 1.9081730462010351,
|
|
"grad_norm": 17.815448615194278,
|
|
"learning_rate": 3.199923779015068e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1674,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14930
|
|
},
|
|
{
|
|
"epoch": 1.9094510831363025,
|
|
"grad_norm": 9.167336092535807,
|
|
"learning_rate": 3.111594202384937e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1453,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 14940
|
|
},
|
|
{
|
|
"epoch": 1.91072912007157,
|
|
"grad_norm": 15.831129982327335,
|
|
"learning_rate": 3.0244932240323095e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1571,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0,
|
|
"step": 14950
|
|
},
|
|
{
|
|
"epoch": 1.9120071570068375,
|
|
"grad_norm": 10.519254389733671,
|
|
"learning_rate": 2.938621277400355e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1096.0,
|
|
"loss": 0.1486,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.625,
|
|
"step": 14960
|
|
},
|
|
{
|
|
"epoch": 1.913285193942105,
|
|
"grad_norm": 12.955231124203314,
|
|
"learning_rate": 2.8539787898163302e-09,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.125,
|
|
"step": 14970
|
|
},
|
|
{
|
|
"epoch": 1.9145632308773723,
|
|
"grad_norm": 14.75785834916094,
|
|
"learning_rate": 2.7705661824891922e-09,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14980
|
|
},
|
|
{
|
|
"epoch": 1.9158412678126397,
|
|
"grad_norm": 11.390718009084507,
|
|
"learning_rate": 2.6883838705076265e-09,
|
|
"logits/chosen": -2.90625,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1499,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.75,
|
|
"rewards/margins": 4.1875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 14990
|
|
},
|
|
{
|
|
"epoch": 1.9171193047479074,
|
|
"grad_norm": 9.648512751461917,
|
|
"learning_rate": 2.607432262837966e-09,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1498,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15000
|
|
},
|
|
{
|
|
"epoch": 1.9183973416831748,
|
|
"grad_norm": 10.772529811188837,
|
|
"learning_rate": 2.5277117623221654e-09,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1080.0,
|
|
"loss": 0.1565,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.15625,
|
|
"rewards/rejected": -9.5,
|
|
"step": 15010
|
|
},
|
|
{
|
|
"epoch": 1.9196753786184422,
|
|
"grad_norm": 11.968154707014914,
|
|
"learning_rate": 2.4492227656757736e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1557,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.21875,
|
|
"rewards/rejected": -9.75,
|
|
"step": 15020
|
|
},
|
|
{
|
|
"epoch": 1.9209534155537096,
|
|
"grad_norm": 12.502294203547288,
|
|
"learning_rate": 2.3719656634859642e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15030
|
|
},
|
|
{
|
|
"epoch": 1.922231452488977,
|
|
"grad_norm": 8.959430299711036,
|
|
"learning_rate": 2.2959408402096202e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1575,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15040
|
|
},
|
|
{
|
|
"epoch": 1.9235094894242444,
|
|
"grad_norm": 10.710421147055664,
|
|
"learning_rate": 2.2211486741713635e-09,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1467,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 15050
|
|
},
|
|
{
|
|
"epoch": 1.9247875263595118,
|
|
"grad_norm": 11.312570556276544,
|
|
"learning_rate": 2.1475895375617226e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1636,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 15060
|
|
},
|
|
{
|
|
"epoch": 1.9260655632947792,
|
|
"grad_norm": 12.268507744807593,
|
|
"learning_rate": 2.0752637964352727e-09,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 15070
|
|
},
|
|
{
|
|
"epoch": 1.9273436002300466,
|
|
"grad_norm": 8.086827719122715,
|
|
"learning_rate": 2.0041718107088046e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1201,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.8125,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 15080
|
|
},
|
|
{
|
|
"epoch": 1.928621637165314,
|
|
"grad_norm": 10.64959916901656,
|
|
"learning_rate": 1.9343139341595204e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1603,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15090
|
|
},
|
|
{
|
|
"epoch": 1.9298996741005814,
|
|
"grad_norm": 11.142977135987163,
|
|
"learning_rate": 1.865690514423257e-09,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1431,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15100
|
|
},
|
|
{
|
|
"epoch": 1.9311777110358488,
|
|
"grad_norm": 13.70615553224399,
|
|
"learning_rate": 1.7983018929929039e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -736.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1518,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15110
|
|
},
|
|
{
|
|
"epoch": 1.9324557479711164,
|
|
"grad_norm": 17.93121904701502,
|
|
"learning_rate": 1.7321484052164325e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1596,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15120
|
|
},
|
|
{
|
|
"epoch": 1.9337337849063838,
|
|
"grad_norm": 11.71594907844557,
|
|
"learning_rate": 1.6672303802954812e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1679,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.65625,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15130
|
|
},
|
|
{
|
|
"epoch": 1.9350118218416512,
|
|
"grad_norm": 9.520084659517018,
|
|
"learning_rate": 1.603548141283606e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1493,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.0,
|
|
"rewards/rejected": -9.625,
|
|
"step": 15140
|
|
},
|
|
{
|
|
"epoch": 1.9362898587769186,
|
|
"grad_norm": 7.489687357486243,
|
|
"learning_rate": 1.5411020050846434e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1541,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.53125,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15150
|
|
},
|
|
{
|
|
"epoch": 1.9375678957121862,
|
|
"grad_norm": 14.027073581679955,
|
|
"learning_rate": 1.4798922824512116e-09,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.154,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15160
|
|
},
|
|
{
|
|
"epoch": 1.9388459326474536,
|
|
"grad_norm": 8.37642421142221,
|
|
"learning_rate": 1.4199192779831282e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1431,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15170
|
|
},
|
|
{
|
|
"epoch": 1.940123969582721,
|
|
"grad_norm": 10.76061449217435,
|
|
"learning_rate": 1.3611832901258558e-09,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.146,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.25,
|
|
"step": 15180
|
|
},
|
|
{
|
|
"epoch": 1.9414020065179884,
|
|
"grad_norm": 13.39651566636036,
|
|
"learning_rate": 1.3036846111690869e-09,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1502,
|
|
"rewards/accuracies": 0.8687499761581421,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 15190
|
|
},
|
|
{
|
|
"epoch": 1.9426800434532558,
|
|
"grad_norm": 16.455139746642296,
|
|
"learning_rate": 1.247423527245217e-09,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -676.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1469,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 15200
|
|
},
|
|
{
|
|
"epoch": 1.9439580803885232,
|
|
"grad_norm": 13.102136764965527,
|
|
"learning_rate": 1.192400318328013e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.921875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.135,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15210
|
|
},
|
|
{
|
|
"epoch": 1.9452361173237906,
|
|
"grad_norm": 14.701240584022592,
|
|
"learning_rate": 1.138615258231168e-09,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -756.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.90625,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15220
|
|
},
|
|
{
|
|
"epoch": 1.946514154259058,
|
|
"grad_norm": 10.082463775879928,
|
|
"learning_rate": 1.086068614606861e-09,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1408,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15230
|
|
},
|
|
{
|
|
"epoch": 1.9477921911943255,
|
|
"grad_norm": 11.289323866928568,
|
|
"learning_rate": 1.0347606489445881e-09,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1522,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -5.71875,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15240
|
|
},
|
|
{
|
|
"epoch": 1.9490702281295929,
|
|
"grad_norm": 10.055014544286356,
|
|
"learning_rate": 9.846916165696938e-10,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -732.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1408,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15250
|
|
},
|
|
{
|
|
"epoch": 1.9503482650648603,
|
|
"grad_norm": 15.40463618483445,
|
|
"learning_rate": 9.358617666422875e-10,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1509,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15260
|
|
},
|
|
{
|
|
"epoch": 1.9516263020001277,
|
|
"grad_norm": 9.301535906865904,
|
|
"learning_rate": 8.882713421557997e-10,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.78125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15270
|
|
},
|
|
{
|
|
"epoch": 1.9529043389353953,
|
|
"grad_norm": 11.068625667949027,
|
|
"learning_rate": 8.419205799359009e-10,
|
|
"logits/chosen": -3.15625,
|
|
"logits/rejected": -2.953125,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.75,
|
|
"step": 15280
|
|
},
|
|
{
|
|
"epoch": 1.9541823758706627,
|
|
"grad_norm": 10.016924649285729,
|
|
"learning_rate": 7.968097106393345e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -680.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 5.0,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 15290
|
|
},
|
|
{
|
|
"epoch": 1.95546041280593,
|
|
"grad_norm": 13.01372115011469,
|
|
"learning_rate": 7.529389587526413e-10,
|
|
"logits/chosen": -3.140625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1491,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.75,
|
|
"rewards/rejected": -10.25,
|
|
"step": 15300
|
|
},
|
|
{
|
|
"epoch": 1.9567384497411975,
|
|
"grad_norm": 10.0163137391518,
|
|
"learning_rate": 7.103085425912147e-10,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.671875,
|
|
"logps/chosen": -668.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1503,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.125,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15310
|
|
},
|
|
{
|
|
"epoch": 1.958016486676465,
|
|
"grad_norm": 16.423064610005266,
|
|
"learning_rate": 6.689186742980524e-10,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15320
|
|
},
|
|
{
|
|
"epoch": 1.9592945236117325,
|
|
"grad_norm": 13.916020179023214,
|
|
"learning_rate": 6.287695598428399e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1542,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15330
|
|
},
|
|
{
|
|
"epoch": 1.960572560547,
|
|
"grad_norm": 9.588268395911374,
|
|
"learning_rate": 5.898613990208412e-10,
|
|
"logits/chosen": -2.984375,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1403,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.625,
|
|
"rewards/margins": 4.125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 15340
|
|
},
|
|
{
|
|
"epoch": 1.9618505974822673,
|
|
"grad_norm": 13.784001571321943,
|
|
"learning_rate": 5.521943854518984e-10,
|
|
"logits/chosen": -2.921875,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -728.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1405,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.5625,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15350
|
|
},
|
|
{
|
|
"epoch": 1.9631286344175347,
|
|
"grad_norm": 9.205782297316095,
|
|
"learning_rate": 5.157687065795446e-10,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1602,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15360
|
|
},
|
|
{
|
|
"epoch": 1.9644066713528021,
|
|
"grad_norm": 13.261409322792284,
|
|
"learning_rate": 4.805845436699763e-10,
|
|
"logits/chosen": -2.953125,
|
|
"logits/rejected": -2.71875,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1592,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15370
|
|
},
|
|
{
|
|
"epoch": 1.9656847082880695,
|
|
"grad_norm": 13.788978001801002,
|
|
"learning_rate": 4.4664207181119295e-10,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -700.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1638,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.125,
|
|
"step": 15380
|
|
},
|
|
{
|
|
"epoch": 1.966962745223337,
|
|
"grad_norm": 14.237004738621629,
|
|
"learning_rate": 4.1394145991219267e-10,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.90625,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": -5.4375,
|
|
"rewards/margins": 4.5,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15390
|
|
},
|
|
{
|
|
"epoch": 1.9682407821586043,
|
|
"grad_norm": 10.140151106144453,
|
|
"learning_rate": 3.8248287070200003e-10,
|
|
"logits/chosen": -2.9375,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -708.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1583,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.34375,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15400
|
|
},
|
|
{
|
|
"epoch": 1.9695188190938717,
|
|
"grad_norm": 14.432458275606095,
|
|
"learning_rate": 3.5226646072894497e-10,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -692.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.153,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15410
|
|
},
|
|
{
|
|
"epoch": 1.9707968560291391,
|
|
"grad_norm": 11.184958679299635,
|
|
"learning_rate": 3.232923803598575e-10,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1104.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -5.28125,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 15420
|
|
},
|
|
{
|
|
"epoch": 1.9720748929644065,
|
|
"grad_norm": 11.322731302817918,
|
|
"learning_rate": 2.955607737793464e-10,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15430
|
|
},
|
|
{
|
|
"epoch": 1.9733529298996741,
|
|
"grad_norm": 9.0128125279139,
|
|
"learning_rate": 2.690717789890773e-10,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1289,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.40625,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15440
|
|
},
|
|
{
|
|
"epoch": 1.9746309668349415,
|
|
"grad_norm": 7.468856109935237,
|
|
"learning_rate": 2.4382552780696787e-10,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.25,
|
|
"rewards/margins": 4.4375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 15450
|
|
},
|
|
{
|
|
"epoch": 1.975909003770209,
|
|
"grad_norm": 14.22098149523964,
|
|
"learning_rate": 2.1982214586679927e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -712.0,
|
|
"logps/rejected": -1168.0,
|
|
"loss": 0.1495,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.59375,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15460
|
|
},
|
|
{
|
|
"epoch": 1.9771870407054764,
|
|
"grad_norm": 12.531352602719545,
|
|
"learning_rate": 1.9706175261724467e-10,
|
|
"logits/chosen": -3.109375,
|
|
"logits/rejected": -2.859375,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1508,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.5,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15470
|
|
},
|
|
{
|
|
"epoch": 1.978465077640744,
|
|
"grad_norm": 12.129161571300925,
|
|
"learning_rate": 1.7554446132159172e-10,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1534,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.8125,
|
|
"step": 15480
|
|
},
|
|
{
|
|
"epoch": 1.9797431145760114,
|
|
"grad_norm": 10.829943019749386,
|
|
"learning_rate": 1.5527037905699315e-10,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1176.0,
|
|
"loss": 0.1454,
|
|
"rewards/accuracies": 0.956250011920929,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.84375,
|
|
"rewards/rejected": -10.3125,
|
|
"step": 15490
|
|
},
|
|
{
|
|
"epoch": 1.9810211515112788,
|
|
"grad_norm": 10.449380680038518,
|
|
"learning_rate": 1.362396067139393e-10,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1609,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 15500
|
|
},
|
|
{
|
|
"epoch": 1.9822991884465462,
|
|
"grad_norm": 10.15870473477103,
|
|
"learning_rate": 1.1845223899586975e-10,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.734375,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1184.0,
|
|
"loss": 0.1576,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.78125,
|
|
"rewards/rejected": -10.25,
|
|
"step": 15510
|
|
},
|
|
{
|
|
"epoch": 1.9835772253818136,
|
|
"grad_norm": 12.471330269650029,
|
|
"learning_rate": 1.019083644185903e-10,
|
|
"logits/chosen": -3.0625,
|
|
"logits/rejected": -2.796875,
|
|
"logps/chosen": -688.0,
|
|
"logps/rejected": -1112.0,
|
|
"loss": 0.1533,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.375,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.625,
|
|
"step": 15520
|
|
},
|
|
{
|
|
"epoch": 1.984855262317081,
|
|
"grad_norm": 10.493625321839762,
|
|
"learning_rate": 8.660806530991216e-11,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.59375,
|
|
"logps/chosen": -684.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1667,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.3125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 15530
|
|
},
|
|
{
|
|
"epoch": 1.9861332992523484,
|
|
"grad_norm": 17.377116192588073,
|
|
"learning_rate": 7.255141780918018e-11,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.84375,
|
|
"logps/chosen": -768.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1665,
|
|
"rewards/accuracies": 0.893750011920929,
|
|
"rewards/chosen": -5.8125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15540
|
|
},
|
|
{
|
|
"epoch": 1.9874113361876158,
|
|
"grad_norm": 13.187550536514495,
|
|
"learning_rate": 5.973849186685643e-11,
|
|
"logits/chosen": -2.96875,
|
|
"logits/rejected": -2.8125,
|
|
"logps/chosen": -780.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1823,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -6.03125,
|
|
"rewards/margins": 4.09375,
|
|
"rewards/rejected": -10.125,
|
|
"step": 15550
|
|
},
|
|
{
|
|
"epoch": 1.9886893731228832,
|
|
"grad_norm": 11.526002181186742,
|
|
"learning_rate": 4.816935124435373e-11,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.6875,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1216.0,
|
|
"loss": 0.1346,
|
|
"rewards/accuracies": 0.9437500238418579,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 5.0,
|
|
"rewards/rejected": -10.375,
|
|
"step": 15560
|
|
},
|
|
{
|
|
"epoch": 1.9899674100581506,
|
|
"grad_norm": 11.545572776265107,
|
|
"learning_rate": 3.7844053513536035e-11,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.875,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1742,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.375,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15570
|
|
},
|
|
{
|
|
"epoch": 1.991245446993418,
|
|
"grad_norm": 13.213488743746863,
|
|
"learning_rate": 2.8762650056468607e-11,
|
|
"logits/chosen": -3.0,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -704.0,
|
|
"logps/rejected": -1144.0,
|
|
"loss": 0.1423,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": -5.40625,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15580
|
|
},
|
|
{
|
|
"epoch": 1.9925234839286854,
|
|
"grad_norm": 9.895766976087293,
|
|
"learning_rate": 2.0925186065223756e-11,
|
|
"logits/chosen": -3.078125,
|
|
"logits/rejected": -2.765625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1705,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -5.34375,
|
|
"rewards/margins": 4.6875,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 15590
|
|
},
|
|
{
|
|
"epoch": 1.993801520863953,
|
|
"grad_norm": 10.674668242208217,
|
|
"learning_rate": 1.433170054163102e-11,
|
|
"logits/chosen": -3.015625,
|
|
"logits/rejected": -2.65625,
|
|
"logps/chosen": -696.0,
|
|
"logps/rejected": -1120.0,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": -5.46875,
|
|
"rewards/margins": 4.28125,
|
|
"rewards/rejected": -9.75,
|
|
"step": 15600
|
|
},
|
|
{
|
|
"epoch": 1.9950795577992204,
|
|
"grad_norm": 9.530695253521662,
|
|
"learning_rate": 8.982226296999629e-12,
|
|
"logits/chosen": -3.125,
|
|
"logits/rejected": -2.828125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1136.0,
|
|
"loss": 0.1513,
|
|
"rewards/accuracies": 0.8812500238418579,
|
|
"rewards/chosen": -5.59375,
|
|
"rewards/margins": 4.3125,
|
|
"rewards/rejected": -9.875,
|
|
"step": 15610
|
|
},
|
|
{
|
|
"epoch": 1.9963575947344878,
|
|
"grad_norm": 20.579796060334516,
|
|
"learning_rate": 4.876789952146243e-12,
|
|
"logits/chosen": -3.09375,
|
|
"logits/rejected": -2.78125,
|
|
"logps/chosen": -720.0,
|
|
"logps/rejected": -1152.0,
|
|
"loss": 0.1676,
|
|
"rewards/accuracies": 0.9312499761581421,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.46875,
|
|
"rewards/rejected": -10.0,
|
|
"step": 15620
|
|
},
|
|
{
|
|
"epoch": 1.9976356316697552,
|
|
"grad_norm": 9.627868429654509,
|
|
"learning_rate": 2.015411937061895e-12,
|
|
"logits/chosen": -3.03125,
|
|
"logits/rejected": -2.890625,
|
|
"logps/chosen": -672.0,
|
|
"logps/rejected": -1128.0,
|
|
"loss": 0.1395,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": -5.21875,
|
|
"rewards/margins": 4.71875,
|
|
"rewards/rejected": -9.9375,
|
|
"step": 15630
|
|
},
|
|
{
|
|
"epoch": 1.9989136686050228,
|
|
"grad_norm": 10.344279691753776,
|
|
"learning_rate": 3.981064908842313e-13,
|
|
"logits/chosen": -3.046875,
|
|
"logits/rejected": -2.75,
|
|
"logps/chosen": -716.0,
|
|
"logps/rejected": -1160.0,
|
|
"loss": 0.1478,
|
|
"rewards/accuracies": 0.918749988079071,
|
|
"rewards/chosen": -5.53125,
|
|
"rewards/margins": 4.5625,
|
|
"rewards/rejected": -10.0625,
|
|
"step": 15640
|
|
},
|
|
{
|
|
"epoch": 1.9999360981532366,
|
|
"step": 15648,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.0,
|
|
"train_runtime": 42.011,
|
|
"train_samples_per_second": 23839.372,
|
|
"train_steps_per_second": 186.237
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 7824,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 10000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|