3889 lines
134 KiB
JSON
3889 lines
134 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2133,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"entropy": 1.2380126953125,
|
||
|
|
"epoch": 0.004690294022806555,
|
||
|
|
"grad_norm": 62.6229133605957,
|
||
|
|
"learning_rate": 2.102803738317757e-08,
|
||
|
|
"logits/chosen": 0.13142231863153445,
|
||
|
|
"logits/rejected": 0.5568989329185723,
|
||
|
|
"logps/chosen": -488.3,
|
||
|
|
"logps/rejected": -610.840625,
|
||
|
|
"loss": 0.6997802734375,
|
||
|
|
"mean_token_accuracy": 0.7084598530083894,
|
||
|
|
"num_tokens": 1702319.0,
|
||
|
|
"rewards/accuracies": 0.27421875,
|
||
|
|
"rewards/chosen": -0.0020932674407958983,
|
||
|
|
"rewards/margins": -0.0007374286651611328,
|
||
|
|
"rewards/rejected": -0.0013402223587036133,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2425048828125,
|
||
|
|
"epoch": 0.00938058804561311,
|
||
|
|
"grad_norm": 58.14165496826172,
|
||
|
|
"learning_rate": 4.4392523364485976e-08,
|
||
|
|
"logits/chosen": 0.21829557273294725,
|
||
|
|
"logits/rejected": 0.6628808946931506,
|
||
|
|
"logps/chosen": -464.21875,
|
||
|
|
"logps/rejected": -625.34375,
|
||
|
|
"loss": 0.70186767578125,
|
||
|
|
"mean_token_accuracy": 0.7171314246952534,
|
||
|
|
"num_tokens": 3396761.0,
|
||
|
|
"rewards/accuracies": 0.2765625,
|
||
|
|
"rewards/chosen": 0.0008783265948295593,
|
||
|
|
"rewards/margins": -0.007377481460571289,
|
||
|
|
"rewards/rejected": 0.008276915550231934,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2296875,
|
||
|
|
"epoch": 0.014070882068419664,
|
||
|
|
"grad_norm": 59.69734573364258,
|
||
|
|
"learning_rate": 6.775700934579438e-08,
|
||
|
|
"logits/chosen": 0.16846038564981053,
|
||
|
|
"logits/rejected": 0.5656773570114122,
|
||
|
|
"logps/chosen": -477.23125,
|
||
|
|
"logps/rejected": -592.425,
|
||
|
|
"loss": 0.69720458984375,
|
||
|
|
"mean_token_accuracy": 0.7160559518262744,
|
||
|
|
"num_tokens": 5098223.0,
|
||
|
|
"rewards/accuracies": 0.2640625,
|
||
|
|
"rewards/chosen": 0.0025088131427764893,
|
||
|
|
"rewards/margins": 0.0023264408111572264,
|
||
|
|
"rewards/rejected": 0.00020335018634796143,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25751953125,
|
||
|
|
"epoch": 0.01876117609122622,
|
||
|
|
"grad_norm": 58.34840774536133,
|
||
|
|
"learning_rate": 9.11214953271028e-08,
|
||
|
|
"logits/chosen": 0.19244852821179498,
|
||
|
|
"logits/rejected": 0.5710635891111505,
|
||
|
|
"logps/chosen": -456.75625,
|
||
|
|
"logps/rejected": -600.471875,
|
||
|
|
"loss": 0.69652099609375,
|
||
|
|
"mean_token_accuracy": 0.7064009141176939,
|
||
|
|
"num_tokens": 6711358.0,
|
||
|
|
"rewards/accuracies": 0.29140625,
|
||
|
|
"rewards/chosen": 0.0020108461380004884,
|
||
|
|
"rewards/margins": 0.0012389898300170898,
|
||
|
|
"rewards/rejected": 0.0007735252380371094,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2514892578125,
|
||
|
|
"epoch": 0.023451470114032775,
|
||
|
|
"grad_norm": 67.314453125,
|
||
|
|
"learning_rate": 1.1448598130841122e-07,
|
||
|
|
"logits/chosen": 0.12212445211997296,
|
||
|
|
"logits/rejected": 0.4953063663441955,
|
||
|
|
"logps/chosen": -449.971875,
|
||
|
|
"logps/rejected": -597.55,
|
||
|
|
"loss": 0.692919921875,
|
||
|
|
"mean_token_accuracy": 0.7137674799188971,
|
||
|
|
"num_tokens": 8355688.0,
|
||
|
|
"rewards/accuracies": 0.3140625,
|
||
|
|
"rewards/chosen": 0.0004706621170043945,
|
||
|
|
"rewards/margins": 0.017674684524536133,
|
||
|
|
"rewards/rejected": -0.017192131280899046,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.29541015625,
|
||
|
|
"epoch": 0.028141764136839328,
|
||
|
|
"grad_norm": 73.55916595458984,
|
||
|
|
"learning_rate": 1.378504672897196e-07,
|
||
|
|
"logits/chosen": 0.09788538624036117,
|
||
|
|
"logits/rejected": 0.5997201450063375,
|
||
|
|
"logps/chosen": -483.2125,
|
||
|
|
"logps/rejected": -697.359375,
|
||
|
|
"loss": 0.687384033203125,
|
||
|
|
"mean_token_accuracy": 0.7136357419192791,
|
||
|
|
"num_tokens": 10102836.0,
|
||
|
|
"rewards/accuracies": 0.3359375,
|
||
|
|
"rewards/chosen": 0.0074482947587966915,
|
||
|
|
"rewards/margins": 0.024856948852539064,
|
||
|
|
"rewards/rejected": -0.017406320571899413,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.32314453125,
|
||
|
|
"epoch": 0.032832058159645884,
|
||
|
|
"grad_norm": 78.66187286376953,
|
||
|
|
"learning_rate": 1.61214953271028e-07,
|
||
|
|
"logits/chosen": 0.1396756814227043,
|
||
|
|
"logits/rejected": 0.6164610300295253,
|
||
|
|
"logps/chosen": -519.153125,
|
||
|
|
"logps/rejected": -679.1875,
|
||
|
|
"loss": 0.6819580078125,
|
||
|
|
"mean_token_accuracy": 0.7015179498121142,
|
||
|
|
"num_tokens": 11789869.0,
|
||
|
|
"rewards/accuracies": 0.3953125,
|
||
|
|
"rewards/chosen": 0.00875704288482666,
|
||
|
|
"rewards/margins": 0.0570986270904541,
|
||
|
|
"rewards/rejected": -0.04829350709915161,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2225830078125,
|
||
|
|
"epoch": 0.03752235218245244,
|
||
|
|
"grad_norm": 58.509803771972656,
|
||
|
|
"learning_rate": 1.8457943925233644e-07,
|
||
|
|
"logits/chosen": 0.14060797812788714,
|
||
|
|
"logits/rejected": 0.564689663234594,
|
||
|
|
"logps/chosen": -405.990625,
|
||
|
|
"logps/rejected": -560.575,
|
||
|
|
"loss": 0.67740478515625,
|
||
|
|
"mean_token_accuracy": 0.7172680467367172,
|
||
|
|
"num_tokens": 13410425.0,
|
||
|
|
"rewards/accuracies": 0.42734375,
|
||
|
|
"rewards/chosen": 0.019669818878173827,
|
||
|
|
"rewards/margins": 0.06200284957885742,
|
||
|
|
"rewards/rejected": -0.042318296432495114,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2392822265625,
|
||
|
|
"epoch": 0.04221264620525899,
|
||
|
|
"grad_norm": 56.662208557128906,
|
||
|
|
"learning_rate": 2.0794392523364485e-07,
|
||
|
|
"logits/chosen": 0.06250881791098087,
|
||
|
|
"logits/rejected": 0.5188753865520929,
|
||
|
|
"logps/chosen": -479.88125,
|
||
|
|
"logps/rejected": -573.56875,
|
||
|
|
"loss": 0.66741943359375,
|
||
|
|
"mean_token_accuracy": 0.7083912387490272,
|
||
|
|
"num_tokens": 15075240.0,
|
||
|
|
"rewards/accuracies": 0.48515625,
|
||
|
|
"rewards/chosen": 0.018572616577148437,
|
||
|
|
"rewards/margins": 0.07655124664306641,
|
||
|
|
"rewards/rejected": -0.058021259307861325,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.282275390625,
|
||
|
|
"epoch": 0.04690294022806555,
|
||
|
|
"grad_norm": 48.26545715332031,
|
||
|
|
"learning_rate": 2.3130841121495325e-07,
|
||
|
|
"logits/chosen": 0.01315832664804976,
|
||
|
|
"logits/rejected": 0.4849252921361867,
|
||
|
|
"logps/chosen": -465.8,
|
||
|
|
"logps/rejected": -594.171875,
|
||
|
|
"loss": 0.65252685546875,
|
||
|
|
"mean_token_accuracy": 0.7040323404595256,
|
||
|
|
"num_tokens": 16654416.0,
|
||
|
|
"rewards/accuracies": 0.51796875,
|
||
|
|
"rewards/chosen": 0.00697178840637207,
|
||
|
|
"rewards/margins": 0.1246042251586914,
|
||
|
|
"rewards/rejected": -0.11760797500610351,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2884765625,
|
||
|
|
"epoch": 0.0515932342508721,
|
||
|
|
"grad_norm": 47.12166213989258,
|
||
|
|
"learning_rate": 2.546728971962617e-07,
|
||
|
|
"logits/chosen": -0.009655450046421743,
|
||
|
|
"logits/rejected": 0.5041717435221977,
|
||
|
|
"logps/chosen": -481.08125,
|
||
|
|
"logps/rejected": -682.14375,
|
||
|
|
"loss": 0.630169677734375,
|
||
|
|
"mean_token_accuracy": 0.7061939595267177,
|
||
|
|
"num_tokens": 18380326.0,
|
||
|
|
"rewards/accuracies": 0.5640625,
|
||
|
|
"rewards/chosen": -0.0007643461227416992,
|
||
|
|
"rewards/margins": 0.24774904251098634,
|
||
|
|
"rewards/rejected": -0.24857692718505858,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.245703125,
|
||
|
|
"epoch": 0.056283528273678655,
|
||
|
|
"grad_norm": 104.83141326904297,
|
||
|
|
"learning_rate": 2.780373831775701e-07,
|
||
|
|
"logits/chosen": 0.07013991718436058,
|
||
|
|
"logits/rejected": 0.4994907462181204,
|
||
|
|
"logps/chosen": -465.61875,
|
||
|
|
"logps/rejected": -618.36875,
|
||
|
|
"loss": 0.629339599609375,
|
||
|
|
"mean_token_accuracy": 0.7171272663399577,
|
||
|
|
"num_tokens": 20071329.0,
|
||
|
|
"rewards/accuracies": 0.5921875,
|
||
|
|
"rewards/chosen": -0.009405803680419923,
|
||
|
|
"rewards/margins": 0.30645027160644533,
|
||
|
|
"rewards/rejected": -0.3159339904785156,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.27783203125,
|
||
|
|
"epoch": 0.06097382229648521,
|
||
|
|
"grad_norm": 51.31402587890625,
|
||
|
|
"learning_rate": 3.014018691588785e-07,
|
||
|
|
"logits/chosen": -0.11757746507376968,
|
||
|
|
"logits/rejected": 0.3011634539394827,
|
||
|
|
"logps/chosen": -484.26875,
|
||
|
|
"logps/rejected": -646.71875,
|
||
|
|
"loss": 0.61903076171875,
|
||
|
|
"mean_token_accuracy": 0.7081391334533691,
|
||
|
|
"num_tokens": 21795525.0,
|
||
|
|
"rewards/accuracies": 0.59296875,
|
||
|
|
"rewards/chosen": -0.02664012908935547,
|
||
|
|
"rewards/margins": 0.32644290924072267,
|
||
|
|
"rewards/rejected": -0.3529624938964844,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2402099609375,
|
||
|
|
"epoch": 0.06566411631929177,
|
||
|
|
"grad_norm": 46.31311798095703,
|
||
|
|
"learning_rate": 3.247663551401869e-07,
|
||
|
|
"logits/chosen": -0.1286943394512165,
|
||
|
|
"logits/rejected": 0.2729156901183648,
|
||
|
|
"logps/chosen": -454.53125,
|
||
|
|
"logps/rejected": -620.6125,
|
||
|
|
"loss": 0.60274658203125,
|
||
|
|
"mean_token_accuracy": 0.7122334752231836,
|
||
|
|
"num_tokens": 23425866.0,
|
||
|
|
"rewards/accuracies": 0.6171875,
|
||
|
|
"rewards/chosen": -0.00044622421264648435,
|
||
|
|
"rewards/margins": 0.4336221694946289,
|
||
|
|
"rewards/rejected": -0.43410415649414064,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.31734619140625,
|
||
|
|
"epoch": 0.07035441034209831,
|
||
|
|
"grad_norm": 58.106231689453125,
|
||
|
|
"learning_rate": 3.481308411214953e-07,
|
||
|
|
"logits/chosen": -0.1213149723170385,
|
||
|
|
"logits/rejected": 0.3410684163822822,
|
||
|
|
"logps/chosen": -504.925,
|
||
|
|
"logps/rejected": -746.7875,
|
||
|
|
"loss": 0.6043121337890625,
|
||
|
|
"mean_token_accuracy": 0.7165910258889199,
|
||
|
|
"num_tokens": 25164538.0,
|
||
|
|
"rewards/accuracies": 0.64140625,
|
||
|
|
"rewards/chosen": -0.05042829513549805,
|
||
|
|
"rewards/margins": 0.5049018859863281,
|
||
|
|
"rewards/rejected": -0.5555450439453125,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.196240234375,
|
||
|
|
"epoch": 0.07504470436490487,
|
||
|
|
"grad_norm": 41.2678108215332,
|
||
|
|
"learning_rate": 3.7149532710280375e-07,
|
||
|
|
"logits/chosen": -0.20468518942131192,
|
||
|
|
"logits/rejected": 0.24485665882537527,
|
||
|
|
"logps/chosen": -453.096875,
|
||
|
|
"logps/rejected": -595.925,
|
||
|
|
"loss": 0.5784027099609375,
|
||
|
|
"mean_token_accuracy": 0.7159464564174414,
|
||
|
|
"num_tokens": 26805458.0,
|
||
|
|
"rewards/accuracies": 0.6375,
|
||
|
|
"rewards/chosen": -0.012469863891601563,
|
||
|
|
"rewards/margins": 0.47882614135742185,
|
||
|
|
"rewards/rejected": -0.49114646911621096,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3654052734375,
|
||
|
|
"epoch": 0.07973499838771143,
|
||
|
|
"grad_norm": 49.10939407348633,
|
||
|
|
"learning_rate": 3.9485981308411216e-07,
|
||
|
|
"logits/chosen": -0.35195917230947377,
|
||
|
|
"logits/rejected": 0.3429667283637607,
|
||
|
|
"logps/chosen": -479.825,
|
||
|
|
"logps/rejected": -772.64375,
|
||
|
|
"loss": 0.6050064086914062,
|
||
|
|
"mean_token_accuracy": 0.7037448784336447,
|
||
|
|
"num_tokens": 28484655.0,
|
||
|
|
"rewards/accuracies": 0.63359375,
|
||
|
|
"rewards/chosen": -0.06340255737304687,
|
||
|
|
"rewards/margins": 0.5980648040771485,
|
||
|
|
"rewards/rejected": -0.6617424011230468,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1974609375,
|
||
|
|
"epoch": 0.08442529241051798,
|
||
|
|
"grad_norm": 54.43376922607422,
|
||
|
|
"learning_rate": 4.1822429906542056e-07,
|
||
|
|
"logits/chosen": -0.25779980917979717,
|
||
|
|
"logits/rejected": 0.31058484361609595,
|
||
|
|
"logps/chosen": -471.9875,
|
||
|
|
"logps/rejected": -559.44375,
|
||
|
|
"loss": 0.600372314453125,
|
||
|
|
"mean_token_accuracy": 0.7057331206277013,
|
||
|
|
"num_tokens": 30109849.0,
|
||
|
|
"rewards/accuracies": 0.63359375,
|
||
|
|
"rewards/chosen": -0.08329944610595703,
|
||
|
|
"rewards/margins": 0.5749111175537109,
|
||
|
|
"rewards/rejected": -0.6581335067749023,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.286376953125,
|
||
|
|
"epoch": 0.08911558643332454,
|
||
|
|
"grad_norm": 42.45182800292969,
|
||
|
|
"learning_rate": 4.4158878504672897e-07,
|
||
|
|
"logits/chosen": -0.35561966840299924,
|
||
|
|
"logits/rejected": 0.15353543618014337,
|
||
|
|
"logps/chosen": -463.621875,
|
||
|
|
"logps/rejected": -656.0375,
|
||
|
|
"loss": 0.5772369384765625,
|
||
|
|
"mean_token_accuracy": 0.7057704044505954,
|
||
|
|
"num_tokens": 31744162.0,
|
||
|
|
"rewards/accuracies": 0.64921875,
|
||
|
|
"rewards/chosen": -0.1075469970703125,
|
||
|
|
"rewards/margins": 0.6416389465332031,
|
||
|
|
"rewards/rejected": -0.7492580413818359,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3052490234375,
|
||
|
|
"epoch": 0.0938058804561311,
|
||
|
|
"grad_norm": 62.58420181274414,
|
||
|
|
"learning_rate": 4.6495327102803737e-07,
|
||
|
|
"logits/chosen": -0.35697863410661446,
|
||
|
|
"logits/rejected": 0.20630076585241763,
|
||
|
|
"logps/chosen": -465.2,
|
||
|
|
"logps/rejected": -731.7375,
|
||
|
|
"loss": 0.557763671875,
|
||
|
|
"mean_token_accuracy": 0.7047198623418808,
|
||
|
|
"num_tokens": 33449741.0,
|
||
|
|
"rewards/accuracies": 0.665625,
|
||
|
|
"rewards/chosen": -0.0918426513671875,
|
||
|
|
"rewards/margins": 0.718874740600586,
|
||
|
|
"rewards/rejected": -0.8103523254394531,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.24949951171875,
|
||
|
|
"epoch": 0.09849617447893765,
|
||
|
|
"grad_norm": 38.690673828125,
|
||
|
|
"learning_rate": 4.883177570093458e-07,
|
||
|
|
"logits/chosen": -0.36467700401264636,
|
||
|
|
"logits/rejected": 0.16843314232397516,
|
||
|
|
"logps/chosen": -500.925,
|
||
|
|
"logps/rejected": -669.7,
|
||
|
|
"loss": 0.558367919921875,
|
||
|
|
"mean_token_accuracy": 0.7056573690846563,
|
||
|
|
"num_tokens": 35141213.0,
|
||
|
|
"rewards/accuracies": 0.678125,
|
||
|
|
"rewards/chosen": -0.13154408931732178,
|
||
|
|
"rewards/margins": 0.7080429077148438,
|
||
|
|
"rewards/rejected": -0.8396020889282226,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.26572265625,
|
||
|
|
"epoch": 0.1031864685017442,
|
||
|
|
"grad_norm": 52.72610855102539,
|
||
|
|
"learning_rate": 4.986972381448671e-07,
|
||
|
|
"logits/chosen": -0.3342161703831604,
|
||
|
|
"logits/rejected": 0.13690835608174162,
|
||
|
|
"logps/chosen": -473.95,
|
||
|
|
"logps/rejected": -683.790625,
|
||
|
|
"loss": 0.55330810546875,
|
||
|
|
"mean_token_accuracy": 0.7077068831771612,
|
||
|
|
"num_tokens": 36812333.0,
|
||
|
|
"rewards/accuracies": 0.675,
|
||
|
|
"rewards/chosen": -0.11907577514648438,
|
||
|
|
"rewards/margins": 0.7173862457275391,
|
||
|
|
"rewards/rejected": -0.8365554809570312,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.18297119140625,
|
||
|
|
"epoch": 0.10787676252455075,
|
||
|
|
"grad_norm": 39.5418586730957,
|
||
|
|
"learning_rate": 4.960917144346014e-07,
|
||
|
|
"logits/chosen": -0.3157396531611962,
|
||
|
|
"logits/rejected": 0.06412190059466064,
|
||
|
|
"logps/chosen": -460.26875,
|
||
|
|
"logps/rejected": -588.134375,
|
||
|
|
"loss": 0.55792236328125,
|
||
|
|
"mean_token_accuracy": 0.7146372459828854,
|
||
|
|
"num_tokens": 38463618.0,
|
||
|
|
"rewards/accuracies": 0.67265625,
|
||
|
|
"rewards/chosen": -0.12630558013916016,
|
||
|
|
"rewards/margins": 0.7576118469238281,
|
||
|
|
"rewards/rejected": -0.8838109970092773,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2109130859375,
|
||
|
|
"epoch": 0.11256705654735731,
|
||
|
|
"grad_norm": 42.412132263183594,
|
||
|
|
"learning_rate": 4.934861907243356e-07,
|
||
|
|
"logits/chosen": -0.2700029816915189,
|
||
|
|
"logits/rejected": 0.16311413886610973,
|
||
|
|
"logps/chosen": -465.3375,
|
||
|
|
"logps/rejected": -638.7125,
|
||
|
|
"loss": 0.5756942749023437,
|
||
|
|
"mean_token_accuracy": 0.7144404657185077,
|
||
|
|
"num_tokens": 40153509.0,
|
||
|
|
"rewards/accuracies": 0.66015625,
|
||
|
|
"rewards/chosen": -0.17855072021484375,
|
||
|
|
"rewards/margins": 0.8050395965576171,
|
||
|
|
"rewards/rejected": -0.983447265625,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2478759765625,
|
||
|
|
"epoch": 0.11725735057016387,
|
||
|
|
"grad_norm": 52.07088851928711,
|
||
|
|
"learning_rate": 4.908806670140698e-07,
|
||
|
|
"logits/chosen": -0.30485234342876655,
|
||
|
|
"logits/rejected": 0.15554822631006052,
|
||
|
|
"logps/chosen": -502.325,
|
||
|
|
"logps/rejected": -663.3875,
|
||
|
|
"loss": 0.560797119140625,
|
||
|
|
"mean_token_accuracy": 0.7079653844237328,
|
||
|
|
"num_tokens": 41888438.0,
|
||
|
|
"rewards/accuracies": 0.68828125,
|
||
|
|
"rewards/chosen": -0.08403854370117188,
|
||
|
|
"rewards/margins": 0.7736038208007813,
|
||
|
|
"rewards/rejected": -0.8577094078063965,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.264404296875,
|
||
|
|
"epoch": 0.12194764459297042,
|
||
|
|
"grad_norm": 42.2757682800293,
|
||
|
|
"learning_rate": 4.88275143303804e-07,
|
||
|
|
"logits/chosen": -0.26204608927871825,
|
||
|
|
"logits/rejected": 0.16244758132757325,
|
||
|
|
"logps/chosen": -505.00625,
|
||
|
|
"logps/rejected": -656.80625,
|
||
|
|
"loss": 0.577191162109375,
|
||
|
|
"mean_token_accuracy": 0.7026191920042038,
|
||
|
|
"num_tokens": 43585062.0,
|
||
|
|
"rewards/accuracies": 0.66171875,
|
||
|
|
"rewards/chosen": -0.12207617759704589,
|
||
|
|
"rewards/margins": 0.7578861236572265,
|
||
|
|
"rewards/rejected": -0.8799530029296875,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2654296875,
|
||
|
|
"epoch": 0.12663793861577696,
|
||
|
|
"grad_norm": 37.81559371948242,
|
||
|
|
"learning_rate": 4.856696195935383e-07,
|
||
|
|
"logits/chosen": -0.23835901714841606,
|
||
|
|
"logits/rejected": 0.2375799555447144,
|
||
|
|
"logps/chosen": -470.54375,
|
||
|
|
"logps/rejected": -673.821875,
|
||
|
|
"loss": 0.550146484375,
|
||
|
|
"mean_token_accuracy": 0.7080739632248878,
|
||
|
|
"num_tokens": 45300333.0,
|
||
|
|
"rewards/accuracies": 0.66484375,
|
||
|
|
"rewards/chosen": -0.06469240188598632,
|
||
|
|
"rewards/margins": 0.7684661865234375,
|
||
|
|
"rewards/rejected": -0.8330526351928711,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.279296875,
|
||
|
|
"epoch": 0.13132823263858354,
|
||
|
|
"grad_norm": 90.60016632080078,
|
||
|
|
"learning_rate": 4.830640958832725e-07,
|
||
|
|
"logits/chosen": -0.21629677708878328,
|
||
|
|
"logits/rejected": 0.3163977187624283,
|
||
|
|
"logps/chosen": -472.6375,
|
||
|
|
"logps/rejected": -703.2375,
|
||
|
|
"loss": 0.5496139526367188,
|
||
|
|
"mean_token_accuracy": 0.7126924116164446,
|
||
|
|
"num_tokens": 47012468.0,
|
||
|
|
"rewards/accuracies": 0.675,
|
||
|
|
"rewards/chosen": -0.1156158447265625,
|
||
|
|
"rewards/margins": 0.8513694763183594,
|
||
|
|
"rewards/rejected": -0.9671096801757812,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2152099609375,
|
||
|
|
"epoch": 0.13601852666139008,
|
||
|
|
"grad_norm": 45.61990737915039,
|
||
|
|
"learning_rate": 4.804585721730068e-07,
|
||
|
|
"logits/chosen": -0.22120995301749558,
|
||
|
|
"logits/rejected": 0.15842283279794905,
|
||
|
|
"logps/chosen": -464.7625,
|
||
|
|
"logps/rejected": -603.35625,
|
||
|
|
"loss": 0.5582733154296875,
|
||
|
|
"mean_token_accuracy": 0.7125538818538188,
|
||
|
|
"num_tokens": 48630480.0,
|
||
|
|
"rewards/accuracies": 0.69140625,
|
||
|
|
"rewards/chosen": -0.1512277603149414,
|
||
|
|
"rewards/margins": 0.7555118560791015,
|
||
|
|
"rewards/rejected": -0.9074989318847656,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3152099609375,
|
||
|
|
"epoch": 0.14070882068419663,
|
||
|
|
"grad_norm": 48.600868225097656,
|
||
|
|
"learning_rate": 4.77853048462741e-07,
|
||
|
|
"logits/chosen": -0.2612698061733851,
|
||
|
|
"logits/rejected": 0.27788530891313673,
|
||
|
|
"logps/chosen": -461.784375,
|
||
|
|
"logps/rejected": -702.3375,
|
||
|
|
"loss": 0.5443603515625,
|
||
|
|
"mean_token_accuracy": 0.7086123580113053,
|
||
|
|
"num_tokens": 50268655.0,
|
||
|
|
"rewards/accuracies": 0.696875,
|
||
|
|
"rewards/chosen": -0.11336288452148438,
|
||
|
|
"rewards/margins": 0.8591796875,
|
||
|
|
"rewards/rejected": -0.9724113464355468,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25264892578125,
|
||
|
|
"epoch": 0.1453991147070032,
|
||
|
|
"grad_norm": 55.53269958496094,
|
||
|
|
"learning_rate": 4.752475247524752e-07,
|
||
|
|
"logits/chosen": -0.29287300945492223,
|
||
|
|
"logits/rejected": 0.16384651433880326,
|
||
|
|
"logps/chosen": -474.85625,
|
||
|
|
"logps/rejected": -628.31875,
|
||
|
|
"loss": 0.5508621215820313,
|
||
|
|
"mean_token_accuracy": 0.7046513512730599,
|
||
|
|
"num_tokens": 51907656.0,
|
||
|
|
"rewards/accuracies": 0.67265625,
|
||
|
|
"rewards/chosen": -0.0640228271484375,
|
||
|
|
"rewards/margins": 0.7980010986328125,
|
||
|
|
"rewards/rejected": -0.8622137069702148,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.22216796875,
|
||
|
|
"epoch": 0.15008940872980975,
|
||
|
|
"grad_norm": 42.761417388916016,
|
||
|
|
"learning_rate": 4.726420010422095e-07,
|
||
|
|
"logits/chosen": -0.25384103775604866,
|
||
|
|
"logits/rejected": 0.22921325218368754,
|
||
|
|
"logps/chosen": -459.175,
|
||
|
|
"logps/rejected": -641.70625,
|
||
|
|
"loss": 0.5528564453125,
|
||
|
|
"mean_token_accuracy": 0.7093667019158602,
|
||
|
|
"num_tokens": 53593177.0,
|
||
|
|
"rewards/accuracies": 0.68828125,
|
||
|
|
"rewards/chosen": -0.13062238693237305,
|
||
|
|
"rewards/margins": 0.8838020324707031,
|
||
|
|
"rewards/rejected": -1.0145111083984375,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.268212890625,
|
||
|
|
"epoch": 0.1547797027526163,
|
||
|
|
"grad_norm": 54.23225021362305,
|
||
|
|
"learning_rate": 4.7003647733194366e-07,
|
||
|
|
"logits/chosen": -0.2853499345335391,
|
||
|
|
"logits/rejected": 0.26540551703707715,
|
||
|
|
"logps/chosen": -485.53125,
|
||
|
|
"logps/rejected": -700.928125,
|
||
|
|
"loss": 0.5279586791992188,
|
||
|
|
"mean_token_accuracy": 0.7084382649511098,
|
||
|
|
"num_tokens": 55282207.0,
|
||
|
|
"rewards/accuracies": 0.69609375,
|
||
|
|
"rewards/chosen": -0.11557159423828126,
|
||
|
|
"rewards/margins": 0.9410869598388671,
|
||
|
|
"rewards/rejected": -1.0567031860351563,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.276025390625,
|
||
|
|
"epoch": 0.15946999677542287,
|
||
|
|
"grad_norm": 46.780303955078125,
|
||
|
|
"learning_rate": 4.6743095362167794e-07,
|
||
|
|
"logits/chosen": -0.24054302220386106,
|
||
|
|
"logits/rejected": 0.19597009005313973,
|
||
|
|
"logps/chosen": -500.8625,
|
||
|
|
"logps/rejected": -639.490625,
|
||
|
|
"loss": 0.5339645385742188,
|
||
|
|
"mean_token_accuracy": 0.7041525453329086,
|
||
|
|
"num_tokens": 56953401.0,
|
||
|
|
"rewards/accuracies": 0.70078125,
|
||
|
|
"rewards/chosen": -0.06949176788330078,
|
||
|
|
"rewards/margins": 0.908062744140625,
|
||
|
|
"rewards/rejected": -0.977838134765625,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.23541259765625,
|
||
|
|
"epoch": 0.1641602907982294,
|
||
|
|
"grad_norm": 40.72328567504883,
|
||
|
|
"learning_rate": 4.6482542991141217e-07,
|
||
|
|
"logits/chosen": -0.3234164875089639,
|
||
|
|
"logits/rejected": 0.045452649283336415,
|
||
|
|
"logps/chosen": -488.621875,
|
||
|
|
"logps/rejected": -626.525,
|
||
|
|
"loss": 0.5379608154296875,
|
||
|
|
"mean_token_accuracy": 0.7138542663305998,
|
||
|
|
"num_tokens": 58589968.0,
|
||
|
|
"rewards/accuracies": 0.6859375,
|
||
|
|
"rewards/chosen": -0.15412139892578125,
|
||
|
|
"rewards/margins": 0.9932891845703125,
|
||
|
|
"rewards/rejected": -1.147345733642578,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2406005859375,
|
||
|
|
"epoch": 0.16885058482103596,
|
||
|
|
"grad_norm": 75.66559600830078,
|
||
|
|
"learning_rate": 4.6221990620114644e-07,
|
||
|
|
"logits/chosen": -0.2896619681440237,
|
||
|
|
"logits/rejected": 0.14065292768913057,
|
||
|
|
"logps/chosen": -470.790625,
|
||
|
|
"logps/rejected": -633.634375,
|
||
|
|
"loss": 0.53887939453125,
|
||
|
|
"mean_token_accuracy": 0.7099913412705063,
|
||
|
|
"num_tokens": 60306336.0,
|
||
|
|
"rewards/accuracies": 0.6796875,
|
||
|
|
"rewards/chosen": -0.11739025115966797,
|
||
|
|
"rewards/margins": 0.9522857666015625,
|
||
|
|
"rewards/rejected": -1.07008056640625,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2303955078125,
|
||
|
|
"epoch": 0.17354087884384253,
|
||
|
|
"grad_norm": 44.52969741821289,
|
||
|
|
"learning_rate": 4.596143824908806e-07,
|
||
|
|
"logits/chosen": -0.3331617374520748,
|
||
|
|
"logits/rejected": 0.14742571109351904,
|
||
|
|
"logps/chosen": -439.86875,
|
||
|
|
"logps/rejected": -650.50625,
|
||
|
|
"loss": 0.5416946411132812,
|
||
|
|
"mean_token_accuracy": 0.7148818030953408,
|
||
|
|
"num_tokens": 61945687.0,
|
||
|
|
"rewards/accuracies": 0.678125,
|
||
|
|
"rewards/chosen": -0.0981292724609375,
|
||
|
|
"rewards/margins": 0.8962696075439454,
|
||
|
|
"rewards/rejected": -0.9943016052246094,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.19935302734375,
|
||
|
|
"epoch": 0.17823117286664908,
|
||
|
|
"grad_norm": 63.72592544555664,
|
||
|
|
"learning_rate": 4.570088587806149e-07,
|
||
|
|
"logits/chosen": -0.3539870810775333,
|
||
|
|
"logits/rejected": 0.0757170174215348,
|
||
|
|
"logps/chosen": -486.1875,
|
||
|
|
"logps/rejected": -613.4,
|
||
|
|
"loss": 0.5257133483886719,
|
||
|
|
"mean_token_accuracy": 0.70884978659451,
|
||
|
|
"num_tokens": 63623916.0,
|
||
|
|
"rewards/accuracies": 0.6984375,
|
||
|
|
"rewards/chosen": -0.15736675262451172,
|
||
|
|
"rewards/margins": 0.9691802978515625,
|
||
|
|
"rewards/rejected": -1.1265029907226562,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2439208984375,
|
||
|
|
"epoch": 0.18292146688945563,
|
||
|
|
"grad_norm": 54.821414947509766,
|
||
|
|
"learning_rate": 4.544033350703491e-07,
|
||
|
|
"logits/chosen": -0.3400324315735825,
|
||
|
|
"logits/rejected": 0.1258022766480983,
|
||
|
|
"logps/chosen": -510.83125,
|
||
|
|
"logps/rejected": -640.46875,
|
||
|
|
"loss": 0.5287796020507812,
|
||
|
|
"mean_token_accuracy": 0.706388033926487,
|
||
|
|
"num_tokens": 65288521.0,
|
||
|
|
"rewards/accuracies": 0.6828125,
|
||
|
|
"rewards/chosen": -0.19127464294433594,
|
||
|
|
"rewards/margins": 0.9656356811523438,
|
||
|
|
"rewards/rejected": -1.15693359375,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.264453125,
|
||
|
|
"epoch": 0.1876117609122622,
|
||
|
|
"grad_norm": 41.4052848815918,
|
||
|
|
"learning_rate": 4.517978113600834e-07,
|
||
|
|
"logits/chosen": -0.2976419383716833,
|
||
|
|
"logits/rejected": 0.12311620193998565,
|
||
|
|
"logps/chosen": -494.665625,
|
||
|
|
"logps/rejected": -675.7125,
|
||
|
|
"loss": 0.5210617065429688,
|
||
|
|
"mean_token_accuracy": 0.7038448538631201,
|
||
|
|
"num_tokens": 66997155.0,
|
||
|
|
"rewards/accuracies": 0.69921875,
|
||
|
|
"rewards/chosen": -0.16325674057006836,
|
||
|
|
"rewards/margins": 1.0209686279296875,
|
||
|
|
"rewards/rejected": -1.184283447265625,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2718505859375,
|
||
|
|
"epoch": 0.19230205493506874,
|
||
|
|
"grad_norm": 45.782466888427734,
|
||
|
|
"learning_rate": 4.491922876498176e-07,
|
||
|
|
"logits/chosen": -0.36419587700943057,
|
||
|
|
"logits/rejected": 0.09847667197489854,
|
||
|
|
"logps/chosen": -486.46875,
|
||
|
|
"logps/rejected": -711.55625,
|
||
|
|
"loss": 0.5182464599609375,
|
||
|
|
"mean_token_accuracy": 0.7099555041640997,
|
||
|
|
"num_tokens": 68686843.0,
|
||
|
|
"rewards/accuracies": 0.69296875,
|
||
|
|
"rewards/chosen": -0.14240741729736328,
|
||
|
|
"rewards/margins": 1.0581985473632813,
|
||
|
|
"rewards/rejected": -1.2014225006103516,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.33214111328125,
|
||
|
|
"epoch": 0.1969923489578753,
|
||
|
|
"grad_norm": 40.87909698486328,
|
||
|
|
"learning_rate": 4.465867639395518e-07,
|
||
|
|
"logits/chosen": -0.3484446268901798,
|
||
|
|
"logits/rejected": 0.2495398141798188,
|
||
|
|
"logps/chosen": -444.46875,
|
||
|
|
"logps/rejected": -750.2875,
|
||
|
|
"loss": 0.528167724609375,
|
||
|
|
"mean_token_accuracy": 0.7072742033749819,
|
||
|
|
"num_tokens": 70338632.0,
|
||
|
|
"rewards/accuracies": 0.71171875,
|
||
|
|
"rewards/chosen": -0.1199951171875,
|
||
|
|
"rewards/margins": 1.0843963623046875,
|
||
|
|
"rewards/rejected": -1.2041954040527343,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.277685546875,
|
||
|
|
"epoch": 0.20168264298068186,
|
||
|
|
"grad_norm": 45.7933235168457,
|
||
|
|
"learning_rate": 4.439812402292861e-07,
|
||
|
|
"logits/chosen": -0.32845346277744014,
|
||
|
|
"logits/rejected": 0.05965557834924514,
|
||
|
|
"logps/chosen": -453.76875,
|
||
|
|
"logps/rejected": -691.446875,
|
||
|
|
"loss": 0.5423171997070313,
|
||
|
|
"mean_token_accuracy": 0.711910505220294,
|
||
|
|
"num_tokens": 72020519.0,
|
||
|
|
"rewards/accuracies": 0.68515625,
|
||
|
|
"rewards/chosen": -0.13662586212158204,
|
||
|
|
"rewards/margins": 1.0402114868164063,
|
||
|
|
"rewards/rejected": -1.1772491455078125,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.237890625,
|
||
|
|
"epoch": 0.2063729370034884,
|
||
|
|
"grad_norm": 49.85696029663086,
|
||
|
|
"learning_rate": 4.4137571651902026e-07,
|
||
|
|
"logits/chosen": -0.32513024103298,
|
||
|
|
"logits/rejected": 0.10268295673836134,
|
||
|
|
"logps/chosen": -483.075,
|
||
|
|
"logps/rejected": -637.725,
|
||
|
|
"loss": 0.5276473999023438,
|
||
|
|
"mean_token_accuracy": 0.710579814761877,
|
||
|
|
"num_tokens": 73698147.0,
|
||
|
|
"rewards/accuracies": 0.70390625,
|
||
|
|
"rewards/chosen": -0.11184616088867187,
|
||
|
|
"rewards/margins": 1.006769561767578,
|
||
|
|
"rewards/rejected": -1.118536376953125,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.237841796875,
|
||
|
|
"epoch": 0.21106323102629496,
|
||
|
|
"grad_norm": 42.590423583984375,
|
||
|
|
"learning_rate": 4.3877019280875453e-07,
|
||
|
|
"logits/chosen": -0.35171392704646287,
|
||
|
|
"logits/rejected": 0.17345263794266536,
|
||
|
|
"logps/chosen": -464.83125,
|
||
|
|
"logps/rejected": -657.575,
|
||
|
|
"loss": 0.5287338256835937,
|
||
|
|
"mean_token_accuracy": 0.7119293544441462,
|
||
|
|
"num_tokens": 75384108.0,
|
||
|
|
"rewards/accuracies": 0.68203125,
|
||
|
|
"rewards/chosen": -0.10481681823730468,
|
||
|
|
"rewards/margins": 1.0241561889648438,
|
||
|
|
"rewards/rejected": -1.1294021606445312,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.261865234375,
|
||
|
|
"epoch": 0.2157535250491015,
|
||
|
|
"grad_norm": 42.76150131225586,
|
||
|
|
"learning_rate": 4.3616466909848876e-07,
|
||
|
|
"logits/chosen": -0.18868256814838458,
|
||
|
|
"logits/rejected": 0.3261802948037552,
|
||
|
|
"logps/chosen": -476.89375,
|
||
|
|
"logps/rejected": -704.0625,
|
||
|
|
"loss": 0.5230316162109375,
|
||
|
|
"mean_token_accuracy": 0.7178037509322166,
|
||
|
|
"num_tokens": 77095077.0,
|
||
|
|
"rewards/accuracies": 0.7046875,
|
||
|
|
"rewards/chosen": -0.11411991119384765,
|
||
|
|
"rewards/margins": 1.1168342590332032,
|
||
|
|
"rewards/rejected": -1.2309730529785157,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2946533203125,
|
||
|
|
"epoch": 0.22044381907190808,
|
||
|
|
"grad_norm": 63.289634704589844,
|
||
|
|
"learning_rate": 4.3355914538822304e-07,
|
||
|
|
"logits/chosen": -0.2706520637717097,
|
||
|
|
"logits/rejected": 0.2805214967390196,
|
||
|
|
"logps/chosen": -481.93125,
|
||
|
|
"logps/rejected": -709.6875,
|
||
|
|
"loss": 0.5318412780761719,
|
||
|
|
"mean_token_accuracy": 0.7098235335201025,
|
||
|
|
"num_tokens": 78734566.0,
|
||
|
|
"rewards/accuracies": 0.690625,
|
||
|
|
"rewards/chosen": -0.1305774688720703,
|
||
|
|
"rewards/margins": 1.0886627197265626,
|
||
|
|
"rewards/rejected": -1.2193614959716796,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1965087890625,
|
||
|
|
"epoch": 0.22513411309471462,
|
||
|
|
"grad_norm": 43.6287841796875,
|
||
|
|
"learning_rate": 4.309536216779572e-07,
|
||
|
|
"logits/chosen": -0.33620425129897835,
|
||
|
|
"logits/rejected": 0.16308440620940146,
|
||
|
|
"logps/chosen": -424.91875,
|
||
|
|
"logps/rejected": -613.725,
|
||
|
|
"loss": 0.5122772216796875,
|
||
|
|
"mean_token_accuracy": 0.7142355434596539,
|
||
|
|
"num_tokens": 80328183.0,
|
||
|
|
"rewards/accuracies": 0.7171875,
|
||
|
|
"rewards/chosen": -0.1288132667541504,
|
||
|
|
"rewards/margins": 1.0819381713867187,
|
||
|
|
"rewards/rejected": -1.2110176086425781,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2611328125,
|
||
|
|
"epoch": 0.22982440711752117,
|
||
|
|
"grad_norm": 67.60916137695312,
|
||
|
|
"learning_rate": 4.283480979676915e-07,
|
||
|
|
"logits/chosen": -0.28429071871907674,
|
||
|
|
"logits/rejected": 0.16387335536423142,
|
||
|
|
"logps/chosen": -468.709375,
|
||
|
|
"logps/rejected": -726.628125,
|
||
|
|
"loss": 0.5209976196289062,
|
||
|
|
"mean_token_accuracy": 0.7114728279411793,
|
||
|
|
"num_tokens": 82035476.0,
|
||
|
|
"rewards/accuracies": 0.6984375,
|
||
|
|
"rewards/chosen": -0.09270973205566406,
|
||
|
|
"rewards/margins": 1.1327011108398437,
|
||
|
|
"rewards/rejected": -1.225337600708008,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2105712890625,
|
||
|
|
"epoch": 0.23451470114032774,
|
||
|
|
"grad_norm": 45.96240234375,
|
||
|
|
"learning_rate": 4.257425742574257e-07,
|
||
|
|
"logits/chosen": -0.2853582343226048,
|
||
|
|
"logits/rejected": 0.11035681701698016,
|
||
|
|
"logps/chosen": -459.8875,
|
||
|
|
"logps/rejected": -625.590625,
|
||
|
|
"loss": 0.55081787109375,
|
||
|
|
"mean_token_accuracy": 0.7172987235710024,
|
||
|
|
"num_tokens": 83724940.0,
|
||
|
|
"rewards/accuracies": 0.690625,
|
||
|
|
"rewards/chosen": -0.16839942932128907,
|
||
|
|
"rewards/margins": 1.0144439697265626,
|
||
|
|
"rewards/rejected": -1.1823104858398437,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2220703125,
|
||
|
|
"epoch": 0.2392049951631343,
|
||
|
|
"grad_norm": 291.7288818359375,
|
||
|
|
"learning_rate": 4.2313705054716e-07,
|
||
|
|
"logits/chosen": -0.3063585163455549,
|
||
|
|
"logits/rejected": 0.2143574987077231,
|
||
|
|
"logps/chosen": -453.5375,
|
||
|
|
"logps/rejected": -601.79375,
|
||
|
|
"loss": 0.5269645690917969,
|
||
|
|
"mean_token_accuracy": 0.7116759425029159,
|
||
|
|
"num_tokens": 85405053.0,
|
||
|
|
"rewards/accuracies": 0.68046875,
|
||
|
|
"rewards/chosen": -0.10627117156982421,
|
||
|
|
"rewards/margins": 1.0517967224121094,
|
||
|
|
"rewards/rejected": -1.1577285766601562,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.186474609375,
|
||
|
|
"epoch": 0.24389528918594083,
|
||
|
|
"grad_norm": 49.72892761230469,
|
||
|
|
"learning_rate": 4.2053152683689417e-07,
|
||
|
|
"logits/chosen": -0.35408713711909,
|
||
|
|
"logits/rejected": 0.08242981100965566,
|
||
|
|
"logps/chosen": -483.99375,
|
||
|
|
"logps/rejected": -616.715625,
|
||
|
|
"loss": 0.5304771423339844,
|
||
|
|
"mean_token_accuracy": 0.7096766533330083,
|
||
|
|
"num_tokens": 87075810.0,
|
||
|
|
"rewards/accuracies": 0.7109375,
|
||
|
|
"rewards/chosen": -0.1855466842651367,
|
||
|
|
"rewards/margins": 1.030268096923828,
|
||
|
|
"rewards/rejected": -1.2156890869140624,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2286376953125,
|
||
|
|
"epoch": 0.2485855832087474,
|
||
|
|
"grad_norm": 41.44746398925781,
|
||
|
|
"learning_rate": 4.1792600312662845e-07,
|
||
|
|
"logits/chosen": -0.42685112278879933,
|
||
|
|
"logits/rejected": 0.028443143035817398,
|
||
|
|
"logps/chosen": -472.765625,
|
||
|
|
"logps/rejected": -648.171875,
|
||
|
|
"loss": 0.5234077453613282,
|
||
|
|
"mean_token_accuracy": 0.7097115091979503,
|
||
|
|
"num_tokens": 88775166.0,
|
||
|
|
"rewards/accuracies": 0.6984375,
|
||
|
|
"rewards/chosen": -0.17693424224853516,
|
||
|
|
"rewards/margins": 1.093129539489746,
|
||
|
|
"rewards/rejected": -1.2699623107910156,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.24697265625,
|
||
|
|
"epoch": 0.2532758772315539,
|
||
|
|
"grad_norm": 67.51264190673828,
|
||
|
|
"learning_rate": 4.153204794163627e-07,
|
||
|
|
"logits/chosen": -0.30632683199230437,
|
||
|
|
"logits/rejected": 0.09340387012174516,
|
||
|
|
"logps/chosen": -474.659375,
|
||
|
|
"logps/rejected": -663.43125,
|
||
|
|
"loss": 0.5192657470703125,
|
||
|
|
"mean_token_accuracy": 0.7120445661246777,
|
||
|
|
"num_tokens": 90483291.0,
|
||
|
|
"rewards/accuracies": 0.7078125,
|
||
|
|
"rewards/chosen": -0.1671518325805664,
|
||
|
|
"rewards/margins": 1.0724746704101562,
|
||
|
|
"rewards/rejected": -1.23973388671875,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.254345703125,
|
||
|
|
"epoch": 0.2579661712543605,
|
||
|
|
"grad_norm": 45.99155807495117,
|
||
|
|
"learning_rate": 4.1271495570609696e-07,
|
||
|
|
"logits/chosen": -0.23550229351542282,
|
||
|
|
"logits/rejected": 0.1451573755983327,
|
||
|
|
"logps/chosen": -510.16875,
|
||
|
|
"logps/rejected": -668.446875,
|
||
|
|
"loss": 0.5291549682617187,
|
||
|
|
"mean_token_accuracy": 0.7142287530004978,
|
||
|
|
"num_tokens": 92221920.0,
|
||
|
|
"rewards/accuracies": 0.68671875,
|
||
|
|
"rewards/chosen": -0.22687950134277343,
|
||
|
|
"rewards/margins": 1.0468917846679688,
|
||
|
|
"rewards/rejected": -1.2740165710449218,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2295654296875,
|
||
|
|
"epoch": 0.2626564652771671,
|
||
|
|
"grad_norm": 45.227359771728516,
|
||
|
|
"learning_rate": 4.1010943199583113e-07,
|
||
|
|
"logits/chosen": -0.28406279958537584,
|
||
|
|
"logits/rejected": 0.1589292666080559,
|
||
|
|
"logps/chosen": -460.29375,
|
||
|
|
"logps/rejected": -637.91875,
|
||
|
|
"loss": 0.54854736328125,
|
||
|
|
"mean_token_accuracy": 0.7158264957368374,
|
||
|
|
"num_tokens": 93930470.0,
|
||
|
|
"rewards/accuracies": 0.684375,
|
||
|
|
"rewards/chosen": -0.1995685577392578,
|
||
|
|
"rewards/margins": 1.00377197265625,
|
||
|
|
"rewards/rejected": -1.2029953002929688,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3091552734375,
|
||
|
|
"epoch": 0.2673467592999736,
|
||
|
|
"grad_norm": 41.385459899902344,
|
||
|
|
"learning_rate": 4.0750390828556536e-07,
|
||
|
|
"logits/chosen": -0.2780874213954464,
|
||
|
|
"logits/rejected": 0.19283398683678324,
|
||
|
|
"logps/chosen": -493.2875,
|
||
|
|
"logps/rejected": -737.296875,
|
||
|
|
"loss": 0.514007568359375,
|
||
|
|
"mean_token_accuracy": 0.7038853816688061,
|
||
|
|
"num_tokens": 95674576.0,
|
||
|
|
"rewards/accuracies": 0.70859375,
|
||
|
|
"rewards/chosen": -0.10278053283691406,
|
||
|
|
"rewards/margins": 1.14259033203125,
|
||
|
|
"rewards/rejected": -1.2454254150390625,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2052001953125,
|
||
|
|
"epoch": 0.27203705332278016,
|
||
|
|
"grad_norm": 40.99026107788086,
|
||
|
|
"learning_rate": 4.0489838457529964e-07,
|
||
|
|
"logits/chosen": -0.33785028651093185,
|
||
|
|
"logits/rejected": 0.1845951028028979,
|
||
|
|
"logps/chosen": -474.328125,
|
||
|
|
"logps/rejected": -641.23125,
|
||
|
|
"loss": 0.5077590942382812,
|
||
|
|
"mean_token_accuracy": 0.7113756321370601,
|
||
|
|
"num_tokens": 97360867.0,
|
||
|
|
"rewards/accuracies": 0.71796875,
|
||
|
|
"rewards/chosen": -0.12218704223632812,
|
||
|
|
"rewards/margins": 1.0552566528320313,
|
||
|
|
"rewards/rejected": -1.17777099609375,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2611083984375,
|
||
|
|
"epoch": 0.27672734734558674,
|
||
|
|
"grad_norm": 49.85224151611328,
|
||
|
|
"learning_rate": 4.022928608650338e-07,
|
||
|
|
"logits/chosen": -0.3494676075196345,
|
||
|
|
"logits/rejected": 0.17322246770157881,
|
||
|
|
"logps/chosen": -486.14375,
|
||
|
|
"logps/rejected": -657.125,
|
||
|
|
"loss": 0.53740234375,
|
||
|
|
"mean_token_accuracy": 0.70248509272933,
|
||
|
|
"num_tokens": 99032070.0,
|
||
|
|
"rewards/accuracies": 0.6875,
|
||
|
|
"rewards/chosen": -0.19557952880859375,
|
||
|
|
"rewards/margins": 1.0435150146484375,
|
||
|
|
"rewards/rejected": -1.2389533996582032,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.219482421875,
|
||
|
|
"epoch": 0.28141764136839326,
|
||
|
|
"grad_norm": 44.720306396484375,
|
||
|
|
"learning_rate": 3.996873371547681e-07,
|
||
|
|
"logits/chosen": -0.3348284376731146,
|
||
|
|
"logits/rejected": 0.042711471660991984,
|
||
|
|
"logps/chosen": -475.38125,
|
||
|
|
"logps/rejected": -685.36875,
|
||
|
|
"loss": 0.5462005615234375,
|
||
|
|
"mean_token_accuracy": 0.7143606793135404,
|
||
|
|
"num_tokens": 100746662.0,
|
||
|
|
"rewards/accuracies": 0.69296875,
|
||
|
|
"rewards/chosen": -0.20562248229980468,
|
||
|
|
"rewards/margins": 1.092462921142578,
|
||
|
|
"rewards/rejected": -1.2979248046875,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.28480224609375,
|
||
|
|
"epoch": 0.28610793539119983,
|
||
|
|
"grad_norm": 42.72548294067383,
|
||
|
|
"learning_rate": 3.970818134445023e-07,
|
||
|
|
"logits/chosen": -0.39000245026929997,
|
||
|
|
"logits/rejected": 0.23665966765112026,
|
||
|
|
"logps/chosen": -449.0,
|
||
|
|
"logps/rejected": -775.440625,
|
||
|
|
"loss": 0.47276611328125,
|
||
|
|
"mean_token_accuracy": 0.71767991296947,
|
||
|
|
"num_tokens": 102474099.0,
|
||
|
|
"rewards/accuracies": 0.725,
|
||
|
|
"rewards/chosen": -0.17837677001953126,
|
||
|
|
"rewards/margins": 1.2704742431640625,
|
||
|
|
"rewards/rejected": -1.4485992431640624,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.24482421875,
|
||
|
|
"epoch": 0.2907982294140064,
|
||
|
|
"grad_norm": 58.969539642333984,
|
||
|
|
"learning_rate": 3.944762897342366e-07,
|
||
|
|
"logits/chosen": -0.27561446925400485,
|
||
|
|
"logits/rejected": 0.08348458751869889,
|
||
|
|
"logps/chosen": -479.00625,
|
||
|
|
"logps/rejected": -641.2,
|
||
|
|
"loss": 0.5534393310546875,
|
||
|
|
"mean_token_accuracy": 0.7048881415277719,
|
||
|
|
"num_tokens": 104130552.0,
|
||
|
|
"rewards/accuracies": 0.70234375,
|
||
|
|
"rewards/chosen": -0.22642822265625,
|
||
|
|
"rewards/margins": 1.002734375,
|
||
|
|
"rewards/rejected": -1.2297664642333985,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.201513671875,
|
||
|
|
"epoch": 0.2954885234368129,
|
||
|
|
"grad_norm": 39.077484130859375,
|
||
|
|
"learning_rate": 3.9187076602397077e-07,
|
||
|
|
"logits/chosen": -0.301554321263415,
|
||
|
|
"logits/rejected": 0.21063951345062448,
|
||
|
|
"logps/chosen": -446.575,
|
||
|
|
"logps/rejected": -647.35625,
|
||
|
|
"loss": 0.49993896484375,
|
||
|
|
"mean_token_accuracy": 0.7235580589622259,
|
||
|
|
"num_tokens": 105816792.0,
|
||
|
|
"rewards/accuracies": 0.721875,
|
||
|
|
"rewards/chosen": -0.1338104248046875,
|
||
|
|
"rewards/margins": 1.1701812744140625,
|
||
|
|
"rewards/rejected": -1.3043838500976563,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2219970703125,
|
||
|
|
"epoch": 0.3001788174596195,
|
||
|
|
"grad_norm": 67.47047424316406,
|
||
|
|
"learning_rate": 3.8926524231370505e-07,
|
||
|
|
"logits/chosen": -0.2605092185300394,
|
||
|
|
"logits/rejected": 0.03782858556336989,
|
||
|
|
"logps/chosen": -487.1375,
|
||
|
|
"logps/rejected": -570.540625,
|
||
|
|
"loss": 0.5586784362792969,
|
||
|
|
"mean_token_accuracy": 0.7060683678835631,
|
||
|
|
"num_tokens": 107434278.0,
|
||
|
|
"rewards/accuracies": 0.684375,
|
||
|
|
"rewards/chosen": -0.20991897583007812,
|
||
|
|
"rewards/margins": 0.9743408203125,
|
||
|
|
"rewards/rejected": -1.1841949462890624,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.23212890625,
|
||
|
|
"epoch": 0.30486911148242607,
|
||
|
|
"grad_norm": 43.978355407714844,
|
||
|
|
"learning_rate": 3.866597186034393e-07,
|
||
|
|
"logits/chosen": -0.3408145959077753,
|
||
|
|
"logits/rejected": 0.05489902388052849,
|
||
|
|
"logps/chosen": -469.74375,
|
||
|
|
"logps/rejected": -648.896875,
|
||
|
|
"loss": 0.530322265625,
|
||
|
|
"mean_token_accuracy": 0.7166027266532182,
|
||
|
|
"num_tokens": 109108061.0,
|
||
|
|
"rewards/accuracies": 0.7,
|
||
|
|
"rewards/chosen": -0.18619918823242188,
|
||
|
|
"rewards/margins": 1.049748992919922,
|
||
|
|
"rewards/rejected": -1.2358810424804687,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2399169921875,
|
||
|
|
"epoch": 0.3095594055052326,
|
||
|
|
"grad_norm": 43.92350387573242,
|
||
|
|
"learning_rate": 3.8405419489317355e-07,
|
||
|
|
"logits/chosen": -0.3518982698881636,
|
||
|
|
"logits/rejected": 0.15893815329437216,
|
||
|
|
"logps/chosen": -458.603125,
|
||
|
|
"logps/rejected": -696.175,
|
||
|
|
"loss": 0.5294532775878906,
|
||
|
|
"mean_token_accuracy": 0.7165129277855158,
|
||
|
|
"num_tokens": 110839489.0,
|
||
|
|
"rewards/accuracies": 0.7078125,
|
||
|
|
"rewards/chosen": -0.16309404373168945,
|
||
|
|
"rewards/margins": 1.1441436767578126,
|
||
|
|
"rewards/rejected": -1.30704345703125,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2388427734375,
|
||
|
|
"epoch": 0.31424969952803916,
|
||
|
|
"grad_norm": 45.37200164794922,
|
||
|
|
"learning_rate": 3.8144867118290773e-07,
|
||
|
|
"logits/chosen": -0.36028611481310185,
|
||
|
|
"logits/rejected": 0.1766612002287472,
|
||
|
|
"logps/chosen": -477.196875,
|
||
|
|
"logps/rejected": -692.225,
|
||
|
|
"loss": 0.48761749267578125,
|
||
|
|
"mean_token_accuracy": 0.7143629230558872,
|
||
|
|
"num_tokens": 112533599.0,
|
||
|
|
"rewards/accuracies": 0.7140625,
|
||
|
|
"rewards/chosen": -0.15464000701904296,
|
||
|
|
"rewards/margins": 1.1968528747558593,
|
||
|
|
"rewards/rejected": -1.350921630859375,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2490478515625,
|
||
|
|
"epoch": 0.31893999355084574,
|
||
|
|
"grad_norm": 50.02637481689453,
|
||
|
|
"learning_rate": 3.78843147472642e-07,
|
||
|
|
"logits/chosen": -0.43763616453963217,
|
||
|
|
"logits/rejected": 0.03516448731243771,
|
||
|
|
"logps/chosen": -473.65,
|
||
|
|
"logps/rejected": -714.50625,
|
||
|
|
"loss": 0.5369621276855469,
|
||
|
|
"mean_token_accuracy": 0.711085844412446,
|
||
|
|
"num_tokens": 114210162.0,
|
||
|
|
"rewards/accuracies": 0.675,
|
||
|
|
"rewards/chosen": -0.20833778381347656,
|
||
|
|
"rewards/margins": 1.1489297866821289,
|
||
|
|
"rewards/rejected": -1.357391357421875,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2511962890625,
|
||
|
|
"epoch": 0.32363028757365225,
|
||
|
|
"grad_norm": 43.783878326416016,
|
||
|
|
"learning_rate": 3.7623762376237623e-07,
|
||
|
|
"logits/chosen": -0.4307592521929092,
|
||
|
|
"logits/rejected": 0.07158832410380966,
|
||
|
|
"logps/chosen": -449.75,
|
||
|
|
"logps/rejected": -669.51875,
|
||
|
|
"loss": 0.5142929077148437,
|
||
|
|
"mean_token_accuracy": 0.709177390858531,
|
||
|
|
"num_tokens": 115807028.0,
|
||
|
|
"rewards/accuracies": 0.709375,
|
||
|
|
"rewards/chosen": -0.2126626491546631,
|
||
|
|
"rewards/margins": 1.17650146484375,
|
||
|
|
"rewards/rejected": -1.389113998413086,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.275048828125,
|
||
|
|
"epoch": 0.3283205815964588,
|
||
|
|
"grad_norm": 38.663330078125,
|
||
|
|
"learning_rate": 3.7363210005211046e-07,
|
||
|
|
"logits/chosen": -0.3787208537319539,
|
||
|
|
"logits/rejected": 0.12219470055302668,
|
||
|
|
"logps/chosen": -470.571875,
|
||
|
|
"logps/rejected": -723.1125,
|
||
|
|
"loss": 0.5110260009765625,
|
||
|
|
"mean_token_accuracy": 0.711947762966156,
|
||
|
|
"num_tokens": 117509330.0,
|
||
|
|
"rewards/accuracies": 0.71796875,
|
||
|
|
"rewards/chosen": -0.15255508422851563,
|
||
|
|
"rewards/margins": 1.1452316284179687,
|
||
|
|
"rewards/rejected": -1.2983154296875,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.290673828125,
|
||
|
|
"epoch": 0.3330108756192654,
|
||
|
|
"grad_norm": 50.86594772338867,
|
||
|
|
"learning_rate": 3.710265763418447e-07,
|
||
|
|
"logits/chosen": -0.36973986713310913,
|
||
|
|
"logits/rejected": 0.15082829922912258,
|
||
|
|
"logps/chosen": -490.76875,
|
||
|
|
"logps/rejected": -683.375,
|
||
|
|
"loss": 0.5232513427734375,
|
||
|
|
"mean_token_accuracy": 0.7003827568143606,
|
||
|
|
"num_tokens": 119183888.0,
|
||
|
|
"rewards/accuracies": 0.703125,
|
||
|
|
"rewards/chosen": -0.2656074523925781,
|
||
|
|
"rewards/margins": 1.10184326171875,
|
||
|
|
"rewards/rejected": -1.3671539306640625,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2464599609375,
|
||
|
|
"epoch": 0.3377011696420719,
|
||
|
|
"grad_norm": 77.96504974365234,
|
||
|
|
"learning_rate": 3.684210526315789e-07,
|
||
|
|
"logits/chosen": -0.31842311219187563,
|
||
|
|
"logits/rejected": 0.1074574384256495,
|
||
|
|
"logps/chosen": -481.25625,
|
||
|
|
"logps/rejected": -687.425,
|
||
|
|
"loss": 0.5421051025390625,
|
||
|
|
"mean_token_accuracy": 0.7150287635624408,
|
||
|
|
"num_tokens": 120851520.0,
|
||
|
|
"rewards/accuracies": 0.68203125,
|
||
|
|
"rewards/chosen": -0.22847366333007812,
|
||
|
|
"rewards/margins": 1.0441268920898437,
|
||
|
|
"rewards/rejected": -1.2725669860839843,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2686279296875,
|
||
|
|
"epoch": 0.3423914636648785,
|
||
|
|
"grad_norm": 47.35414505004883,
|
||
|
|
"learning_rate": 3.658155289213132e-07,
|
||
|
|
"logits/chosen": -0.3355926860262508,
|
||
|
|
"logits/rejected": 0.18558617957879314,
|
||
|
|
"logps/chosen": -471.8875,
|
||
|
|
"logps/rejected": -713.5375,
|
||
|
|
"loss": 0.5218536376953125,
|
||
|
|
"mean_token_accuracy": 0.7111153559759259,
|
||
|
|
"num_tokens": 122550214.0,
|
||
|
|
"rewards/accuracies": 0.70234375,
|
||
|
|
"rewards/chosen": -0.22301788330078126,
|
||
|
|
"rewards/margins": 1.0697059631347656,
|
||
|
|
"rewards/rejected": -1.2926933288574218,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.28505859375,
|
||
|
|
"epoch": 0.34708175768768507,
|
||
|
|
"grad_norm": 49.15150451660156,
|
||
|
|
"learning_rate": 3.6321000521104736e-07,
|
||
|
|
"logits/chosen": -0.3326699481098578,
|
||
|
|
"logits/rejected": 0.01767519845961821,
|
||
|
|
"logps/chosen": -478.29375,
|
||
|
|
"logps/rejected": -686.0125,
|
||
|
|
"loss": 0.560009765625,
|
||
|
|
"mean_token_accuracy": 0.7098246956244111,
|
||
|
|
"num_tokens": 124254418.0,
|
||
|
|
"rewards/accuracies": 0.678125,
|
||
|
|
"rewards/chosen": -0.24724063873291016,
|
||
|
|
"rewards/margins": 1.1002532958984375,
|
||
|
|
"rewards/rejected": -1.3478904724121095,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2275146484375,
|
||
|
|
"epoch": 0.3517720517104916,
|
||
|
|
"grad_norm": 64.82447814941406,
|
||
|
|
"learning_rate": 3.6060448150078164e-07,
|
||
|
|
"logits/chosen": -0.3452819294543103,
|
||
|
|
"logits/rejected": 0.07315509678314808,
|
||
|
|
"logps/chosen": -433.484375,
|
||
|
|
"logps/rejected": -634.0625,
|
||
|
|
"loss": 0.5266708374023438,
|
||
|
|
"mean_token_accuracy": 0.7124956920742989,
|
||
|
|
"num_tokens": 125879625.0,
|
||
|
|
"rewards/accuracies": 0.69921875,
|
||
|
|
"rewards/chosen": -0.21740396022796632,
|
||
|
|
"rewards/margins": 1.1657791137695312,
|
||
|
|
"rewards/rejected": -1.3828399658203125,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2758056640625,
|
||
|
|
"epoch": 0.35646234573329816,
|
||
|
|
"grad_norm": 44.01753616333008,
|
||
|
|
"learning_rate": 3.5799895779051587e-07,
|
||
|
|
"logits/chosen": -0.3970313479144509,
|
||
|
|
"logits/rejected": -0.013435611206957002,
|
||
|
|
"logps/chosen": -483.08125,
|
||
|
|
"logps/rejected": -656.5125,
|
||
|
|
"loss": 0.5375274658203125,
|
||
|
|
"mean_token_accuracy": 0.6993119914084673,
|
||
|
|
"num_tokens": 127473944.0,
|
||
|
|
"rewards/accuracies": 0.68671875,
|
||
|
|
"rewards/chosen": -0.2423839569091797,
|
||
|
|
"rewards/margins": 1.0922943115234376,
|
||
|
|
"rewards/rejected": -1.3347213745117188,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2775634765625,
|
||
|
|
"epoch": 0.36115263975610473,
|
||
|
|
"grad_norm": 49.41606140136719,
|
||
|
|
"learning_rate": 3.5539343408025015e-07,
|
||
|
|
"logits/chosen": -0.38184818044372293,
|
||
|
|
"logits/rejected": 0.1429431066083195,
|
||
|
|
"logps/chosen": -501.05625,
|
||
|
|
"logps/rejected": -693.2625,
|
||
|
|
"loss": 0.5124237060546875,
|
||
|
|
"mean_token_accuracy": 0.7008192248642444,
|
||
|
|
"num_tokens": 129129963.0,
|
||
|
|
"rewards/accuracies": 0.71171875,
|
||
|
|
"rewards/chosen": -0.22407178878784179,
|
||
|
|
"rewards/margins": 1.2268234252929688,
|
||
|
|
"rewards/rejected": -1.450592041015625,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.280322265625,
|
||
|
|
"epoch": 0.36584293377891125,
|
||
|
|
"grad_norm": 46.2809944152832,
|
||
|
|
"learning_rate": 3.527879103699843e-07,
|
||
|
|
"logits/chosen": -0.25291314200945403,
|
||
|
|
"logits/rejected": 0.14969483951627313,
|
||
|
|
"logps/chosen": -501.875,
|
||
|
|
"logps/rejected": -715.396875,
|
||
|
|
"loss": 0.5315521240234375,
|
||
|
|
"mean_token_accuracy": 0.7103803750127554,
|
||
|
|
"num_tokens": 130895409.0,
|
||
|
|
"rewards/accuracies": 0.70078125,
|
||
|
|
"rewards/chosen": -0.20745162963867186,
|
||
|
|
"rewards/margins": 1.1037071228027344,
|
||
|
|
"rewards/rejected": -1.310882568359375,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2568115234375,
|
||
|
|
"epoch": 0.3705332278017178,
|
||
|
|
"grad_norm": 38.061893463134766,
|
||
|
|
"learning_rate": 3.501823866597186e-07,
|
||
|
|
"logits/chosen": -0.29507836911014856,
|
||
|
|
"logits/rejected": 0.09863675737702185,
|
||
|
|
"logps/chosen": -479.525,
|
||
|
|
"logps/rejected": -677.296875,
|
||
|
|
"loss": 0.5366546630859375,
|
||
|
|
"mean_token_accuracy": 0.713270222209394,
|
||
|
|
"num_tokens": 132594219.0,
|
||
|
|
"rewards/accuracies": 0.68125,
|
||
|
|
"rewards/chosen": -0.21154518127441407,
|
||
|
|
"rewards/margins": 1.1149787902832031,
|
||
|
|
"rewards/rejected": -1.3264785766601563,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.26630859375,
|
||
|
|
"epoch": 0.3752235218245244,
|
||
|
|
"grad_norm": 40.844886779785156,
|
||
|
|
"learning_rate": 3.4757686294945283e-07,
|
||
|
|
"logits/chosen": -0.32828585106273067,
|
||
|
|
"logits/rejected": 0.13410074141871198,
|
||
|
|
"logps/chosen": -486.453125,
|
||
|
|
"logps/rejected": -650.13125,
|
||
|
|
"loss": 0.5410888671875,
|
||
|
|
"mean_token_accuracy": 0.7061129916459322,
|
||
|
|
"num_tokens": 134231292.0,
|
||
|
|
"rewards/accuracies": 0.7046875,
|
||
|
|
"rewards/chosen": -0.21239662170410156,
|
||
|
|
"rewards/margins": 1.0793701171875,
|
||
|
|
"rewards/rejected": -1.2920806884765625,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1828125,
|
||
|
|
"epoch": 0.3799138158473309,
|
||
|
|
"grad_norm": 36.530025482177734,
|
||
|
|
"learning_rate": 3.449713392391871e-07,
|
||
|
|
"logits/chosen": -0.2730355243405189,
|
||
|
|
"logits/rejected": 0.09156594890767208,
|
||
|
|
"logps/chosen": -460.49375,
|
||
|
|
"logps/rejected": -573.3875,
|
||
|
|
"loss": 0.5164207458496094,
|
||
|
|
"mean_token_accuracy": 0.7155072972178459,
|
||
|
|
"num_tokens": 135893775.0,
|
||
|
|
"rewards/accuracies": 0.6984375,
|
||
|
|
"rewards/chosen": -0.12217826843261718,
|
||
|
|
"rewards/margins": 0.9473831176757812,
|
||
|
|
"rewards/rejected": -1.0695075988769531,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.22552490234375,
|
||
|
|
"epoch": 0.3846041098701375,
|
||
|
|
"grad_norm": 40.324920654296875,
|
||
|
|
"learning_rate": 3.423658155289213e-07,
|
||
|
|
"logits/chosen": -0.2511466705433654,
|
||
|
|
"logits/rejected": 0.22135653984727344,
|
||
|
|
"logps/chosen": -459.48125,
|
||
|
|
"logps/rejected": -658.15,
|
||
|
|
"loss": 0.54027099609375,
|
||
|
|
"mean_token_accuracy": 0.7177877865731717,
|
||
|
|
"num_tokens": 137573025.0,
|
||
|
|
"rewards/accuracies": 0.6875,
|
||
|
|
"rewards/chosen": -0.17630014419555665,
|
||
|
|
"rewards/margins": 1.01343994140625,
|
||
|
|
"rewards/rejected": -1.1897003173828125,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2394775390625,
|
||
|
|
"epoch": 0.38929440389294406,
|
||
|
|
"grad_norm": 44.12002182006836,
|
||
|
|
"learning_rate": 3.397602918186555e-07,
|
||
|
|
"logits/chosen": -0.28912138485015315,
|
||
|
|
"logits/rejected": 0.19754984016879085,
|
||
|
|
"logps/chosen": -475.20625,
|
||
|
|
"logps/rejected": -668.40625,
|
||
|
|
"loss": 0.5154266357421875,
|
||
|
|
"mean_token_accuracy": 0.7126389849931002,
|
||
|
|
"num_tokens": 139224712.0,
|
||
|
|
"rewards/accuracies": 0.71171875,
|
||
|
|
"rewards/chosen": -0.1871318817138672,
|
||
|
|
"rewards/margins": 1.09332275390625,
|
||
|
|
"rewards/rejected": -1.2801509857177735,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.246630859375,
|
||
|
|
"epoch": 0.3939846979157506,
|
||
|
|
"grad_norm": 87.65414428710938,
|
||
|
|
"learning_rate": 3.371547681083898e-07,
|
||
|
|
"logits/chosen": -0.2749478759605691,
|
||
|
|
"logits/rejected": 0.17435728309129989,
|
||
|
|
"logps/chosen": -494.425,
|
||
|
|
"logps/rejected": -700.06875,
|
||
|
|
"loss": 0.5238235473632813,
|
||
|
|
"mean_token_accuracy": 0.7123918626457453,
|
||
|
|
"num_tokens": 140969200.0,
|
||
|
|
"rewards/accuracies": 0.71796875,
|
||
|
|
"rewards/chosen": -0.21366329193115235,
|
||
|
|
"rewards/margins": 1.1942451477050782,
|
||
|
|
"rewards/rejected": -1.407720947265625,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.230517578125,
|
||
|
|
"epoch": 0.39867499193855716,
|
||
|
|
"grad_norm": 81.68108367919922,
|
||
|
|
"learning_rate": 3.34549244398124e-07,
|
||
|
|
"logits/chosen": -0.3473418693014164,
|
||
|
|
"logits/rejected": 0.0888236907473366,
|
||
|
|
"logps/chosen": -476.23125,
|
||
|
|
"logps/rejected": -655.90625,
|
||
|
|
"loss": 0.5314712524414062,
|
||
|
|
"mean_token_accuracy": 0.7124969633296132,
|
||
|
|
"num_tokens": 142707119.0,
|
||
|
|
"rewards/accuracies": 0.7,
|
||
|
|
"rewards/chosen": -0.2162473678588867,
|
||
|
|
"rewards/margins": 1.093731689453125,
|
||
|
|
"rewards/rejected": -1.3099411010742188,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.322705078125,
|
||
|
|
"epoch": 0.40336528596136373,
|
||
|
|
"grad_norm": 59.91925048828125,
|
||
|
|
"learning_rate": 3.3194372068785824e-07,
|
||
|
|
"logits/chosen": -0.3529681888952906,
|
||
|
|
"logits/rejected": 0.23290446857518904,
|
||
|
|
"logps/chosen": -443.8625,
|
||
|
|
"logps/rejected": -743.39375,
|
||
|
|
"loss": 0.5114044189453125,
|
||
|
|
"mean_token_accuracy": 0.7079902265220881,
|
||
|
|
"num_tokens": 144391043.0,
|
||
|
|
"rewards/accuracies": 0.7296875,
|
||
|
|
"rewards/chosen": -0.2095262050628662,
|
||
|
|
"rewards/margins": 1.2830490112304687,
|
||
|
|
"rewards/rejected": -1.492266845703125,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.27998046875,
|
||
|
|
"epoch": 0.40805557998417025,
|
||
|
|
"grad_norm": 40.34415054321289,
|
||
|
|
"learning_rate": 3.2933819697759247e-07,
|
||
|
|
"logits/chosen": -0.38396973862868355,
|
||
|
|
"logits/rejected": 0.1605909461102152,
|
||
|
|
"logps/chosen": -454.6,
|
||
|
|
"logps/rejected": -662.209375,
|
||
|
|
"loss": 0.52490234375,
|
||
|
|
"mean_token_accuracy": 0.7066729519516229,
|
||
|
|
"num_tokens": 145997144.0,
|
||
|
|
"rewards/accuracies": 0.71484375,
|
||
|
|
"rewards/chosen": -0.1993091583251953,
|
||
|
|
"rewards/margins": 1.155511474609375,
|
||
|
|
"rewards/rejected": -1.3549407958984374,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.243115234375,
|
||
|
|
"epoch": 0.4127458740069768,
|
||
|
|
"grad_norm": 43.68743133544922,
|
||
|
|
"learning_rate": 3.2673267326732674e-07,
|
||
|
|
"logits/chosen": -0.3361740698074229,
|
||
|
|
"logits/rejected": 0.07762473854104804,
|
||
|
|
"logps/chosen": -509.29375,
|
||
|
|
"logps/rejected": -656.60625,
|
||
|
|
"loss": 0.4995269775390625,
|
||
|
|
"mean_token_accuracy": 0.7099218437448144,
|
||
|
|
"num_tokens": 147695183.0,
|
||
|
|
"rewards/accuracies": 0.70859375,
|
||
|
|
"rewards/chosen": -0.23754005432128905,
|
||
|
|
"rewards/margins": 1.1845123291015625,
|
||
|
|
"rewards/rejected": -1.4216781616210938,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.23310546875,
|
||
|
|
"epoch": 0.4174361680297834,
|
||
|
|
"grad_norm": 80.83831024169922,
|
||
|
|
"learning_rate": 3.241271495570609e-07,
|
||
|
|
"logits/chosen": -0.3442709798320729,
|
||
|
|
"logits/rejected": 0.10712556906402045,
|
||
|
|
"logps/chosen": -499.9625,
|
||
|
|
"logps/rejected": -665.31875,
|
||
|
|
"loss": 0.5188461303710937,
|
||
|
|
"mean_token_accuracy": 0.707976047694683,
|
||
|
|
"num_tokens": 149444689.0,
|
||
|
|
"rewards/accuracies": 0.7109375,
|
||
|
|
"rewards/chosen": -0.2080381393432617,
|
||
|
|
"rewards/margins": 1.1563003540039063,
|
||
|
|
"rewards/rejected": -1.3644058227539062,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.278857421875,
|
||
|
|
"epoch": 0.4221264620525899,
|
||
|
|
"grad_norm": 46.61259460449219,
|
||
|
|
"learning_rate": 3.215216258467952e-07,
|
||
|
|
"logits/chosen": -0.3779255074786172,
|
||
|
|
"logits/rejected": 0.13898465945471042,
|
||
|
|
"logps/chosen": -463.884375,
|
||
|
|
"logps/rejected": -697.05625,
|
||
|
|
"loss": 0.509967041015625,
|
||
|
|
"mean_token_accuracy": 0.7144387796521187,
|
||
|
|
"num_tokens": 151137840.0,
|
||
|
|
"rewards/accuracies": 0.72734375,
|
||
|
|
"rewards/chosen": -0.22325286865234376,
|
||
|
|
"rewards/margins": 1.138238525390625,
|
||
|
|
"rewards/rejected": -1.3617431640625,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2478271484375,
|
||
|
|
"epoch": 0.4268167560753965,
|
||
|
|
"grad_norm": 57.145328521728516,
|
||
|
|
"learning_rate": 3.189161021365294e-07,
|
||
|
|
"logits/chosen": -0.4029233645848759,
|
||
|
|
"logits/rejected": 0.07330459311441123,
|
||
|
|
"logps/chosen": -447.553125,
|
||
|
|
"logps/rejected": -658.375,
|
||
|
|
"loss": 0.536572265625,
|
||
|
|
"mean_token_accuracy": 0.7051212705671788,
|
||
|
|
"num_tokens": 152711567.0,
|
||
|
|
"rewards/accuracies": 0.70546875,
|
||
|
|
"rewards/chosen": -0.22397823333740235,
|
||
|
|
"rewards/margins": 1.013479232788086,
|
||
|
|
"rewards/rejected": -1.2374153137207031,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.212109375,
|
||
|
|
"epoch": 0.431507050098203,
|
||
|
|
"grad_norm": 45.5133171081543,
|
||
|
|
"learning_rate": 3.163105784262637e-07,
|
||
|
|
"logits/chosen": -0.3665716216140561,
|
||
|
|
"logits/rejected": 0.03931592270211523,
|
||
|
|
"logps/chosen": -450.85625,
|
||
|
|
"logps/rejected": -642.99375,
|
||
|
|
"loss": 0.5301528930664062,
|
||
|
|
"mean_token_accuracy": 0.7148423504084349,
|
||
|
|
"num_tokens": 154353646.0,
|
||
|
|
"rewards/accuracies": 0.69140625,
|
||
|
|
"rewards/chosen": -0.18856353759765626,
|
||
|
|
"rewards/margins": 1.0661237716674805,
|
||
|
|
"rewards/rejected": -1.2548416137695313,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.214599609375,
|
||
|
|
"epoch": 0.4361973441210096,
|
||
|
|
"grad_norm": 43.24400329589844,
|
||
|
|
"learning_rate": 3.137050547159979e-07,
|
||
|
|
"logits/chosen": -0.30179459918123197,
|
||
|
|
"logits/rejected": 0.09087620740651976,
|
||
|
|
"logps/chosen": -494.99375,
|
||
|
|
"logps/rejected": -610.909375,
|
||
|
|
"loss": 0.5239151000976563,
|
||
|
|
"mean_token_accuracy": 0.7096556110307575,
|
||
|
|
"num_tokens": 156025668.0,
|
||
|
|
"rewards/accuracies": 0.70703125,
|
||
|
|
"rewards/chosen": -0.2380859375,
|
||
|
|
"rewards/margins": 1.0573944091796874,
|
||
|
|
"rewards/rejected": -1.295379638671875,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.207666015625,
|
||
|
|
"epoch": 0.44088763814381615,
|
||
|
|
"grad_norm": 72.49191284179688,
|
||
|
|
"learning_rate": 3.1109953100573216e-07,
|
||
|
|
"logits/chosen": -0.29770645267053186,
|
||
|
|
"logits/rejected": 0.15372499151275804,
|
||
|
|
"logps/chosen": -484.06875,
|
||
|
|
"logps/rejected": -627.015625,
|
||
|
|
"loss": 0.5368232727050781,
|
||
|
|
"mean_token_accuracy": 0.7107538601383567,
|
||
|
|
"num_tokens": 157761123.0,
|
||
|
|
"rewards/accuracies": 0.703125,
|
||
|
|
"rewards/chosen": -0.27497024536132814,
|
||
|
|
"rewards/margins": 1.0991371154785157,
|
||
|
|
"rewards/rejected": -1.3735931396484375,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.22421875,
|
||
|
|
"epoch": 0.44557793216662267,
|
||
|
|
"grad_norm": 44.017452239990234,
|
||
|
|
"learning_rate": 3.084940072954664e-07,
|
||
|
|
"logits/chosen": -0.4168874535254143,
|
||
|
|
"logits/rejected": 0.011802939570460098,
|
||
|
|
"logps/chosen": -430.26875,
|
||
|
|
"logps/rejected": -586.91875,
|
||
|
|
"loss": 0.5136375427246094,
|
||
|
|
"mean_token_accuracy": 0.7067252658307552,
|
||
|
|
"num_tokens": 159297345.0,
|
||
|
|
"rewards/accuracies": 0.7140625,
|
||
|
|
"rewards/chosen": -0.20767440795898437,
|
||
|
|
"rewards/margins": 1.0971954345703125,
|
||
|
|
"rewards/rejected": -1.30452880859375,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25185546875,
|
||
|
|
"epoch": 0.45026822618942924,
|
||
|
|
"grad_norm": 66.41146850585938,
|
||
|
|
"learning_rate": 3.058884835852006e-07,
|
||
|
|
"logits/chosen": -0.3064041353585244,
|
||
|
|
"logits/rejected": 0.22961672692960927,
|
||
|
|
"logps/chosen": -475.13125,
|
||
|
|
"logps/rejected": -759.978125,
|
||
|
|
"loss": 0.5192276000976562,
|
||
|
|
"mean_token_accuracy": 0.7137241207063199,
|
||
|
|
"num_tokens": 161048146.0,
|
||
|
|
"rewards/accuracies": 0.71171875,
|
||
|
|
"rewards/chosen": -0.2347585678100586,
|
||
|
|
"rewards/margins": 1.1838924407958984,
|
||
|
|
"rewards/rejected": -1.4190101623535156,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2472900390625,
|
||
|
|
"epoch": 0.4549585202122358,
|
||
|
|
"grad_norm": 56.4462776184082,
|
||
|
|
"learning_rate": 3.0328295987493483e-07,
|
||
|
|
"logits/chosen": -0.3383007687161007,
|
||
|
|
"logits/rejected": 0.13007250379762667,
|
||
|
|
"logps/chosen": -473.79375,
|
||
|
|
"logps/rejected": -676.871875,
|
||
|
|
"loss": 0.5186408996582031,
|
||
|
|
"mean_token_accuracy": 0.7102607268840074,
|
||
|
|
"num_tokens": 162759295.0,
|
||
|
|
"rewards/accuracies": 0.6984375,
|
||
|
|
"rewards/chosen": -0.22657947540283202,
|
||
|
|
"rewards/margins": 1.1189689636230469,
|
||
|
|
"rewards/rejected": -1.3456756591796875,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.30703125,
|
||
|
|
"epoch": 0.45964881423504234,
|
||
|
|
"grad_norm": 50.83198165893555,
|
||
|
|
"learning_rate": 3.0067743616466906e-07,
|
||
|
|
"logits/chosen": -0.2545741993101498,
|
||
|
|
"logits/rejected": 0.15262439804664002,
|
||
|
|
"logps/chosen": -483.86875,
|
||
|
|
"logps/rejected": -712.11875,
|
||
|
|
"loss": 0.5216323852539062,
|
||
|
|
"mean_token_accuracy": 0.7064540114253759,
|
||
|
|
"num_tokens": 164446081.0,
|
||
|
|
"rewards/accuracies": 0.7078125,
|
||
|
|
"rewards/chosen": -0.20740795135498047,
|
||
|
|
"rewards/margins": 1.1302799224853515,
|
||
|
|
"rewards/rejected": -1.337384033203125,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2359375,
|
||
|
|
"epoch": 0.4643391082578489,
|
||
|
|
"grad_norm": 55.23470687866211,
|
||
|
|
"learning_rate": 2.9807191245440334e-07,
|
||
|
|
"logits/chosen": -0.30537551995678974,
|
||
|
|
"logits/rejected": 0.11711747734812943,
|
||
|
|
"logps/chosen": -465.8,
|
||
|
|
"logps/rejected": -614.971875,
|
||
|
|
"loss": 0.5563156127929687,
|
||
|
|
"mean_token_accuracy": 0.7104039791971445,
|
||
|
|
"num_tokens": 166020948.0,
|
||
|
|
"rewards/accuracies": 0.67109375,
|
||
|
|
"rewards/chosen": -0.21481266021728515,
|
||
|
|
"rewards/margins": 0.9331451416015625,
|
||
|
|
"rewards/rejected": -1.1478729248046875,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2370849609375,
|
||
|
|
"epoch": 0.4690294022806555,
|
||
|
|
"grad_norm": 42.55168914794922,
|
||
|
|
"learning_rate": 2.954663887441375e-07,
|
||
|
|
"logits/chosen": -0.26711619092213956,
|
||
|
|
"logits/rejected": 0.21803520041534127,
|
||
|
|
"logps/chosen": -469.85,
|
||
|
|
"logps/rejected": -661.95625,
|
||
|
|
"loss": 0.506292724609375,
|
||
|
|
"mean_token_accuracy": 0.71516153793782,
|
||
|
|
"num_tokens": 167714489.0,
|
||
|
|
"rewards/accuracies": 0.72265625,
|
||
|
|
"rewards/chosen": -0.22079696655273437,
|
||
|
|
"rewards/margins": 1.1295989990234374,
|
||
|
|
"rewards/rejected": -1.3502197265625,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.26083984375,
|
||
|
|
"epoch": 0.473719696303462,
|
||
|
|
"grad_norm": 57.51393508911133,
|
||
|
|
"learning_rate": 2.928608650338718e-07,
|
||
|
|
"logits/chosen": -0.3097135444020215,
|
||
|
|
"logits/rejected": 0.0631521841392775,
|
||
|
|
"logps/chosen": -485.55625,
|
||
|
|
"logps/rejected": -617.9875,
|
||
|
|
"loss": 0.5513885498046875,
|
||
|
|
"mean_token_accuracy": 0.7011837583035231,
|
||
|
|
"num_tokens": 169396307.0,
|
||
|
|
"rewards/accuracies": 0.67265625,
|
||
|
|
"rewards/chosen": -0.2332773208618164,
|
||
|
|
"rewards/margins": 0.9677375793457031,
|
||
|
|
"rewards/rejected": -1.2006851196289063,
|
||
|
|
"step": 1010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.258544921875,
|
||
|
|
"epoch": 0.4784099903262686,
|
||
|
|
"grad_norm": 47.03550720214844,
|
||
|
|
"learning_rate": 2.90255341323606e-07,
|
||
|
|
"logits/chosen": -0.3139030744878589,
|
||
|
|
"logits/rejected": 0.1747507997422271,
|
||
|
|
"logps/chosen": -476.775,
|
||
|
|
"logps/rejected": -688.54375,
|
||
|
|
"loss": 0.5033882141113282,
|
||
|
|
"mean_token_accuracy": 0.7097974337637425,
|
||
|
|
"num_tokens": 171061030.0,
|
||
|
|
"rewards/accuracies": 0.72578125,
|
||
|
|
"rewards/chosen": -0.20766181945800782,
|
||
|
|
"rewards/margins": 1.085690689086914,
|
||
|
|
"rewards/rejected": -1.2930282592773437,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2618408203125,
|
||
|
|
"epoch": 0.48310028434907515,
|
||
|
|
"grad_norm": 37.497474670410156,
|
||
|
|
"learning_rate": 2.876498176133403e-07,
|
||
|
|
"logits/chosen": -0.35845184599210705,
|
||
|
|
"logits/rejected": 0.06585844424002088,
|
||
|
|
"logps/chosen": -487.5375,
|
||
|
|
"logps/rejected": -678.8,
|
||
|
|
"loss": 0.4933441162109375,
|
||
|
|
"mean_token_accuracy": 0.7051137916743755,
|
||
|
|
"num_tokens": 172742451.0,
|
||
|
|
"rewards/accuracies": 0.72578125,
|
||
|
|
"rewards/chosen": -0.22617607116699218,
|
||
|
|
"rewards/margins": 1.1918342590332032,
|
||
|
|
"rewards/rejected": -1.4184478759765624,
|
||
|
|
"step": 1030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2374267578125,
|
||
|
|
"epoch": 0.48779057837188167,
|
||
|
|
"grad_norm": 43.61983871459961,
|
||
|
|
"learning_rate": 2.8504429390307447e-07,
|
||
|
|
"logits/chosen": -0.2910774125262612,
|
||
|
|
"logits/rejected": 0.15164186582642813,
|
||
|
|
"logps/chosen": -468.56875,
|
||
|
|
"logps/rejected": -712.00625,
|
||
|
|
"loss": 0.5211807250976562,
|
||
|
|
"mean_token_accuracy": 0.7186769794672727,
|
||
|
|
"num_tokens": 174461002.0,
|
||
|
|
"rewards/accuracies": 0.70390625,
|
||
|
|
"rewards/chosen": -0.23565731048583985,
|
||
|
|
"rewards/margins": 1.1290267944335937,
|
||
|
|
"rewards/rejected": -1.3653778076171874,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.237646484375,
|
||
|
|
"epoch": 0.49248087239468824,
|
||
|
|
"grad_norm": 41.82542419433594,
|
||
|
|
"learning_rate": 2.8243877019280875e-07,
|
||
|
|
"logits/chosen": -0.27986107767208945,
|
||
|
|
"logits/rejected": 0.09014670592249303,
|
||
|
|
"logps/chosen": -482.040625,
|
||
|
|
"logps/rejected": -618.11875,
|
||
|
|
"loss": 0.5121337890625,
|
||
|
|
"mean_token_accuracy": 0.7039212234318256,
|
||
|
|
"num_tokens": 176123752.0,
|
||
|
|
"rewards/accuracies": 0.7125,
|
||
|
|
"rewards/chosen": -0.23637866973876953,
|
||
|
|
"rewards/margins": 1.126165008544922,
|
||
|
|
"rewards/rejected": -1.3623748779296876,
|
||
|
|
"step": 1050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.244970703125,
|
||
|
|
"epoch": 0.4971711664174948,
|
||
|
|
"grad_norm": 39.264984130859375,
|
||
|
|
"learning_rate": 2.79833246482543e-07,
|
||
|
|
"logits/chosen": -0.3118509453919952,
|
||
|
|
"logits/rejected": 0.171299993947704,
|
||
|
|
"logps/chosen": -454.53125,
|
||
|
|
"logps/rejected": -675.275,
|
||
|
|
"loss": 0.5181777954101563,
|
||
|
|
"mean_token_accuracy": 0.7140839375555516,
|
||
|
|
"num_tokens": 177838032.0,
|
||
|
|
"rewards/accuracies": 0.69296875,
|
||
|
|
"rewards/chosen": -0.19367313385009766,
|
||
|
|
"rewards/margins": 1.2017547607421875,
|
||
|
|
"rewards/rejected": -1.39564208984375,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1814453125,
|
||
|
|
"epoch": 0.5018614604403013,
|
||
|
|
"grad_norm": 43.66828536987305,
|
||
|
|
"learning_rate": 2.7722772277227726e-07,
|
||
|
|
"logits/chosen": -0.35000077578470384,
|
||
|
|
"logits/rejected": 0.07725395072725653,
|
||
|
|
"logps/chosen": -449.09375,
|
||
|
|
"logps/rejected": -622.125,
|
||
|
|
"loss": 0.4893054962158203,
|
||
|
|
"mean_token_accuracy": 0.7206394921988248,
|
||
|
|
"num_tokens": 179504581.0,
|
||
|
|
"rewards/accuracies": 0.72421875,
|
||
|
|
"rewards/chosen": -0.19243621826171875,
|
||
|
|
"rewards/margins": 1.165966796875,
|
||
|
|
"rewards/rejected": -1.3583648681640625,
|
||
|
|
"step": 1070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25322265625,
|
||
|
|
"epoch": 0.5065517544631079,
|
||
|
|
"grad_norm": 40.902252197265625,
|
||
|
|
"learning_rate": 2.7462219906201143e-07,
|
||
|
|
"logits/chosen": -0.33450535889940103,
|
||
|
|
"logits/rejected": 0.1168974359193452,
|
||
|
|
"logps/chosen": -470.7125,
|
||
|
|
"logps/rejected": -658.0875,
|
||
|
|
"loss": 0.5172897338867187,
|
||
|
|
"mean_token_accuracy": 0.7043697636574506,
|
||
|
|
"num_tokens": 181183445.0,
|
||
|
|
"rewards/accuracies": 0.7140625,
|
||
|
|
"rewards/chosen": -0.2109607696533203,
|
||
|
|
"rewards/margins": 1.235137939453125,
|
||
|
|
"rewards/rejected": -1.4460359573364259,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3098876953125,
|
||
|
|
"epoch": 0.5112420484859145,
|
||
|
|
"grad_norm": 47.25940704345703,
|
||
|
|
"learning_rate": 2.7201667535174566e-07,
|
||
|
|
"logits/chosen": -0.29794072833065727,
|
||
|
|
"logits/rejected": 0.10716472846404641,
|
||
|
|
"logps/chosen": -498.059375,
|
||
|
|
"logps/rejected": -685.2625,
|
||
|
|
"loss": 0.526763916015625,
|
||
|
|
"mean_token_accuracy": 0.6980478201061487,
|
||
|
|
"num_tokens": 182829730.0,
|
||
|
|
"rewards/accuracies": 0.71015625,
|
||
|
|
"rewards/chosen": -0.2504600524902344,
|
||
|
|
"rewards/margins": 1.2077743530273437,
|
||
|
|
"rewards/rejected": -1.457940673828125,
|
||
|
|
"step": 1090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.32303466796875,
|
||
|
|
"epoch": 0.515932342508721,
|
||
|
|
"grad_norm": 39.55305862426758,
|
||
|
|
"learning_rate": 2.6941115164147994e-07,
|
||
|
|
"logits/chosen": -0.36416835318792495,
|
||
|
|
"logits/rejected": 0.10446686581728493,
|
||
|
|
"logps/chosen": -481.640625,
|
||
|
|
"logps/rejected": -717.20625,
|
||
|
|
"loss": 0.5190200805664062,
|
||
|
|
"mean_token_accuracy": 0.7050384001806378,
|
||
|
|
"num_tokens": 184507519.0,
|
||
|
|
"rewards/accuracies": 0.728125,
|
||
|
|
"rewards/chosen": -0.24545488357543946,
|
||
|
|
"rewards/margins": 1.1905990600585938,
|
||
|
|
"rewards/rejected": -1.4363555908203125,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25341796875,
|
||
|
|
"epoch": 0.5206226365315275,
|
||
|
|
"grad_norm": 39.871578216552734,
|
||
|
|
"learning_rate": 2.6680562793121416e-07,
|
||
|
|
"logits/chosen": -0.3013125689062654,
|
||
|
|
"logits/rejected": 0.10314716436853602,
|
||
|
|
"logps/chosen": -473.5625,
|
||
|
|
"logps/rejected": -651.525,
|
||
|
|
"loss": 0.5050193786621093,
|
||
|
|
"mean_token_accuracy": 0.7060678452253342,
|
||
|
|
"num_tokens": 186153120.0,
|
||
|
|
"rewards/accuracies": 0.703125,
|
||
|
|
"rewards/chosen": -0.1920490264892578,
|
||
|
|
"rewards/margins": 1.227564239501953,
|
||
|
|
"rewards/rejected": -1.4199142456054688,
|
||
|
|
"step": 1110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.176513671875,
|
||
|
|
"epoch": 0.5253129305543341,
|
||
|
|
"grad_norm": 52.36431121826172,
|
||
|
|
"learning_rate": 2.642001042209484e-07,
|
||
|
|
"logits/chosen": -0.26450315077369746,
|
||
|
|
"logits/rejected": 0.15473802955973542,
|
||
|
|
"logps/chosen": -464.03125,
|
||
|
|
"logps/rejected": -627.96875,
|
||
|
|
"loss": 0.49540557861328127,
|
||
|
|
"mean_token_accuracy": 0.7205847401171923,
|
||
|
|
"num_tokens": 187806398.0,
|
||
|
|
"rewards/accuracies": 0.71796875,
|
||
|
|
"rewards/chosen": -0.2411958694458008,
|
||
|
|
"rewards/margins": 1.1548843383789062,
|
||
|
|
"rewards/rejected": -1.395941162109375,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.31982421875,
|
||
|
|
"epoch": 0.5300032245771407,
|
||
|
|
"grad_norm": 36.76437759399414,
|
||
|
|
"learning_rate": 2.615945805106826e-07,
|
||
|
|
"logits/chosen": -0.3149445633982369,
|
||
|
|
"logits/rejected": 0.16251844820700828,
|
||
|
|
"logps/chosen": -476.3375,
|
||
|
|
"logps/rejected": -757.70625,
|
||
|
|
"loss": 0.5259773254394531,
|
||
|
|
"mean_token_accuracy": 0.7150143172591925,
|
||
|
|
"num_tokens": 189521715.0,
|
||
|
|
"rewards/accuracies": 0.71953125,
|
||
|
|
"rewards/chosen": -0.2303621292114258,
|
||
|
|
"rewards/margins": 1.315435791015625,
|
||
|
|
"rewards/rejected": -1.5461624145507813,
|
||
|
|
"step": 1130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2891357421875,
|
||
|
|
"epoch": 0.5346935185999472,
|
||
|
|
"grad_norm": 70.96980285644531,
|
||
|
|
"learning_rate": 2.589890568004169e-07,
|
||
|
|
"logits/chosen": -0.3271291111287221,
|
||
|
|
"logits/rejected": 0.16675223605322737,
|
||
|
|
"logps/chosen": -500.31875,
|
||
|
|
"logps/rejected": -694.71875,
|
||
|
|
"loss": 0.5380020141601562,
|
||
|
|
"mean_token_accuracy": 0.705069973319769,
|
||
|
|
"num_tokens": 191236695.0,
|
||
|
|
"rewards/accuracies": 0.7203125,
|
||
|
|
"rewards/chosen": -0.3011770248413086,
|
||
|
|
"rewards/margins": 1.1962921142578125,
|
||
|
|
"rewards/rejected": -1.4982635498046875,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2275634765625,
|
||
|
|
"epoch": 0.5393838126227538,
|
||
|
|
"grad_norm": 48.03555679321289,
|
||
|
|
"learning_rate": 2.5638353309015107e-07,
|
||
|
|
"logits/chosen": -0.30602316091402987,
|
||
|
|
"logits/rejected": 0.08515583556015452,
|
||
|
|
"logps/chosen": -465.7125,
|
||
|
|
"logps/rejected": -623.03125,
|
||
|
|
"loss": 0.4871429443359375,
|
||
|
|
"mean_token_accuracy": 0.7122127812355756,
|
||
|
|
"num_tokens": 192930148.0,
|
||
|
|
"rewards/accuracies": 0.71875,
|
||
|
|
"rewards/chosen": -0.2552909851074219,
|
||
|
|
"rewards/margins": 1.3664947509765626,
|
||
|
|
"rewards/rejected": -1.6219619750976562,
|
||
|
|
"step": 1150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.312890625,
|
||
|
|
"epoch": 0.5440741066455603,
|
||
|
|
"grad_norm": 39.23597717285156,
|
||
|
|
"learning_rate": 2.5377800937988535e-07,
|
||
|
|
"logits/chosen": -0.34489304897849415,
|
||
|
|
"logits/rejected": 0.1796444057444734,
|
||
|
|
"logps/chosen": -510.19375,
|
||
|
|
"logps/rejected": -763.225,
|
||
|
|
"loss": 0.5116714477539063,
|
||
|
|
"mean_token_accuracy": 0.7016687668859959,
|
||
|
|
"num_tokens": 194613883.0,
|
||
|
|
"rewards/accuracies": 0.709375,
|
||
|
|
"rewards/chosen": -0.2514598846435547,
|
||
|
|
"rewards/margins": 1.2324905395507812,
|
||
|
|
"rewards/rejected": -1.4837749481201172,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.28082275390625,
|
||
|
|
"epoch": 0.5487644006683668,
|
||
|
|
"grad_norm": 69.99358367919922,
|
||
|
|
"learning_rate": 2.511724856696196e-07,
|
||
|
|
"logits/chosen": -0.27411585619664347,
|
||
|
|
"logits/rejected": 0.28711497107316053,
|
||
|
|
"logps/chosen": -474.628125,
|
||
|
|
"logps/rejected": -750.44375,
|
||
|
|
"loss": 0.5183349609375,
|
||
|
|
"mean_token_accuracy": 0.7180445723235607,
|
||
|
|
"num_tokens": 196335064.0,
|
||
|
|
"rewards/accuracies": 0.71953125,
|
||
|
|
"rewards/chosen": -0.289007568359375,
|
||
|
|
"rewards/margins": 1.1853988647460938,
|
||
|
|
"rewards/rejected": -1.474517822265625,
|
||
|
|
"step": 1170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.265185546875,
|
||
|
|
"epoch": 0.5534546946911735,
|
||
|
|
"grad_norm": 36.792850494384766,
|
||
|
|
"learning_rate": 2.485669619593538e-07,
|
||
|
|
"logits/chosen": -0.3359410788666688,
|
||
|
|
"logits/rejected": 0.05803117348722396,
|
||
|
|
"logps/chosen": -470.0328125,
|
||
|
|
"logps/rejected": -689.440625,
|
||
|
|
"loss": 0.5088897705078125,
|
||
|
|
"mean_token_accuracy": 0.7110123753547668,
|
||
|
|
"num_tokens": 198019334.0,
|
||
|
|
"rewards/accuracies": 0.70234375,
|
||
|
|
"rewards/chosen": -0.2291189193725586,
|
||
|
|
"rewards/margins": 1.2735336303710938,
|
||
|
|
"rewards/rejected": -1.5025585174560547,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.111865234375,
|
||
|
|
"epoch": 0.55814498871398,
|
||
|
|
"grad_norm": 39.022945404052734,
|
||
|
|
"learning_rate": 2.45961438249088e-07,
|
||
|
|
"logits/chosen": -0.3723478462502887,
|
||
|
|
"logits/rejected": 0.07380545194565284,
|
||
|
|
"logps/chosen": -442.759375,
|
||
|
|
"logps/rejected": -580.7875,
|
||
|
|
"loss": 0.4936191558837891,
|
||
|
|
"mean_token_accuracy": 0.7239787954837084,
|
||
|
|
"num_tokens": 199692142.0,
|
||
|
|
"rewards/accuracies": 0.7171875,
|
||
|
|
"rewards/chosen": -0.2774223327636719,
|
||
|
|
"rewards/margins": 1.1883934020996094,
|
||
|
|
"rewards/rejected": -1.4662437438964844,
|
||
|
|
"step": 1190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2387451171875,
|
||
|
|
"epoch": 0.5628352827367865,
|
||
|
|
"grad_norm": 48.250648498535156,
|
||
|
|
"learning_rate": 2.433559145388223e-07,
|
||
|
|
"logits/chosen": -0.37183548958390833,
|
||
|
|
"logits/rejected": 0.058249644272995095,
|
||
|
|
"logps/chosen": -463.95,
|
||
|
|
"logps/rejected": -668.440625,
|
||
|
|
"loss": 0.5038101196289062,
|
||
|
|
"mean_token_accuracy": 0.7077918142080307,
|
||
|
|
"num_tokens": 201359937.0,
|
||
|
|
"rewards/accuracies": 0.71328125,
|
||
|
|
"rewards/chosen": -0.25881156921386717,
|
||
|
|
"rewards/margins": 1.1958583831787108,
|
||
|
|
"rewards/rejected": -1.4544029235839844,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2365234375,
|
||
|
|
"epoch": 0.5675255767595931,
|
||
|
|
"grad_norm": 57.87482833862305,
|
||
|
|
"learning_rate": 2.4075039082855653e-07,
|
||
|
|
"logits/chosen": -0.3253422542766312,
|
||
|
|
"logits/rejected": 0.11797782865500077,
|
||
|
|
"logps/chosen": -433.44375,
|
||
|
|
"logps/rejected": -660.53125,
|
||
|
|
"loss": 0.5135528564453125,
|
||
|
|
"mean_token_accuracy": 0.7127903729677201,
|
||
|
|
"num_tokens": 203050869.0,
|
||
|
|
"rewards/accuracies": 0.69140625,
|
||
|
|
"rewards/chosen": -0.21519794464111328,
|
||
|
|
"rewards/margins": 1.4135284423828125,
|
||
|
|
"rewards/rejected": -1.630401611328125,
|
||
|
|
"step": 1210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.23800048828125,
|
||
|
|
"epoch": 0.5722158707823997,
|
||
|
|
"grad_norm": 43.60979080200195,
|
||
|
|
"learning_rate": 2.3814486711829076e-07,
|
||
|
|
"logits/chosen": -0.4246529642592443,
|
||
|
|
"logits/rejected": 0.08977297602859086,
|
||
|
|
"logps/chosen": -451.81875,
|
||
|
|
"logps/rejected": -692.68125,
|
||
|
|
"loss": 0.4846954345703125,
|
||
|
|
"mean_token_accuracy": 0.7116952534765005,
|
||
|
|
"num_tokens": 204729703.0,
|
||
|
|
"rewards/accuracies": 0.72578125,
|
||
|
|
"rewards/chosen": -0.3041818618774414,
|
||
|
|
"rewards/margins": 1.2798492431640625,
|
||
|
|
"rewards/rejected": -1.5844802856445312,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.32520751953125,
|
||
|
|
"epoch": 0.5769061648052062,
|
||
|
|
"grad_norm": 52.88212585449219,
|
||
|
|
"learning_rate": 2.35539343408025e-07,
|
||
|
|
"logits/chosen": -0.4144738956031821,
|
||
|
|
"logits/rejected": 0.1442082849155881,
|
||
|
|
"logps/chosen": -495.73125,
|
||
|
|
"logps/rejected": -764.740625,
|
||
|
|
"loss": 0.505401611328125,
|
||
|
|
"mean_token_accuracy": 0.707651786506176,
|
||
|
|
"num_tokens": 206446007.0,
|
||
|
|
"rewards/accuracies": 0.7171875,
|
||
|
|
"rewards/chosen": -0.37430610656738283,
|
||
|
|
"rewards/margins": 1.3635223388671875,
|
||
|
|
"rewards/rejected": -1.738238525390625,
|
||
|
|
"step": 1230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.250390625,
|
||
|
|
"epoch": 0.5815964588280128,
|
||
|
|
"grad_norm": 47.2264404296875,
|
||
|
|
"learning_rate": 2.3293381969775924e-07,
|
||
|
|
"logits/chosen": -0.34146382107676954,
|
||
|
|
"logits/rejected": 0.16612224711888873,
|
||
|
|
"logps/chosen": -488.14375,
|
||
|
|
"logps/rejected": -695.575,
|
||
|
|
"loss": 0.510650634765625,
|
||
|
|
"mean_token_accuracy": 0.7113540126010776,
|
||
|
|
"num_tokens": 208151679.0,
|
||
|
|
"rewards/accuracies": 0.71171875,
|
||
|
|
"rewards/chosen": -0.2607236862182617,
|
||
|
|
"rewards/margins": 1.2523696899414063,
|
||
|
|
"rewards/rejected": -1.5134803771972656,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2435302734375,
|
||
|
|
"epoch": 0.5862867528508193,
|
||
|
|
"grad_norm": 43.1168327331543,
|
||
|
|
"learning_rate": 2.303282959874935e-07,
|
||
|
|
"logits/chosen": -0.28546225309033735,
|
||
|
|
"logits/rejected": 0.2084978743788679,
|
||
|
|
"logps/chosen": -489.084375,
|
||
|
|
"logps/rejected": -698.959375,
|
||
|
|
"loss": 0.4846649169921875,
|
||
|
|
"mean_token_accuracy": 0.7133546834811568,
|
||
|
|
"num_tokens": 209904565.0,
|
||
|
|
"rewards/accuracies": 0.7296875,
|
||
|
|
"rewards/chosen": -0.23052234649658204,
|
||
|
|
"rewards/margins": 1.3557510375976562,
|
||
|
|
"rewards/rejected": -1.586376953125,
|
||
|
|
"step": 1250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.304833984375,
|
||
|
|
"epoch": 0.5909770468736258,
|
||
|
|
"grad_norm": 46.38766098022461,
|
||
|
|
"learning_rate": 2.2772277227722772e-07,
|
||
|
|
"logits/chosen": -0.3320207102770688,
|
||
|
|
"logits/rejected": 0.15268373542131383,
|
||
|
|
"logps/chosen": -497.2,
|
||
|
|
"logps/rejected": -721.1125,
|
||
|
|
"loss": 0.4992431640625,
|
||
|
|
"mean_token_accuracy": 0.7041572637856006,
|
||
|
|
"num_tokens": 211634743.0,
|
||
|
|
"rewards/accuracies": 0.721875,
|
||
|
|
"rewards/chosen": -0.23194313049316406,
|
||
|
|
"rewards/margins": 1.245294189453125,
|
||
|
|
"rewards/rejected": -1.477325439453125,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3220458984375,
|
||
|
|
"epoch": 0.5956673408964325,
|
||
|
|
"grad_norm": 49.88144302368164,
|
||
|
|
"learning_rate": 2.2511724856696197e-07,
|
||
|
|
"logits/chosen": -0.36437243560616833,
|
||
|
|
"logits/rejected": 0.3099788081189955,
|
||
|
|
"logps/chosen": -487.084375,
|
||
|
|
"logps/rejected": -765.0875,
|
||
|
|
"loss": 0.5212089538574218,
|
||
|
|
"mean_token_accuracy": 0.6997947275638581,
|
||
|
|
"num_tokens": 213353842.0,
|
||
|
|
"rewards/accuracies": 0.7046875,
|
||
|
|
"rewards/chosen": -0.24524993896484376,
|
||
|
|
"rewards/margins": 1.2819038391113282,
|
||
|
|
"rewards/rejected": -1.5273468017578125,
|
||
|
|
"step": 1270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2190673828125,
|
||
|
|
"epoch": 0.600357634919239,
|
||
|
|
"grad_norm": 58.774261474609375,
|
||
|
|
"learning_rate": 2.225117248566962e-07,
|
||
|
|
"logits/chosen": -0.2944029906126194,
|
||
|
|
"logits/rejected": 0.12548050740456665,
|
||
|
|
"logps/chosen": -478.509375,
|
||
|
|
"logps/rejected": -663.2625,
|
||
|
|
"loss": 0.49728050231933596,
|
||
|
|
"mean_token_accuracy": 0.7116608984768391,
|
||
|
|
"num_tokens": 215023462.0,
|
||
|
|
"rewards/accuracies": 0.69609375,
|
||
|
|
"rewards/chosen": -0.23260726928710937,
|
||
|
|
"rewards/margins": 1.20821533203125,
|
||
|
|
"rewards/rejected": -1.4408126831054688,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.312060546875,
|
||
|
|
"epoch": 0.6050479289420455,
|
||
|
|
"grad_norm": 50.618873596191406,
|
||
|
|
"learning_rate": 2.1990620114643042e-07,
|
||
|
|
"logits/chosen": -0.4199054390668423,
|
||
|
|
"logits/rejected": 0.18468027054402908,
|
||
|
|
"logps/chosen": -486.25,
|
||
|
|
"logps/rejected": -709.20625,
|
||
|
|
"loss": 0.5122344970703125,
|
||
|
|
"mean_token_accuracy": 0.6997991988435388,
|
||
|
|
"num_tokens": 216700271.0,
|
||
|
|
"rewards/accuracies": 0.6953125,
|
||
|
|
"rewards/chosen": -0.21283988952636718,
|
||
|
|
"rewards/margins": 1.145751953125,
|
||
|
|
"rewards/rejected": -1.3585968017578125,
|
||
|
|
"step": 1290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25576171875,
|
||
|
|
"epoch": 0.6097382229648521,
|
||
|
|
"grad_norm": 52.30256652832031,
|
||
|
|
"learning_rate": 2.1730067743616465e-07,
|
||
|
|
"logits/chosen": -0.33569222926382486,
|
||
|
|
"logits/rejected": 0.1868876063062059,
|
||
|
|
"logps/chosen": -454.11875,
|
||
|
|
"logps/rejected": -698.68125,
|
||
|
|
"loss": 0.48102340698242185,
|
||
|
|
"mean_token_accuracy": 0.7133908785879612,
|
||
|
|
"num_tokens": 218349883.0,
|
||
|
|
"rewards/accuracies": 0.71640625,
|
||
|
|
"rewards/chosen": -0.17691356539726258,
|
||
|
|
"rewards/margins": 1.347509765625,
|
||
|
|
"rewards/rejected": -1.5247505187988282,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25462646484375,
|
||
|
|
"epoch": 0.6144285169876587,
|
||
|
|
"grad_norm": 64.008544921875,
|
||
|
|
"learning_rate": 2.1469515372589887e-07,
|
||
|
|
"logits/chosen": -0.38799636342554056,
|
||
|
|
"logits/rejected": 0.05200148625905195,
|
||
|
|
"logps/chosen": -492.275,
|
||
|
|
"logps/rejected": -644.978125,
|
||
|
|
"loss": 0.5071174621582031,
|
||
|
|
"mean_token_accuracy": 0.7041159149259328,
|
||
|
|
"num_tokens": 220019139.0,
|
||
|
|
"rewards/accuracies": 0.69921875,
|
||
|
|
"rewards/chosen": -0.21364364624023438,
|
||
|
|
"rewards/margins": 1.1556907653808595,
|
||
|
|
"rewards/rejected": -1.3692604064941407,
|
||
|
|
"step": 1310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2711181640625,
|
||
|
|
"epoch": 0.6191188110104652,
|
||
|
|
"grad_norm": 48.48934555053711,
|
||
|
|
"learning_rate": 2.1208963001563313e-07,
|
||
|
|
"logits/chosen": -0.3561808660204896,
|
||
|
|
"logits/rejected": 0.13831664785273107,
|
||
|
|
"logps/chosen": -500.71875,
|
||
|
|
"logps/rejected": -694.2375,
|
||
|
|
"loss": 0.5029388427734375,
|
||
|
|
"mean_token_accuracy": 0.7105904638767242,
|
||
|
|
"num_tokens": 221722944.0,
|
||
|
|
"rewards/accuracies": 0.69375,
|
||
|
|
"rewards/chosen": -0.23125057220458983,
|
||
|
|
"rewards/margins": 1.2435516357421874,
|
||
|
|
"rewards/rejected": -1.4749588012695312,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.19822998046875,
|
||
|
|
"epoch": 0.6238091050332718,
|
||
|
|
"grad_norm": 42.350685119628906,
|
||
|
|
"learning_rate": 2.0948410630536735e-07,
|
||
|
|
"logits/chosen": -0.37555533940911084,
|
||
|
|
"logits/rejected": -0.011265530235249926,
|
||
|
|
"logps/chosen": -457.528125,
|
||
|
|
"logps/rejected": -591.6,
|
||
|
|
"loss": 0.5330169677734375,
|
||
|
|
"mean_token_accuracy": 0.7136769216507673,
|
||
|
|
"num_tokens": 223404308.0,
|
||
|
|
"rewards/accuracies": 0.6921875,
|
||
|
|
"rewards/chosen": -0.3046894073486328,
|
||
|
|
"rewards/margins": 1.0406204223632813,
|
||
|
|
"rewards/rejected": -1.3457130432128905,
|
||
|
|
"step": 1330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.22252197265625,
|
||
|
|
"epoch": 0.6284993990560783,
|
||
|
|
"grad_norm": 45.17627716064453,
|
||
|
|
"learning_rate": 2.068785825951016e-07,
|
||
|
|
"logits/chosen": -0.28848142084641326,
|
||
|
|
"logits/rejected": 0.12333503271507404,
|
||
|
|
"logps/chosen": -498.05,
|
||
|
|
"logps/rejected": -652.503125,
|
||
|
|
"loss": 0.4917594909667969,
|
||
|
|
"mean_token_accuracy": 0.7163054622709751,
|
||
|
|
"num_tokens": 225127215.0,
|
||
|
|
"rewards/accuracies": 0.7203125,
|
||
|
|
"rewards/chosen": -0.24520225524902345,
|
||
|
|
"rewards/margins": 1.2569358825683594,
|
||
|
|
"rewards/rejected": -1.5024093627929687,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3197998046875,
|
||
|
|
"epoch": 0.6331896930788848,
|
||
|
|
"grad_norm": 78.06428527832031,
|
||
|
|
"learning_rate": 2.0427305888483583e-07,
|
||
|
|
"logits/chosen": -0.363285197439468,
|
||
|
|
"logits/rejected": 0.12140866192187372,
|
||
|
|
"logps/chosen": -479.86875,
|
||
|
|
"logps/rejected": -726.678125,
|
||
|
|
"loss": 0.5196853637695312,
|
||
|
|
"mean_token_accuracy": 0.6993409872055054,
|
||
|
|
"num_tokens": 226798538.0,
|
||
|
|
"rewards/accuracies": 0.71171875,
|
||
|
|
"rewards/chosen": -0.2906646728515625,
|
||
|
|
"rewards/margins": 1.2201629638671876,
|
||
|
|
"rewards/rejected": -1.51029052734375,
|
||
|
|
"step": 1350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.263818359375,
|
||
|
|
"epoch": 0.6378799871016915,
|
||
|
|
"grad_norm": 45.774662017822266,
|
||
|
|
"learning_rate": 2.0166753517457009e-07,
|
||
|
|
"logits/chosen": -0.3178116434676858,
|
||
|
|
"logits/rejected": 0.13615178149001414,
|
||
|
|
"logps/chosen": -483.6,
|
||
|
|
"logps/rejected": -678.51875,
|
||
|
|
"loss": 0.5075515747070313,
|
||
|
|
"mean_token_accuracy": 0.7105112101882696,
|
||
|
|
"num_tokens": 228525211.0,
|
||
|
|
"rewards/accuracies": 0.71953125,
|
||
|
|
"rewards/chosen": -0.22887954711914063,
|
||
|
|
"rewards/margins": 1.2723709106445313,
|
||
|
|
"rewards/rejected": -1.5004180908203124,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.28734130859375,
|
||
|
|
"epoch": 0.642570281124498,
|
||
|
|
"grad_norm": 45.82160568237305,
|
||
|
|
"learning_rate": 1.990620114643043e-07,
|
||
|
|
"logits/chosen": -0.2266978024313957,
|
||
|
|
"logits/rejected": 0.3092070634916513,
|
||
|
|
"logps/chosen": -473.2625,
|
||
|
|
"logps/rejected": -772.3,
|
||
|
|
"loss": 0.49246978759765625,
|
||
|
|
"mean_token_accuracy": 0.7207242472097277,
|
||
|
|
"num_tokens": 230277952.0,
|
||
|
|
"rewards/accuracies": 0.72734375,
|
||
|
|
"rewards/chosen": -0.18862266540527345,
|
||
|
|
"rewards/margins": 1.2539337158203125,
|
||
|
|
"rewards/rejected": -1.4429153442382812,
|
||
|
|
"step": 1370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2372802734375,
|
||
|
|
"epoch": 0.6472605751473045,
|
||
|
|
"grad_norm": 53.567955017089844,
|
||
|
|
"learning_rate": 1.9645648775403857e-07,
|
||
|
|
"logits/chosen": -0.3122498323546022,
|
||
|
|
"logits/rejected": -0.029102683684336005,
|
||
|
|
"logps/chosen": -456.4125,
|
||
|
|
"logps/rejected": -621.05625,
|
||
|
|
"loss": 0.5261543273925782,
|
||
|
|
"mean_token_accuracy": 0.7144070915877819,
|
||
|
|
"num_tokens": 231910230.0,
|
||
|
|
"rewards/accuracies": 0.70234375,
|
||
|
|
"rewards/chosen": -0.16479625701904296,
|
||
|
|
"rewards/margins": 1.0987884521484375,
|
||
|
|
"rewards/rejected": -1.263460922241211,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25101318359375,
|
||
|
|
"epoch": 0.6519508691701111,
|
||
|
|
"grad_norm": 54.65653610229492,
|
||
|
|
"learning_rate": 1.938509640437728e-07,
|
||
|
|
"logits/chosen": -0.2595864928279747,
|
||
|
|
"logits/rejected": 0.17200187501163847,
|
||
|
|
"logps/chosen": -457.83125,
|
||
|
|
"logps/rejected": -694.93125,
|
||
|
|
"loss": 0.5121467590332032,
|
||
|
|
"mean_token_accuracy": 0.7173424143344164,
|
||
|
|
"num_tokens": 233587330.0,
|
||
|
|
"rewards/accuracies": 0.7234375,
|
||
|
|
"rewards/chosen": -0.223736572265625,
|
||
|
|
"rewards/margins": 1.3258369445800782,
|
||
|
|
"rewards/rejected": -1.5491180419921875,
|
||
|
|
"step": 1390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2558837890625,
|
||
|
|
"epoch": 0.6566411631929177,
|
||
|
|
"grad_norm": 50.95151138305664,
|
||
|
|
"learning_rate": 1.9124544033350704e-07,
|
||
|
|
"logits/chosen": -0.2936878858134837,
|
||
|
|
"logits/rejected": 0.12141680482417419,
|
||
|
|
"logps/chosen": -477.040625,
|
||
|
|
"logps/rejected": -639.909375,
|
||
|
|
"loss": 0.5095062255859375,
|
||
|
|
"mean_token_accuracy": 0.708953882008791,
|
||
|
|
"num_tokens": 235270410.0,
|
||
|
|
"rewards/accuracies": 0.71015625,
|
||
|
|
"rewards/chosen": -0.20118942260742187,
|
||
|
|
"rewards/margins": 1.2124801635742188,
|
||
|
|
"rewards/rejected": -1.4136474609375,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3095947265625,
|
||
|
|
"epoch": 0.6613314572157242,
|
||
|
|
"grad_norm": 40.7271728515625,
|
||
|
|
"learning_rate": 1.8863991662324127e-07,
|
||
|
|
"logits/chosen": -0.34175182540570026,
|
||
|
|
"logits/rejected": 0.13167951761138544,
|
||
|
|
"logps/chosen": -459.971875,
|
||
|
|
"logps/rejected": -683.640625,
|
||
|
|
"loss": 0.5016754150390625,
|
||
|
|
"mean_token_accuracy": 0.7081727286800742,
|
||
|
|
"num_tokens": 236914943.0,
|
||
|
|
"rewards/accuracies": 0.71640625,
|
||
|
|
"rewards/chosen": -0.19021434783935548,
|
||
|
|
"rewards/margins": 1.188072967529297,
|
||
|
|
"rewards/rejected": -1.378137969970703,
|
||
|
|
"step": 1410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2114013671875,
|
||
|
|
"epoch": 0.6660217512385308,
|
||
|
|
"grad_norm": 39.70977783203125,
|
||
|
|
"learning_rate": 1.8603439291297552e-07,
|
||
|
|
"logits/chosen": -0.24609394416242006,
|
||
|
|
"logits/rejected": 0.1799542514096824,
|
||
|
|
"logps/chosen": -494.9625,
|
||
|
|
"logps/rejected": -643.74375,
|
||
|
|
"loss": 0.5288177490234375,
|
||
|
|
"mean_token_accuracy": 0.7189813105389475,
|
||
|
|
"num_tokens": 238709572.0,
|
||
|
|
"rewards/accuracies": 0.70625,
|
||
|
|
"rewards/chosen": -0.2049093246459961,
|
||
|
|
"rewards/margins": 1.2056396484375,
|
||
|
|
"rewards/rejected": -1.4105159759521484,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.21826171875,
|
||
|
|
"epoch": 0.6707120452613373,
|
||
|
|
"grad_norm": 66.35521697998047,
|
||
|
|
"learning_rate": 1.8342886920270972e-07,
|
||
|
|
"logits/chosen": -0.32402960394939234,
|
||
|
|
"logits/rejected": 0.06577563256399063,
|
||
|
|
"logps/chosen": -470.8625,
|
||
|
|
"logps/rejected": -613.003125,
|
||
|
|
"loss": 0.5125747680664062,
|
||
|
|
"mean_token_accuracy": 0.7137641754001379,
|
||
|
|
"num_tokens": 240329577.0,
|
||
|
|
"rewards/accuracies": 0.721875,
|
||
|
|
"rewards/chosen": -0.28786163330078124,
|
||
|
|
"rewards/margins": 1.20198974609375,
|
||
|
|
"rewards/rejected": -1.490332794189453,
|
||
|
|
"step": 1430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2763671875,
|
||
|
|
"epoch": 0.6754023392841438,
|
||
|
|
"grad_norm": 59.820579528808594,
|
||
|
|
"learning_rate": 1.8082334549244395e-07,
|
||
|
|
"logits/chosen": -0.2621199295266179,
|
||
|
|
"logits/rejected": 0.21548721230861387,
|
||
|
|
"logps/chosen": -488.2375,
|
||
|
|
"logps/rejected": -711.73125,
|
||
|
|
"loss": 0.5426666259765625,
|
||
|
|
"mean_token_accuracy": 0.7102572401985526,
|
||
|
|
"num_tokens": 242037983.0,
|
||
|
|
"rewards/accuracies": 0.6796875,
|
||
|
|
"rewards/chosen": -0.2579639434814453,
|
||
|
|
"rewards/margins": 1.1012718200683593,
|
||
|
|
"rewards/rejected": -1.3592147827148438,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2553955078125,
|
||
|
|
"epoch": 0.6800926333069505,
|
||
|
|
"grad_norm": 39.72897720336914,
|
||
|
|
"learning_rate": 1.782178217821782e-07,
|
||
|
|
"logits/chosen": -0.3045310204445476,
|
||
|
|
"logits/rejected": 0.11346873189208409,
|
||
|
|
"logps/chosen": -502.8375,
|
||
|
|
"logps/rejected": -687.9125,
|
||
|
|
"loss": 0.5121414184570312,
|
||
|
|
"mean_token_accuracy": 0.7075636066496372,
|
||
|
|
"num_tokens": 243740239.0,
|
||
|
|
"rewards/accuracies": 0.7234375,
|
||
|
|
"rewards/chosen": -0.26166362762451173,
|
||
|
|
"rewards/margins": 1.3748176574707032,
|
||
|
|
"rewards/rejected": -1.6358539581298828,
|
||
|
|
"step": 1450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2446044921875,
|
||
|
|
"epoch": 0.684782927329757,
|
||
|
|
"grad_norm": 42.146484375,
|
||
|
|
"learning_rate": 1.7561229807191243e-07,
|
||
|
|
"logits/chosen": -0.3356412574221423,
|
||
|
|
"logits/rejected": 0.10238671794143098,
|
||
|
|
"logps/chosen": -451.0875,
|
||
|
|
"logps/rejected": -697.11875,
|
||
|
|
"loss": 0.5116485595703125,
|
||
|
|
"mean_token_accuracy": 0.7145647119730711,
|
||
|
|
"num_tokens": 245400678.0,
|
||
|
|
"rewards/accuracies": 0.72109375,
|
||
|
|
"rewards/chosen": -0.2481781005859375,
|
||
|
|
"rewards/margins": 1.1638092041015624,
|
||
|
|
"rewards/rejected": -1.4118682861328125,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.185888671875,
|
||
|
|
"epoch": 0.6894732213525635,
|
||
|
|
"grad_norm": 47.214019775390625,
|
||
|
|
"learning_rate": 1.7300677436164668e-07,
|
||
|
|
"logits/chosen": -0.39723945328086835,
|
||
|
|
"logits/rejected": -0.04166929725172916,
|
||
|
|
"logps/chosen": -467.621875,
|
||
|
|
"logps/rejected": -568.09375,
|
||
|
|
"loss": 0.4959739685058594,
|
||
|
|
"mean_token_accuracy": 0.7094458827748895,
|
||
|
|
"num_tokens": 247054384.0,
|
||
|
|
"rewards/accuracies": 0.703125,
|
||
|
|
"rewards/chosen": -0.20417518615722657,
|
||
|
|
"rewards/margins": 1.1270927429199218,
|
||
|
|
"rewards/rejected": -1.3313568115234375,
|
||
|
|
"step": 1470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2665283203125,
|
||
|
|
"epoch": 0.6941635153753701,
|
||
|
|
"grad_norm": 53.28609848022461,
|
||
|
|
"learning_rate": 1.704012506513809e-07,
|
||
|
|
"logits/chosen": -0.3926752625519089,
|
||
|
|
"logits/rejected": 0.12300171587715411,
|
||
|
|
"logps/chosen": -490.3625,
|
||
|
|
"logps/rejected": -751.45,
|
||
|
|
"loss": 0.5226806640625,
|
||
|
|
"mean_token_accuracy": 0.7115387700498104,
|
||
|
|
"num_tokens": 248780570.0,
|
||
|
|
"rewards/accuracies": 0.709375,
|
||
|
|
"rewards/chosen": -0.31193199157714846,
|
||
|
|
"rewards/margins": 1.3558914184570312,
|
||
|
|
"rewards/rejected": -1.6674148559570312,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3110107421875,
|
||
|
|
"epoch": 0.6988538093981767,
|
||
|
|
"grad_norm": 44.8239860534668,
|
||
|
|
"learning_rate": 1.6779572694111516e-07,
|
||
|
|
"logits/chosen": -0.4324692173902081,
|
||
|
|
"logits/rejected": 0.07500056479665548,
|
||
|
|
"logps/chosen": -438.93125,
|
||
|
|
"logps/rejected": -697.46875,
|
||
|
|
"loss": 0.4912841796875,
|
||
|
|
"mean_token_accuracy": 0.7042546521872282,
|
||
|
|
"num_tokens": 250392447.0,
|
||
|
|
"rewards/accuracies": 0.73125,
|
||
|
|
"rewards/chosen": -0.21766271591186523,
|
||
|
|
"rewards/margins": 1.302496337890625,
|
||
|
|
"rewards/rejected": -1.5202301025390625,
|
||
|
|
"step": 1490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2731201171875,
|
||
|
|
"epoch": 0.7035441034209832,
|
||
|
|
"grad_norm": 59.22135543823242,
|
||
|
|
"learning_rate": 1.651902032308494e-07,
|
||
|
|
"logits/chosen": -0.3529709127225384,
|
||
|
|
"logits/rejected": 0.1258731407236429,
|
||
|
|
"logps/chosen": -452.975,
|
||
|
|
"logps/rejected": -722.65625,
|
||
|
|
"loss": 0.49285736083984377,
|
||
|
|
"mean_token_accuracy": 0.7159934408962727,
|
||
|
|
"num_tokens": 252045693.0,
|
||
|
|
"rewards/accuracies": 0.69765625,
|
||
|
|
"rewards/chosen": -0.19012088775634767,
|
||
|
|
"rewards/margins": 1.233636474609375,
|
||
|
|
"rewards/rejected": -1.4240127563476563,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.21953125,
|
||
|
|
"epoch": 0.7082343974437898,
|
||
|
|
"grad_norm": 67.9064712524414,
|
||
|
|
"learning_rate": 1.6258467952058364e-07,
|
||
|
|
"logits/chosen": -0.355636732334131,
|
||
|
|
"logits/rejected": 0.06523703115240106,
|
||
|
|
"logps/chosen": -484.0,
|
||
|
|
"logps/rejected": -613.578125,
|
||
|
|
"loss": 0.5271133422851563,
|
||
|
|
"mean_token_accuracy": 0.7069945391267538,
|
||
|
|
"num_tokens": 253694458.0,
|
||
|
|
"rewards/accuracies": 0.70078125,
|
||
|
|
"rewards/chosen": -0.21510238647460939,
|
||
|
|
"rewards/margins": 1.0788116455078125,
|
||
|
|
"rewards/rejected": -1.2938308715820312,
|
||
|
|
"step": 1510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2014404296875,
|
||
|
|
"epoch": 0.7129246914665963,
|
||
|
|
"grad_norm": 40.9355583190918,
|
||
|
|
"learning_rate": 1.5997915581031787e-07,
|
||
|
|
"logits/chosen": -0.40950678159563625,
|
||
|
|
"logits/rejected": 0.01910210709722237,
|
||
|
|
"logps/chosen": -454.46875,
|
||
|
|
"logps/rejected": -634.725,
|
||
|
|
"loss": 0.5015480041503906,
|
||
|
|
"mean_token_accuracy": 0.7128090810030698,
|
||
|
|
"num_tokens": 255410705.0,
|
||
|
|
"rewards/accuracies": 0.7109375,
|
||
|
|
"rewards/chosen": -0.24631481170654296,
|
||
|
|
"rewards/margins": 1.258221435546875,
|
||
|
|
"rewards/rejected": -1.5049667358398438,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.241748046875,
|
||
|
|
"epoch": 0.7176149854894028,
|
||
|
|
"grad_norm": 43.45293426513672,
|
||
|
|
"learning_rate": 1.5737363210005212e-07,
|
||
|
|
"logits/chosen": -0.3803298971013801,
|
||
|
|
"logits/rejected": -0.022503258323848245,
|
||
|
|
"logps/chosen": -469.95625,
|
||
|
|
"logps/rejected": -660.88125,
|
||
|
|
"loss": 0.5067474365234375,
|
||
|
|
"mean_token_accuracy": 0.7098428962752223,
|
||
|
|
"num_tokens": 257032747.0,
|
||
|
|
"rewards/accuracies": 0.7046875,
|
||
|
|
"rewards/chosen": -0.20492134094238282,
|
||
|
|
"rewards/margins": 1.2111480712890625,
|
||
|
|
"rewards/rejected": -1.4155357360839844,
|
||
|
|
"step": 1530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2524169921875,
|
||
|
|
"epoch": 0.7223052795122095,
|
||
|
|
"grad_norm": 46.686073303222656,
|
||
|
|
"learning_rate": 1.5476810838978635e-07,
|
||
|
|
"logits/chosen": -0.409221722079539,
|
||
|
|
"logits/rejected": 0.037211792533006396,
|
||
|
|
"logps/chosen": -465.66875,
|
||
|
|
"logps/rejected": -655.08515625,
|
||
|
|
"loss": 0.49529037475585935,
|
||
|
|
"mean_token_accuracy": 0.7079074520617723,
|
||
|
|
"num_tokens": 258717701.0,
|
||
|
|
"rewards/accuracies": 0.73125,
|
||
|
|
"rewards/chosen": -0.18844895362854003,
|
||
|
|
"rewards/margins": 1.2114707946777343,
|
||
|
|
"rewards/rejected": -1.4001953125,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.20712890625,
|
||
|
|
"epoch": 0.726995573535016,
|
||
|
|
"grad_norm": 41.296077728271484,
|
||
|
|
"learning_rate": 1.521625846795206e-07,
|
||
|
|
"logits/chosen": -0.31212731107450276,
|
||
|
|
"logits/rejected": 0.11191561789900543,
|
||
|
|
"logps/chosen": -480.353125,
|
||
|
|
"logps/rejected": -633.00625,
|
||
|
|
"loss": 0.503009033203125,
|
||
|
|
"mean_token_accuracy": 0.717727231234312,
|
||
|
|
"num_tokens": 260394661.0,
|
||
|
|
"rewards/accuracies": 0.7265625,
|
||
|
|
"rewards/chosen": -0.2229745864868164,
|
||
|
|
"rewards/margins": 1.2388389587402344,
|
||
|
|
"rewards/rejected": -1.461846160888672,
|
||
|
|
"step": 1550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.279541015625,
|
||
|
|
"epoch": 0.7316858675578225,
|
||
|
|
"grad_norm": 40.81024932861328,
|
||
|
|
"learning_rate": 1.495570609692548e-07,
|
||
|
|
"logits/chosen": -0.36629913463706937,
|
||
|
|
"logits/rejected": 0.12231380401378948,
|
||
|
|
"logps/chosen": -463.809375,
|
||
|
|
"logps/rejected": -651.1375,
|
||
|
|
"loss": 0.5072137832641601,
|
||
|
|
"mean_token_accuracy": 0.70716253798455,
|
||
|
|
"num_tokens": 262044098.0,
|
||
|
|
"rewards/accuracies": 0.71484375,
|
||
|
|
"rewards/chosen": -0.16593360900878906,
|
||
|
|
"rewards/margins": 1.2185470581054687,
|
||
|
|
"rewards/rejected": -1.384647750854492,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2444580078125,
|
||
|
|
"epoch": 0.7363761615806291,
|
||
|
|
"grad_norm": 47.42740249633789,
|
||
|
|
"learning_rate": 1.4695153725898905e-07,
|
||
|
|
"logits/chosen": -0.33029111726362215,
|
||
|
|
"logits/rejected": 0.16555770502337477,
|
||
|
|
"logps/chosen": -491.6,
|
||
|
|
"logps/rejected": -660.21875,
|
||
|
|
"loss": 0.526171875,
|
||
|
|
"mean_token_accuracy": 0.7089153520762921,
|
||
|
|
"num_tokens": 263781303.0,
|
||
|
|
"rewards/accuracies": 0.7046875,
|
||
|
|
"rewards/chosen": -0.20786285400390625,
|
||
|
|
"rewards/margins": 1.1246536254882813,
|
||
|
|
"rewards/rejected": -1.33277587890625,
|
||
|
|
"step": 1570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2452880859375,
|
||
|
|
"epoch": 0.7410664556034356,
|
||
|
|
"grad_norm": 36.83119201660156,
|
||
|
|
"learning_rate": 1.4434601354872328e-07,
|
||
|
|
"logits/chosen": -0.35121305578205475,
|
||
|
|
"logits/rejected": 0.11945853415718628,
|
||
|
|
"logps/chosen": -461.51875,
|
||
|
|
"logps/rejected": -655.359375,
|
||
|
|
"loss": 0.482196044921875,
|
||
|
|
"mean_token_accuracy": 0.7106880776584148,
|
||
|
|
"num_tokens": 265433617.0,
|
||
|
|
"rewards/accuracies": 0.72265625,
|
||
|
|
"rewards/chosen": -0.16603565216064453,
|
||
|
|
"rewards/margins": 1.304248046875,
|
||
|
|
"rewards/rejected": -1.470250701904297,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2256103515625,
|
||
|
|
"epoch": 0.7457567496262422,
|
||
|
|
"grad_norm": 39.94232177734375,
|
||
|
|
"learning_rate": 1.417404898384575e-07,
|
||
|
|
"logits/chosen": -0.3938522607414349,
|
||
|
|
"logits/rejected": 0.11833697097822693,
|
||
|
|
"logps/chosen": -456.45625,
|
||
|
|
"logps/rejected": -635.05,
|
||
|
|
"loss": 0.5009937286376953,
|
||
|
|
"mean_token_accuracy": 0.7109608668833971,
|
||
|
|
"num_tokens": 267110073.0,
|
||
|
|
"rewards/accuracies": 0.7171875,
|
||
|
|
"rewards/chosen": -0.17589912414550782,
|
||
|
|
"rewards/margins": 1.1410980224609375,
|
||
|
|
"rewards/rejected": -1.3171512603759765,
|
||
|
|
"step": 1590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2601318359375,
|
||
|
|
"epoch": 0.7504470436490488,
|
||
|
|
"grad_norm": 43.43891525268555,
|
||
|
|
"learning_rate": 1.3913496612819176e-07,
|
||
|
|
"logits/chosen": -0.31867154075075216,
|
||
|
|
"logits/rejected": 0.07912379150382028,
|
||
|
|
"logps/chosen": -460.015625,
|
||
|
|
"logps/rejected": -635.6125,
|
||
|
|
"loss": 0.5057296752929688,
|
||
|
|
"mean_token_accuracy": 0.7127839136868716,
|
||
|
|
"num_tokens": 268708942.0,
|
||
|
|
"rewards/accuracies": 0.72109375,
|
||
|
|
"rewards/chosen": -0.19757213592529296,
|
||
|
|
"rewards/margins": 1.215704345703125,
|
||
|
|
"rewards/rejected": -1.413494873046875,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2163818359375,
|
||
|
|
"epoch": 0.7551373376718553,
|
||
|
|
"grad_norm": 44.50661087036133,
|
||
|
|
"learning_rate": 1.3652944241792598e-07,
|
||
|
|
"logits/chosen": -0.414186569736156,
|
||
|
|
"logits/rejected": -0.032987588264573646,
|
||
|
|
"logps/chosen": -501.94375,
|
||
|
|
"logps/rejected": -625.9125,
|
||
|
|
"loss": 0.5188003540039062,
|
||
|
|
"mean_token_accuracy": 0.7055005624890327,
|
||
|
|
"num_tokens": 270406707.0,
|
||
|
|
"rewards/accuracies": 0.7,
|
||
|
|
"rewards/chosen": -0.21747322082519532,
|
||
|
|
"rewards/margins": 1.1645050048828125,
|
||
|
|
"rewards/rejected": -1.3811737060546876,
|
||
|
|
"step": 1610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2386474609375,
|
||
|
|
"epoch": 0.7598276316946618,
|
||
|
|
"grad_norm": 40.76424026489258,
|
||
|
|
"learning_rate": 1.3392391870766024e-07,
|
||
|
|
"logits/chosen": -0.25317166552302806,
|
||
|
|
"logits/rejected": 0.25054843549323225,
|
||
|
|
"logps/chosen": -448.25625,
|
||
|
|
"logps/rejected": -678.025,
|
||
|
|
"loss": 0.498260498046875,
|
||
|
|
"mean_token_accuracy": 0.7153599586337804,
|
||
|
|
"num_tokens": 272057284.0,
|
||
|
|
"rewards/accuracies": 0.71328125,
|
||
|
|
"rewards/chosen": -0.15067138671875,
|
||
|
|
"rewards/margins": 1.1843612670898438,
|
||
|
|
"rewards/rejected": -1.3350822448730468,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3083251953125,
|
||
|
|
"epoch": 0.7645179257174685,
|
||
|
|
"grad_norm": 53.002323150634766,
|
||
|
|
"learning_rate": 1.3131839499739446e-07,
|
||
|
|
"logits/chosen": -0.3471587077953908,
|
||
|
|
"logits/rejected": 0.19919105227987713,
|
||
|
|
"logps/chosen": -494.4125,
|
||
|
|
"logps/rejected": -728.68125,
|
||
|
|
"loss": 0.4993560791015625,
|
||
|
|
"mean_token_accuracy": 0.7017248619347811,
|
||
|
|
"num_tokens": 273777667.0,
|
||
|
|
"rewards/accuracies": 0.7359375,
|
||
|
|
"rewards/chosen": -0.22601699829101562,
|
||
|
|
"rewards/margins": 1.2609451293945313,
|
||
|
|
"rewards/rejected": -1.4867431640625,
|
||
|
|
"step": 1630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.24683837890625,
|
||
|
|
"epoch": 0.769208219740275,
|
||
|
|
"grad_norm": 58.614864349365234,
|
||
|
|
"learning_rate": 1.2871287128712872e-07,
|
||
|
|
"logits/chosen": -0.35487965619433126,
|
||
|
|
"logits/rejected": 0.08004843726996642,
|
||
|
|
"logps/chosen": -459.94375,
|
||
|
|
"logps/rejected": -656.10625,
|
||
|
|
"loss": 0.4921421051025391,
|
||
|
|
"mean_token_accuracy": 0.7122417978942395,
|
||
|
|
"num_tokens": 275509944.0,
|
||
|
|
"rewards/accuracies": 0.73125,
|
||
|
|
"rewards/chosen": -0.21575851440429689,
|
||
|
|
"rewards/margins": 1.220342254638672,
|
||
|
|
"rewards/rejected": -1.435931396484375,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2380615234375,
|
||
|
|
"epoch": 0.7738985137630815,
|
||
|
|
"grad_norm": 46.248634338378906,
|
||
|
|
"learning_rate": 1.2610734757686294e-07,
|
||
|
|
"logits/chosen": -0.378603477523175,
|
||
|
|
"logits/rejected": 0.0143330288482811,
|
||
|
|
"logps/chosen": -473.325,
|
||
|
|
"logps/rejected": -637.475,
|
||
|
|
"loss": 0.5132255554199219,
|
||
|
|
"mean_token_accuracy": 0.7066951882094145,
|
||
|
|
"num_tokens": 277153043.0,
|
||
|
|
"rewards/accuracies": 0.721875,
|
||
|
|
"rewards/chosen": -0.2193842887878418,
|
||
|
|
"rewards/margins": 1.1281280517578125,
|
||
|
|
"rewards/rejected": -1.3469314575195312,
|
||
|
|
"step": 1650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2247314453125,
|
||
|
|
"epoch": 0.7785888077858881,
|
||
|
|
"grad_norm": 49.24350357055664,
|
||
|
|
"learning_rate": 1.2350182386659717e-07,
|
||
|
|
"logits/chosen": -0.41227246676183515,
|
||
|
|
"logits/rejected": 0.047900230891089454,
|
||
|
|
"logps/chosen": -448.90625,
|
||
|
|
"logps/rejected": -652.575,
|
||
|
|
"loss": 0.49638519287109373,
|
||
|
|
"mean_token_accuracy": 0.707376803830266,
|
||
|
|
"num_tokens": 278795974.0,
|
||
|
|
"rewards/accuracies": 0.71796875,
|
||
|
|
"rewards/chosen": -0.19539546966552734,
|
||
|
|
"rewards/margins": 1.2829010009765625,
|
||
|
|
"rewards/rejected": -1.47823486328125,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1978515625,
|
||
|
|
"epoch": 0.7832791018086946,
|
||
|
|
"grad_norm": 48.88486862182617,
|
||
|
|
"learning_rate": 1.2089630015633142e-07,
|
||
|
|
"logits/chosen": -0.34792149701180813,
|
||
|
|
"logits/rejected": 0.0740143644661438,
|
||
|
|
"logps/chosen": -448.4125,
|
||
|
|
"logps/rejected": -628.1125,
|
||
|
|
"loss": 0.49319000244140626,
|
||
|
|
"mean_token_accuracy": 0.7200800001621246,
|
||
|
|
"num_tokens": 280435349.0,
|
||
|
|
"rewards/accuracies": 0.734375,
|
||
|
|
"rewards/chosen": -0.17760887145996093,
|
||
|
|
"rewards/margins": 1.242255401611328,
|
||
|
|
"rewards/rejected": -1.419854736328125,
|
||
|
|
"step": 1670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.205029296875,
|
||
|
|
"epoch": 0.7879693958315012,
|
||
|
|
"grad_norm": 43.86332702636719,
|
||
|
|
"learning_rate": 1.1829077644606565e-07,
|
||
|
|
"logits/chosen": -0.29688106453686836,
|
||
|
|
"logits/rejected": 0.18034652960629277,
|
||
|
|
"logps/chosen": -475.58125,
|
||
|
|
"logps/rejected": -648.65,
|
||
|
|
"loss": 0.48222007751464846,
|
||
|
|
"mean_token_accuracy": 0.714630151540041,
|
||
|
|
"num_tokens": 282172052.0,
|
||
|
|
"rewards/accuracies": 0.72734375,
|
||
|
|
"rewards/chosen": -0.19398794174194336,
|
||
|
|
"rewards/margins": 1.296198272705078,
|
||
|
|
"rewards/rejected": -1.490453338623047,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.28875732421875,
|
||
|
|
"epoch": 0.7926596898543078,
|
||
|
|
"grad_norm": 44.04449462890625,
|
||
|
|
"learning_rate": 1.1568525273579989e-07,
|
||
|
|
"logits/chosen": -0.3517104921207329,
|
||
|
|
"logits/rejected": 0.10797833504461599,
|
||
|
|
"logps/chosen": -506.24375,
|
||
|
|
"logps/rejected": -716.54375,
|
||
|
|
"loss": 0.5166778564453125,
|
||
|
|
"mean_token_accuracy": 0.7019935846328735,
|
||
|
|
"num_tokens": 283802991.0,
|
||
|
|
"rewards/accuracies": 0.70703125,
|
||
|
|
"rewards/chosen": -0.24034361839294432,
|
||
|
|
"rewards/margins": 1.193011474609375,
|
||
|
|
"rewards/rejected": -1.4332107543945312,
|
||
|
|
"step": 1690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2384033203125,
|
||
|
|
"epoch": 0.7973499838771143,
|
||
|
|
"grad_norm": 42.062034606933594,
|
||
|
|
"learning_rate": 1.1307972902553413e-07,
|
||
|
|
"logits/chosen": -0.3546815150553647,
|
||
|
|
"logits/rejected": 0.009532807889907468,
|
||
|
|
"logps/chosen": -462.965625,
|
||
|
|
"logps/rejected": -640.890625,
|
||
|
|
"loss": 0.4919563293457031,
|
||
|
|
"mean_token_accuracy": 0.7082353591918945,
|
||
|
|
"num_tokens": 285483282.0,
|
||
|
|
"rewards/accuracies": 0.74140625,
|
||
|
|
"rewards/chosen": -0.19779624938964843,
|
||
|
|
"rewards/margins": 1.1956352233886718,
|
||
|
|
"rewards/rejected": -1.3938270568847657,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.23271484375,
|
||
|
|
"epoch": 0.8020402778999208,
|
||
|
|
"grad_norm": 44.65546798706055,
|
||
|
|
"learning_rate": 1.1047420531526837e-07,
|
||
|
|
"logits/chosen": -0.3493073648480609,
|
||
|
|
"logits/rejected": 0.07239800461008125,
|
||
|
|
"logps/chosen": -459.8,
|
||
|
|
"logps/rejected": -687.525,
|
||
|
|
"loss": 0.5149833679199218,
|
||
|
|
"mean_token_accuracy": 0.7182338822633028,
|
||
|
|
"num_tokens": 287194708.0,
|
||
|
|
"rewards/accuracies": 0.70859375,
|
||
|
|
"rewards/chosen": -0.2390056610107422,
|
||
|
|
"rewards/margins": 1.19747314453125,
|
||
|
|
"rewards/rejected": -1.4365234375,
|
||
|
|
"step": 1710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2440673828125,
|
||
|
|
"epoch": 0.8067305719227275,
|
||
|
|
"grad_norm": 44.430301666259766,
|
||
|
|
"learning_rate": 1.078686816050026e-07,
|
||
|
|
"logits/chosen": -0.33688168437130167,
|
||
|
|
"logits/rejected": 0.22037569268858842,
|
||
|
|
"logps/chosen": -453.78125,
|
||
|
|
"logps/rejected": -688.84375,
|
||
|
|
"loss": 0.46872520446777344,
|
||
|
|
"mean_token_accuracy": 0.7180781122297049,
|
||
|
|
"num_tokens": 288892457.0,
|
||
|
|
"rewards/accuracies": 0.7390625,
|
||
|
|
"rewards/chosen": -0.1388528823852539,
|
||
|
|
"rewards/margins": 1.2692779541015624,
|
||
|
|
"rewards/rejected": -1.4086166381835938,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2156494140625,
|
||
|
|
"epoch": 0.811420865945534,
|
||
|
|
"grad_norm": 52.477691650390625,
|
||
|
|
"learning_rate": 1.0526315789473683e-07,
|
||
|
|
"logits/chosen": -0.34269354405153873,
|
||
|
|
"logits/rejected": 0.033022382885200416,
|
||
|
|
"logps/chosen": -451.6875,
|
||
|
|
"logps/rejected": -617.4875,
|
||
|
|
"loss": 0.4788845062255859,
|
||
|
|
"mean_token_accuracy": 0.7164073050022125,
|
||
|
|
"num_tokens": 290522032.0,
|
||
|
|
"rewards/accuracies": 0.72734375,
|
||
|
|
"rewards/chosen": -0.18807697296142578,
|
||
|
|
"rewards/margins": 1.2641963958740234,
|
||
|
|
"rewards/rejected": -1.4524368286132812,
|
||
|
|
"step": 1730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.21414794921875,
|
||
|
|
"epoch": 0.8161111599683405,
|
||
|
|
"grad_norm": 41.159706115722656,
|
||
|
|
"learning_rate": 1.0265763418447107e-07,
|
||
|
|
"logits/chosen": -0.3360969872494596,
|
||
|
|
"logits/rejected": 0.1577603664545836,
|
||
|
|
"logps/chosen": -471.64375,
|
||
|
|
"logps/rejected": -631.603125,
|
||
|
|
"loss": 0.4908496856689453,
|
||
|
|
"mean_token_accuracy": 0.7176168402656913,
|
||
|
|
"num_tokens": 292202781.0,
|
||
|
|
"rewards/accuracies": 0.7140625,
|
||
|
|
"rewards/chosen": -0.1997955322265625,
|
||
|
|
"rewards/margins": 1.258979034423828,
|
||
|
|
"rewards/rejected": -1.4585357666015626,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2014404296875,
|
||
|
|
"epoch": 0.8208014539911471,
|
||
|
|
"grad_norm": 44.04994583129883,
|
||
|
|
"learning_rate": 1.0005211047420531e-07,
|
||
|
|
"logits/chosen": -0.35303433846999394,
|
||
|
|
"logits/rejected": 0.0034600471369497406,
|
||
|
|
"logps/chosen": -498.23125,
|
||
|
|
"logps/rejected": -624.215625,
|
||
|
|
"loss": 0.5030181884765625,
|
||
|
|
"mean_token_accuracy": 0.7130729466676712,
|
||
|
|
"num_tokens": 293904790.0,
|
||
|
|
"rewards/accuracies": 0.73828125,
|
||
|
|
"rewards/chosen": -0.2209268569946289,
|
||
|
|
"rewards/margins": 1.241543960571289,
|
||
|
|
"rewards/rejected": -1.4627880096435546,
|
||
|
|
"step": 1750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2294189453125,
|
||
|
|
"epoch": 0.8254917480139536,
|
||
|
|
"grad_norm": 39.74637222290039,
|
||
|
|
"learning_rate": 9.744658676393955e-08,
|
||
|
|
"logits/chosen": -0.3580387652304702,
|
||
|
|
"logits/rejected": 0.07639393565541747,
|
||
|
|
"logps/chosen": -462.471875,
|
||
|
|
"logps/rejected": -637.53125,
|
||
|
|
"loss": 0.5074844360351562,
|
||
|
|
"mean_token_accuracy": 0.712321732006967,
|
||
|
|
"num_tokens": 295577189.0,
|
||
|
|
"rewards/accuracies": 0.7,
|
||
|
|
"rewards/chosen": -0.18819160461425782,
|
||
|
|
"rewards/margins": 1.1423080444335938,
|
||
|
|
"rewards/rejected": -1.3303741455078124,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.31600341796875,
|
||
|
|
"epoch": 0.8301820420367602,
|
||
|
|
"grad_norm": 57.74708557128906,
|
||
|
|
"learning_rate": 9.484106305367379e-08,
|
||
|
|
"logits/chosen": -0.347444101216201,
|
||
|
|
"logits/rejected": 0.1768859150063874,
|
||
|
|
"logps/chosen": -539.6375,
|
||
|
|
"logps/rejected": -739.56875,
|
||
|
|
"loss": 0.5145782470703125,
|
||
|
|
"mean_token_accuracy": 0.6992437416687608,
|
||
|
|
"num_tokens": 297311062.0,
|
||
|
|
"rewards/accuracies": 0.721875,
|
||
|
|
"rewards/chosen": -0.24378433227539062,
|
||
|
|
"rewards/margins": 1.293054962158203,
|
||
|
|
"rewards/rejected": -1.536578369140625,
|
||
|
|
"step": 1770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.202099609375,
|
||
|
|
"epoch": 0.8348723360595668,
|
||
|
|
"grad_norm": 69.78555297851562,
|
||
|
|
"learning_rate": 9.223553934340803e-08,
|
||
|
|
"logits/chosen": -0.3747393354513525,
|
||
|
|
"logits/rejected": 0.10190953988080428,
|
||
|
|
"logps/chosen": -457.48125,
|
||
|
|
"logps/rejected": -642.8875,
|
||
|
|
"loss": 0.4973968505859375,
|
||
|
|
"mean_token_accuracy": 0.7205143462866545,
|
||
|
|
"num_tokens": 298981777.0,
|
||
|
|
"rewards/accuracies": 0.72578125,
|
||
|
|
"rewards/chosen": -0.2223785400390625,
|
||
|
|
"rewards/margins": 1.3282341003417968,
|
||
|
|
"rewards/rejected": -1.550653076171875,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.23270263671875,
|
||
|
|
"epoch": 0.8395626300823733,
|
||
|
|
"grad_norm": 46.602272033691406,
|
||
|
|
"learning_rate": 8.963001563314227e-08,
|
||
|
|
"logits/chosen": -0.385768303897686,
|
||
|
|
"logits/rejected": 0.06807283903709165,
|
||
|
|
"logps/chosen": -484.478125,
|
||
|
|
"logps/rejected": -650.49375,
|
||
|
|
"loss": 0.5179962158203125,
|
||
|
|
"mean_token_accuracy": 0.7067975364625454,
|
||
|
|
"num_tokens": 300670752.0,
|
||
|
|
"rewards/accuracies": 0.71484375,
|
||
|
|
"rewards/chosen": -0.20882492065429686,
|
||
|
|
"rewards/margins": 1.1724075317382812,
|
||
|
|
"rewards/rejected": -1.3815521240234374,
|
||
|
|
"step": 1790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2193115234375,
|
||
|
|
"epoch": 0.8442529241051798,
|
||
|
|
"grad_norm": 42.331512451171875,
|
||
|
|
"learning_rate": 8.702449192287648e-08,
|
||
|
|
"logits/chosen": -0.33325068389407,
|
||
|
|
"logits/rejected": 0.12141434221741103,
|
||
|
|
"logps/chosen": -460.55625,
|
||
|
|
"logps/rejected": -658.896875,
|
||
|
|
"loss": 0.490618896484375,
|
||
|
|
"mean_token_accuracy": 0.715523487702012,
|
||
|
|
"num_tokens": 302408053.0,
|
||
|
|
"rewards/accuracies": 0.73046875,
|
||
|
|
"rewards/chosen": -0.14317150115966798,
|
||
|
|
"rewards/margins": 1.2409027099609375,
|
||
|
|
"rewards/rejected": -1.383633804321289,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2162841796875,
|
||
|
|
"epoch": 0.8489432181279863,
|
||
|
|
"grad_norm": 38.72737503051758,
|
||
|
|
"learning_rate": 8.441896821261072e-08,
|
||
|
|
"logits/chosen": -0.3193038072324029,
|
||
|
|
"logits/rejected": 0.09296690443437472,
|
||
|
|
"logps/chosen": -473.86875,
|
||
|
|
"logps/rejected": -688.78125,
|
||
|
|
"loss": 0.5184173583984375,
|
||
|
|
"mean_token_accuracy": 0.7158959347754716,
|
||
|
|
"num_tokens": 304140722.0,
|
||
|
|
"rewards/accuracies": 0.7109375,
|
||
|
|
"rewards/chosen": -0.2360006332397461,
|
||
|
|
"rewards/margins": 1.1482696533203125,
|
||
|
|
"rewards/rejected": -1.3845916748046876,
|
||
|
|
"step": 1810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.21527099609375,
|
||
|
|
"epoch": 0.853633512150793,
|
||
|
|
"grad_norm": 45.180023193359375,
|
||
|
|
"learning_rate": 8.181344450234496e-08,
|
||
|
|
"logits/chosen": -0.3929161294335359,
|
||
|
|
"logits/rejected": 0.15350176675456656,
|
||
|
|
"logps/chosen": -483.09375,
|
||
|
|
"logps/rejected": -646.66875,
|
||
|
|
"loss": 0.4897247314453125,
|
||
|
|
"mean_token_accuracy": 0.7121978150680661,
|
||
|
|
"num_tokens": 305822256.0,
|
||
|
|
"rewards/accuracies": 0.72265625,
|
||
|
|
"rewards/chosen": -0.18443603515625,
|
||
|
|
"rewards/margins": 1.2927734375,
|
||
|
|
"rewards/rejected": -1.47667236328125,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.1842041015625,
|
||
|
|
"epoch": 0.8583238061735995,
|
||
|
|
"grad_norm": 48.51628112792969,
|
||
|
|
"learning_rate": 7.92079207920792e-08,
|
||
|
|
"logits/chosen": -0.3951759013914627,
|
||
|
|
"logits/rejected": -0.06797198136206242,
|
||
|
|
"logps/chosen": -448.1,
|
||
|
|
"logps/rejected": -632.9,
|
||
|
|
"loss": 0.5045974731445313,
|
||
|
|
"mean_token_accuracy": 0.7215316914021969,
|
||
|
|
"num_tokens": 307531146.0,
|
||
|
|
"rewards/accuracies": 0.7125,
|
||
|
|
"rewards/chosen": -0.2178356170654297,
|
||
|
|
"rewards/margins": 1.1380462646484375,
|
||
|
|
"rewards/rejected": -1.356158447265625,
|
||
|
|
"step": 1830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.26640625,
|
||
|
|
"epoch": 0.863014100196406,
|
||
|
|
"grad_norm": 39.41709899902344,
|
||
|
|
"learning_rate": 7.660239708181344e-08,
|
||
|
|
"logits/chosen": -0.38910823828454666,
|
||
|
|
"logits/rejected": 0.13045722797837564,
|
||
|
|
"logps/chosen": -468.475,
|
||
|
|
"logps/rejected": -673.340625,
|
||
|
|
"loss": 0.47516441345214844,
|
||
|
|
"mean_token_accuracy": 0.7065185211598873,
|
||
|
|
"num_tokens": 309213627.0,
|
||
|
|
"rewards/accuracies": 0.72265625,
|
||
|
|
"rewards/chosen": -0.20412101745605468,
|
||
|
|
"rewards/margins": 1.3498138427734374,
|
||
|
|
"rewards/rejected": -1.55379638671875,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.221923828125,
|
||
|
|
"epoch": 0.8677043942192126,
|
||
|
|
"grad_norm": 47.1512565612793,
|
||
|
|
"learning_rate": 7.399687337154768e-08,
|
||
|
|
"logits/chosen": -0.3544700164600053,
|
||
|
|
"logits/rejected": 0.013244739388591794,
|
||
|
|
"logps/chosen": -472.465625,
|
||
|
|
"logps/rejected": -669.546875,
|
||
|
|
"loss": 0.5173255920410156,
|
||
|
|
"mean_token_accuracy": 0.714903998747468,
|
||
|
|
"num_tokens": 310921408.0,
|
||
|
|
"rewards/accuracies": 0.6953125,
|
||
|
|
"rewards/chosen": -0.25082285404205323,
|
||
|
|
"rewards/margins": 1.2442939758300782,
|
||
|
|
"rewards/rejected": -1.49520263671875,
|
||
|
|
"step": 1850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2416259765625,
|
||
|
|
"epoch": 0.8723946882420192,
|
||
|
|
"grad_norm": 55.58268356323242,
|
||
|
|
"learning_rate": 7.139134966128191e-08,
|
||
|
|
"logits/chosen": -0.42932233723869945,
|
||
|
|
"logits/rejected": 0.006661953719145919,
|
||
|
|
"logps/chosen": -485.2625,
|
||
|
|
"logps/rejected": -650.05625,
|
||
|
|
"loss": 0.5058219909667969,
|
||
|
|
"mean_token_accuracy": 0.7036891637369991,
|
||
|
|
"num_tokens": 312542177.0,
|
||
|
|
"rewards/accuracies": 0.7109375,
|
||
|
|
"rewards/chosen": -0.18791580200195312,
|
||
|
|
"rewards/margins": 1.2160125732421876,
|
||
|
|
"rewards/rejected": -1.403900146484375,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3213134765625,
|
||
|
|
"epoch": 0.8770849822648257,
|
||
|
|
"grad_norm": 66.06620788574219,
|
||
|
|
"learning_rate": 6.878582595101615e-08,
|
||
|
|
"logits/chosen": -0.34825132475588183,
|
||
|
|
"logits/rejected": 0.0558576905787664,
|
||
|
|
"logps/chosen": -517.925,
|
||
|
|
"logps/rejected": -723.8125,
|
||
|
|
"loss": 0.5183258056640625,
|
||
|
|
"mean_token_accuracy": 0.6997947439551353,
|
||
|
|
"num_tokens": 314220483.0,
|
||
|
|
"rewards/accuracies": 0.71953125,
|
||
|
|
"rewards/chosen": -0.23440055847167968,
|
||
|
|
"rewards/margins": 1.1802268981933595,
|
||
|
|
"rewards/rejected": -1.41419677734375,
|
||
|
|
"step": 1870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2500732421875,
|
||
|
|
"epoch": 0.8817752762876323,
|
||
|
|
"grad_norm": 33.63868713378906,
|
||
|
|
"learning_rate": 6.618030224075039e-08,
|
||
|
|
"logits/chosen": -0.3358364325160193,
|
||
|
|
"logits/rejected": 0.1604155617960564,
|
||
|
|
"logps/chosen": -442.221875,
|
||
|
|
"logps/rejected": -656.95625,
|
||
|
|
"loss": 0.49998931884765624,
|
||
|
|
"mean_token_accuracy": 0.7081999473273755,
|
||
|
|
"num_tokens": 315871466.0,
|
||
|
|
"rewards/accuracies": 0.7265625,
|
||
|
|
"rewards/chosen": -0.18980827331542968,
|
||
|
|
"rewards/margins": 1.1712890625,
|
||
|
|
"rewards/rejected": -1.360552978515625,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.231689453125,
|
||
|
|
"epoch": 0.8864655703104388,
|
||
|
|
"grad_norm": 43.70984649658203,
|
||
|
|
"learning_rate": 6.357477853048463e-08,
|
||
|
|
"logits/chosen": -0.3156704533986772,
|
||
|
|
"logits/rejected": 0.046957729068871246,
|
||
|
|
"logps/chosen": -472.4375,
|
||
|
|
"logps/rejected": -613.6625,
|
||
|
|
"loss": 0.49947547912597656,
|
||
|
|
"mean_token_accuracy": 0.7070603730157018,
|
||
|
|
"num_tokens": 317495293.0,
|
||
|
|
"rewards/accuracies": 0.73046875,
|
||
|
|
"rewards/chosen": -0.2524730682373047,
|
||
|
|
"rewards/margins": 1.140589141845703,
|
||
|
|
"rewards/rejected": -1.3932937622070312,
|
||
|
|
"step": 1890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.292578125,
|
||
|
|
"epoch": 0.8911558643332453,
|
||
|
|
"grad_norm": 45.049705505371094,
|
||
|
|
"learning_rate": 6.096925482021887e-08,
|
||
|
|
"logits/chosen": -0.4336513052648586,
|
||
|
|
"logits/rejected": 0.0579325140142804,
|
||
|
|
"logps/chosen": -474.175,
|
||
|
|
"logps/rejected": -707.24375,
|
||
|
|
"loss": 0.4972557067871094,
|
||
|
|
"mean_token_accuracy": 0.7035533275455237,
|
||
|
|
"num_tokens": 319170192.0,
|
||
|
|
"rewards/accuracies": 0.7078125,
|
||
|
|
"rewards/chosen": -0.2193094253540039,
|
||
|
|
"rewards/margins": 1.2396873474121093,
|
||
|
|
"rewards/rejected": -1.4588394165039062,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.204248046875,
|
||
|
|
"epoch": 0.895846158356052,
|
||
|
|
"grad_norm": 40.22047805786133,
|
||
|
|
"learning_rate": 5.836373110995309e-08,
|
||
|
|
"logits/chosen": -0.3028328678177533,
|
||
|
|
"logits/rejected": 0.12352224319010548,
|
||
|
|
"logps/chosen": -486.69375,
|
||
|
|
"logps/rejected": -669.490625,
|
||
|
|
"loss": 0.5012863159179688,
|
||
|
|
"mean_token_accuracy": 0.7151582084596158,
|
||
|
|
"num_tokens": 320877211.0,
|
||
|
|
"rewards/accuracies": 0.70625,
|
||
|
|
"rewards/chosen": -0.19364986419677735,
|
||
|
|
"rewards/margins": 1.2327484130859374,
|
||
|
|
"rewards/rejected": -1.4259002685546875,
|
||
|
|
"step": 1910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.215087890625,
|
||
|
|
"epoch": 0.9005364523788585,
|
||
|
|
"grad_norm": 87.88335418701172,
|
||
|
|
"learning_rate": 5.575820739968733e-08,
|
||
|
|
"logits/chosen": -0.37968428679909233,
|
||
|
|
"logits/rejected": 0.042592474067907934,
|
||
|
|
"logps/chosen": -451.80625,
|
||
|
|
"logps/rejected": -589.6625,
|
||
|
|
"loss": 0.5153121948242188,
|
||
|
|
"mean_token_accuracy": 0.711966947093606,
|
||
|
|
"num_tokens": 322486844.0,
|
||
|
|
"rewards/accuracies": 0.72265625,
|
||
|
|
"rewards/chosen": -0.227374267578125,
|
||
|
|
"rewards/margins": 1.1719680786132813,
|
||
|
|
"rewards/rejected": -1.3991973876953125,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2044921875,
|
||
|
|
"epoch": 0.905226746401665,
|
||
|
|
"grad_norm": 54.445945739746094,
|
||
|
|
"learning_rate": 5.315268368942157e-08,
|
||
|
|
"logits/chosen": -0.3291715983156379,
|
||
|
|
"logits/rejected": 0.01717249772363879,
|
||
|
|
"logps/chosen": -481.8125,
|
||
|
|
"logps/rejected": -601.60625,
|
||
|
|
"loss": 0.5008369445800781,
|
||
|
|
"mean_token_accuracy": 0.7127945670858026,
|
||
|
|
"num_tokens": 324157116.0,
|
||
|
|
"rewards/accuracies": 0.71015625,
|
||
|
|
"rewards/chosen": -0.18655776977539062,
|
||
|
|
"rewards/margins": 1.2192840576171875,
|
||
|
|
"rewards/rejected": -1.4057220458984374,
|
||
|
|
"step": 1930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2319091796875,
|
||
|
|
"epoch": 0.9099170404244716,
|
||
|
|
"grad_norm": 47.28034973144531,
|
||
|
|
"learning_rate": 5.054715997915581e-08,
|
||
|
|
"logits/chosen": -0.37527385347360676,
|
||
|
|
"logits/rejected": -0.05364750899153496,
|
||
|
|
"logps/chosen": -463.29375,
|
||
|
|
"logps/rejected": -625.09375,
|
||
|
|
"loss": 0.49831314086914064,
|
||
|
|
"mean_token_accuracy": 0.7146999042481184,
|
||
|
|
"num_tokens": 325827616.0,
|
||
|
|
"rewards/accuracies": 0.73671875,
|
||
|
|
"rewards/chosen": -0.1948223114013672,
|
||
|
|
"rewards/margins": 1.2142105102539062,
|
||
|
|
"rewards/rejected": -1.4088592529296875,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.28642578125,
|
||
|
|
"epoch": 0.9146073344472782,
|
||
|
|
"grad_norm": 39.064815521240234,
|
||
|
|
"learning_rate": 4.794163626889004e-08,
|
||
|
|
"logits/chosen": -0.3388733610444481,
|
||
|
|
"logits/rejected": 0.12303869993057844,
|
||
|
|
"logps/chosen": -498.6375,
|
||
|
|
"logps/rejected": -695.2,
|
||
|
|
"loss": 0.5198501586914063,
|
||
|
|
"mean_token_accuracy": 0.7057459820061922,
|
||
|
|
"num_tokens": 327526585.0,
|
||
|
|
"rewards/accuracies": 0.709375,
|
||
|
|
"rewards/chosen": -0.20029373168945314,
|
||
|
|
"rewards/margins": 1.1490142822265625,
|
||
|
|
"rewards/rejected": -1.3491470336914062,
|
||
|
|
"step": 1950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.280712890625,
|
||
|
|
"epoch": 0.9192976284700847,
|
||
|
|
"grad_norm": 43.056724548339844,
|
||
|
|
"learning_rate": 4.533611255862428e-08,
|
||
|
|
"logits/chosen": -0.33125824228419354,
|
||
|
|
"logits/rejected": 0.06951288466580796,
|
||
|
|
"logps/chosen": -505.46875,
|
||
|
|
"logps/rejected": -757.3375,
|
||
|
|
"loss": 0.49418106079101565,
|
||
|
|
"mean_token_accuracy": 0.7125601809471845,
|
||
|
|
"num_tokens": 329344689.0,
|
||
|
|
"rewards/accuracies": 0.71875,
|
||
|
|
"rewards/chosen": -0.2517139434814453,
|
||
|
|
"rewards/margins": 1.2828144073486327,
|
||
|
|
"rewards/rejected": -1.534075927734375,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.3103759765625,
|
||
|
|
"epoch": 0.9239879224928913,
|
||
|
|
"grad_norm": 47.633358001708984,
|
||
|
|
"learning_rate": 4.273058884835852e-08,
|
||
|
|
"logits/chosen": -0.32412154662737075,
|
||
|
|
"logits/rejected": 0.14630080084089,
|
||
|
|
"logps/chosen": -507.8625,
|
||
|
|
"logps/rejected": -787.490625,
|
||
|
|
"loss": 0.49291152954101564,
|
||
|
|
"mean_token_accuracy": 0.7110428418964148,
|
||
|
|
"num_tokens": 331076977.0,
|
||
|
|
"rewards/accuracies": 0.72265625,
|
||
|
|
"rewards/chosen": -0.2450836181640625,
|
||
|
|
"rewards/margins": 1.400872802734375,
|
||
|
|
"rewards/rejected": -1.6462799072265626,
|
||
|
|
"step": 1970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2950927734375,
|
||
|
|
"epoch": 0.9286782165156978,
|
||
|
|
"grad_norm": 40.30205154418945,
|
||
|
|
"learning_rate": 4.0125065138092755e-08,
|
||
|
|
"logits/chosen": -0.3315387981538188,
|
||
|
|
"logits/rejected": 0.1416249235004452,
|
||
|
|
"logps/chosen": -485.65,
|
||
|
|
"logps/rejected": -699.46875,
|
||
|
|
"loss": 0.4833381652832031,
|
||
|
|
"mean_token_accuracy": 0.7057482942938804,
|
||
|
|
"num_tokens": 332736113.0,
|
||
|
|
"rewards/accuracies": 0.72890625,
|
||
|
|
"rewards/chosen": -0.2063751220703125,
|
||
|
|
"rewards/margins": 1.2729263305664062,
|
||
|
|
"rewards/rejected": -1.4789794921875,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.30361328125,
|
||
|
|
"epoch": 0.9333685105385043,
|
||
|
|
"grad_norm": 51.24271774291992,
|
||
|
|
"learning_rate": 3.751954142782699e-08,
|
||
|
|
"logits/chosen": -0.3614101444479402,
|
||
|
|
"logits/rejected": 0.12883004161401715,
|
||
|
|
"logps/chosen": -457.2375,
|
||
|
|
"logps/rejected": -743.24375,
|
||
|
|
"loss": 0.50074462890625,
|
||
|
|
"mean_token_accuracy": 0.7107256408780813,
|
||
|
|
"num_tokens": 334425608.0,
|
||
|
|
"rewards/accuracies": 0.72109375,
|
||
|
|
"rewards/chosen": -0.21077451705932618,
|
||
|
|
"rewards/margins": 1.164959716796875,
|
||
|
|
"rewards/rejected": -1.3759368896484374,
|
||
|
|
"step": 1990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.240966796875,
|
||
|
|
"epoch": 0.938058804561311,
|
||
|
|
"grad_norm": 40.48645782470703,
|
||
|
|
"learning_rate": 3.491401771756123e-08,
|
||
|
|
"logits/chosen": -0.42722963460593266,
|
||
|
|
"logits/rejected": 0.08344267947642378,
|
||
|
|
"logps/chosen": -463.0375,
|
||
|
|
"logps/rejected": -612.61875,
|
||
|
|
"loss": 0.4973747253417969,
|
||
|
|
"mean_token_accuracy": 0.7031021565198898,
|
||
|
|
"num_tokens": 336040370.0,
|
||
|
|
"rewards/accuracies": 0.7234375,
|
||
|
|
"rewards/chosen": -0.19219436645507812,
|
||
|
|
"rewards/margins": 1.2013168334960938,
|
||
|
|
"rewards/rejected": -1.3940261840820312,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2700927734375,
|
||
|
|
"epoch": 0.9427490985841175,
|
||
|
|
"grad_norm": 43.718074798583984,
|
||
|
|
"learning_rate": 3.230849400729546e-08,
|
||
|
|
"logits/chosen": -0.33091222728590197,
|
||
|
|
"logits/rejected": 0.18359424973817012,
|
||
|
|
"logps/chosen": -492.859375,
|
||
|
|
"logps/rejected": -667.65,
|
||
|
|
"loss": 0.4870887756347656,
|
||
|
|
"mean_token_accuracy": 0.7044433875009417,
|
||
|
|
"num_tokens": 337738884.0,
|
||
|
|
"rewards/accuracies": 0.71953125,
|
||
|
|
"rewards/chosen": -0.20358028411865234,
|
||
|
|
"rewards/margins": 1.360540771484375,
|
||
|
|
"rewards/rejected": -1.56429443359375,
|
||
|
|
"step": 2010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.25888671875,
|
||
|
|
"epoch": 0.947439392606924,
|
||
|
|
"grad_norm": 48.371150970458984,
|
||
|
|
"learning_rate": 2.97029702970297e-08,
|
||
|
|
"logits/chosen": -0.3781629436293919,
|
||
|
|
"logits/rejected": 0.05601582152754293,
|
||
|
|
"logps/chosen": -474.40625,
|
||
|
|
"logps/rejected": -704.209375,
|
||
|
|
"loss": 0.5054107666015625,
|
||
|
|
"mean_token_accuracy": 0.7089995898306369,
|
||
|
|
"num_tokens": 339374987.0,
|
||
|
|
"rewards/accuracies": 0.7078125,
|
||
|
|
"rewards/chosen": -0.2509334564208984,
|
||
|
|
"rewards/margins": 1.4051788330078125,
|
||
|
|
"rewards/rejected": -1.65648193359375,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2791015625,
|
||
|
|
"epoch": 0.9521296866297306,
|
||
|
|
"grad_norm": 45.81821823120117,
|
||
|
|
"learning_rate": 2.709744658676394e-08,
|
||
|
|
"logits/chosen": -0.3524803996629535,
|
||
|
|
"logits/rejected": 0.16845931554752744,
|
||
|
|
"logps/chosen": -458.95625,
|
||
|
|
"logps/rejected": -714.40625,
|
||
|
|
"loss": 0.5151969909667968,
|
||
|
|
"mean_token_accuracy": 0.70955845490098,
|
||
|
|
"num_tokens": 341081547.0,
|
||
|
|
"rewards/accuracies": 0.68984375,
|
||
|
|
"rewards/chosen": -0.19017089605331422,
|
||
|
|
"rewards/margins": 1.173492431640625,
|
||
|
|
"rewards/rejected": -1.364105224609375,
|
||
|
|
"step": 2030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2907958984375,
|
||
|
|
"epoch": 0.9568199806525371,
|
||
|
|
"grad_norm": 41.41585922241211,
|
||
|
|
"learning_rate": 2.4491922876498173e-08,
|
||
|
|
"logits/chosen": -0.4542369755451925,
|
||
|
|
"logits/rejected": -0.058119418372802836,
|
||
|
|
"logps/chosen": -481.484375,
|
||
|
|
"logps/rejected": -672.94375,
|
||
|
|
"loss": 0.49808197021484374,
|
||
|
|
"mean_token_accuracy": 0.7006991088390351,
|
||
|
|
"num_tokens": 342748476.0,
|
||
|
|
"rewards/accuracies": 0.73359375,
|
||
|
|
"rewards/chosen": -0.15017204284667968,
|
||
|
|
"rewards/margins": 1.2583206176757813,
|
||
|
|
"rewards/rejected": -1.4087432861328124,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2560546875,
|
||
|
|
"epoch": 0.9615102746753437,
|
||
|
|
"grad_norm": 47.81901931762695,
|
||
|
|
"learning_rate": 2.1886399166232413e-08,
|
||
|
|
"logits/chosen": -0.35172637393118933,
|
||
|
|
"logits/rejected": 0.08463381700840975,
|
||
|
|
"logps/chosen": -490.5375,
|
||
|
|
"logps/rejected": -701.64375,
|
||
|
|
"loss": 0.4924041748046875,
|
||
|
|
"mean_token_accuracy": 0.7111782263964415,
|
||
|
|
"num_tokens": 344461836.0,
|
||
|
|
"rewards/accuracies": 0.7203125,
|
||
|
|
"rewards/chosen": -0.19839134216308593,
|
||
|
|
"rewards/margins": 1.3799209594726562,
|
||
|
|
"rewards/rejected": -1.57762451171875,
|
||
|
|
"step": 2050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2580810546875,
|
||
|
|
"epoch": 0.9662005686981503,
|
||
|
|
"grad_norm": 42.42911911010742,
|
||
|
|
"learning_rate": 1.928087545596665e-08,
|
||
|
|
"logits/chosen": -0.3737677612104985,
|
||
|
|
"logits/rejected": 0.0983303444858018,
|
||
|
|
"logps/chosen": -433.90625,
|
||
|
|
"logps/rejected": -661.34375,
|
||
|
|
"loss": 0.484228515625,
|
||
|
|
"mean_token_accuracy": 0.7108163654804229,
|
||
|
|
"num_tokens": 346122958.0,
|
||
|
|
"rewards/accuracies": 0.71875,
|
||
|
|
"rewards/chosen": -0.13962364196777344,
|
||
|
|
"rewards/margins": 1.2540679931640626,
|
||
|
|
"rewards/rejected": -1.3932327270507812,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2074462890625,
|
||
|
|
"epoch": 0.9708908627209568,
|
||
|
|
"grad_norm": 47.9688720703125,
|
||
|
|
"learning_rate": 1.6675351745700885e-08,
|
||
|
|
"logits/chosen": -0.3346860115742126,
|
||
|
|
"logits/rejected": 0.13174351469037554,
|
||
|
|
"logps/chosen": -463.53125,
|
||
|
|
"logps/rejected": -660.0875,
|
||
|
|
"loss": 0.5093505859375,
|
||
|
|
"mean_token_accuracy": 0.7194312229752541,
|
||
|
|
"num_tokens": 347810695.0,
|
||
|
|
"rewards/accuracies": 0.6984375,
|
||
|
|
"rewards/chosen": -0.22800559997558595,
|
||
|
|
"rewards/margins": 1.2211044311523438,
|
||
|
|
"rewards/rejected": -1.4488922119140626,
|
||
|
|
"step": 2070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.229345703125,
|
||
|
|
"epoch": 0.9755811567437633,
|
||
|
|
"grad_norm": 41.37725830078125,
|
||
|
|
"learning_rate": 1.4069828035435123e-08,
|
||
|
|
"logits/chosen": -0.3518116668610031,
|
||
|
|
"logits/rejected": 0.1489972835310932,
|
||
|
|
"logps/chosen": -456.98125,
|
||
|
|
"logps/rejected": -670.26875,
|
||
|
|
"loss": 0.5096603393554687,
|
||
|
|
"mean_token_accuracy": 0.71624949388206,
|
||
|
|
"num_tokens": 349502012.0,
|
||
|
|
"rewards/accuracies": 0.70546875,
|
||
|
|
"rewards/chosen": -0.2621330261230469,
|
||
|
|
"rewards/margins": 1.2914962768554688,
|
||
|
|
"rewards/rejected": -1.5537750244140625,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2279052734375,
|
||
|
|
"epoch": 0.98027145076657,
|
||
|
|
"grad_norm": 51.86234664916992,
|
||
|
|
"learning_rate": 1.1464304325169358e-08,
|
||
|
|
"logits/chosen": -0.3000658016784428,
|
||
|
|
"logits/rejected": 0.17493088714280344,
|
||
|
|
"logps/chosen": -478.8,
|
||
|
|
"logps/rejected": -717.7,
|
||
|
|
"loss": 0.505853271484375,
|
||
|
|
"mean_token_accuracy": 0.7180974125862122,
|
||
|
|
"num_tokens": 351224924.0,
|
||
|
|
"rewards/accuracies": 0.7078125,
|
||
|
|
"rewards/chosen": -0.2635894775390625,
|
||
|
|
"rewards/margins": 1.29547119140625,
|
||
|
|
"rewards/rejected": -1.55958251953125,
|
||
|
|
"step": 2090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2322509765625,
|
||
|
|
"epoch": 0.9849617447893765,
|
||
|
|
"grad_norm": 51.60932159423828,
|
||
|
|
"learning_rate": 8.858780614903594e-09,
|
||
|
|
"logits/chosen": -0.35039016367491677,
|
||
|
|
"logits/rejected": 0.10474107382362959,
|
||
|
|
"logps/chosen": -484.2,
|
||
|
|
"logps/rejected": -661.559375,
|
||
|
|
"loss": 0.5166488647460937,
|
||
|
|
"mean_token_accuracy": 0.7098216339945793,
|
||
|
|
"num_tokens": 352931771.0,
|
||
|
|
"rewards/accuracies": 0.68828125,
|
||
|
|
"rewards/chosen": -0.22312393188476562,
|
||
|
|
"rewards/margins": 1.132904052734375,
|
||
|
|
"rewards/rejected": -1.35577392578125,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.155126953125,
|
||
|
|
"epoch": 0.989652038812183,
|
||
|
|
"grad_norm": 46.09328842163086,
|
||
|
|
"learning_rate": 6.253256904637832e-09,
|
||
|
|
"logits/chosen": -0.3290858068608348,
|
||
|
|
"logits/rejected": -0.02093571722635537,
|
||
|
|
"logps/chosen": -485.83125,
|
||
|
|
"logps/rejected": -543.66875,
|
||
|
|
"loss": 0.5270843505859375,
|
||
|
|
"mean_token_accuracy": 0.713743943348527,
|
||
|
|
"num_tokens": 354621334.0,
|
||
|
|
"rewards/accuracies": 0.7046875,
|
||
|
|
"rewards/chosen": -0.19669809341430664,
|
||
|
|
"rewards/margins": 1.0803466796875,
|
||
|
|
"rewards/rejected": -1.276715087890625,
|
||
|
|
"step": 2110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2471435546875,
|
||
|
|
"epoch": 0.9943423328349896,
|
||
|
|
"grad_norm": 65.90406036376953,
|
||
|
|
"learning_rate": 3.6477331943720682e-09,
|
||
|
|
"logits/chosen": -0.20736095787263226,
|
||
|
|
"logits/rejected": 0.1461316692738893,
|
||
|
|
"logps/chosen": -506.425,
|
||
|
|
"logps/rejected": -653.98125,
|
||
|
|
"loss": 0.5250595092773438,
|
||
|
|
"mean_token_accuracy": 0.7124230587854982,
|
||
|
|
"num_tokens": 356312751.0,
|
||
|
|
"rewards/accuracies": 0.71640625,
|
||
|
|
"rewards/chosen": -0.22593250274658203,
|
||
|
|
"rewards/margins": 1.2058555603027343,
|
||
|
|
"rewards/rejected": -1.4319067001342773,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.2607177734375,
|
||
|
|
"epoch": 0.9990326268577961,
|
||
|
|
"grad_norm": 44.84026336669922,
|
||
|
|
"learning_rate": 1.0422094841063053e-09,
|
||
|
|
"logits/chosen": -0.3921418517965115,
|
||
|
|
"logits/rejected": -0.05872409453138806,
|
||
|
|
"logps/chosen": -464.925,
|
||
|
|
"logps/rejected": -629.39375,
|
||
|
|
"loss": 0.49729080200195314,
|
||
|
|
"mean_token_accuracy": 0.7027096889913083,
|
||
|
|
"num_tokens": 357948612.0,
|
||
|
|
"rewards/accuracies": 0.72421875,
|
||
|
|
"rewards/chosen": -0.2333770751953125,
|
||
|
|
"rewards/margins": 1.1683845520019531,
|
||
|
|
"rewards/rejected": -1.4017159461975097,
|
||
|
|
"step": 2130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"entropy": 1.178030303030303,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"logits/chosen": -0.30206525592803823,
|
||
|
|
"logits/rejected": -0.0929858402560999,
|
||
|
|
"logps/chosen": -472.09090909090907,
|
||
|
|
"logps/rejected": -557.5757575757576,
|
||
|
|
"mean_token_accuracy": 0.7097648002884605,
|
||
|
|
"num_tokens": 358277178.0,
|
||
|
|
"rewards/accuracies": 0.7234848484848485,
|
||
|
|
"rewards/chosen": -0.24461780894886365,
|
||
|
|
"rewards/margins": 1.0697687322443181,
|
||
|
|
"rewards/rejected": -1.3149266098484849,
|
||
|
|
"step": 2133,
|
||
|
|
"total_flos": 4.975817267087409e+18,
|
||
|
|
"train_loss": 0.527936527665322,
|
||
|
|
"train_runtime": 22045.0555,
|
||
|
|
"train_samples_per_second": 12.379,
|
||
|
|
"train_steps_per_second": 0.097
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 2133,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 4.975817267087409e+18,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|