5844 lines
203 KiB
JSON
5844 lines
203 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.6155917425310937,
|
|
"eval_steps": 20,
|
|
"global_step": 700,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004615976407231696,
|
|
"grad_norm": 92.4314081607968,
|
|
"learning_rate": 1.1494252873563218e-08,
|
|
"logits/chosen": -1.3403388261795044,
|
|
"logits/rejected": -1.3443610668182373,
|
|
"logps/chosen": -48.98606872558594,
|
|
"logps/rejected": -52.890384674072266,
|
|
"loss": 0.6931,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.009231952814463392,
|
|
"grad_norm": 114.27394093227154,
|
|
"learning_rate": 2.2988505747126436e-08,
|
|
"logits/chosen": -1.3453574180603027,
|
|
"logits/rejected": -1.3622318506240845,
|
|
"logps/chosen": -39.51582336425781,
|
|
"logps/rejected": -55.267478942871094,
|
|
"loss": 0.6984,
|
|
"rewards/accuracies": 0.4444444477558136,
|
|
"rewards/chosen": -0.0065580871887505054,
|
|
"rewards/margins": -0.004221578594297171,
|
|
"rewards/rejected": -0.002336508594453335,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.01384792922169509,
|
|
"grad_norm": 150.47548740563838,
|
|
"learning_rate": 3.448275862068965e-08,
|
|
"logits/chosen": -1.3104500770568848,
|
|
"logits/rejected": -1.3256760835647583,
|
|
"logps/chosen": -46.711997985839844,
|
|
"logps/rejected": -61.08738327026367,
|
|
"loss": 0.7077,
|
|
"rewards/accuracies": 0.4583333432674408,
|
|
"rewards/chosen": -0.015531142242252827,
|
|
"rewards/margins": -0.022088024765253067,
|
|
"rewards/rejected": 0.006556881591677666,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.018463905628926785,
|
|
"grad_norm": 127.64708744370179,
|
|
"learning_rate": 4.597701149425287e-08,
|
|
"logits/chosen": -1.3497395515441895,
|
|
"logits/rejected": -1.3723570108413696,
|
|
"logps/chosen": -50.4114875793457,
|
|
"logps/rejected": -67.92998504638672,
|
|
"loss": 0.7139,
|
|
"rewards/accuracies": 0.4027777910232544,
|
|
"rewards/chosen": 0.04585569351911545,
|
|
"rewards/margins": -0.033931903541088104,
|
|
"rewards/rejected": 0.07978759706020355,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.023079882036158482,
|
|
"grad_norm": 104.28603257129862,
|
|
"learning_rate": 5.747126436781609e-08,
|
|
"logits/chosen": -1.3193544149398804,
|
|
"logits/rejected": -1.3253015279769897,
|
|
"logps/chosen": -48.21293258666992,
|
|
"logps/rejected": -55.63939666748047,
|
|
"loss": 0.6759,
|
|
"rewards/accuracies": 0.6527777910232544,
|
|
"rewards/chosen": 0.09531965851783752,
|
|
"rewards/margins": 0.03943055123090744,
|
|
"rewards/rejected": 0.055889103561639786,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.02769585844339018,
|
|
"grad_norm": 122.92866276312793,
|
|
"learning_rate": 6.89655172413793e-08,
|
|
"logits/chosen": -1.356438159942627,
|
|
"logits/rejected": -1.3778866529464722,
|
|
"logps/chosen": -47.38197326660156,
|
|
"logps/rejected": -62.85205841064453,
|
|
"loss": 0.7012,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.021549424156546593,
|
|
"rewards/margins": -0.00793980248272419,
|
|
"rewards/rejected": 0.029489226639270782,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.032311834850621876,
|
|
"grad_norm": 141.4940238366865,
|
|
"learning_rate": 8.045977011494252e-08,
|
|
"logits/chosen": -1.2811020612716675,
|
|
"logits/rejected": -1.3018286228179932,
|
|
"logps/chosen": -53.21059799194336,
|
|
"logps/rejected": -68.97090148925781,
|
|
"loss": 0.6877,
|
|
"rewards/accuracies": 0.4583333432674408,
|
|
"rewards/chosen": -0.0111711286008358,
|
|
"rewards/margins": 0.018631484359502792,
|
|
"rewards/rejected": -0.029802612960338593,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.03692781125785357,
|
|
"grad_norm": 108.13024807755075,
|
|
"learning_rate": 9.195402298850574e-08,
|
|
"logits/chosen": -1.3367334604263306,
|
|
"logits/rejected": -1.3522446155548096,
|
|
"logps/chosen": -40.02373504638672,
|
|
"logps/rejected": -54.60912322998047,
|
|
"loss": 0.702,
|
|
"rewards/accuracies": 0.4166666567325592,
|
|
"rewards/chosen": 0.010513358749449253,
|
|
"rewards/margins": -0.013667477294802666,
|
|
"rewards/rejected": 0.024180836975574493,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.04154378766508527,
|
|
"grad_norm": 90.94991906906556,
|
|
"learning_rate": 1.0344827586206897e-07,
|
|
"logits/chosen": -1.2363929748535156,
|
|
"logits/rejected": -1.2396348714828491,
|
|
"logps/chosen": -56.863731384277344,
|
|
"logps/rejected": -53.349342346191406,
|
|
"loss": 0.6938,
|
|
"rewards/accuracies": 0.4583333432674408,
|
|
"rewards/chosen": 0.07457832247018814,
|
|
"rewards/margins": 0.004799458663910627,
|
|
"rewards/rejected": 0.06977886706590652,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.046159764072316964,
|
|
"grad_norm": 166.09887872402663,
|
|
"learning_rate": 1.1494252873563217e-07,
|
|
"logits/chosen": -1.2872830629348755,
|
|
"logits/rejected": -1.317036747932434,
|
|
"logps/chosen": -49.23244857788086,
|
|
"logps/rejected": -71.62715911865234,
|
|
"loss": 0.7087,
|
|
"rewards/accuracies": 0.5694444179534912,
|
|
"rewards/chosen": 0.02608451619744301,
|
|
"rewards/margins": -0.02054634317755699,
|
|
"rewards/rejected": 0.046630859375,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.046159764072316964,
|
|
"eval_logits/chosen": -1.262330412864685,
|
|
"eval_logits/rejected": -1.273974895477295,
|
|
"eval_logps/chosen": -48.664798736572266,
|
|
"eval_logps/rejected": -56.1088752746582,
|
|
"eval_loss": 0.6998714804649353,
|
|
"eval_rewards/accuracies": 0.4228110611438751,
|
|
"eval_rewards/chosen": -0.01568525843322277,
|
|
"eval_rewards/margins": -0.007799813989549875,
|
|
"eval_rewards/rejected": -0.007885444909334183,
|
|
"eval_runtime": 231.8501,
|
|
"eval_samples_per_second": 7.479,
|
|
"eval_steps_per_second": 1.872,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05077574047954866,
|
|
"grad_norm": 141.4312370714434,
|
|
"learning_rate": 1.2643678160919542e-07,
|
|
"logits/chosen": -1.328560709953308,
|
|
"logits/rejected": -1.3631365299224854,
|
|
"logps/chosen": -50.79507827758789,
|
|
"logps/rejected": -79.58642578125,
|
|
"loss": 0.6838,
|
|
"rewards/accuracies": 0.5277777910232544,
|
|
"rewards/chosen": 0.04927082732319832,
|
|
"rewards/margins": 0.02934443950653076,
|
|
"rewards/rejected": 0.019926389679312706,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.05539171688678036,
|
|
"grad_norm": 106.70546600458962,
|
|
"learning_rate": 1.379310344827586e-07,
|
|
"logits/chosen": -1.335903286933899,
|
|
"logits/rejected": -1.3434231281280518,
|
|
"logps/chosen": -59.29114532470703,
|
|
"logps/rejected": -62.07218933105469,
|
|
"loss": 0.6961,
|
|
"rewards/accuracies": 0.5138888955116272,
|
|
"rewards/chosen": 0.00791182741522789,
|
|
"rewards/margins": -0.0008803076343610883,
|
|
"rewards/rejected": 0.008792135864496231,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.06000769329401205,
|
|
"grad_norm": 84.5108710571048,
|
|
"learning_rate": 1.4942528735632184e-07,
|
|
"logits/chosen": -1.315495491027832,
|
|
"logits/rejected": -1.314201831817627,
|
|
"logps/chosen": -52.26453399658203,
|
|
"logps/rejected": -46.654151916503906,
|
|
"loss": 0.6879,
|
|
"rewards/accuracies": 0.5972222089767456,
|
|
"rewards/chosen": 0.10577751696109772,
|
|
"rewards/margins": 0.01635124906897545,
|
|
"rewards/rejected": 0.08942626416683197,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.06462366970124375,
|
|
"grad_norm": 142.3451166395631,
|
|
"learning_rate": 1.6091954022988505e-07,
|
|
"logits/chosen": -1.3120254278182983,
|
|
"logits/rejected": -1.3418428897857666,
|
|
"logps/chosen": -54.30976486206055,
|
|
"logps/rejected": -76.96250915527344,
|
|
"loss": 0.6796,
|
|
"rewards/accuracies": 0.4722222089767456,
|
|
"rewards/chosen": 0.14193940162658691,
|
|
"rewards/margins": 0.03795723244547844,
|
|
"rewards/rejected": 0.10398217290639877,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.06923964610847544,
|
|
"grad_norm": 85.45105537711353,
|
|
"learning_rate": 1.7241379310344828e-07,
|
|
"logits/chosen": -1.339949131011963,
|
|
"logits/rejected": -1.35366952419281,
|
|
"logps/chosen": -47.45890808105469,
|
|
"logps/rejected": -56.32393264770508,
|
|
"loss": 0.685,
|
|
"rewards/accuracies": 0.5,
|
|
"rewards/chosen": 0.12981660664081573,
|
|
"rewards/margins": 0.022233910858631134,
|
|
"rewards/rejected": 0.1075827032327652,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.07385562251570714,
|
|
"grad_norm": 87.30098240848938,
|
|
"learning_rate": 1.839080459770115e-07,
|
|
"logits/chosen": -1.3168249130249023,
|
|
"logits/rejected": -1.3266100883483887,
|
|
"logps/chosen": -48.27603530883789,
|
|
"logps/rejected": -54.10696792602539,
|
|
"loss": 0.6693,
|
|
"rewards/accuracies": 0.5833333134651184,
|
|
"rewards/chosen": 0.2039007544517517,
|
|
"rewards/margins": 0.05477040261030197,
|
|
"rewards/rejected": 0.14913035929203033,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.07847159892293884,
|
|
"grad_norm": 81.90822350206311,
|
|
"learning_rate": 1.9540229885057472e-07,
|
|
"logits/chosen": -1.3052334785461426,
|
|
"logits/rejected": -1.3256360292434692,
|
|
"logps/chosen": -44.5953483581543,
|
|
"logps/rejected": -61.29960250854492,
|
|
"loss": 0.674,
|
|
"rewards/accuracies": 0.5555555820465088,
|
|
"rewards/chosen": 0.29883071780204773,
|
|
"rewards/margins": 0.04845905303955078,
|
|
"rewards/rejected": 0.25037166476249695,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.08308757533017054,
|
|
"grad_norm": 97.62737031971352,
|
|
"learning_rate": 2.0689655172413793e-07,
|
|
"logits/chosen": -1.3107632398605347,
|
|
"logits/rejected": -1.3140422105789185,
|
|
"logps/chosen": -51.13896179199219,
|
|
"logps/rejected": -48.718727111816406,
|
|
"loss": 0.6724,
|
|
"rewards/accuracies": 0.5972222089767456,
|
|
"rewards/chosen": 0.2992264926433563,
|
|
"rewards/margins": 0.05020047724246979,
|
|
"rewards/rejected": 0.24902603030204773,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.08770355173740223,
|
|
"grad_norm": 91.70503719282425,
|
|
"learning_rate": 2.1839080459770114e-07,
|
|
"logits/chosen": -1.258926510810852,
|
|
"logits/rejected": -1.2699991464614868,
|
|
"logps/chosen": -50.59396743774414,
|
|
"logps/rejected": -56.2684326171875,
|
|
"loss": 0.6819,
|
|
"rewards/accuracies": 0.5138888955116272,
|
|
"rewards/chosen": 0.33299052715301514,
|
|
"rewards/margins": 0.038206882774829865,
|
|
"rewards/rejected": 0.29478365182876587,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.09231952814463393,
|
|
"grad_norm": 92.44581774007628,
|
|
"learning_rate": 2.2988505747126435e-07,
|
|
"logits/chosen": -1.3053722381591797,
|
|
"logits/rejected": -1.316298007965088,
|
|
"logps/chosen": -52.49648666381836,
|
|
"logps/rejected": -56.09816360473633,
|
|
"loss": 0.6665,
|
|
"rewards/accuracies": 0.5555555820465088,
|
|
"rewards/chosen": 0.38449087738990784,
|
|
"rewards/margins": 0.07676863670349121,
|
|
"rewards/rejected": 0.30772221088409424,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.09231952814463393,
|
|
"eval_logits/chosen": -1.2573766708374023,
|
|
"eval_logits/rejected": -1.269149899482727,
|
|
"eval_logps/chosen": -47.605804443359375,
|
|
"eval_logps/rejected": -55.2476806640625,
|
|
"eval_loss": 0.6606337428092957,
|
|
"eval_rewards/accuracies": 0.5725806355476379,
|
|
"eval_rewards/chosen": 0.5138096213340759,
|
|
"eval_rewards/margins": 0.09109989553689957,
|
|
"eval_rewards/rejected": 0.42270979285240173,
|
|
"eval_runtime": 227.103,
|
|
"eval_samples_per_second": 7.635,
|
|
"eval_steps_per_second": 1.911,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.09693550455186563,
|
|
"grad_norm": 83.51334071319128,
|
|
"learning_rate": 2.413793103448276e-07,
|
|
"logits/chosen": -1.3424174785614014,
|
|
"logits/rejected": -1.3526452779769897,
|
|
"logps/chosen": -44.44143295288086,
|
|
"logps/rejected": -49.79873275756836,
|
|
"loss": 0.6451,
|
|
"rewards/accuracies": 0.6527777910232544,
|
|
"rewards/chosen": 0.5364899039268494,
|
|
"rewards/margins": 0.11971965432167053,
|
|
"rewards/rejected": 0.4167703092098236,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.10155148095909731,
|
|
"grad_norm": 97.54181471785779,
|
|
"learning_rate": 2.5287356321839084e-07,
|
|
"logits/chosen": -1.307891845703125,
|
|
"logits/rejected": -1.3404256105422974,
|
|
"logps/chosen": -50.57395935058594,
|
|
"logps/rejected": -79.52447509765625,
|
|
"loss": 0.6618,
|
|
"rewards/accuracies": 0.5138888955116272,
|
|
"rewards/chosen": 0.5593165159225464,
|
|
"rewards/margins": 0.12356305122375488,
|
|
"rewards/rejected": 0.4357534646987915,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.10616745736632902,
|
|
"grad_norm": 87.61515534490086,
|
|
"learning_rate": 2.64367816091954e-07,
|
|
"logits/chosen": -1.2781007289886475,
|
|
"logits/rejected": -1.2948905229568481,
|
|
"logps/chosen": -55.20244216918945,
|
|
"logps/rejected": -57.26641845703125,
|
|
"loss": 0.656,
|
|
"rewards/accuracies": 0.5833333134651184,
|
|
"rewards/chosen": 0.5990750193595886,
|
|
"rewards/margins": 0.11623137444257736,
|
|
"rewards/rejected": 0.4828437268733978,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.11078343377356072,
|
|
"grad_norm": 89.38653936134894,
|
|
"learning_rate": 2.758620689655172e-07,
|
|
"logits/chosen": -1.2956469058990479,
|
|
"logits/rejected": -1.3047106266021729,
|
|
"logps/chosen": -48.0570068359375,
|
|
"logps/rejected": -54.5909423828125,
|
|
"loss": 0.6482,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 0.6316156983375549,
|
|
"rewards/margins": 0.15523308515548706,
|
|
"rewards/rejected": 0.47638261318206787,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.1153994101807924,
|
|
"grad_norm": 80.50515017031447,
|
|
"learning_rate": 2.873563218390804e-07,
|
|
"logits/chosen": -1.3206638097763062,
|
|
"logits/rejected": -1.328073501586914,
|
|
"logps/chosen": -51.81482696533203,
|
|
"logps/rejected": -51.81681442260742,
|
|
"loss": 0.6387,
|
|
"rewards/accuracies": 0.6527777910232544,
|
|
"rewards/chosen": 0.7185304164886475,
|
|
"rewards/margins": 0.1627696007490158,
|
|
"rewards/rejected": 0.5557608008384705,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.1200153865880241,
|
|
"grad_norm": 81.34880289712139,
|
|
"learning_rate": 2.988505747126437e-07,
|
|
"logits/chosen": -1.3584266901016235,
|
|
"logits/rejected": -1.3828377723693848,
|
|
"logps/chosen": -44.66862869262695,
|
|
"logps/rejected": -66.46587371826172,
|
|
"loss": 0.6441,
|
|
"rewards/accuracies": 0.5416666865348816,
|
|
"rewards/chosen": 0.8013312816619873,
|
|
"rewards/margins": 0.21720875799655914,
|
|
"rewards/rejected": 0.5841224789619446,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.1246313629952558,
|
|
"grad_norm": 77.01376791928342,
|
|
"learning_rate": 3.103448275862069e-07,
|
|
"logits/chosen": -1.4568628072738647,
|
|
"logits/rejected": -1.4819716215133667,
|
|
"logps/chosen": -47.80024719238281,
|
|
"logps/rejected": -64.31399536132812,
|
|
"loss": 0.6113,
|
|
"rewards/accuracies": 0.6388888955116272,
|
|
"rewards/chosen": 0.7414547204971313,
|
|
"rewards/margins": 0.2719371020793915,
|
|
"rewards/rejected": 0.4695175588130951,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.1292473394024875,
|
|
"grad_norm": 72.36047181825025,
|
|
"learning_rate": 3.218390804597701e-07,
|
|
"logits/chosen": -1.2487589120864868,
|
|
"logits/rejected": -1.250648021697998,
|
|
"logps/chosen": -44.55437469482422,
|
|
"logps/rejected": -49.30759811401367,
|
|
"loss": 0.6265,
|
|
"rewards/accuracies": 0.6527777910232544,
|
|
"rewards/chosen": 0.8203690052032471,
|
|
"rewards/margins": 0.1877826303243637,
|
|
"rewards/rejected": 0.6325862407684326,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.1338633158097192,
|
|
"grad_norm": 82.28322419724,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": -1.3345168828964233,
|
|
"logits/rejected": -1.3384166955947876,
|
|
"logps/chosen": -46.937095642089844,
|
|
"logps/rejected": -48.4022216796875,
|
|
"loss": 0.5824,
|
|
"rewards/accuracies": 0.6388888955116272,
|
|
"rewards/chosen": 0.8726701736450195,
|
|
"rewards/margins": 0.2923206090927124,
|
|
"rewards/rejected": 0.5803494453430176,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.13847929221695088,
|
|
"grad_norm": 71.91030915826812,
|
|
"learning_rate": 3.4482758620689656e-07,
|
|
"logits/chosen": -1.326303243637085,
|
|
"logits/rejected": -1.3416494131088257,
|
|
"logps/chosen": -45.597774505615234,
|
|
"logps/rejected": -55.49925994873047,
|
|
"loss": 0.5933,
|
|
"rewards/accuracies": 0.7083333134651184,
|
|
"rewards/chosen": 0.8803545236587524,
|
|
"rewards/margins": 0.32467857003211975,
|
|
"rewards/rejected": 0.5556759834289551,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.13847929221695088,
|
|
"eval_logits/chosen": -1.2449160814285278,
|
|
"eval_logits/rejected": -1.2571001052856445,
|
|
"eval_logps/chosen": -46.695552825927734,
|
|
"eval_logps/rejected": -54.76215362548828,
|
|
"eval_loss": 0.6056556105613708,
|
|
"eval_rewards/accuracies": 0.6440092325210571,
|
|
"eval_rewards/chosen": 0.9689397215843201,
|
|
"eval_rewards/margins": 0.30346596240997314,
|
|
"eval_rewards/rejected": 0.6654736995697021,
|
|
"eval_runtime": 226.9584,
|
|
"eval_samples_per_second": 7.64,
|
|
"eval_steps_per_second": 1.912,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.1430952686241826,
|
|
"grad_norm": 73.53474176254309,
|
|
"learning_rate": 3.5632183908045977e-07,
|
|
"logits/chosen": -1.302392840385437,
|
|
"logits/rejected": -1.3110175132751465,
|
|
"logps/chosen": -47.1639289855957,
|
|
"logps/rejected": -52.83234786987305,
|
|
"loss": 0.6038,
|
|
"rewards/accuracies": 0.7361111044883728,
|
|
"rewards/chosen": 0.9415748119354248,
|
|
"rewards/margins": 0.2688363194465637,
|
|
"rewards/rejected": 0.6727384924888611,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.14771124503141428,
|
|
"grad_norm": 72.82764174123524,
|
|
"learning_rate": 3.67816091954023e-07,
|
|
"logits/chosen": -1.3600918054580688,
|
|
"logits/rejected": -1.3759901523590088,
|
|
"logps/chosen": -48.68782043457031,
|
|
"logps/rejected": -59.76481246948242,
|
|
"loss": 0.5893,
|
|
"rewards/accuracies": 0.6666666865348816,
|
|
"rewards/chosen": 1.036276936531067,
|
|
"rewards/margins": 0.393592894077301,
|
|
"rewards/rejected": 0.6426840424537659,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.152327221438646,
|
|
"grad_norm": 72.4502448635078,
|
|
"learning_rate": 3.793103448275862e-07,
|
|
"logits/chosen": -1.254841923713684,
|
|
"logits/rejected": -1.2786000967025757,
|
|
"logps/chosen": -45.10692596435547,
|
|
"logps/rejected": -67.49703216552734,
|
|
"loss": 0.5441,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 1.195844292640686,
|
|
"rewards/margins": 0.5360373854637146,
|
|
"rewards/rejected": 0.6598069667816162,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.15694319784587768,
|
|
"grad_norm": 137.30984216926402,
|
|
"learning_rate": 3.9080459770114945e-07,
|
|
"logits/chosen": -1.3896088600158691,
|
|
"logits/rejected": -1.4264814853668213,
|
|
"logps/chosen": -41.05742645263672,
|
|
"logps/rejected": -65.78131103515625,
|
|
"loss": 0.6531,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 1.2528166770935059,
|
|
"rewards/margins": 0.4981537461280823,
|
|
"rewards/rejected": 0.7546629905700684,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.16155917425310937,
|
|
"grad_norm": 72.56751313040455,
|
|
"learning_rate": 4.0229885057471266e-07,
|
|
"logits/chosen": -1.3160932064056396,
|
|
"logits/rejected": -1.3216156959533691,
|
|
"logps/chosen": -42.512996673583984,
|
|
"logps/rejected": -46.83217239379883,
|
|
"loss": 0.5056,
|
|
"rewards/accuracies": 0.7361111044883728,
|
|
"rewards/chosen": 1.5907378196716309,
|
|
"rewards/margins": 0.5790587067604065,
|
|
"rewards/rejected": 1.0116791725158691,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.16617515066034108,
|
|
"grad_norm": 78.13005318225228,
|
|
"learning_rate": 4.1379310344827586e-07,
|
|
"logits/chosen": -1.2581322193145752,
|
|
"logits/rejected": -1.2822985649108887,
|
|
"logps/chosen": -44.267303466796875,
|
|
"logps/rejected": -71.07489013671875,
|
|
"loss": 0.5435,
|
|
"rewards/accuracies": 0.7361111044883728,
|
|
"rewards/chosen": 1.4014304876327515,
|
|
"rewards/margins": 0.723700761795044,
|
|
"rewards/rejected": 0.6777297258377075,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.17079112706757277,
|
|
"grad_norm": 56.428824716251846,
|
|
"learning_rate": 4.25287356321839e-07,
|
|
"logits/chosen": -1.2850017547607422,
|
|
"logits/rejected": -1.2932665348052979,
|
|
"logps/chosen": -52.70983123779297,
|
|
"logps/rejected": -53.61183166503906,
|
|
"loss": 0.6011,
|
|
"rewards/accuracies": 0.7083333134651184,
|
|
"rewards/chosen": 1.2770977020263672,
|
|
"rewards/margins": 0.5247661471366882,
|
|
"rewards/rejected": 0.7523314952850342,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.17540710347480445,
|
|
"grad_norm": 80.88401808498448,
|
|
"learning_rate": 4.367816091954023e-07,
|
|
"logits/chosen": -1.2536025047302246,
|
|
"logits/rejected": -1.2705798149108887,
|
|
"logps/chosen": -47.25906753540039,
|
|
"logps/rejected": -60.58863067626953,
|
|
"loss": 0.5302,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 1.4430739879608154,
|
|
"rewards/margins": 0.5440190434455872,
|
|
"rewards/rejected": 0.8990550637245178,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.18002307988203617,
|
|
"grad_norm": 68.76597330947016,
|
|
"learning_rate": 4.482758620689655e-07,
|
|
"logits/chosen": -1.2450529336929321,
|
|
"logits/rejected": -1.2557368278503418,
|
|
"logps/chosen": -42.16405487060547,
|
|
"logps/rejected": -52.293785095214844,
|
|
"loss": 0.5623,
|
|
"rewards/accuracies": 0.5972222089767456,
|
|
"rewards/chosen": 1.489140510559082,
|
|
"rewards/margins": 0.561518669128418,
|
|
"rewards/rejected": 0.9276217222213745,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.18463905628926786,
|
|
"grad_norm": 74.75348151064274,
|
|
"learning_rate": 4.597701149425287e-07,
|
|
"logits/chosen": -1.3210065364837646,
|
|
"logits/rejected": -1.3294503688812256,
|
|
"logps/chosen": -42.845787048339844,
|
|
"logps/rejected": -46.24819564819336,
|
|
"loss": 0.5108,
|
|
"rewards/accuracies": 0.6944444179534912,
|
|
"rewards/chosen": 1.1906384229660034,
|
|
"rewards/margins": 0.6210441589355469,
|
|
"rewards/rejected": 0.5695942640304565,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.18463905628926786,
|
|
"eval_logits/chosen": -1.226657509803772,
|
|
"eval_logits/rejected": -1.2391773462295532,
|
|
"eval_logps/chosen": -46.65249252319336,
|
|
"eval_logps/rejected": -55.351402282714844,
|
|
"eval_loss": 0.5300613045692444,
|
|
"eval_rewards/accuracies": 0.7142857313156128,
|
|
"eval_rewards/chosen": 0.9904682636260986,
|
|
"eval_rewards/margins": 0.6196123957633972,
|
|
"eval_rewards/rejected": 0.3708558976650238,
|
|
"eval_runtime": 227.2212,
|
|
"eval_samples_per_second": 7.631,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.18925503269649954,
|
|
"grad_norm": 60.96010208828231,
|
|
"learning_rate": 4.712643678160919e-07,
|
|
"logits/chosen": -1.2890104055404663,
|
|
"logits/rejected": -1.3069978952407837,
|
|
"logps/chosen": -50.1616096496582,
|
|
"logps/rejected": -61.80992126464844,
|
|
"loss": 0.485,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 0.9283716678619385,
|
|
"rewards/margins": 0.7194375991821289,
|
|
"rewards/rejected": 0.20893406867980957,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.19387100910373126,
|
|
"grad_norm": 58.75128856791501,
|
|
"learning_rate": 4.827586206896552e-07,
|
|
"logits/chosen": -1.282674789428711,
|
|
"logits/rejected": -1.3157954216003418,
|
|
"logps/chosen": -42.50400161743164,
|
|
"logps/rejected": -75.54652404785156,
|
|
"loss": 0.4729,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.7927578091621399,
|
|
"rewards/margins": 1.0220049619674683,
|
|
"rewards/rejected": -0.22924719750881195,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.19848698551096294,
|
|
"grad_norm": 69.45296124181107,
|
|
"learning_rate": 4.942528735632184e-07,
|
|
"logits/chosen": -1.2970733642578125,
|
|
"logits/rejected": -1.3046212196350098,
|
|
"logps/chosen": -48.21321487426758,
|
|
"logps/rejected": -59.63574981689453,
|
|
"loss": 0.4753,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.7275898456573486,
|
|
"rewards/margins": 0.7579395771026611,
|
|
"rewards/rejected": -0.030349718406796455,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.20310296191819463,
|
|
"grad_norm": 57.934663164568036,
|
|
"learning_rate": 4.999979670146248e-07,
|
|
"logits/chosen": -1.322100043296814,
|
|
"logits/rejected": -1.333228588104248,
|
|
"logps/chosen": -51.94272232055664,
|
|
"logps/rejected": -62.809814453125,
|
|
"loss": 0.4698,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.5794834494590759,
|
|
"rewards/margins": 0.9340064525604248,
|
|
"rewards/rejected": -0.3545229136943817,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.20771893832542634,
|
|
"grad_norm": 67.05839544195749,
|
|
"learning_rate": 4.99981703330008e-07,
|
|
"logits/chosen": -1.2547653913497925,
|
|
"logits/rejected": -1.2667738199234009,
|
|
"logps/chosen": -44.88441467285156,
|
|
"logps/rejected": -55.610042572021484,
|
|
"loss": 0.4928,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.672675371170044,
|
|
"rewards/margins": 0.72029709815979,
|
|
"rewards/rejected": -0.04762159287929535,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.21233491473265803,
|
|
"grad_norm": 51.75653250091916,
|
|
"learning_rate": 4.99949177018813e-07,
|
|
"logits/chosen": -1.3547184467315674,
|
|
"logits/rejected": -1.3632615804672241,
|
|
"logps/chosen": -41.517826080322266,
|
|
"logps/rejected": -52.83651351928711,
|
|
"loss": 0.3985,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.8576219081878662,
|
|
"rewards/margins": 1.0123066902160645,
|
|
"rewards/rejected": -0.15468484163284302,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.21695089113988972,
|
|
"grad_norm": 81.64528533284856,
|
|
"learning_rate": 4.999003901970474e-07,
|
|
"logits/chosen": -1.3031115531921387,
|
|
"logits/rejected": -1.3127225637435913,
|
|
"logps/chosen": -54.79065704345703,
|
|
"logps/rejected": -54.33130645751953,
|
|
"loss": 0.5699,
|
|
"rewards/accuracies": 0.7222222089767456,
|
|
"rewards/chosen": 0.6079578995704651,
|
|
"rewards/margins": 0.6609423160552979,
|
|
"rewards/rejected": -0.05298437178134918,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.22156686754712143,
|
|
"grad_norm": 50.92778720915042,
|
|
"learning_rate": 4.998353460385512e-07,
|
|
"logits/chosen": -1.2541792392730713,
|
|
"logits/rejected": -1.2738375663757324,
|
|
"logps/chosen": -45.45362091064453,
|
|
"logps/rejected": -64.12794494628906,
|
|
"loss": 0.4024,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.9434598088264465,
|
|
"rewards/margins": 1.2422370910644531,
|
|
"rewards/rejected": -0.29877743124961853,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.22618284395435312,
|
|
"grad_norm": 56.275721433856205,
|
|
"learning_rate": 4.997540487747892e-07,
|
|
"logits/chosen": -1.2653698921203613,
|
|
"logits/rejected": -1.2802023887634277,
|
|
"logps/chosen": -44.68564987182617,
|
|
"logps/rejected": -66.80543518066406,
|
|
"loss": 0.4761,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.8570264577865601,
|
|
"rewards/margins": 1.2518484592437744,
|
|
"rewards/rejected": -0.39482197165489197,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.2307988203615848,
|
|
"grad_norm": 52.70179467657195,
|
|
"learning_rate": 4.996565036945769e-07,
|
|
"logits/chosen": -1.2993725538253784,
|
|
"logits/rejected": -1.3049986362457275,
|
|
"logps/chosen": -50.757686614990234,
|
|
"logps/rejected": -53.39494323730469,
|
|
"loss": 0.474,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.6508947014808655,
|
|
"rewards/margins": 0.9926181435585022,
|
|
"rewards/rejected": -0.3417234420776367,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.2307988203615848,
|
|
"eval_logits/chosen": -1.2166502475738525,
|
|
"eval_logits/rejected": -1.2277939319610596,
|
|
"eval_logps/chosen": -46.99800109863281,
|
|
"eval_logps/rejected": -56.576629638671875,
|
|
"eval_loss": 0.45405343174934387,
|
|
"eval_rewards/accuracies": 0.7511520981788635,
|
|
"eval_rewards/chosen": 0.8177129626274109,
|
|
"eval_rewards/margins": 1.0594747066497803,
|
|
"eval_rewards/rejected": -0.24176181852817535,
|
|
"eval_runtime": 227.4246,
|
|
"eval_samples_per_second": 7.625,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.23541479676881652,
|
|
"grad_norm": 44.630871285034665,
|
|
"learning_rate": 4.995427171437356e-07,
|
|
"logits/chosen": -1.2710050344467163,
|
|
"logits/rejected": -1.2925546169281006,
|
|
"logps/chosen": -44.29911422729492,
|
|
"logps/rejected": -63.83744430541992,
|
|
"loss": 0.3944,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.7817994356155396,
|
|
"rewards/margins": 1.3069223165512085,
|
|
"rewards/rejected": -0.5251227617263794,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.2400307731760482,
|
|
"grad_norm": 63.36947457687719,
|
|
"learning_rate": 4.994126965246796e-07,
|
|
"logits/chosen": -1.281785488128662,
|
|
"logits/rejected": -1.2921262979507446,
|
|
"logps/chosen": -45.61968994140625,
|
|
"logps/rejected": -57.57981872558594,
|
|
"loss": 0.4314,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.837566077709198,
|
|
"rewards/margins": 1.084651231765747,
|
|
"rewards/rejected": -0.247085303068161,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.24464674958327992,
|
|
"grad_norm": 64.3165584011508,
|
|
"learning_rate": 4.992664502959351e-07,
|
|
"logits/chosen": -1.2655751705169678,
|
|
"logits/rejected": -1.3007822036743164,
|
|
"logps/chosen": -42.23821258544922,
|
|
"logps/rejected": -85.14391326904297,
|
|
"loss": 0.3478,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.9540318250656128,
|
|
"rewards/margins": 1.9801336526870728,
|
|
"rewards/rejected": -1.0261015892028809,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.2492627259905116,
|
|
"grad_norm": 68.33245668142573,
|
|
"learning_rate": 4.991039879715898e-07,
|
|
"logits/chosen": -1.2239530086517334,
|
|
"logits/rejected": -1.2506436109542847,
|
|
"logps/chosen": -48.302852630615234,
|
|
"logps/rejected": -70.37635803222656,
|
|
"loss": 0.4095,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 1.1005271673202515,
|
|
"rewards/margins": 1.5407756567001343,
|
|
"rewards/rejected": -0.4402484893798828,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.2538787023977433,
|
|
"grad_norm": 41.58256253343475,
|
|
"learning_rate": 4.989253201206736e-07,
|
|
"logits/chosen": -1.3317803144454956,
|
|
"logits/rejected": -1.3334723711013794,
|
|
"logps/chosen": -47.94260787963867,
|
|
"logps/rejected": -48.03237533569336,
|
|
"loss": 0.4524,
|
|
"rewards/accuracies": 0.6944444179534912,
|
|
"rewards/chosen": 0.9174912571907043,
|
|
"rewards/margins": 1.0279741287231445,
|
|
"rewards/rejected": -0.11048289388418198,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.258494678804975,
|
|
"grad_norm": 66.23818262108296,
|
|
"learning_rate": 4.987304583664712e-07,
|
|
"logits/chosen": -1.2193766832351685,
|
|
"logits/rejected": -1.2288120985031128,
|
|
"logps/chosen": -55.089717864990234,
|
|
"logps/rejected": -61.21225357055664,
|
|
"loss": 0.4449,
|
|
"rewards/accuracies": 0.7222222089767456,
|
|
"rewards/chosen": 0.7610146999359131,
|
|
"rewards/margins": 1.0799916982650757,
|
|
"rewards/rejected": -0.3189769983291626,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.26311065521220667,
|
|
"grad_norm": 58.36439834992655,
|
|
"learning_rate": 4.985194153857662e-07,
|
|
"logits/chosen": -1.3395094871520996,
|
|
"logits/rejected": -1.3416942358016968,
|
|
"logps/chosen": -43.976890563964844,
|
|
"logps/rejected": -45.82760238647461,
|
|
"loss": 0.4929,
|
|
"rewards/accuracies": 0.7222222089767456,
|
|
"rewards/chosen": 0.4750349223613739,
|
|
"rewards/margins": 0.913371741771698,
|
|
"rewards/rejected": -0.4383367896080017,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.2677266316194384,
|
|
"grad_norm": 41.441657797600875,
|
|
"learning_rate": 4.982922049080163e-07,
|
|
"logits/chosen": -1.3572431802749634,
|
|
"logits/rejected": -1.3625115156173706,
|
|
"logps/chosen": -42.45515441894531,
|
|
"logps/rejected": -49.73173522949219,
|
|
"loss": 0.3691,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.41610708832740784,
|
|
"rewards/margins": 1.2800379991531372,
|
|
"rewards/rejected": -0.8639309406280518,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.2723426080266701,
|
|
"grad_norm": 60.30435878433939,
|
|
"learning_rate": 4.980488417144599e-07,
|
|
"logits/chosen": -1.2863659858703613,
|
|
"logits/rejected": -1.3199315071105957,
|
|
"logps/chosen": -48.62416076660156,
|
|
"logps/rejected": -85.73699188232422,
|
|
"loss": 0.4597,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.11831729859113693,
|
|
"rewards/margins": 1.6947745084762573,
|
|
"rewards/rejected": -1.5764573812484741,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.27695858443390176,
|
|
"grad_norm": 41.355652091388535,
|
|
"learning_rate": 4.977893416371544e-07,
|
|
"logits/chosen": -1.2884269952774048,
|
|
"logits/rejected": -1.2976269721984863,
|
|
"logps/chosen": -41.79518127441406,
|
|
"logps/rejected": -54.925662994384766,
|
|
"loss": 0.3826,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.0362311452627182,
|
|
"rewards/margins": 1.4945807456970215,
|
|
"rewards/rejected": -1.4583497047424316,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.27695858443390176,
|
|
"eval_logits/chosen": -1.207366704940796,
|
|
"eval_logits/rejected": -1.2173371315002441,
|
|
"eval_logps/chosen": -48.50833511352539,
|
|
"eval_logps/rejected": -58.64011764526367,
|
|
"eval_loss": 0.4093886911869049,
|
|
"eval_rewards/accuracies": 0.7724654674530029,
|
|
"eval_rewards/chosen": 0.06255079805850983,
|
|
"eval_rewards/margins": 1.336057424545288,
|
|
"eval_rewards/rejected": -1.27350652217865,
|
|
"eval_runtime": 227.1307,
|
|
"eval_samples_per_second": 7.634,
|
|
"eval_steps_per_second": 1.911,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.28157456084113347,
|
|
"grad_norm": 57.25843545889309,
|
|
"learning_rate": 4.975137215579469e-07,
|
|
"logits/chosen": -1.1866450309753418,
|
|
"logits/rejected": -1.186094045639038,
|
|
"logps/chosen": -55.403018951416016,
|
|
"logps/rejected": -51.23255920410156,
|
|
"loss": 0.4266,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.05289599671959877,
|
|
"rewards/margins": 1.1414172649383545,
|
|
"rewards/rejected": -1.0885213613510132,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.2861905372483652,
|
|
"grad_norm": 39.62322228531182,
|
|
"learning_rate": 4.972219994073755e-07,
|
|
"logits/chosen": -1.18235182762146,
|
|
"logits/rejected": -1.2118382453918457,
|
|
"logps/chosen": -48.611637115478516,
|
|
"logps/rejected": -79.41681671142578,
|
|
"loss": 0.3892,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": -0.14360421895980835,
|
|
"rewards/margins": 1.8774958848953247,
|
|
"rewards/rejected": -2.0211000442504883,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.2908065136555969,
|
|
"grad_norm": 52.17921097346343,
|
|
"learning_rate": 4.969141941635025e-07,
|
|
"logits/chosen": -1.2253869771957397,
|
|
"logits/rejected": -1.2435510158538818,
|
|
"logps/chosen": -50.45375061035156,
|
|
"logps/rejected": -69.55599212646484,
|
|
"loss": 0.4746,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.31797370314598083,
|
|
"rewards/margins": 1.7364860773086548,
|
|
"rewards/rejected": -2.054459571838379,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.29542249006282856,
|
|
"grad_norm": 70.38261738253976,
|
|
"learning_rate": 4.965903258506806e-07,
|
|
"logits/chosen": -1.1796499490737915,
|
|
"logits/rejected": -1.1957367658615112,
|
|
"logps/chosen": -49.15879440307617,
|
|
"logps/rejected": -73.3575210571289,
|
|
"loss": 0.3346,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.04160241410136223,
|
|
"rewards/margins": 1.7600233554840088,
|
|
"rewards/rejected": -1.7184207439422607,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.30003846647006027,
|
|
"grad_norm": 57.934452383223125,
|
|
"learning_rate": 4.962504155382493e-07,
|
|
"logits/chosen": -1.3256597518920898,
|
|
"logits/rejected": -1.3279542922973633,
|
|
"logps/chosen": -44.50282287597656,
|
|
"logps/rejected": -48.98247146606445,
|
|
"loss": 0.382,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.2423497587442398,
|
|
"rewards/margins": 1.2484761476516724,
|
|
"rewards/rejected": -1.0061264038085938,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.304654442877292,
|
|
"grad_norm": 58.71983276565311,
|
|
"learning_rate": 4.958944853391652e-07,
|
|
"logits/chosen": -1.1831316947937012,
|
|
"logits/rejected": -1.1885439157485962,
|
|
"logps/chosen": -45.726505279541016,
|
|
"logps/rejected": -54.00067138671875,
|
|
"loss": 0.4078,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 0.6056569814682007,
|
|
"rewards/margins": 1.2579783201217651,
|
|
"rewards/rejected": -0.6523212790489197,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.30927041928452365,
|
|
"grad_norm": 50.9411946225504,
|
|
"learning_rate": 4.955225584085624e-07,
|
|
"logits/chosen": -1.3628097772598267,
|
|
"logits/rejected": -1.3736504316329956,
|
|
"logps/chosen": -44.589229583740234,
|
|
"logps/rejected": -60.06134796142578,
|
|
"loss": 0.4245,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 0.8993210196495056,
|
|
"rewards/margins": 1.6077146530151367,
|
|
"rewards/rejected": -0.7083935737609863,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.31388639569175536,
|
|
"grad_norm": 53.494824083866554,
|
|
"learning_rate": 4.951346589422467e-07,
|
|
"logits/chosen": -1.2143707275390625,
|
|
"logits/rejected": -1.2256660461425781,
|
|
"logps/chosen": -44.73698425292969,
|
|
"logps/rejected": -63.74873733520508,
|
|
"loss": 0.4379,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.0820810794830322,
|
|
"rewards/margins": 1.6512118577957153,
|
|
"rewards/rejected": -0.5691307187080383,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.3185023720989871,
|
|
"grad_norm": 87.56798544187119,
|
|
"learning_rate": 4.94730812175122e-07,
|
|
"logits/chosen": -1.338615894317627,
|
|
"logits/rejected": -1.344118595123291,
|
|
"logps/chosen": -45.98448181152344,
|
|
"logps/rejected": -49.50959396362305,
|
|
"loss": 0.4706,
|
|
"rewards/accuracies": 0.6805555820465088,
|
|
"rewards/chosen": 1.05098295211792,
|
|
"rewards/margins": 1.33613121509552,
|
|
"rewards/rejected": -0.2851482033729553,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.32311834850621873,
|
|
"grad_norm": 68.21005063884613,
|
|
"learning_rate": 4.943110443795476e-07,
|
|
"logits/chosen": -1.258334755897522,
|
|
"logits/rejected": -1.2637797594070435,
|
|
"logps/chosen": -50.91215515136719,
|
|
"logps/rejected": -52.46604537963867,
|
|
"loss": 0.4759,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.7942720055580139,
|
|
"rewards/margins": 1.164951205253601,
|
|
"rewards/rejected": -0.37067916989326477,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.32311834850621873,
|
|
"eval_logits/chosen": -1.1890074014663696,
|
|
"eval_logits/rejected": -1.1990076303482056,
|
|
"eval_logps/chosen": -47.355777740478516,
|
|
"eval_logps/rejected": -58.06831359863281,
|
|
"eval_loss": 0.38173291087150574,
|
|
"eval_rewards/accuracies": 0.7718893885612488,
|
|
"eval_rewards/chosen": 0.6388264298439026,
|
|
"eval_rewards/margins": 1.6264294385910034,
|
|
"eval_rewards/rejected": -0.987602949142456,
|
|
"eval_runtime": 227.2808,
|
|
"eval_samples_per_second": 7.629,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.32773432491345045,
|
|
"grad_norm": 50.18122842606284,
|
|
"learning_rate": 4.938753828636297e-07,
|
|
"logits/chosen": -1.2129461765289307,
|
|
"logits/rejected": -1.21940279006958,
|
|
"logps/chosen": -53.97553253173828,
|
|
"logps/rejected": -53.66658020019531,
|
|
"loss": 0.4623,
|
|
"rewards/accuracies": 0.6805555820465088,
|
|
"rewards/chosen": 0.7083945274353027,
|
|
"rewards/margins": 1.2968156337738037,
|
|
"rewards/rejected": -0.5884211659431458,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.33235030132068216,
|
|
"grad_norm": 50.59660648914643,
|
|
"learning_rate": 4.934238559694447e-07,
|
|
"logits/chosen": -1.1950477361679077,
|
|
"logits/rejected": -1.2141423225402832,
|
|
"logps/chosen": -47.11637878417969,
|
|
"logps/rejected": -65.34754943847656,
|
|
"loss": 0.3506,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.5280824899673462,
|
|
"rewards/margins": 1.7753053903579712,
|
|
"rewards/rejected": -1.2472230195999146,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.3369662777279138,
|
|
"grad_norm": 48.072274874250304,
|
|
"learning_rate": 4.929564930711957e-07,
|
|
"logits/chosen": -1.281104564666748,
|
|
"logits/rejected": -1.2873430252075195,
|
|
"logps/chosen": -46.95094299316406,
|
|
"logps/rejected": -52.25088882446289,
|
|
"loss": 0.3785,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.3459606468677521,
|
|
"rewards/margins": 1.2943564653396606,
|
|
"rewards/rejected": -0.9483956098556519,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.34158225413514554,
|
|
"grad_norm": 45.84138607827678,
|
|
"learning_rate": 4.924733245733008e-07,
|
|
"logits/chosen": -1.168983817100525,
|
|
"logits/rejected": -1.1675101518630981,
|
|
"logps/chosen": -53.398841857910156,
|
|
"logps/rejected": -48.5836181640625,
|
|
"loss": 0.372,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 0.4610249102115631,
|
|
"rewards/margins": 1.1595698595046997,
|
|
"rewards/rejected": -0.6985449194908142,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.34619823054237725,
|
|
"grad_norm": 41.47855724733255,
|
|
"learning_rate": 4.91974381908416e-07,
|
|
"logits/chosen": -1.3004454374313354,
|
|
"logits/rejected": -1.3239775896072388,
|
|
"logps/chosen": -44.39426803588867,
|
|
"logps/rejected": -69.07200622558594,
|
|
"loss": 0.3055,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.01821739226579666,
|
|
"rewards/margins": 2.1426663398742676,
|
|
"rewards/rejected": -2.124448776245117,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.3508142069496089,
|
|
"grad_norm": 38.94525175640219,
|
|
"learning_rate": 4.914596975353898e-07,
|
|
"logits/chosen": -1.263897180557251,
|
|
"logits/rejected": -1.2764997482299805,
|
|
"logps/chosen": -47.19563674926758,
|
|
"logps/rejected": -57.78141403198242,
|
|
"loss": 0.386,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.055071063339710236,
|
|
"rewards/margins": 1.5383036136627197,
|
|
"rewards/rejected": -1.4832323789596558,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.3554301833568406,
|
|
"grad_norm": 66.86847446450086,
|
|
"learning_rate": 4.909293049371519e-07,
|
|
"logits/chosen": -1.2072829008102417,
|
|
"logits/rejected": -1.2103402614593506,
|
|
"logps/chosen": -55.665191650390625,
|
|
"logps/rejected": -53.18820571899414,
|
|
"loss": 0.4106,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.15528617799282074,
|
|
"rewards/margins": 1.4208853244781494,
|
|
"rewards/rejected": -1.2655991315841675,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.36004615976407234,
|
|
"grad_norm": 58.39718753271016,
|
|
"learning_rate": 4.903832386185343e-07,
|
|
"logits/chosen": -1.246012568473816,
|
|
"logits/rejected": -1.2516732215881348,
|
|
"logps/chosen": -52.729427337646484,
|
|
"logps/rejected": -53.41749572753906,
|
|
"loss": 0.4389,
|
|
"rewards/accuracies": 0.7361111044883728,
|
|
"rewards/chosen": 0.07690320909023285,
|
|
"rewards/margins": 1.1573313474655151,
|
|
"rewards/rejected": -1.080428123474121,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.364662136171304,
|
|
"grad_norm": 48.60983015034901,
|
|
"learning_rate": 4.89821534104028e-07,
|
|
"logits/chosen": -1.2771211862564087,
|
|
"logits/rejected": -1.2919840812683105,
|
|
"logps/chosen": -50.660606384277344,
|
|
"logps/rejected": -68.34771728515625,
|
|
"loss": 0.3141,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.26177337765693665,
|
|
"rewards/margins": 2.4097442626953125,
|
|
"rewards/rejected": -2.1479711532592773,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.3692781125785357,
|
|
"grad_norm": 53.41098297744381,
|
|
"learning_rate": 4.892442279354698e-07,
|
|
"logits/chosen": -1.2370442152023315,
|
|
"logits/rejected": -1.2537869215011597,
|
|
"logps/chosen": -52.96739959716797,
|
|
"logps/rejected": -71.32413482666016,
|
|
"loss": 0.3591,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": -0.09003262966871262,
|
|
"rewards/margins": 1.8999587297439575,
|
|
"rewards/rejected": -1.989991545677185,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.3692781125785357,
|
|
"eval_logits/chosen": -1.1776655912399292,
|
|
"eval_logits/rejected": -1.1867269277572632,
|
|
"eval_logps/chosen": -48.62409973144531,
|
|
"eval_logps/rejected": -59.5378303527832,
|
|
"eval_loss": 0.3585492968559265,
|
|
"eval_rewards/accuracies": 0.7908986210823059,
|
|
"eval_rewards/chosen": 0.0046647959388792515,
|
|
"eval_rewards/margins": 1.7270255088806152,
|
|
"eval_rewards/rejected": -1.7223609685897827,
|
|
"eval_runtime": 226.9778,
|
|
"eval_samples_per_second": 7.64,
|
|
"eval_steps_per_second": 1.912,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.3738940889857674,
|
|
"grad_norm": 48.30891742529218,
|
|
"learning_rate": 4.886513576696673e-07,
|
|
"logits/chosen": -1.2570397853851318,
|
|
"logits/rejected": -1.2753015756607056,
|
|
"logps/chosen": -50.679901123046875,
|
|
"logps/rejected": -69.58779907226562,
|
|
"loss": 0.3731,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.10189250111579895,
|
|
"rewards/margins": 1.991517424583435,
|
|
"rewards/rejected": -2.093410015106201,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.3785100653929991,
|
|
"grad_norm": 38.94107066608042,
|
|
"learning_rate": 4.880429618759543e-07,
|
|
"logits/chosen": -1.300181269645691,
|
|
"logits/rejected": -1.309295892715454,
|
|
"logps/chosen": -54.4401969909668,
|
|
"logps/rejected": -57.44432830810547,
|
|
"loss": 0.4199,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": -0.0659228190779686,
|
|
"rewards/margins": 1.5560578107833862,
|
|
"rewards/rejected": -1.6219807863235474,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.3831260418002308,
|
|
"grad_norm": 42.007218912580726,
|
|
"learning_rate": 4.874190801336817e-07,
|
|
"logits/chosen": -1.2063199281692505,
|
|
"logits/rejected": -1.2187817096710205,
|
|
"logps/chosen": -52.59967041015625,
|
|
"logps/rejected": -62.30234909057617,
|
|
"loss": 0.3373,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.0022201803512871265,
|
|
"rewards/margins": 1.9247175455093384,
|
|
"rewards/rejected": -1.9269376993179321,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.3877420182074625,
|
|
"grad_norm": 41.8692578527763,
|
|
"learning_rate": 4.867797530296431e-07,
|
|
"logits/chosen": -1.2532522678375244,
|
|
"logits/rejected": -1.2671799659729004,
|
|
"logps/chosen": -54.77018737792969,
|
|
"logps/rejected": -65.7540512084961,
|
|
"loss": 0.2993,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": -0.03232141211628914,
|
|
"rewards/margins": 2.12778377532959,
|
|
"rewards/rejected": -2.1601054668426514,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.39235799461469417,
|
|
"grad_norm": 34.82958012225073,
|
|
"learning_rate": 4.861250221554343e-07,
|
|
"logits/chosen": -1.208885669708252,
|
|
"logits/rejected": -1.2256441116333008,
|
|
"logps/chosen": -43.97615051269531,
|
|
"logps/rejected": -71.41213989257812,
|
|
"loss": 0.3062,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.07487620413303375,
|
|
"rewards/margins": 2.4894187450408936,
|
|
"rewards/rejected": -2.4145421981811523,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.3969739710219259,
|
|
"grad_norm": 31.38457100645333,
|
|
"learning_rate": 4.854549301047476e-07,
|
|
"logits/chosen": -1.232684850692749,
|
|
"logits/rejected": -1.2298487424850464,
|
|
"logps/chosen": -51.118751525878906,
|
|
"logps/rejected": -52.634708404541016,
|
|
"loss": 0.3429,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.21625421941280365,
|
|
"rewards/margins": 1.5239174365997314,
|
|
"rewards/rejected": -1.3076633214950562,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.4015899474291576,
|
|
"grad_norm": 51.50376706566093,
|
|
"learning_rate": 4.847695204706005e-07,
|
|
"logits/chosen": -1.26514732837677,
|
|
"logits/rejected": -1.266494631767273,
|
|
"logps/chosen": -46.0894889831543,
|
|
"logps/rejected": -49.9418830871582,
|
|
"loss": 0.3849,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.47246673703193665,
|
|
"rewards/margins": 1.3556917905807495,
|
|
"rewards/rejected": -0.88322514295578,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.40620592383638926,
|
|
"grad_norm": 39.155946696884016,
|
|
"learning_rate": 4.840688378425e-07,
|
|
"logits/chosen": -1.1615080833435059,
|
|
"logits/rejected": -1.1706922054290771,
|
|
"logps/chosen": -54.5849609375,
|
|
"logps/rejected": -67.2327651977539,
|
|
"loss": 0.2596,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.5416942238807678,
|
|
"rewards/margins": 2.2614989280700684,
|
|
"rewards/rejected": -1.7198045253753662,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.410821900243621,
|
|
"grad_norm": 41.671444907690415,
|
|
"learning_rate": 4.833529278035422e-07,
|
|
"logits/chosen": -1.30013108253479,
|
|
"logits/rejected": -1.3356281518936157,
|
|
"logps/chosen": -45.55073547363281,
|
|
"logps/rejected": -83.93589782714844,
|
|
"loss": 0.3114,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.6670578718185425,
|
|
"rewards/margins": 3.266463279724121,
|
|
"rewards/rejected": -2.599405527114868,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.4154378766508527,
|
|
"grad_norm": 39.182516678430176,
|
|
"learning_rate": 4.826218369274459e-07,
|
|
"logits/chosen": -1.1979715824127197,
|
|
"logits/rejected": -1.2180922031402588,
|
|
"logps/chosen": -46.9954948425293,
|
|
"logps/rejected": -76.03369140625,
|
|
"loss": 0.3212,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.5715588927268982,
|
|
"rewards/margins": 2.937516212463379,
|
|
"rewards/rejected": -2.3659567832946777,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.4154378766508527,
|
|
"eval_logits/chosen": -1.169126272201538,
|
|
"eval_logits/rejected": -1.1776955127716064,
|
|
"eval_logps/chosen": -46.824092864990234,
|
|
"eval_logps/rejected": -58.37122344970703,
|
|
"eval_loss": 0.33698517084121704,
|
|
"eval_rewards/accuracies": 0.7857142686843872,
|
|
"eval_rewards/chosen": 0.9046696424484253,
|
|
"eval_rewards/margins": 2.043729782104492,
|
|
"eval_rewards/rejected": -1.139060139656067,
|
|
"eval_runtime": 227.2193,
|
|
"eval_samples_per_second": 7.631,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.42005385305808435,
|
|
"grad_norm": 34.66268965474155,
|
|
"learning_rate": 4.818756127755237e-07,
|
|
"logits/chosen": -1.245609164237976,
|
|
"logits/rejected": -1.2478203773498535,
|
|
"logps/chosen": -44.18710708618164,
|
|
"logps/rejected": -48.15581130981445,
|
|
"loss": 0.2919,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 1.065934658050537,
|
|
"rewards/margins": 1.712632656097412,
|
|
"rewards/rejected": -0.6466982364654541,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.42466982946531606,
|
|
"grad_norm": 48.14030342071319,
|
|
"learning_rate": 4.811143038935873e-07,
|
|
"logits/chosen": -1.1818798780441284,
|
|
"logits/rejected": -1.189144253730774,
|
|
"logps/chosen": -51.0610237121582,
|
|
"logps/rejected": -54.243797302246094,
|
|
"loss": 0.3902,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 1.165444254875183,
|
|
"rewards/margins": 1.9199776649475098,
|
|
"rewards/rejected": -0.7545332908630371,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.4292858058725478,
|
|
"grad_norm": 45.03104875996758,
|
|
"learning_rate": 4.803379598087899e-07,
|
|
"logits/chosen": -1.2626315355300903,
|
|
"logits/rejected": -1.2647953033447266,
|
|
"logps/chosen": -47.61143493652344,
|
|
"logps/rejected": -46.91539764404297,
|
|
"loss": 0.3381,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.8671760559082031,
|
|
"rewards/margins": 1.6762652397155762,
|
|
"rewards/rejected": -0.8090891242027283,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.43390178227977944,
|
|
"grad_norm": 63.031567731654626,
|
|
"learning_rate": 4.795466310264034e-07,
|
|
"logits/chosen": -1.18837308883667,
|
|
"logits/rejected": -1.2106761932373047,
|
|
"logps/chosen": -45.96682357788086,
|
|
"logps/rejected": -75.37088012695312,
|
|
"loss": 0.4522,
|
|
"rewards/accuracies": 0.7222222089767456,
|
|
"rewards/chosen": 0.41675278544425964,
|
|
"rewards/margins": 2.4167346954345703,
|
|
"rewards/rejected": -1.9999819993972778,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.43851775868701115,
|
|
"grad_norm": 30.590120071737818,
|
|
"learning_rate": 4.787403690265335e-07,
|
|
"logits/chosen": -1.2696727514266968,
|
|
"logits/rejected": -1.2787107229232788,
|
|
"logps/chosen": -46.919532775878906,
|
|
"logps/rejected": -57.34629440307617,
|
|
"loss": 0.3259,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.6651458144187927,
|
|
"rewards/margins": 2.023615837097168,
|
|
"rewards/rejected": -1.358469843864441,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.44313373509424286,
|
|
"grad_norm": 65.00881113396572,
|
|
"learning_rate": 4.779192262607702e-07,
|
|
"logits/chosen": -1.1799297332763672,
|
|
"logits/rejected": -1.1909632682800293,
|
|
"logps/chosen": -51.90432357788086,
|
|
"logps/rejected": -67.85346221923828,
|
|
"loss": 0.3613,
|
|
"rewards/accuracies": 0.7361111044883728,
|
|
"rewards/chosen": 0.8239190578460693,
|
|
"rewards/margins": 2.4376637935638428,
|
|
"rewards/rejected": -1.6137449741363525,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.4477497115014745,
|
|
"grad_norm": 45.51586779707086,
|
|
"learning_rate": 4.770832561487758e-07,
|
|
"logits/chosen": -1.273619294166565,
|
|
"logits/rejected": -1.2734426259994507,
|
|
"logps/chosen": -51.00930404663086,
|
|
"logps/rejected": -50.82846450805664,
|
|
"loss": 0.2704,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.6313294172286987,
|
|
"rewards/margins": 1.9725594520568848,
|
|
"rewards/rejected": -1.3412299156188965,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.45236568790870624,
|
|
"grad_norm": 59.66597715979004,
|
|
"learning_rate": 4.762325130748097e-07,
|
|
"logits/chosen": -1.1289526224136353,
|
|
"logits/rejected": -1.1318069696426392,
|
|
"logps/chosen": -56.16923522949219,
|
|
"logps/rejected": -53.92496109008789,
|
|
"loss": 0.3301,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.5848253965377808,
|
|
"rewards/margins": 1.7668564319610596,
|
|
"rewards/rejected": -1.1820309162139893,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.45698166431593795,
|
|
"grad_norm": 27.39596009585691,
|
|
"learning_rate": 4.7536705238418995e-07,
|
|
"logits/chosen": -1.2408547401428223,
|
|
"logits/rejected": -1.2465531826019287,
|
|
"logps/chosen": -51.9420280456543,
|
|
"logps/rejected": -59.4866943359375,
|
|
"loss": 0.2815,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.4031212031841278,
|
|
"rewards/margins": 2.25166654586792,
|
|
"rewards/rejected": -1.8485453128814697,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.4615976407231696,
|
|
"grad_norm": 55.89572206023644,
|
|
"learning_rate": 4.7448693037969336e-07,
|
|
"logits/chosen": -1.221846103668213,
|
|
"logits/rejected": -1.229733943939209,
|
|
"logps/chosen": -51.18299102783203,
|
|
"logps/rejected": -57.01997375488281,
|
|
"loss": 0.3319,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.33577215671539307,
|
|
"rewards/margins": 1.9459080696105957,
|
|
"rewards/rejected": -1.6101359128952026,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.4615976407231696,
|
|
"eval_logits/chosen": -1.1536659002304077,
|
|
"eval_logits/rejected": -1.1624053716659546,
|
|
"eval_logps/chosen": -47.9106559753418,
|
|
"eval_logps/rejected": -59.707550048828125,
|
|
"eval_loss": 0.3186802566051483,
|
|
"eval_rewards/accuracies": 0.8012672662734985,
|
|
"eval_rewards/chosen": 0.3613872528076172,
|
|
"eval_rewards/margins": 2.168609857559204,
|
|
"eval_rewards/rejected": -1.807222604751587,
|
|
"eval_runtime": 227.3404,
|
|
"eval_samples_per_second": 7.627,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.4662136171304013,
|
|
"grad_norm": 47.53414859111433,
|
|
"learning_rate": 4.735922043178923e-07,
|
|
"logits/chosen": -1.130042552947998,
|
|
"logits/rejected": -1.1414666175842285,
|
|
"logps/chosen": -50.747615814208984,
|
|
"logps/rejected": -67.29204559326172,
|
|
"loss": 0.2861,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.3648765981197357,
|
|
"rewards/margins": 2.223259449005127,
|
|
"rewards/rejected": -1.8583827018737793,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.47082959353763304,
|
|
"grad_norm": 45.15212642774012,
|
|
"learning_rate": 4.7268293240543017e-07,
|
|
"logits/chosen": -1.2075278759002686,
|
|
"logits/rejected": -1.2145761251449585,
|
|
"logps/chosen": -50.80036163330078,
|
|
"logps/rejected": -64.42232513427734,
|
|
"loss": 0.3686,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.3706679344177246,
|
|
"rewards/margins": 2.0169572830200195,
|
|
"rewards/rejected": -1.646289348602295,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.4754455699448647,
|
|
"grad_norm": 51.8551798480754,
|
|
"learning_rate": 4.717591737952344e-07,
|
|
"logits/chosen": -1.225889801979065,
|
|
"logits/rejected": -1.2406290769577026,
|
|
"logps/chosen": -42.99085235595703,
|
|
"logps/rejected": -63.13933563232422,
|
|
"loss": 0.31,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.2732833921909332,
|
|
"rewards/margins": 2.319049835205078,
|
|
"rewards/rejected": -2.045766592025757,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.4800615463520964,
|
|
"grad_norm": 45.48745944142938,
|
|
"learning_rate": 4.7082098858266837e-07,
|
|
"logits/chosen": -1.2216678857803345,
|
|
"logits/rejected": -1.2374674081802368,
|
|
"logps/chosen": -39.35933303833008,
|
|
"logps/rejected": -69.94754791259766,
|
|
"loss": 0.3897,
|
|
"rewards/accuracies": 0.7083333134651184,
|
|
"rewards/chosen": 0.15072119235992432,
|
|
"rewards/margins": 2.5561957359313965,
|
|
"rewards/rejected": -2.4054746627807617,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.4846775227593281,
|
|
"grad_norm": 23.330292154413335,
|
|
"learning_rate": 4.698684378016222e-07,
|
|
"logits/chosen": -1.235012412071228,
|
|
"logits/rejected": -1.2410335540771484,
|
|
"logps/chosen": -51.16822052001953,
|
|
"logps/rejected": -67.03107452392578,
|
|
"loss": 0.2774,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.5210573673248291,
|
|
"rewards/margins": 2.396049976348877,
|
|
"rewards/rejected": -1.8749926090240479,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.48929349916655984,
|
|
"grad_norm": 44.38100018119066,
|
|
"learning_rate": 4.6890158342054174e-07,
|
|
"logits/chosen": -1.2579890489578247,
|
|
"logits/rejected": -1.2686585187911987,
|
|
"logps/chosen": -43.69232177734375,
|
|
"logps/rejected": -59.38545608520508,
|
|
"loss": 0.3209,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.6999004483222961,
|
|
"rewards/margins": 2.572566032409668,
|
|
"rewards/rejected": -1.8726658821105957,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.4939094755737915,
|
|
"grad_norm": 48.05758145861009,
|
|
"learning_rate": 4.679204883383973e-07,
|
|
"logits/chosen": -1.291759729385376,
|
|
"logits/rejected": -1.311813235282898,
|
|
"logps/chosen": -42.26872253417969,
|
|
"logps/rejected": -75.81320190429688,
|
|
"loss": 0.2963,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.7384893894195557,
|
|
"rewards/margins": 3.459284782409668,
|
|
"rewards/rejected": -2.7207956314086914,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.4985254519810232,
|
|
"grad_norm": 49.231820908366814,
|
|
"learning_rate": 4.669252163805919e-07,
|
|
"logits/chosen": -1.2944141626358032,
|
|
"logits/rejected": -1.3142738342285156,
|
|
"logps/chosen": -45.93452453613281,
|
|
"logps/rejected": -61.74488830566406,
|
|
"loss": 0.3873,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.7554634809494019,
|
|
"rewards/margins": 2.4260568618774414,
|
|
"rewards/rejected": -1.6705933809280396,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.5031414283882549,
|
|
"grad_norm": 39.371230612811594,
|
|
"learning_rate": 4.65915832294809e-07,
|
|
"logits/chosen": -1.2367289066314697,
|
|
"logits/rejected": -1.2460105419158936,
|
|
"logps/chosen": -43.46434783935547,
|
|
"logps/rejected": -65.94113159179688,
|
|
"loss": 0.3123,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.8203165531158447,
|
|
"rewards/margins": 2.6564347743988037,
|
|
"rewards/rejected": -1.836118221282959,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.5077574047954866,
|
|
"grad_norm": 36.504230004493856,
|
|
"learning_rate": 4.6489240174680026e-07,
|
|
"logits/chosen": -1.274338722229004,
|
|
"logits/rejected": -1.274038314819336,
|
|
"logps/chosen": -46.72291564941406,
|
|
"logps/rejected": -46.98860549926758,
|
|
"loss": 0.3653,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.7758186459541321,
|
|
"rewards/margins": 1.619003415107727,
|
|
"rewards/rejected": -0.8431846499443054,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.5077574047954866,
|
|
"eval_logits/chosen": -1.1508095264434814,
|
|
"eval_logits/rejected": -1.158846139907837,
|
|
"eval_logps/chosen": -47.08699035644531,
|
|
"eval_logps/rejected": -59.3014030456543,
|
|
"eval_loss": 0.30850934982299805,
|
|
"eval_rewards/accuracies": 0.8029953837394714,
|
|
"eval_rewards/chosen": 0.7732176184654236,
|
|
"eval_rewards/margins": 2.377366065979004,
|
|
"eval_rewards/rejected": -1.6041483879089355,
|
|
"eval_runtime": 227.3591,
|
|
"eval_samples_per_second": 7.627,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.5123733812027182,
|
|
"grad_norm": 58.78605593626179,
|
|
"learning_rate": 4.638549913161138e-07,
|
|
"logits/chosen": -1.163451910018921,
|
|
"logits/rejected": -1.1646764278411865,
|
|
"logps/chosen": -52.2976188659668,
|
|
"logps/rejected": -53.40987777709961,
|
|
"loss": 0.2728,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.7431490421295166,
|
|
"rewards/margins": 2.2729909420013428,
|
|
"rewards/rejected": -1.5298418998718262,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.51698935760995,
|
|
"grad_norm": 47.481692611666446,
|
|
"learning_rate": 4.6280366849176267e-07,
|
|
"logits/chosen": -1.1689667701721191,
|
|
"logits/rejected": -1.176836371421814,
|
|
"logps/chosen": -48.97719192504883,
|
|
"logps/rejected": -53.70207977294922,
|
|
"loss": 0.3088,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.4173290729522705,
|
|
"rewards/margins": 1.9205392599105835,
|
|
"rewards/rejected": -1.503210186958313,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.5216053340171817,
|
|
"grad_norm": 26.23879949976686,
|
|
"learning_rate": 4.6173850166783446e-07,
|
|
"logits/chosen": -1.1005306243896484,
|
|
"logits/rejected": -1.1028097867965698,
|
|
"logps/chosen": -48.85009002685547,
|
|
"logps/rejected": -60.73183822631836,
|
|
"loss": 0.2688,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.3439117670059204,
|
|
"rewards/margins": 2.138643741607666,
|
|
"rewards/rejected": -1.7947319746017456,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.5262213104244133,
|
|
"grad_norm": 42.68538113603284,
|
|
"learning_rate": 4.606595601390417e-07,
|
|
"logits/chosen": -1.1647553443908691,
|
|
"logits/rejected": -1.1835236549377441,
|
|
"logps/chosen": -47.58196258544922,
|
|
"logps/rejected": -73.24917602539062,
|
|
"loss": 0.2677,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.032037004828453064,
|
|
"rewards/margins": 2.9576616287231445,
|
|
"rewards/rejected": -2.9256248474121094,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.5308372868316451,
|
|
"grad_norm": 47.39085343776366,
|
|
"learning_rate": 4.595669140962143e-07,
|
|
"logits/chosen": -1.2832393646240234,
|
|
"logits/rejected": -1.3168139457702637,
|
|
"logps/chosen": -42.34735870361328,
|
|
"logps/rejected": -91.76000213623047,
|
|
"loss": 0.3269,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.22074511647224426,
|
|
"rewards/margins": 3.9149208068847656,
|
|
"rewards/rejected": -4.1356658935546875,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.5354532632388768,
|
|
"grad_norm": 34.48196631675489,
|
|
"learning_rate": 4.5846063462173284e-07,
|
|
"logits/chosen": -1.2154712677001953,
|
|
"logits/rejected": -1.2207145690917969,
|
|
"logps/chosen": -46.80333709716797,
|
|
"logps/rejected": -61.56758499145508,
|
|
"loss": 0.299,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.07934803515672684,
|
|
"rewards/margins": 2.2728195190429688,
|
|
"rewards/rejected": -2.1934714317321777,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.5400692396461084,
|
|
"grad_norm": 53.84175279608908,
|
|
"learning_rate": 4.573407936849044e-07,
|
|
"logits/chosen": -1.235826015472412,
|
|
"logits/rejected": -1.2379093170166016,
|
|
"logps/chosen": -53.666229248046875,
|
|
"logps/rejected": -56.39564895629883,
|
|
"loss": 0.34,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.15499740839004517,
|
|
"rewards/margins": 2.093020439147949,
|
|
"rewards/rejected": -1.9380230903625488,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.5446852160533402,
|
|
"grad_norm": 55.22566434624621,
|
|
"learning_rate": 4.5620746413728063e-07,
|
|
"logits/chosen": -1.168860912322998,
|
|
"logits/rejected": -1.167116641998291,
|
|
"logps/chosen": -59.95442199707031,
|
|
"logps/rejected": -55.52897644042969,
|
|
"loss": 0.2556,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.048602912575006485,
|
|
"rewards/margins": 2.1055731773376465,
|
|
"rewards/rejected": -2.0569701194763184,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.5493011924605719,
|
|
"grad_norm": 40.26544195874405,
|
|
"learning_rate": 4.550607197079185e-07,
|
|
"logits/chosen": -1.1609958410263062,
|
|
"logits/rejected": -1.158744215965271,
|
|
"logps/chosen": -46.247127532958984,
|
|
"logps/rejected": -53.768096923828125,
|
|
"loss": 0.2776,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.5583436489105225,
|
|
"rewards/margins": 1.9103565216064453,
|
|
"rewards/rejected": -1.3520128726959229,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.5539171688678035,
|
|
"grad_norm": 22.296733757945297,
|
|
"learning_rate": 4.5390063499858353e-07,
|
|
"logits/chosen": -1.143466591835022,
|
|
"logits/rejected": -1.1510720252990723,
|
|
"logps/chosen": -56.236507415771484,
|
|
"logps/rejected": -72.3839111328125,
|
|
"loss": 0.2216,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 0.30002227425575256,
|
|
"rewards/margins": 2.8752281665802,
|
|
"rewards/rejected": -2.5752060413360596,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5539171688678035,
|
|
"eval_logits/chosen": -1.1655231714248657,
|
|
"eval_logits/rejected": -1.1714258193969727,
|
|
"eval_logps/chosen": -47.62031173706055,
|
|
"eval_logps/rejected": -60.05898666381836,
|
|
"eval_loss": 0.2956756353378296,
|
|
"eval_rewards/accuracies": 0.8093317747116089,
|
|
"eval_rewards/chosen": 0.5065575242042542,
|
|
"eval_rewards/margins": 2.4894962310791016,
|
|
"eval_rewards/rejected": -1.9829388856887817,
|
|
"eval_runtime": 227.4458,
|
|
"eval_samples_per_second": 7.624,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5585331452750353,
|
|
"grad_norm": 25.00341489978113,
|
|
"learning_rate": 4.5272728547889687e-07,
|
|
"logits/chosen": -1.2413771152496338,
|
|
"logits/rejected": -1.2534655332565308,
|
|
"logps/chosen": -51.89939880371094,
|
|
"logps/rejected": -61.60024642944336,
|
|
"loss": 0.2157,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.444425106048584,
|
|
"rewards/margins": 3.039713144302368,
|
|
"rewards/rejected": -2.595287799835205,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 0.5631491216822669,
|
|
"grad_norm": 59.288877946216886,
|
|
"learning_rate": 4.5154074748142535e-07,
|
|
"logits/chosen": -1.1979435682296753,
|
|
"logits/rejected": -1.200371265411377,
|
|
"logps/chosen": -51.88737106323242,
|
|
"logps/rejected": -62.96666717529297,
|
|
"loss": 0.3125,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.536859929561615,
|
|
"rewards/margins": 2.3762218952178955,
|
|
"rewards/rejected": -1.8393617868423462,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 0.5677650980894986,
|
|
"grad_norm": 49.81197105903562,
|
|
"learning_rate": 4.503410981967158e-07,
|
|
"logits/chosen": -1.1927361488342285,
|
|
"logits/rejected": -1.2023940086364746,
|
|
"logps/chosen": -43.52791213989258,
|
|
"logps/rejected": -68.50804138183594,
|
|
"loss": 0.3784,
|
|
"rewards/accuracies": 0.7083333134651184,
|
|
"rewards/chosen": 0.9149570465087891,
|
|
"rewards/margins": 2.78161883354187,
|
|
"rewards/rejected": -1.8666616678237915,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 0.5723810744967304,
|
|
"grad_norm": 36.729769896842384,
|
|
"learning_rate": 4.4912841566827333e-07,
|
|
"logits/chosen": -1.2108688354492188,
|
|
"logits/rejected": -1.2207088470458984,
|
|
"logps/chosen": -47.57191848754883,
|
|
"logps/rejected": -65.24411010742188,
|
|
"loss": 0.2484,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 1.3121000528335571,
|
|
"rewards/margins": 2.9495744705200195,
|
|
"rewards/rejected": -1.6374740600585938,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 0.576997050903962,
|
|
"grad_norm": 52.94413715920982,
|
|
"learning_rate": 4.4790277878748415e-07,
|
|
"logits/chosen": -1.2431470155715942,
|
|
"logits/rejected": -1.2514811754226685,
|
|
"logps/chosen": -41.430206298828125,
|
|
"logps/rejected": -57.99823760986328,
|
|
"loss": 0.2988,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.8287627696990967,
|
|
"rewards/margins": 2.483140707015991,
|
|
"rewards/rejected": -1.6543784141540527,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.5816130273111938,
|
|
"grad_norm": 35.54104658985613,
|
|
"learning_rate": 4.466642672884835e-07,
|
|
"logits/chosen": -1.1631712913513184,
|
|
"logits/rejected": -1.168829321861267,
|
|
"logps/chosen": -44.09839630126953,
|
|
"logps/rejected": -58.30562973022461,
|
|
"loss": 0.2578,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.6901536583900452,
|
|
"rewards/margins": 2.6523261070251465,
|
|
"rewards/rejected": -1.9621726274490356,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 0.5862290037184255,
|
|
"grad_norm": 37.73217497274028,
|
|
"learning_rate": 4.454129617429682e-07,
|
|
"logits/chosen": -1.2659639120101929,
|
|
"logits/rejected": -1.26548433303833,
|
|
"logps/chosen": -48.81840515136719,
|
|
"logps/rejected": -50.18553161621094,
|
|
"loss": 0.303,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.6821924448013306,
|
|
"rewards/margins": 2.1100261211395264,
|
|
"rewards/rejected": -1.4278337955474854,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 0.5908449801256571,
|
|
"grad_norm": 19.374735161669783,
|
|
"learning_rate": 4.441489435549551e-07,
|
|
"logits/chosen": -1.1399126052856445,
|
|
"logits/rejected": -1.1459710597991943,
|
|
"logps/chosen": -51.82578659057617,
|
|
"logps/rejected": -68.86441040039062,
|
|
"loss": 0.2382,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.7167325615882874,
|
|
"rewards/margins": 3.1428277492523193,
|
|
"rewards/rejected": -2.426095485687256,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 0.5954609565328889,
|
|
"grad_norm": 41.763567555359415,
|
|
"learning_rate": 4.4287229495548573e-07,
|
|
"logits/chosen": -1.183684229850769,
|
|
"logits/rejected": -1.190173625946045,
|
|
"logps/chosen": -53.23274230957031,
|
|
"logps/rejected": -66.342041015625,
|
|
"loss": 0.2552,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.7042552828788757,
|
|
"rewards/margins": 3.324800729751587,
|
|
"rewards/rejected": -2.6205453872680664,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 0.6000769329401205,
|
|
"grad_norm": 53.08605142258815,
|
|
"learning_rate": 4.415830989972761e-07,
|
|
"logits/chosen": -1.1716980934143066,
|
|
"logits/rejected": -1.1756532192230225,
|
|
"logps/chosen": -48.24959182739258,
|
|
"logps/rejected": -57.04353713989258,
|
|
"loss": 0.2986,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.6846582889556885,
|
|
"rewards/margins": 2.4726505279541016,
|
|
"rewards/rejected": -1.7879924774169922,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.6000769329401205,
|
|
"eval_logits/chosen": -1.1839433908462524,
|
|
"eval_logits/rejected": -1.1875815391540527,
|
|
"eval_logps/chosen": -47.51332092285156,
|
|
"eval_logps/rejected": -60.25726318359375,
|
|
"eval_loss": 0.2889851927757263,
|
|
"eval_rewards/accuracies": 0.8139401078224182,
|
|
"eval_rewards/chosen": 0.5600550174713135,
|
|
"eval_rewards/margins": 2.6421334743499756,
|
|
"eval_rewards/rejected": -2.082078218460083,
|
|
"eval_runtime": 227.3437,
|
|
"eval_samples_per_second": 7.627,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.6046929093473522,
|
|
"grad_norm": 40.94616402060229,
|
|
"learning_rate": 4.402814395493142e-07,
|
|
"logits/chosen": -1.2763242721557617,
|
|
"logits/rejected": -1.2830497026443481,
|
|
"logps/chosen": -46.679168701171875,
|
|
"logps/rejected": -46.02785873413086,
|
|
"loss": 0.3389,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.5309950113296509,
|
|
"rewards/margins": 1.931261658668518,
|
|
"rewards/rejected": -1.4002668857574463,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 0.609308885754584,
|
|
"grad_norm": 37.45563925423882,
|
|
"learning_rate": 4.3896740129140354e-07,
|
|
"logits/chosen": -1.2274925708770752,
|
|
"logits/rejected": -1.2251402139663696,
|
|
"logps/chosen": -49.84899139404297,
|
|
"logps/rejected": -48.70707702636719,
|
|
"loss": 0.2565,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.42680761218070984,
|
|
"rewards/margins": 2.4617788791656494,
|
|
"rewards/rejected": -2.0349714756011963,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 0.6139248621618156,
|
|
"grad_norm": 39.43164361196429,
|
|
"learning_rate": 4.3764106970865456e-07,
|
|
"logits/chosen": -1.2649712562561035,
|
|
"logits/rejected": -1.2683297395706177,
|
|
"logps/chosen": -42.68367004394531,
|
|
"logps/rejected": -59.22146224975586,
|
|
"loss": 0.3223,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.032459404319524765,
|
|
"rewards/margins": 2.20072603225708,
|
|
"rewards/rejected": -2.168266534805298,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 0.6185408385690473,
|
|
"grad_norm": 38.60665005525895,
|
|
"learning_rate": 4.3630253108592305e-07,
|
|
"logits/chosen": -1.1689542531967163,
|
|
"logits/rejected": -1.1707943677902222,
|
|
"logps/chosen": -55.54972839355469,
|
|
"logps/rejected": -63.90589141845703,
|
|
"loss": 0.2454,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.198521688580513,
|
|
"rewards/margins": 2.9068400859832764,
|
|
"rewards/rejected": -3.1053614616394043,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 0.6231568149762791,
|
|
"grad_norm": 52.06719541190528,
|
|
"learning_rate": 4.3495187250219723e-07,
|
|
"logits/chosen": -1.2435555458068848,
|
|
"logits/rejected": -1.247399926185608,
|
|
"logps/chosen": -44.81391906738281,
|
|
"logps/rejected": -66.15491485595703,
|
|
"loss": 0.3261,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.1677350103855133,
|
|
"rewards/margins": 2.8860220909118652,
|
|
"rewards/rejected": -3.053757429122925,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.6277727913835107,
|
|
"grad_norm": 34.05944822234142,
|
|
"learning_rate": 4.3358918182493253e-07,
|
|
"logits/chosen": -1.1470799446105957,
|
|
"logits/rejected": -1.146039366722107,
|
|
"logps/chosen": -48.27168273925781,
|
|
"logps/rejected": -55.25007629394531,
|
|
"loss": 0.2124,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 0.0416293665766716,
|
|
"rewards/margins": 2.1943235397338867,
|
|
"rewards/rejected": -2.1526942253112793,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 0.6323887677907424,
|
|
"grad_norm": 32.55880144659725,
|
|
"learning_rate": 4.3221454770433554e-07,
|
|
"logits/chosen": -1.2215373516082764,
|
|
"logits/rejected": -1.223750352859497,
|
|
"logps/chosen": -53.73216247558594,
|
|
"logps/rejected": -58.15590286254883,
|
|
"loss": 0.217,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.4631071090698242,
|
|
"rewards/margins": 2.812567949295044,
|
|
"rewards/rejected": -2.3494608402252197,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 0.6370047441979741,
|
|
"grad_norm": 37.29286642220413,
|
|
"learning_rate": 4.308280595675966e-07,
|
|
"logits/chosen": -1.2593313455581665,
|
|
"logits/rejected": -1.2601191997528076,
|
|
"logps/chosen": -51.24105453491211,
|
|
"logps/rejected": -58.06007766723633,
|
|
"loss": 0.305,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.41045814752578735,
|
|
"rewards/margins": 2.3772544860839844,
|
|
"rewards/rejected": -1.9667962789535522,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 0.6416207206052058,
|
|
"grad_norm": 27.73832644868097,
|
|
"learning_rate": 4.2942980761307227e-07,
|
|
"logits/chosen": -1.2309229373931885,
|
|
"logits/rejected": -1.2335686683654785,
|
|
"logps/chosen": -48.84478759765625,
|
|
"logps/rejected": -58.93284225463867,
|
|
"loss": 0.2202,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.5729148387908936,
|
|
"rewards/margins": 2.5622036457061768,
|
|
"rewards/rejected": -1.9892889261245728,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 0.6462366970124375,
|
|
"grad_norm": 49.846247584121116,
|
|
"learning_rate": 4.2801988280441765e-07,
|
|
"logits/chosen": -1.2041369676589966,
|
|
"logits/rejected": -1.2069140672683716,
|
|
"logps/chosen": -50.05432891845703,
|
|
"logps/rejected": -59.74306106567383,
|
|
"loss": 0.2356,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4567064046859741,
|
|
"rewards/margins": 2.9142255783081055,
|
|
"rewards/rejected": -1.4575190544128418,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.6462366970124375,
|
|
"eval_logits/chosen": -1.1684162616729736,
|
|
"eval_logits/rejected": -1.1727546453475952,
|
|
"eval_logps/chosen": -45.66713333129883,
|
|
"eval_logps/rejected": -58.96503829956055,
|
|
"eval_loss": 0.28808632493019104,
|
|
"eval_rewards/accuracies": 0.807603657245636,
|
|
"eval_rewards/chosen": 1.483147144317627,
|
|
"eval_rewards/margins": 2.9191133975982666,
|
|
"eval_rewards/rejected": -1.4359666109085083,
|
|
"eval_runtime": 227.2327,
|
|
"eval_samples_per_second": 7.631,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.6508526734196692,
|
|
"grad_norm": 37.200050161533866,
|
|
"learning_rate": 4.2659837686466813e-07,
|
|
"logits/chosen": -1.2258718013763428,
|
|
"logits/rejected": -1.226365327835083,
|
|
"logps/chosen": -45.65945816040039,
|
|
"logps/rejected": -56.360023498535156,
|
|
"loss": 0.2663,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 1.7170668840408325,
|
|
"rewards/margins": 2.946417808532715,
|
|
"rewards/rejected": -1.2293510437011719,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 0.6554686498269009,
|
|
"grad_norm": 46.91080670819706,
|
|
"learning_rate": 4.25165382270273e-07,
|
|
"logits/chosen": -1.194913387298584,
|
|
"logits/rejected": -1.1949591636657715,
|
|
"logps/chosen": -41.45118713378906,
|
|
"logps/rejected": -49.03273391723633,
|
|
"loss": 0.2576,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 1.5383753776550293,
|
|
"rewards/margins": 2.4985907077789307,
|
|
"rewards/rejected": -0.9602153301239014,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 0.6600846262341326,
|
|
"grad_norm": 38.28106252628991,
|
|
"learning_rate": 4.2372099224507875e-07,
|
|
"logits/chosen": -1.262522578239441,
|
|
"logits/rejected": -1.2750235795974731,
|
|
"logps/chosen": -38.96574401855469,
|
|
"logps/rejected": -67.78875732421875,
|
|
"loss": 0.2815,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 1.0857527256011963,
|
|
"rewards/margins": 3.374871253967285,
|
|
"rewards/rejected": -2.289118766784668,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 0.6647006026413643,
|
|
"grad_norm": 48.61343601009988,
|
|
"learning_rate": 4.2226530075426503e-07,
|
|
"logits/chosen": -1.1528538465499878,
|
|
"logits/rejected": -1.1555874347686768,
|
|
"logps/chosen": -56.5745849609375,
|
|
"logps/rejected": -58.78268051147461,
|
|
"loss": 0.2806,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.9941625595092773,
|
|
"rewards/margins": 2.6789402961730957,
|
|
"rewards/rejected": -1.6847774982452393,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 0.669316579048596,
|
|
"grad_norm": 41.895687140764245,
|
|
"learning_rate": 4.2079840249823106e-07,
|
|
"logits/chosen": -1.1860905885696411,
|
|
"logits/rejected": -1.1894373893737793,
|
|
"logps/chosen": -50.26545715332031,
|
|
"logps/rejected": -72.03532409667969,
|
|
"loss": 0.289,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.2916366159915924,
|
|
"rewards/margins": 3.1359870433807373,
|
|
"rewards/rejected": -2.8443503379821777,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.6739325554558276,
|
|
"grad_norm": 41.288114692753076,
|
|
"learning_rate": 4.193203929064353e-07,
|
|
"logits/chosen": -1.2005698680877686,
|
|
"logits/rejected": -1.2083783149719238,
|
|
"logps/chosen": -51.12635803222656,
|
|
"logps/rejected": -71.51712036132812,
|
|
"loss": 0.345,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.0697176456451416,
|
|
"rewards/margins": 2.9638874530792236,
|
|
"rewards/rejected": -2.894169807434082,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 0.6785485318630594,
|
|
"grad_norm": 42.3623221323418,
|
|
"learning_rate": 4.1783136813118705e-07,
|
|
"logits/chosen": -1.222592830657959,
|
|
"logits/rejected": -1.225614309310913,
|
|
"logps/chosen": -50.67860794067383,
|
|
"logps/rejected": -61.38881301879883,
|
|
"loss": 0.2915,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": -0.12122215330600739,
|
|
"rewards/margins": 2.681208610534668,
|
|
"rewards/rejected": -2.8024303913116455,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 0.6831645082702911,
|
|
"grad_norm": 22.982390981962695,
|
|
"learning_rate": 4.163314250413913e-07,
|
|
"logits/chosen": -1.1687074899673462,
|
|
"logits/rejected": -1.165205955505371,
|
|
"logps/chosen": -46.08445739746094,
|
|
"logps/rejected": -56.52994918823242,
|
|
"loss": 0.1833,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.2055283784866333,
|
|
"rewards/margins": 2.5608911514282227,
|
|
"rewards/rejected": -2.355362892150879,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 0.6877804846775227,
|
|
"grad_norm": 37.84043310920863,
|
|
"learning_rate": 4.1482066121624716e-07,
|
|
"logits/chosen": -1.207397222518921,
|
|
"logits/rejected": -1.208192229270935,
|
|
"logps/chosen": -49.74457550048828,
|
|
"logps/rejected": -50.62785339355469,
|
|
"loss": 0.3247,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.17621606588363647,
|
|
"rewards/margins": 2.258789539337158,
|
|
"rewards/rejected": -2.082573413848877,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 0.6923964610847545,
|
|
"grad_norm": 41.026889158159065,
|
|
"learning_rate": 4.1329917493889933e-07,
|
|
"logits/chosen": -1.3157416582107544,
|
|
"logits/rejected": -1.3190845251083374,
|
|
"logps/chosen": -46.27326965332031,
|
|
"logps/rejected": -61.60360336303711,
|
|
"loss": 0.2407,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.021905170753598213,
|
|
"rewards/margins": 2.830016613006592,
|
|
"rewards/rejected": -2.8081114292144775,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6923964610847545,
|
|
"eval_logits/chosen": -1.1676603555679321,
|
|
"eval_logits/rejected": -1.1713868379592896,
|
|
"eval_logps/chosen": -47.389183044433594,
|
|
"eval_logps/rejected": -60.791683197021484,
|
|
"eval_loss": 0.2743883430957794,
|
|
"eval_rewards/accuracies": 0.8133640289306641,
|
|
"eval_rewards/chosen": 0.6221204400062561,
|
|
"eval_rewards/margins": 2.9714088439941406,
|
|
"eval_rewards/rejected": -2.349287986755371,
|
|
"eval_runtime": 227.521,
|
|
"eval_samples_per_second": 7.621,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6970124374919862,
|
|
"grad_norm": 42.14326526501706,
|
|
"learning_rate": 4.117670651900446e-07,
|
|
"logits/chosen": -1.2038425207138062,
|
|
"logits/rejected": -1.2065904140472412,
|
|
"logps/chosen": -50.9475212097168,
|
|
"logps/rejected": -56.376590728759766,
|
|
"loss": 0.3052,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.660882830619812,
|
|
"rewards/margins": 2.4505326747894287,
|
|
"rewards/rejected": -1.789649486541748,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 0.7016284138992178,
|
|
"grad_norm": 59.036735448430065,
|
|
"learning_rate": 4.1022443164149237e-07,
|
|
"logits/chosen": -1.1808403730392456,
|
|
"logits/rejected": -1.1900469064712524,
|
|
"logps/chosen": -51.46991729736328,
|
|
"logps/rejected": -69.61980438232422,
|
|
"loss": 0.2946,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.5994511842727661,
|
|
"rewards/margins": 3.409363269805908,
|
|
"rewards/rejected": -2.8099122047424316,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 0.7062443903064496,
|
|
"grad_norm": 29.330872231920097,
|
|
"learning_rate": 4.086713746496808e-07,
|
|
"logits/chosen": -1.2124900817871094,
|
|
"logits/rejected": -1.2128535509109497,
|
|
"logps/chosen": -45.35523223876953,
|
|
"logps/rejected": -56.160545349121094,
|
|
"loss": 0.3006,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.7718818187713623,
|
|
"rewards/margins": 2.726104259490967,
|
|
"rewards/rejected": -1.954222559928894,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 0.7108603667136812,
|
|
"grad_norm": 34.74292770737631,
|
|
"learning_rate": 4.0710799524914805e-07,
|
|
"logits/chosen": -1.1383283138275146,
|
|
"logits/rejected": -1.1413955688476562,
|
|
"logps/chosen": -55.99458312988281,
|
|
"logps/rejected": -62.90273666381836,
|
|
"loss": 0.2295,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.6053013205528259,
|
|
"rewards/margins": 2.9940264225006104,
|
|
"rewards/rejected": -2.3887250423431396,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 0.7154763431209129,
|
|
"grad_norm": 30.476243441323902,
|
|
"learning_rate": 4.055343951459592e-07,
|
|
"logits/chosen": -1.191731572151184,
|
|
"logits/rejected": -1.1988056898117065,
|
|
"logps/chosen": -44.046875,
|
|
"logps/rejected": -64.41764068603516,
|
|
"loss": 0.2429,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.2551959455013275,
|
|
"rewards/margins": 3.2936155796051025,
|
|
"rewards/rejected": -3.038419485092163,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.7200923195281447,
|
|
"grad_norm": 30.700879501780552,
|
|
"learning_rate": 4.0395067671108985e-07,
|
|
"logits/chosen": -1.2448992729187012,
|
|
"logits/rejected": -1.2440135478973389,
|
|
"logps/chosen": -41.98812484741211,
|
|
"logps/rejected": -49.34878921508789,
|
|
"loss": 0.2697,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.3203623592853546,
|
|
"rewards/margins": 2.6056876182556152,
|
|
"rewards/rejected": -2.285325050354004,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 0.7247082959353763,
|
|
"grad_norm": 42.617870752566596,
|
|
"learning_rate": 4.0235694297376637e-07,
|
|
"logits/chosen": -1.139160394668579,
|
|
"logits/rejected": -1.1415181159973145,
|
|
"logps/chosen": -58.74495315551758,
|
|
"logps/rejected": -63.99246597290039,
|
|
"loss": 0.2614,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.32423093914985657,
|
|
"rewards/margins": 2.9529130458831787,
|
|
"rewards/rejected": -2.6286821365356445,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 0.729324272342608,
|
|
"grad_norm": 40.78600953952273,
|
|
"learning_rate": 4.0075329761476347e-07,
|
|
"logits/chosen": -1.216194748878479,
|
|
"logits/rejected": -1.2174675464630127,
|
|
"logps/chosen": -50.62156677246094,
|
|
"logps/rejected": -53.62016296386719,
|
|
"loss": 0.2407,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.07387811690568924,
|
|
"rewards/margins": 2.2588860988616943,
|
|
"rewards/rejected": -2.332764148712158,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 0.7339402487498398,
|
|
"grad_norm": 30.002748027873594,
|
|
"learning_rate": 3.991398449596588e-07,
|
|
"logits/chosen": -1.2065101861953735,
|
|
"logits/rejected": -1.211814045906067,
|
|
"logps/chosen": -53.182777404785156,
|
|
"logps/rejected": -66.31592559814453,
|
|
"loss": 0.209,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.0831962302327156,
|
|
"rewards/margins": 3.359541177749634,
|
|
"rewards/rejected": -3.2763442993164062,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 0.7385562251570714,
|
|
"grad_norm": 23.665196371870486,
|
|
"learning_rate": 3.9751668997204647e-07,
|
|
"logits/chosen": -1.1500531435012817,
|
|
"logits/rejected": -1.154016375541687,
|
|
"logps/chosen": -52.478214263916016,
|
|
"logps/rejected": -61.09043884277344,
|
|
"loss": 0.199,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.16100816428661346,
|
|
"rewards/margins": 2.913641929626465,
|
|
"rewards/rejected": -2.752634048461914,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.7385562251570714,
|
|
"eval_logits/chosen": -1.152096152305603,
|
|
"eval_logits/rejected": -1.1563034057617188,
|
|
"eval_logps/chosen": -48.12975311279297,
|
|
"eval_logps/rejected": -61.73065948486328,
|
|
"eval_loss": 0.2689039707183838,
|
|
"eval_rewards/accuracies": 0.8185483813285828,
|
|
"eval_rewards/chosen": 0.2518383860588074,
|
|
"eval_rewards/margins": 3.0706160068511963,
|
|
"eval_rewards/rejected": -2.818777322769165,
|
|
"eval_runtime": 227.7908,
|
|
"eval_samples_per_second": 7.612,
|
|
"eval_steps_per_second": 1.905,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.7431722015643031,
|
|
"grad_norm": 48.30684834494479,
|
|
"learning_rate": 3.958839382467084e-07,
|
|
"logits/chosen": -1.222053050994873,
|
|
"logits/rejected": -1.2289328575134277,
|
|
"logps/chosen": -43.89524459838867,
|
|
"logps/rejected": -56.49114990234375,
|
|
"loss": 0.3092,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.43327876925468445,
|
|
"rewards/margins": 2.904792308807373,
|
|
"rewards/rejected": -2.471513032913208,
|
|
"step": 322
|
|
},
|
|
{
|
|
"epoch": 0.7477881779715349,
|
|
"grad_norm": 42.66239436350315,
|
|
"learning_rate": 3.9424169600274494e-07,
|
|
"logits/chosen": -1.2450088262557983,
|
|
"logits/rejected": -1.24375581741333,
|
|
"logps/chosen": -50.182308197021484,
|
|
"logps/rejected": -56.25563049316406,
|
|
"loss": 0.2956,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.13703547418117523,
|
|
"rewards/margins": 2.3541619777679443,
|
|
"rewards/rejected": -2.4911975860595703,
|
|
"step": 324
|
|
},
|
|
{
|
|
"epoch": 0.7524041543787665,
|
|
"grad_norm": 27.173798633882665,
|
|
"learning_rate": 3.9259007007666436e-07,
|
|
"logits/chosen": -1.212989330291748,
|
|
"logits/rejected": -1.2185275554656982,
|
|
"logps/chosen": -51.97760009765625,
|
|
"logps/rejected": -63.76836395263672,
|
|
"loss": 0.282,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.34300458431243896,
|
|
"rewards/margins": 2.9810492992401123,
|
|
"rewards/rejected": -2.638044595718384,
|
|
"step": 326
|
|
},
|
|
{
|
|
"epoch": 0.7570201307859982,
|
|
"grad_norm": 41.510696079488966,
|
|
"learning_rate": 3.909291679154332e-07,
|
|
"logits/chosen": -1.2237902879714966,
|
|
"logits/rejected": -1.232872486114502,
|
|
"logps/chosen": -48.094669342041016,
|
|
"logps/rejected": -73.24827575683594,
|
|
"loss": 0.3194,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": -0.10229718685150146,
|
|
"rewards/margins": 3.730062246322632,
|
|
"rewards/rejected": -3.832359790802002,
|
|
"step": 328
|
|
},
|
|
{
|
|
"epoch": 0.7616361071932299,
|
|
"grad_norm": 36.28616141840599,
|
|
"learning_rate": 3.892590975694858e-07,
|
|
"logits/chosen": -1.2199351787567139,
|
|
"logits/rejected": -1.2302178144454956,
|
|
"logps/chosen": -44.97592544555664,
|
|
"logps/rejected": -71.27304077148438,
|
|
"loss": 0.23,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.4918098449707031,
|
|
"rewards/margins": 4.322786331176758,
|
|
"rewards/rejected": -3.830976724624634,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.7662520836004616,
|
|
"grad_norm": 23.858921961912298,
|
|
"learning_rate": 3.875799676856952e-07,
|
|
"logits/chosen": -1.1334155797958374,
|
|
"logits/rejected": -1.1350713968276978,
|
|
"logps/chosen": -50.02524948120117,
|
|
"logps/rejected": -62.23411560058594,
|
|
"loss": 0.2071,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.42125022411346436,
|
|
"rewards/margins": 3.270005226135254,
|
|
"rewards/rejected": -2.8487555980682373,
|
|
"step": 332
|
|
},
|
|
{
|
|
"epoch": 0.7708680600076933,
|
|
"grad_norm": 46.04858312353762,
|
|
"learning_rate": 3.858918875003053e-07,
|
|
"logits/chosen": -1.2362879514694214,
|
|
"logits/rejected": -1.2460957765579224,
|
|
"logps/chosen": -48.63706970214844,
|
|
"logps/rejected": -72.94087219238281,
|
|
"loss": 0.2514,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.5030243396759033,
|
|
"rewards/margins": 4.145318031311035,
|
|
"rewards/rejected": -3.64229416847229,
|
|
"step": 334
|
|
},
|
|
{
|
|
"epoch": 0.775484036414925,
|
|
"grad_norm": 21.37867116532646,
|
|
"learning_rate": 3.8419496683182396e-07,
|
|
"logits/chosen": -1.1005958318710327,
|
|
"logits/rejected": -1.1029105186462402,
|
|
"logps/chosen": -46.87886047363281,
|
|
"logps/rejected": -63.63945007324219,
|
|
"loss": 0.1861,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.5341323018074036,
|
|
"rewards/margins": 3.1560137271881104,
|
|
"rewards/rejected": -2.6218814849853516,
|
|
"step": 336
|
|
},
|
|
{
|
|
"epoch": 0.7801000128221567,
|
|
"grad_norm": 34.41230269099275,
|
|
"learning_rate": 3.824893160738792e-07,
|
|
"logits/chosen": -1.1643880605697632,
|
|
"logits/rejected": -1.1722698211669922,
|
|
"logps/chosen": -47.30472946166992,
|
|
"logps/rejected": -66.55563354492188,
|
|
"loss": 0.2727,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.9107217788696289,
|
|
"rewards/margins": 3.5939722061157227,
|
|
"rewards/rejected": -2.683250904083252,
|
|
"step": 338
|
|
},
|
|
{
|
|
"epoch": 0.7847159892293883,
|
|
"grad_norm": 20.704541622004893,
|
|
"learning_rate": 3.8077504618803737e-07,
|
|
"logits/chosen": -1.1662912368774414,
|
|
"logits/rejected": -1.1619421243667603,
|
|
"logps/chosen": -56.10293197631836,
|
|
"logps/rejected": -53.84746170043945,
|
|
"loss": 0.2129,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.9300886988639832,
|
|
"rewards/margins": 2.765589475631714,
|
|
"rewards/rejected": -1.8355004787445068,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7847159892293883,
|
|
"eval_logits/chosen": -1.1408087015151978,
|
|
"eval_logits/rejected": -1.1458781957626343,
|
|
"eval_logps/chosen": -47.36520004272461,
|
|
"eval_logps/rejected": -61.242679595947266,
|
|
"eval_loss": 0.26537051796913147,
|
|
"eval_rewards/accuracies": 0.8185483813285828,
|
|
"eval_rewards/chosen": 0.6341149806976318,
|
|
"eval_rewards/margins": 3.208899736404419,
|
|
"eval_rewards/rejected": -2.574784755706787,
|
|
"eval_runtime": 227.4228,
|
|
"eval_samples_per_second": 7.625,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7893319656366201,
|
|
"grad_norm": 29.91569671776041,
|
|
"learning_rate": 3.7905226869658446e-07,
|
|
"logits/chosen": -1.1567282676696777,
|
|
"logits/rejected": -1.1574435234069824,
|
|
"logps/chosen": -50.19194793701172,
|
|
"logps/rejected": -61.67422866821289,
|
|
"loss": 0.2453,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.956231415271759,
|
|
"rewards/margins": 3.3507864475250244,
|
|
"rewards/rejected": -2.394554853439331,
|
|
"step": 342
|
|
},
|
|
{
|
|
"epoch": 0.7939479420438518,
|
|
"grad_norm": 35.82510237615759,
|
|
"learning_rate": 3.773210956752709e-07,
|
|
"logits/chosen": -1.1932220458984375,
|
|
"logits/rejected": -1.1889605522155762,
|
|
"logps/chosen": -46.48088073730469,
|
|
"logps/rejected": -49.8135986328125,
|
|
"loss": 0.2891,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 0.5073845982551575,
|
|
"rewards/margins": 2.379626989364624,
|
|
"rewards/rejected": -1.8722424507141113,
|
|
"step": 344
|
|
},
|
|
{
|
|
"epoch": 0.7985639184510834,
|
|
"grad_norm": 28.47228811073422,
|
|
"learning_rate": 3.7558163974602093e-07,
|
|
"logits/chosen": -1.1920644044876099,
|
|
"logits/rejected": -1.201475977897644,
|
|
"logps/chosen": -43.6226806640625,
|
|
"logps/rejected": -64.39244842529297,
|
|
"loss": 0.2762,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.10022352635860443,
|
|
"rewards/margins": 3.255174398422241,
|
|
"rewards/rejected": -3.1549510955810547,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 0.8031798948583152,
|
|
"grad_norm": 41.618837869458865,
|
|
"learning_rate": 3.73834014069605e-07,
|
|
"logits/chosen": -1.1160246133804321,
|
|
"logits/rejected": -1.1237598657608032,
|
|
"logps/chosen": -55.9088249206543,
|
|
"logps/rejected": -71.82222747802734,
|
|
"loss": 0.236,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.08166421949863434,
|
|
"rewards/margins": 3.3770830631256104,
|
|
"rewards/rejected": -3.2954187393188477,
|
|
"step": 348
|
|
},
|
|
{
|
|
"epoch": 0.8077958712655469,
|
|
"grad_norm": 34.292113974031,
|
|
"learning_rate": 3.7207833233827914e-07,
|
|
"logits/chosen": -1.221280813217163,
|
|
"logits/rejected": -1.228824257850647,
|
|
"logps/chosen": -51.23641586303711,
|
|
"logps/rejected": -67.17003631591797,
|
|
"loss": 0.2952,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": -0.29954856634140015,
|
|
"rewards/margins": 3.7896947860717773,
|
|
"rewards/rejected": -4.089242935180664,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.8124118476727785,
|
|
"grad_norm": 27.12194701535223,
|
|
"learning_rate": 3.7031470876838786e-07,
|
|
"logits/chosen": -1.1533750295639038,
|
|
"logits/rejected": -1.1603755950927734,
|
|
"logps/chosen": -49.57859802246094,
|
|
"logps/rejected": -74.35897064208984,
|
|
"loss": 0.2616,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.34589090943336487,
|
|
"rewards/margins": 3.77805495262146,
|
|
"rewards/rejected": -4.123946189880371,
|
|
"step": 352
|
|
},
|
|
{
|
|
"epoch": 0.8170278240800103,
|
|
"grad_norm": 33.865318639168144,
|
|
"learning_rate": 3.6854325809293455e-07,
|
|
"logits/chosen": -1.2225959300994873,
|
|
"logits/rejected": -1.2331852912902832,
|
|
"logps/chosen": -42.565975189208984,
|
|
"logps/rejected": -74.49568939208984,
|
|
"loss": 0.2349,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.32742711901664734,
|
|
"rewards/margins": 4.166980743408203,
|
|
"rewards/rejected": -4.494408130645752,
|
|
"step": 354
|
|
},
|
|
{
|
|
"epoch": 0.821643800487242,
|
|
"grad_norm": 44.72468218463322,
|
|
"learning_rate": 3.6676409555411653e-07,
|
|
"logits/chosen": -1.1373627185821533,
|
|
"logits/rejected": -1.1474027633666992,
|
|
"logps/chosen": -51.0811882019043,
|
|
"logps/rejected": -69.32566833496094,
|
|
"loss": 0.2441,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.21452119946479797,
|
|
"rewards/margins": 3.681729555130005,
|
|
"rewards/rejected": -3.8962512016296387,
|
|
"step": 356
|
|
},
|
|
{
|
|
"epoch": 0.8262597768944736,
|
|
"grad_norm": 18.3343181780076,
|
|
"learning_rate": 3.6497733689582866e-07,
|
|
"logits/chosen": -1.1717758178710938,
|
|
"logits/rejected": -1.1715316772460938,
|
|
"logps/chosen": -45.530574798583984,
|
|
"logps/rejected": -56.84959030151367,
|
|
"loss": 0.1942,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.07019754499197006,
|
|
"rewards/margins": 3.02067232131958,
|
|
"rewards/rejected": -2.950474739074707,
|
|
"step": 358
|
|
},
|
|
{
|
|
"epoch": 0.8308757533017054,
|
|
"grad_norm": 35.844182387455,
|
|
"learning_rate": 3.631830983561335e-07,
|
|
"logits/chosen": -1.1136425733566284,
|
|
"logits/rejected": -1.1134952306747437,
|
|
"logps/chosen": -54.07844924926758,
|
|
"logps/rejected": -60.01144027709961,
|
|
"loss": 0.2174,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.009207261726260185,
|
|
"rewards/margins": 3.035071611404419,
|
|
"rewards/rejected": -3.025864601135254,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.8308757533017054,
|
|
"eval_logits/chosen": -1.137829303741455,
|
|
"eval_logits/rejected": -1.142533779144287,
|
|
"eval_logps/chosen": -49.13446807861328,
|
|
"eval_logps/rejected": -63.1205940246582,
|
|
"eval_loss": 0.2611147463321686,
|
|
"eval_rewards/accuracies": 0.8231566548347473,
|
|
"eval_rewards/chosen": -0.25051993131637573,
|
|
"eval_rewards/margins": 3.2632253170013428,
|
|
"eval_rewards/rejected": -3.5137455463409424,
|
|
"eval_runtime": 227.4188,
|
|
"eval_samples_per_second": 7.625,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.835491729708937,
|
|
"grad_norm": 36.48149555986552,
|
|
"learning_rate": 3.613814966596991e-07,
|
|
"logits/chosen": -1.1987216472625732,
|
|
"logits/rejected": -1.2045807838439941,
|
|
"logps/chosen": -49.24374008178711,
|
|
"logps/rejected": -66.37528228759766,
|
|
"loss": 0.2604,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.2955743074417114,
|
|
"rewards/margins": 3.364227533340454,
|
|
"rewards/rejected": -3.659801959991455,
|
|
"step": 362
|
|
},
|
|
{
|
|
"epoch": 0.8401077061161687,
|
|
"grad_norm": 17.123248215692428,
|
|
"learning_rate": 3.595726490102059e-07,
|
|
"logits/chosen": -1.1486543416976929,
|
|
"logits/rejected": -1.155872106552124,
|
|
"logps/chosen": -47.52320098876953,
|
|
"logps/rejected": -72.59225463867188,
|
|
"loss": 0.1309,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": -0.2556496858596802,
|
|
"rewards/margins": 3.9981601238250732,
|
|
"rewards/rejected": -4.253809928894043,
|
|
"step": 364
|
|
},
|
|
{
|
|
"epoch": 0.8447236825234005,
|
|
"grad_norm": 22.608323262045403,
|
|
"learning_rate": 3.577566730827214e-07,
|
|
"logits/chosen": -1.1728885173797607,
|
|
"logits/rejected": -1.1819621324539185,
|
|
"logps/chosen": -47.8609504699707,
|
|
"logps/rejected": -66.17147064208984,
|
|
"loss": 0.2715,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.29853469133377075,
|
|
"rewards/margins": 3.493640661239624,
|
|
"rewards/rejected": -3.792175769805908,
|
|
"step": 366
|
|
},
|
|
{
|
|
"epoch": 0.8493396589306321,
|
|
"grad_norm": 37.233167771755866,
|
|
"learning_rate": 3.559336870160453e-07,
|
|
"logits/chosen": -1.1891751289367676,
|
|
"logits/rejected": -1.193422794342041,
|
|
"logps/chosen": -43.88676834106445,
|
|
"logps/rejected": -61.30946350097656,
|
|
"loss": 0.2259,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": -0.18390725553035736,
|
|
"rewards/margins": 3.307628631591797,
|
|
"rewards/rejected": -3.4915361404418945,
|
|
"step": 368
|
|
},
|
|
{
|
|
"epoch": 0.8539556353378638,
|
|
"grad_norm": 26.22453096528665,
|
|
"learning_rate": 3.541038094050241e-07,
|
|
"logits/chosen": -1.155517339706421,
|
|
"logits/rejected": -1.1603643894195557,
|
|
"logps/chosen": -52.52191162109375,
|
|
"logps/rejected": -73.11701202392578,
|
|
"loss": 0.1787,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.36556607484817505,
|
|
"rewards/margins": 4.345615863800049,
|
|
"rewards/rejected": -4.711181163787842,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.8585716117450956,
|
|
"grad_norm": 39.171509700373235,
|
|
"learning_rate": 3.52267159292835e-07,
|
|
"logits/chosen": -1.1714898347854614,
|
|
"logits/rejected": -1.1753745079040527,
|
|
"logps/chosen": -52.40201950073242,
|
|
"logps/rejected": -71.5418701171875,
|
|
"loss": 0.2399,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.4184909164905548,
|
|
"rewards/margins": 3.973634958267212,
|
|
"rewards/rejected": -4.392125606536865,
|
|
"step": 372
|
|
},
|
|
{
|
|
"epoch": 0.8631875881523272,
|
|
"grad_norm": 23.054835061356226,
|
|
"learning_rate": 3.5042385616324236e-07,
|
|
"logits/chosen": -1.3357490301132202,
|
|
"logits/rejected": -1.345274806022644,
|
|
"logps/chosen": -41.36846923828125,
|
|
"logps/rejected": -68.94818115234375,
|
|
"loss": 0.2237,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.5928651094436646,
|
|
"rewards/margins": 4.040990352630615,
|
|
"rewards/rejected": -4.63385534286499,
|
|
"step": 374
|
|
},
|
|
{
|
|
"epoch": 0.8678035645595589,
|
|
"grad_norm": 23.34114570013692,
|
|
"learning_rate": 3.485740199328244e-07,
|
|
"logits/chosen": -1.1034616231918335,
|
|
"logits/rejected": -1.1045833826065063,
|
|
"logps/chosen": -55.426727294921875,
|
|
"logps/rejected": -59.57293701171875,
|
|
"loss": 0.1942,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.04073745757341385,
|
|
"rewards/margins": 3.197631597518921,
|
|
"rewards/rejected": -3.2383692264556885,
|
|
"step": 376
|
|
},
|
|
{
|
|
"epoch": 0.8724195409667906,
|
|
"grad_norm": 19.26131705604103,
|
|
"learning_rate": 3.4671777094317196e-07,
|
|
"logits/chosen": -1.1123476028442383,
|
|
"logits/rejected": -1.1115697622299194,
|
|
"logps/chosen": -54.29782485961914,
|
|
"logps/rejected": -60.28561782836914,
|
|
"loss": 0.1845,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.4158029556274414,
|
|
"rewards/margins": 3.2452781200408936,
|
|
"rewards/rejected": -3.661081075668335,
|
|
"step": 378
|
|
},
|
|
{
|
|
"epoch": 0.8770355173740223,
|
|
"grad_norm": 40.47170182440331,
|
|
"learning_rate": 3.448552299530595e-07,
|
|
"logits/chosen": -1.1856770515441895,
|
|
"logits/rejected": -1.188659429550171,
|
|
"logps/chosen": -47.90871047973633,
|
|
"logps/rejected": -58.21343231201172,
|
|
"loss": 0.2866,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.3236946761608124,
|
|
"rewards/margins": 3.1452713012695312,
|
|
"rewards/rejected": -3.468966245651245,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.8770355173740223,
|
|
"eval_logits/chosen": -1.1315786838531494,
|
|
"eval_logits/rejected": -1.1364408731460571,
|
|
"eval_logps/chosen": -48.8283576965332,
|
|
"eval_logps/rejected": -63.289493560791016,
|
|
"eval_loss": 0.2587234079837799,
|
|
"eval_rewards/accuracies": 0.820852518081665,
|
|
"eval_rewards/chosen": -0.09746361523866653,
|
|
"eval_rewards/margins": 3.500731945037842,
|
|
"eval_rewards/rejected": -3.5981955528259277,
|
|
"eval_runtime": 227.2598,
|
|
"eval_samples_per_second": 7.63,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.881651493781254,
|
|
"grad_norm": 50.06622297059798,
|
|
"learning_rate": 3.429865181305894e-07,
|
|
"logits/chosen": -1.1800260543823242,
|
|
"logits/rejected": -1.1820400953292847,
|
|
"logps/chosen": -53.985992431640625,
|
|
"logps/rejected": -63.98917770385742,
|
|
"loss": 0.3147,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": -0.11410641670227051,
|
|
"rewards/margins": 3.317570447921753,
|
|
"rewards/rejected": -3.4316766262054443,
|
|
"step": 382
|
|
},
|
|
{
|
|
"epoch": 0.8862674701884857,
|
|
"grad_norm": 50.74505917044877,
|
|
"learning_rate": 3.411117570453091e-07,
|
|
"logits/chosen": -1.1595518589019775,
|
|
"logits/rejected": -1.1614227294921875,
|
|
"logps/chosen": -48.32596969604492,
|
|
"logps/rejected": -61.07521057128906,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.054149970412254333,
|
|
"rewards/margins": 3.22495174407959,
|
|
"rewards/rejected": -3.279102087020874,
|
|
"step": 384
|
|
},
|
|
{
|
|
"epoch": 0.8908834465957174,
|
|
"grad_norm": 33.82726208082807,
|
|
"learning_rate": 3.392310686603025e-07,
|
|
"logits/chosen": -1.2322266101837158,
|
|
"logits/rejected": -1.235073447227478,
|
|
"logps/chosen": -48.9878044128418,
|
|
"logps/rejected": -57.65345001220703,
|
|
"loss": 0.3178,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": -0.5888361930847168,
|
|
"rewards/margins": 2.6325643062591553,
|
|
"rewards/rejected": -3.221400260925293,
|
|
"step": 386
|
|
},
|
|
{
|
|
"epoch": 0.895499423002949,
|
|
"grad_norm": 31.968246601846598,
|
|
"learning_rate": 3.3734457532425554e-07,
|
|
"logits/chosen": -1.14549720287323,
|
|
"logits/rejected": -1.1533808708190918,
|
|
"logps/chosen": -47.60829544067383,
|
|
"logps/rejected": -65.90504455566406,
|
|
"loss": 0.2893,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": -0.4989345967769623,
|
|
"rewards/margins": 3.784000873565674,
|
|
"rewards/rejected": -4.28293514251709,
|
|
"step": 388
|
|
},
|
|
{
|
|
"epoch": 0.9001153994101808,
|
|
"grad_norm": 38.79859580208747,
|
|
"learning_rate": 3.354523997634969e-07,
|
|
"logits/chosen": -1.1520222425460815,
|
|
"logits/rejected": -1.159006953239441,
|
|
"logps/chosen": -51.533203125,
|
|
"logps/rejected": -68.82389068603516,
|
|
"loss": 0.2439,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.5372670292854309,
|
|
"rewards/margins": 3.8664119243621826,
|
|
"rewards/rejected": -4.403678894042969,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.9047313758174125,
|
|
"grad_norm": 48.99149097647652,
|
|
"learning_rate": 3.3355466507401374e-07,
|
|
"logits/chosen": -1.1964970827102661,
|
|
"logits/rejected": -1.195936679840088,
|
|
"logps/chosen": -49.486724853515625,
|
|
"logps/rejected": -51.76087951660156,
|
|
"loss": 0.3225,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": -0.6623955965042114,
|
|
"rewards/margins": 2.538459539413452,
|
|
"rewards/rejected": -3.200855255126953,
|
|
"step": 392
|
|
},
|
|
{
|
|
"epoch": 0.9093473522246441,
|
|
"grad_norm": 34.31601060375262,
|
|
"learning_rate": 3.3165149471344394e-07,
|
|
"logits/chosen": -1.1766917705535889,
|
|
"logits/rejected": -1.1769944429397583,
|
|
"logps/chosen": -50.33845901489258,
|
|
"logps/rejected": -60.9412841796875,
|
|
"loss": 0.2826,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.7761159539222717,
|
|
"rewards/margins": 2.734633445739746,
|
|
"rewards/rejected": -3.510749340057373,
|
|
"step": 394
|
|
},
|
|
{
|
|
"epoch": 0.9139633286318759,
|
|
"grad_norm": 44.03751755180035,
|
|
"learning_rate": 3.297430124930444e-07,
|
|
"logits/chosen": -1.0980035066604614,
|
|
"logits/rejected": -1.098144292831421,
|
|
"logps/chosen": -56.62092590332031,
|
|
"logps/rejected": -61.69611358642578,
|
|
"loss": 0.3528,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -0.423416405916214,
|
|
"rewards/margins": 2.6373236179351807,
|
|
"rewards/rejected": -3.0607402324676514,
|
|
"step": 396
|
|
},
|
|
{
|
|
"epoch": 0.9185793050391076,
|
|
"grad_norm": 20.653109576582093,
|
|
"learning_rate": 3.2782934256963647e-07,
|
|
"logits/chosen": -1.1482434272766113,
|
|
"logits/rejected": -1.1585140228271484,
|
|
"logps/chosen": -52.95622253417969,
|
|
"logps/rejected": -71.6562728881836,
|
|
"loss": 0.2816,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": -0.51191645860672,
|
|
"rewards/margins": 3.753627300262451,
|
|
"rewards/rejected": -4.265543460845947,
|
|
"step": 398
|
|
},
|
|
{
|
|
"epoch": 0.9231952814463392,
|
|
"grad_norm": 33.58507227677532,
|
|
"learning_rate": 3.259106094375289e-07,
|
|
"logits/chosen": -1.1832419633865356,
|
|
"logits/rejected": -1.1917306184768677,
|
|
"logps/chosen": -46.096675872802734,
|
|
"logps/rejected": -71.52079010009766,
|
|
"loss": 0.2501,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.2719300389289856,
|
|
"rewards/margins": 4.045370101928711,
|
|
"rewards/rejected": -4.317299842834473,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.9231952814463392,
|
|
"eval_logits/chosen": -1.1317284107208252,
|
|
"eval_logits/rejected": -1.1366225481033325,
|
|
"eval_logps/chosen": -49.24271011352539,
|
|
"eval_logps/rejected": -63.64417266845703,
|
|
"eval_loss": 0.2558155655860901,
|
|
"eval_rewards/accuracies": 0.820852518081665,
|
|
"eval_rewards/chosen": -0.30464252829551697,
|
|
"eval_rewards/margins": 3.470890522003174,
|
|
"eval_rewards/rejected": -3.775533437728882,
|
|
"eval_runtime": 227.3955,
|
|
"eval_samples_per_second": 7.625,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.927811257853571,
|
|
"grad_norm": 40.05600195486227,
|
|
"learning_rate": 3.239869379204189e-07,
|
|
"logits/chosen": -1.165150761604309,
|
|
"logits/rejected": -1.1658389568328857,
|
|
"logps/chosen": -51.360679626464844,
|
|
"logps/rejected": -65.45414733886719,
|
|
"loss": 0.2124,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.36249446868896484,
|
|
"rewards/margins": 3.780195951461792,
|
|
"rewards/rejected": -4.142690658569336,
|
|
"step": 402
|
|
},
|
|
{
|
|
"epoch": 0.9324272342608027,
|
|
"grad_norm": 39.216765683029095,
|
|
"learning_rate": 3.2205845316327144e-07,
|
|
"logits/chosen": -1.183584213256836,
|
|
"logits/rejected": -1.1847604513168335,
|
|
"logps/chosen": -40.19718933105469,
|
|
"logps/rejected": -51.98136901855469,
|
|
"loss": 0.3544,
|
|
"rewards/accuracies": 0.7361111044883728,
|
|
"rewards/chosen": -0.23264381289482117,
|
|
"rewards/margins": 2.18786883354187,
|
|
"rewards/rejected": -2.4205124378204346,
|
|
"step": 404
|
|
},
|
|
{
|
|
"epoch": 0.9370432106680343,
|
|
"grad_norm": 27.136867213884347,
|
|
"learning_rate": 3.2012528062417845e-07,
|
|
"logits/chosen": -1.1893184185028076,
|
|
"logits/rejected": -1.186366081237793,
|
|
"logps/chosen": -48.742347717285156,
|
|
"logps/rejected": -53.711280822753906,
|
|
"loss": 0.2583,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.4608592987060547,
|
|
"rewards/margins": 2.4421236515045166,
|
|
"rewards/rejected": -2.9029834270477295,
|
|
"step": 406
|
|
},
|
|
{
|
|
"epoch": 0.9416591870752661,
|
|
"grad_norm": 28.572200442333273,
|
|
"learning_rate": 3.1818754606619643e-07,
|
|
"logits/chosen": -1.146033763885498,
|
|
"logits/rejected": -1.154913306236267,
|
|
"logps/chosen": -43.58420944213867,
|
|
"logps/rejected": -65.13819885253906,
|
|
"loss": 0.3209,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": 0.25047793984413147,
|
|
"rewards/margins": 4.219507217407227,
|
|
"rewards/rejected": -3.969028949737549,
|
|
"step": 408
|
|
},
|
|
{
|
|
"epoch": 0.9462751634824977,
|
|
"grad_norm": 22.48708730265553,
|
|
"learning_rate": 3.162453755491655e-07,
|
|
"logits/chosen": -1.2108149528503418,
|
|
"logits/rejected": -1.2209829092025757,
|
|
"logps/chosen": -45.49837112426758,
|
|
"logps/rejected": -68.48489379882812,
|
|
"loss": 0.1921,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.18967992067337036,
|
|
"rewards/margins": 3.9147284030914307,
|
|
"rewards/rejected": -3.725048542022705,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.9508911398897294,
|
|
"grad_norm": 37.73589723314555,
|
|
"learning_rate": 3.142988954215079e-07,
|
|
"logits/chosen": -1.1515933275222778,
|
|
"logits/rejected": -1.1659475564956665,
|
|
"logps/chosen": -48.16081619262695,
|
|
"logps/rejected": -75.52259063720703,
|
|
"loss": 0.2767,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.4500226378440857,
|
|
"rewards/margins": 3.918086051940918,
|
|
"rewards/rejected": -3.4680633544921875,
|
|
"step": 412
|
|
},
|
|
{
|
|
"epoch": 0.9555071162969612,
|
|
"grad_norm": 50.447595273319266,
|
|
"learning_rate": 3.1234823231200925e-07,
|
|
"logits/chosen": -1.1608054637908936,
|
|
"logits/rejected": -1.1741983890533447,
|
|
"logps/chosen": -46.12180709838867,
|
|
"logps/rejected": -76.99250030517578,
|
|
"loss": 0.2659,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.0659295991063118,
|
|
"rewards/margins": 4.484518527984619,
|
|
"rewards/rejected": -4.4185895919799805,
|
|
"step": 414
|
|
},
|
|
{
|
|
"epoch": 0.9601230927041928,
|
|
"grad_norm": 34.31172314274907,
|
|
"learning_rate": 3.1039351312157993e-07,
|
|
"logits/chosen": -1.1714129447937012,
|
|
"logits/rejected": -1.1802603006362915,
|
|
"logps/chosen": -47.3844108581543,
|
|
"logps/rejected": -66.79096221923828,
|
|
"loss": 0.2247,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.06639357656240463,
|
|
"rewards/margins": 3.8744897842407227,
|
|
"rewards/rejected": -3.8080966472625732,
|
|
"step": 416
|
|
},
|
|
{
|
|
"epoch": 0.9647390691114246,
|
|
"grad_norm": 59.97310545853326,
|
|
"learning_rate": 3.0843486501499967e-07,
|
|
"logits/chosen": -1.1815389394760132,
|
|
"logits/rejected": -1.1873387098312378,
|
|
"logps/chosen": -49.0379638671875,
|
|
"logps/rejected": -60.66644287109375,
|
|
"loss": 0.375,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.37162768840789795,
|
|
"rewards/margins": 2.9354913234710693,
|
|
"rewards/rejected": -2.563863515853882,
|
|
"step": 418
|
|
},
|
|
{
|
|
"epoch": 0.9693550455186563,
|
|
"grad_norm": 28.24836123391339,
|
|
"learning_rate": 3.064724154126449e-07,
|
|
"logits/chosen": -1.1865981817245483,
|
|
"logits/rejected": -1.186213731765747,
|
|
"logps/chosen": -49.98203659057617,
|
|
"logps/rejected": -52.87804412841797,
|
|
"loss": 0.231,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.10280448943376541,
|
|
"rewards/margins": 2.791215181350708,
|
|
"rewards/rejected": -2.688410758972168,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.9693550455186563,
|
|
"eval_logits/chosen": -1.1221855878829956,
|
|
"eval_logits/rejected": -1.1280066967010498,
|
|
"eval_logps/chosen": -48.28245544433594,
|
|
"eval_logps/rejected": -62.80416488647461,
|
|
"eval_loss": 0.252390593290329,
|
|
"eval_rewards/accuracies": 0.820852518081665,
|
|
"eval_rewards/chosen": 0.17548592388629913,
|
|
"eval_rewards/margins": 3.531018018722534,
|
|
"eval_rewards/rejected": -3.35553240776062,
|
|
"eval_runtime": 227.4345,
|
|
"eval_samples_per_second": 7.624,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.9739710219258879,
|
|
"grad_norm": 34.078391098200555,
|
|
"learning_rate": 3.045062919821995e-07,
|
|
"logits/chosen": -1.1267274618148804,
|
|
"logits/rejected": -1.139290452003479,
|
|
"logps/chosen": -46.573524475097656,
|
|
"logps/rejected": -72.93567657470703,
|
|
"loss": 0.2995,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.05330763757228851,
|
|
"rewards/margins": 4.125481605529785,
|
|
"rewards/rejected": -4.072174072265625,
|
|
"step": 422
|
|
},
|
|
{
|
|
"epoch": 0.9785869983331197,
|
|
"grad_norm": 40.16987886487936,
|
|
"learning_rate": 3.0253662263034925e-07,
|
|
"logits/chosen": -1.1718653440475464,
|
|
"logits/rejected": -1.1762607097625732,
|
|
"logps/chosen": -51.13752746582031,
|
|
"logps/rejected": -70.25437927246094,
|
|
"loss": 0.2582,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.11435369402170181,
|
|
"rewards/margins": 3.699098825454712,
|
|
"rewards/rejected": -3.8134524822235107,
|
|
"step": 424
|
|
},
|
|
{
|
|
"epoch": 0.9832029747403513,
|
|
"grad_norm": 32.13301323909956,
|
|
"learning_rate": 3.005635354944606e-07,
|
|
"logits/chosen": -1.1121575832366943,
|
|
"logits/rejected": -1.113258957862854,
|
|
"logps/chosen": -53.563053131103516,
|
|
"logps/rejected": -52.64200210571289,
|
|
"loss": 0.2696,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.11980216950178146,
|
|
"rewards/margins": 2.639040231704712,
|
|
"rewards/rejected": -2.7588419914245605,
|
|
"step": 426
|
|
},
|
|
{
|
|
"epoch": 0.987818951147583,
|
|
"grad_norm": 37.16965622890279,
|
|
"learning_rate": 2.9858715893424504e-07,
|
|
"logits/chosen": -1.1091896295547485,
|
|
"logits/rejected": -1.1275534629821777,
|
|
"logps/chosen": -45.88606643676758,
|
|
"logps/rejected": -73.55078125,
|
|
"loss": 0.1871,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.062331780791282654,
|
|
"rewards/margins": 4.677850723266602,
|
|
"rewards/rejected": -4.740182876586914,
|
|
"step": 428
|
|
},
|
|
{
|
|
"epoch": 0.9924349275548148,
|
|
"grad_norm": 26.22123779547593,
|
|
"learning_rate": 2.966076215234082e-07,
|
|
"logits/chosen": -1.066051959991455,
|
|
"logits/rejected": -1.0764615535736084,
|
|
"logps/chosen": -54.595703125,
|
|
"logps/rejected": -72.26995849609375,
|
|
"loss": 0.1937,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.17351490259170532,
|
|
"rewards/margins": 4.106486797332764,
|
|
"rewards/rejected": -3.932971954345703,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.9970509039620464,
|
|
"grad_norm": 33.565390661724486,
|
|
"learning_rate": 2.94625052041286e-07,
|
|
"logits/chosen": -1.1814826726913452,
|
|
"logits/rejected": -1.1842460632324219,
|
|
"logps/chosen": -50.375,
|
|
"logps/rejected": -58.42066955566406,
|
|
"loss": 0.255,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.07062428444623947,
|
|
"rewards/margins": 3.117582082748413,
|
|
"rewards/rejected": -3.188206672668457,
|
|
"step": 432
|
|
},
|
|
{
|
|
"epoch": 1.001666880369278,
|
|
"grad_norm": 21.605417390859568,
|
|
"learning_rate": 2.926395794644665e-07,
|
|
"logits/chosen": -1.1752268075942993,
|
|
"logits/rejected": -1.1771807670593262,
|
|
"logps/chosen": -51.052242279052734,
|
|
"logps/rejected": -61.03368377685547,
|
|
"loss": 0.1838,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.23978914320468903,
|
|
"rewards/margins": 3.637639045715332,
|
|
"rewards/rejected": -3.3978495597839355,
|
|
"step": 434
|
|
},
|
|
{
|
|
"epoch": 1.0062828567765099,
|
|
"grad_norm": 29.001645648697252,
|
|
"learning_rate": 2.906513329583991e-07,
|
|
"logits/chosen": -1.186964511871338,
|
|
"logits/rejected": -1.1928526163101196,
|
|
"logps/chosen": -46.22761535644531,
|
|
"logps/rejected": -62.58315658569336,
|
|
"loss": 0.2362,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": -0.08661065995693207,
|
|
"rewards/margins": 3.6797680854797363,
|
|
"rewards/rejected": -3.76637864112854,
|
|
"step": 436
|
|
},
|
|
{
|
|
"epoch": 1.0108988331837414,
|
|
"grad_norm": 25.079284366199737,
|
|
"learning_rate": 2.886604418689921e-07,
|
|
"logits/chosen": -1.1703391075134277,
|
|
"logits/rejected": -1.1838041543960571,
|
|
"logps/chosen": -44.59703063964844,
|
|
"logps/rejected": -76.01687622070312,
|
|
"loss": 0.2554,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.19546601176261902,
|
|
"rewards/margins": 4.618009567260742,
|
|
"rewards/rejected": -4.813475131988525,
|
|
"step": 438
|
|
},
|
|
{
|
|
"epoch": 1.0155148095909732,
|
|
"grad_norm": 15.893743443702332,
|
|
"learning_rate": 2.866670357141979e-07,
|
|
"logits/chosen": -1.1566696166992188,
|
|
"logits/rejected": -1.1605850458145142,
|
|
"logps/chosen": -50.24718475341797,
|
|
"logps/rejected": -61.23912048339844,
|
|
"loss": 0.2096,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.3380340337753296,
|
|
"rewards/margins": 4.049044609069824,
|
|
"rewards/rejected": -3.711010456085205,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 1.0155148095909732,
|
|
"eval_logits/chosen": -1.1188750267028809,
|
|
"eval_logits/rejected": -1.124423861503601,
|
|
"eval_logps/chosen": -47.42933654785156,
|
|
"eval_logps/rejected": -62.06835174560547,
|
|
"eval_loss": 0.2514457404613495,
|
|
"eval_rewards/accuracies": 0.8237327337265015,
|
|
"eval_rewards/chosen": 0.6020476222038269,
|
|
"eval_rewards/margins": 3.589672088623047,
|
|
"eval_rewards/rejected": -2.9876248836517334,
|
|
"eval_runtime": 227.3665,
|
|
"eval_samples_per_second": 7.626,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 1.020130785998205,
|
|
"grad_norm": 13.432673135223586,
|
|
"learning_rate": 2.8467124417558737e-07,
|
|
"logits/chosen": -1.1185688972473145,
|
|
"logits/rejected": -1.1209557056427002,
|
|
"logps/chosen": -48.8853759765625,
|
|
"logps/rejected": -63.88041305541992,
|
|
"loss": 0.1931,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.8135783076286316,
|
|
"rewards/margins": 4.117891788482666,
|
|
"rewards/rejected": -3.3043136596679688,
|
|
"step": 442
|
|
},
|
|
{
|
|
"epoch": 1.0247467624054365,
|
|
"grad_norm": 26.849141741016137,
|
|
"learning_rate": 2.8267319708991253e-07,
|
|
"logits/chosen": -1.096121907234192,
|
|
"logits/rejected": -1.0977866649627686,
|
|
"logps/chosen": -52.22743225097656,
|
|
"logps/rejected": -55.36363220214844,
|
|
"loss": 0.1924,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 0.6431276202201843,
|
|
"rewards/margins": 3.097036123275757,
|
|
"rewards/rejected": -2.453908681869507,
|
|
"step": 444
|
|
},
|
|
{
|
|
"epoch": 1.0293627388126683,
|
|
"grad_norm": 31.09854358728895,
|
|
"learning_rate": 2.806730244406612e-07,
|
|
"logits/chosen": -1.1671628952026367,
|
|
"logits/rejected": -1.1714211702346802,
|
|
"logps/chosen": -46.41295623779297,
|
|
"logps/rejected": -59.33990478515625,
|
|
"loss": 0.2477,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.5287280082702637,
|
|
"rewards/margins": 3.4002442359924316,
|
|
"rewards/rejected": -2.871516227722168,
|
|
"step": 446
|
|
},
|
|
{
|
|
"epoch": 1.0339787152199,
|
|
"grad_norm": 31.751017976380037,
|
|
"learning_rate": 2.786708563496001e-07,
|
|
"logits/chosen": -1.2408480644226074,
|
|
"logits/rejected": -1.2548807859420776,
|
|
"logps/chosen": -49.92308044433594,
|
|
"logps/rejected": -67.60689544677734,
|
|
"loss": 0.179,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 0.6711897850036621,
|
|
"rewards/margins": 4.588629722595215,
|
|
"rewards/rejected": -3.9174396991729736,
|
|
"step": 448
|
|
},
|
|
{
|
|
"epoch": 1.0385946916271316,
|
|
"grad_norm": 32.2047076869101,
|
|
"learning_rate": 2.7666682306830994e-07,
|
|
"logits/chosen": -1.19577157497406,
|
|
"logits/rejected": -1.194454550743103,
|
|
"logps/chosen": -46.63425827026367,
|
|
"logps/rejected": -49.01081848144531,
|
|
"loss": 0.2547,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.5076774954795837,
|
|
"rewards/margins": 2.832526206970215,
|
|
"rewards/rejected": -2.3248488903045654,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 1.0432106680343634,
|
|
"grad_norm": 28.553486090102076,
|
|
"learning_rate": 2.746610549697119e-07,
|
|
"logits/chosen": -1.1639982461929321,
|
|
"logits/rejected": -1.1696867942810059,
|
|
"logps/chosen": -49.013301849365234,
|
|
"logps/rejected": -65.69493865966797,
|
|
"loss": 0.2036,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.5706920623779297,
|
|
"rewards/margins": 3.6936237812042236,
|
|
"rewards/rejected": -3.122931957244873,
|
|
"step": 452
|
|
},
|
|
{
|
|
"epoch": 1.0478266444415951,
|
|
"grad_norm": 13.92439999757166,
|
|
"learning_rate": 2.7265368253958615e-07,
|
|
"logits/chosen": -1.2167223691940308,
|
|
"logits/rejected": -1.2195782661437988,
|
|
"logps/chosen": -45.21904754638672,
|
|
"logps/rejected": -53.50370407104492,
|
|
"loss": 0.1746,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.7654822468757629,
|
|
"rewards/margins": 3.2732510566711426,
|
|
"rewards/rejected": -2.5077688694000244,
|
|
"step": 454
|
|
},
|
|
{
|
|
"epoch": 1.0524426208488267,
|
|
"grad_norm": 36.82786613306219,
|
|
"learning_rate": 2.706448363680831e-07,
|
|
"logits/chosen": -1.1744914054870605,
|
|
"logits/rejected": -1.1839237213134766,
|
|
"logps/chosen": -47.35738754272461,
|
|
"logps/rejected": -73.86341094970703,
|
|
"loss": 0.1569,
|
|
"rewards/accuracies": 0.9583333134651184,
|
|
"rewards/chosen": 0.696943998336792,
|
|
"rewards/margins": 4.708230018615723,
|
|
"rewards/rejected": -4.011285781860352,
|
|
"step": 456
|
|
},
|
|
{
|
|
"epoch": 1.0570585972560584,
|
|
"grad_norm": 13.78515040845917,
|
|
"learning_rate": 2.686346471412277e-07,
|
|
"logits/chosen": -1.1262328624725342,
|
|
"logits/rejected": -1.1403940916061401,
|
|
"logps/chosen": -49.68544387817383,
|
|
"logps/rejected": -76.74578857421875,
|
|
"loss": 0.1419,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.3619132339954376,
|
|
"rewards/margins": 4.7866668701171875,
|
|
"rewards/rejected": -4.424753665924072,
|
|
"step": 458
|
|
},
|
|
{
|
|
"epoch": 1.0616745736632902,
|
|
"grad_norm": 26.57858405427076,
|
|
"learning_rate": 2.6662324563241805e-07,
|
|
"logits/chosen": -1.2429147958755493,
|
|
"logits/rejected": -1.2455251216888428,
|
|
"logps/chosen": -45.16366195678711,
|
|
"logps/rejected": -57.56427001953125,
|
|
"loss": 0.2357,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.5941984057426453,
|
|
"rewards/margins": 3.2229466438293457,
|
|
"rewards/rejected": -2.6287484169006348,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 1.0616745736632902,
|
|
"eval_logits/chosen": -1.1103266477584839,
|
|
"eval_logits/rejected": -1.1161012649536133,
|
|
"eval_logps/chosen": -47.78204345703125,
|
|
"eval_logps/rejected": -62.643646240234375,
|
|
"eval_loss": 0.2479603886604309,
|
|
"eval_rewards/accuracies": 0.8277649879455566,
|
|
"eval_rewards/chosen": 0.42569395899772644,
|
|
"eval_rewards/margins": 3.700965642929077,
|
|
"eval_rewards/rejected": -3.2752716541290283,
|
|
"eval_runtime": 227.8171,
|
|
"eval_samples_per_second": 7.611,
|
|
"eval_steps_per_second": 1.905,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 1.0662905500705218,
|
|
"grad_norm": 16.869585682992792,
|
|
"learning_rate": 2.6461076269391713e-07,
|
|
"logits/chosen": -1.0661816596984863,
|
|
"logits/rejected": -1.0739065408706665,
|
|
"logps/chosen": -54.75306701660156,
|
|
"logps/rejected": -72.30966186523438,
|
|
"loss": 0.1519,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.5898687839508057,
|
|
"rewards/margins": 4.626287460327148,
|
|
"rewards/rejected": -4.03641939163208,
|
|
"step": 462
|
|
},
|
|
{
|
|
"epoch": 1.0709065264777535,
|
|
"grad_norm": 39.49599828169064,
|
|
"learning_rate": 2.625973292483409e-07,
|
|
"logits/chosen": -1.1013195514678955,
|
|
"logits/rejected": -1.1079678535461426,
|
|
"logps/chosen": -56.018310546875,
|
|
"logps/rejected": -69.99239349365234,
|
|
"loss": 0.2341,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.3379661738872528,
|
|
"rewards/margins": 3.941251516342163,
|
|
"rewards/rejected": -3.603285551071167,
|
|
"step": 464
|
|
},
|
|
{
|
|
"epoch": 1.0755225028849853,
|
|
"grad_norm": 19.11015860441972,
|
|
"learning_rate": 2.6058307628014065e-07,
|
|
"logits/chosen": -1.110822319984436,
|
|
"logits/rejected": -1.1167054176330566,
|
|
"logps/chosen": -53.93489074707031,
|
|
"logps/rejected": -66.05422973632812,
|
|
"loss": 0.1708,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.3638114929199219,
|
|
"rewards/margins": 4.259352207183838,
|
|
"rewards/rejected": -3.895540952682495,
|
|
"step": 466
|
|
},
|
|
{
|
|
"epoch": 1.0801384792922168,
|
|
"grad_norm": 29.43582021185457,
|
|
"learning_rate": 2.5856813482708217e-07,
|
|
"logits/chosen": -1.184598445892334,
|
|
"logits/rejected": -1.1916086673736572,
|
|
"logps/chosen": -49.6500244140625,
|
|
"logps/rejected": -56.30369567871094,
|
|
"loss": 0.2254,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.36434826254844666,
|
|
"rewards/margins": 3.6883039474487305,
|
|
"rewards/rejected": -3.32395601272583,
|
|
"step": 468
|
|
},
|
|
{
|
|
"epoch": 1.0847544556994486,
|
|
"grad_norm": 25.173770275980058,
|
|
"learning_rate": 2.565526359717206e-07,
|
|
"logits/chosen": -1.1290383338928223,
|
|
"logits/rejected": -1.1306836605072021,
|
|
"logps/chosen": -43.971435546875,
|
|
"logps/rejected": -53.389156341552734,
|
|
"loss": 0.289,
|
|
"rewards/accuracies": 0.7777777910232544,
|
|
"rewards/chosen": 0.18997710943222046,
|
|
"rewards/margins": 3.0762457847595215,
|
|
"rewards/rejected": -2.8862688541412354,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 1.0893704321066804,
|
|
"grad_norm": 19.76426072194178,
|
|
"learning_rate": 2.545367108328731e-07,
|
|
"logits/chosen": -1.163740873336792,
|
|
"logits/rejected": -1.1682920455932617,
|
|
"logps/chosen": -49.15140914916992,
|
|
"logps/rejected": -59.53387451171875,
|
|
"loss": 0.187,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.26795661449432373,
|
|
"rewards/margins": 3.4266703128814697,
|
|
"rewards/rejected": -3.1587133407592773,
|
|
"step": 472
|
|
},
|
|
{
|
|
"epoch": 1.0939864085139122,
|
|
"grad_norm": 17.775087193890624,
|
|
"learning_rate": 2.525204905570889e-07,
|
|
"logits/chosen": -1.1204829216003418,
|
|
"logits/rejected": -1.1253838539123535,
|
|
"logps/chosen": -54.046390533447266,
|
|
"logps/rejected": -66.79469299316406,
|
|
"loss": 0.1607,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.39596712589263916,
|
|
"rewards/margins": 4.161929130554199,
|
|
"rewards/rejected": -3.7659616470336914,
|
|
"step": 474
|
|
},
|
|
{
|
|
"epoch": 1.0986023849211437,
|
|
"grad_norm": 24.757995548476888,
|
|
"learning_rate": 2.505041063101171e-07,
|
|
"logits/chosen": -1.1805049180984497,
|
|
"logits/rejected": -1.1901381015777588,
|
|
"logps/chosen": -53.46992492675781,
|
|
"logps/rejected": -59.58029556274414,
|
|
"loss": 0.2762,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.13624940812587738,
|
|
"rewards/margins": 3.440009832382202,
|
|
"rewards/rejected": -3.303760290145874,
|
|
"step": 476
|
|
},
|
|
{
|
|
"epoch": 1.1032183613283755,
|
|
"grad_norm": 19.970980113968885,
|
|
"learning_rate": 2.4848768926837466e-07,
|
|
"logits/chosen": -1.0963982343673706,
|
|
"logits/rejected": -1.112579345703125,
|
|
"logps/chosen": -47.13343811035156,
|
|
"logps/rejected": -87.57925415039062,
|
|
"loss": 0.1717,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.06597856432199478,
|
|
"rewards/margins": 5.439146518707275,
|
|
"rewards/rejected": -5.373167991638184,
|
|
"step": 478
|
|
},
|
|
{
|
|
"epoch": 1.107834337735607,
|
|
"grad_norm": 16.28084665624267,
|
|
"learning_rate": 2.464713706104113e-07,
|
|
"logits/chosen": -1.1157184839248657,
|
|
"logits/rejected": -1.1205692291259766,
|
|
"logps/chosen": -50.34828567504883,
|
|
"logps/rejected": -63.41987609863281,
|
|
"loss": 0.1608,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 0.043437667191028595,
|
|
"rewards/margins": 4.042869567871094,
|
|
"rewards/rejected": -3.99943208694458,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 1.107834337735607,
|
|
"eval_logits/chosen": -1.1090331077575684,
|
|
"eval_logits/rejected": -1.1147438287734985,
|
|
"eval_logps/chosen": -48.225521087646484,
|
|
"eval_logps/rejected": -63.15263748168945,
|
|
"eval_loss": 0.24383509159088135,
|
|
"eval_rewards/accuracies": 0.8294931054115295,
|
|
"eval_rewards/chosen": 0.203952819108963,
|
|
"eval_rewards/margins": 3.733717441558838,
|
|
"eval_rewards/rejected": -3.529764175415039,
|
|
"eval_runtime": 227.3827,
|
|
"eval_samples_per_second": 7.626,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 1.1124503141428388,
|
|
"grad_norm": 22.059822838666445,
|
|
"learning_rate": 2.444552815083767e-07,
|
|
"logits/chosen": -1.1268048286437988,
|
|
"logits/rejected": -1.1286168098449707,
|
|
"logps/chosen": -49.519527435302734,
|
|
"logps/rejected": -52.751312255859375,
|
|
"loss": 0.2264,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.3959563970565796,
|
|
"rewards/margins": 3.2894110679626465,
|
|
"rewards/rejected": -2.8934545516967773,
|
|
"step": 482
|
|
},
|
|
{
|
|
"epoch": 1.1170662905500706,
|
|
"grad_norm": 19.4616654191151,
|
|
"learning_rate": 2.4243955311948693e-07,
|
|
"logits/chosen": -1.1610568761825562,
|
|
"logits/rejected": -1.1703104972839355,
|
|
"logps/chosen": -45.62093734741211,
|
|
"logps/rejected": -71.41989135742188,
|
|
"loss": 0.2218,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.056832365691661835,
|
|
"rewards/margins": 4.676138877868652,
|
|
"rewards/rejected": -4.619307041168213,
|
|
"step": 484
|
|
},
|
|
{
|
|
"epoch": 1.1216822669573023,
|
|
"grad_norm": 29.74478192767247,
|
|
"learning_rate": 2.4042431657749115e-07,
|
|
"logits/chosen": -1.082115650177002,
|
|
"logits/rejected": -1.097312092781067,
|
|
"logps/chosen": -47.262996673583984,
|
|
"logps/rejected": -84.21355438232422,
|
|
"loss": 0.1955,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.2266267091035843,
|
|
"rewards/margins": 4.9980058670043945,
|
|
"rewards/rejected": -4.771378993988037,
|
|
"step": 486
|
|
},
|
|
{
|
|
"epoch": 1.1262982433645339,
|
|
"grad_norm": 34.83946230592783,
|
|
"learning_rate": 2.384097029841419e-07,
|
|
"logits/chosen": -1.1644551753997803,
|
|
"logits/rejected": -1.1694614887237549,
|
|
"logps/chosen": -49.46389389038086,
|
|
"logps/rejected": -59.47831344604492,
|
|
"loss": 0.2086,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.2441619485616684,
|
|
"rewards/margins": 3.580085277557373,
|
|
"rewards/rejected": -3.335923433303833,
|
|
"step": 488
|
|
},
|
|
{
|
|
"epoch": 1.1309142197717656,
|
|
"grad_norm": 21.45284163289699,
|
|
"learning_rate": 2.3639584340066544e-07,
|
|
"logits/chosen": -1.1405658721923828,
|
|
"logits/rejected": -1.146559238433838,
|
|
"logps/chosen": -41.80732727050781,
|
|
"logps/rejected": -62.37071990966797,
|
|
"loss": 0.2166,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.7920265197753906,
|
|
"rewards/margins": 4.414507865905762,
|
|
"rewards/rejected": -3.622481346130371,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 1.1355301961789972,
|
|
"grad_norm": 20.900830046410174,
|
|
"learning_rate": 2.3438286883923539e-07,
|
|
"logits/chosen": -1.164839267730713,
|
|
"logits/rejected": -1.1716415882110596,
|
|
"logps/chosen": -52.77006912231445,
|
|
"logps/rejected": -60.97998046875,
|
|
"loss": 0.2024,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.7002817988395691,
|
|
"rewards/margins": 3.6204490661621094,
|
|
"rewards/rejected": -2.9201676845550537,
|
|
"step": 492
|
|
},
|
|
{
|
|
"epoch": 1.140146172586229,
|
|
"grad_norm": 22.811055008597176,
|
|
"learning_rate": 2.323709102544506e-07,
|
|
"logits/chosen": -1.1385387182235718,
|
|
"logits/rejected": -1.1350369453430176,
|
|
"logps/chosen": -44.97706604003906,
|
|
"logps/rejected": -47.24876403808594,
|
|
"loss": 0.267,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.8821587562561035,
|
|
"rewards/margins": 2.6500444412231445,
|
|
"rewards/rejected": -1.7678859233856201,
|
|
"step": 494
|
|
},
|
|
{
|
|
"epoch": 1.1447621489934607,
|
|
"grad_norm": 25.7099907801656,
|
|
"learning_rate": 2.3036009853481474e-07,
|
|
"logits/chosen": -1.1164131164550781,
|
|
"logits/rejected": -1.124334454536438,
|
|
"logps/chosen": -44.46446228027344,
|
|
"logps/rejected": -66.84891510009766,
|
|
"loss": 0.252,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.42438387870788574,
|
|
"rewards/margins": 4.415454387664795,
|
|
"rewards/rejected": -3.9910707473754883,
|
|
"step": 496
|
|
},
|
|
{
|
|
"epoch": 1.1493781254006925,
|
|
"grad_norm": 24.703959475838438,
|
|
"learning_rate": 2.283505644942223e-07,
|
|
"logits/chosen": -1.1680512428283691,
|
|
"logits/rejected": -1.172341227531433,
|
|
"logps/chosen": -39.898555755615234,
|
|
"logps/rejected": -61.25082778930664,
|
|
"loss": 0.1888,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.8999897837638855,
|
|
"rewards/margins": 4.018680095672607,
|
|
"rewards/rejected": -3.118690252304077,
|
|
"step": 498
|
|
},
|
|
{
|
|
"epoch": 1.153994101807924,
|
|
"grad_norm": 22.54617945872361,
|
|
"learning_rate": 2.2634243886344781e-07,
|
|
"logits/chosen": -1.1353996992111206,
|
|
"logits/rejected": -1.1466150283813477,
|
|
"logps/chosen": -47.095890045166016,
|
|
"logps/rejected": -63.17418670654297,
|
|
"loss": 0.1944,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.0475714206695557,
|
|
"rewards/margins": 4.362390518188477,
|
|
"rewards/rejected": -3.314818859100342,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 1.153994101807924,
|
|
"eval_logits/chosen": -1.1096464395523071,
|
|
"eval_logits/rejected": -1.1153897047042847,
|
|
"eval_logps/chosen": -46.75843811035156,
|
|
"eval_logps/rejected": -61.96721649169922,
|
|
"eval_loss": 0.24556967616081238,
|
|
"eval_rewards/accuracies": 0.8312212228775024,
|
|
"eval_rewards/chosen": 0.9374985098838806,
|
|
"eval_rewards/margins": 3.8745529651641846,
|
|
"eval_rewards/rejected": -2.9370551109313965,
|
|
"eval_runtime": 227.4325,
|
|
"eval_samples_per_second": 7.624,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 1.1586100782151558,
|
|
"grad_norm": 27.18085844791722,
|
|
"learning_rate": 2.2433585228164115e-07,
|
|
"logits/chosen": -1.1491751670837402,
|
|
"logits/rejected": -1.1580781936645508,
|
|
"logps/chosen": -50.64473342895508,
|
|
"logps/rejected": -75.20685577392578,
|
|
"loss": 0.2218,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.8257203698158264,
|
|
"rewards/margins": 5.088500022888184,
|
|
"rewards/rejected": -4.26278018951416,
|
|
"step": 502
|
|
},
|
|
{
|
|
"epoch": 1.1632260546223874,
|
|
"grad_norm": 23.89654504779422,
|
|
"learning_rate": 2.2233093528782938e-07,
|
|
"logits/chosen": -1.1577401161193848,
|
|
"logits/rejected": -1.1688594818115234,
|
|
"logps/chosen": -54.145572662353516,
|
|
"logps/rejected": -66.3442611694336,
|
|
"loss": 0.1751,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 1.085695743560791,
|
|
"rewards/margins": 4.018051624298096,
|
|
"rewards/rejected": -2.9323554039001465,
|
|
"step": 504
|
|
},
|
|
{
|
|
"epoch": 1.1678420310296191,
|
|
"grad_norm": 23.769575263027864,
|
|
"learning_rate": 2.2032781831242367e-07,
|
|
"logits/chosen": -1.1783199310302734,
|
|
"logits/rejected": -1.182464838027954,
|
|
"logps/chosen": -41.669734954833984,
|
|
"logps/rejected": -51.315006256103516,
|
|
"loss": 0.2418,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.9586374759674072,
|
|
"rewards/margins": 3.437027931213379,
|
|
"rewards/rejected": -2.4783899784088135,
|
|
"step": 506
|
|
},
|
|
{
|
|
"epoch": 1.172458007436851,
|
|
"grad_norm": 36.28094297883332,
|
|
"learning_rate": 2.183266316687347e-07,
|
|
"logits/chosen": -1.1632755994796753,
|
|
"logits/rejected": -1.1601239442825317,
|
|
"logps/chosen": -47.738983154296875,
|
|
"logps/rejected": -49.51511001586914,
|
|
"loss": 0.2641,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 1.0999305248260498,
|
|
"rewards/margins": 2.7960052490234375,
|
|
"rewards/rejected": -1.6960747241973877,
|
|
"step": 508
|
|
},
|
|
{
|
|
"epoch": 1.1770739838440827,
|
|
"grad_norm": 18.77553292711027,
|
|
"learning_rate": 2.16327505544495e-07,
|
|
"logits/chosen": -1.1429252624511719,
|
|
"logits/rejected": -1.1522622108459473,
|
|
"logps/chosen": -50.42318344116211,
|
|
"logps/rejected": -66.82962036132812,
|
|
"loss": 0.1438,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 1.0773425102233887,
|
|
"rewards/margins": 4.845379829406738,
|
|
"rewards/rejected": -3.7680368423461914,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 1.1816899602513142,
|
|
"grad_norm": 17.949048045246396,
|
|
"learning_rate": 2.143305699933892e-07,
|
|
"logits/chosen": -1.1755365133285522,
|
|
"logits/rejected": -1.180452823638916,
|
|
"logps/chosen": -43.959930419921875,
|
|
"logps/rejected": -64.32354736328125,
|
|
"loss": 0.2051,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.7747830748558044,
|
|
"rewards/margins": 4.00773811340332,
|
|
"rewards/rejected": -3.23295521736145,
|
|
"step": 512
|
|
},
|
|
{
|
|
"epoch": 1.186305936658546,
|
|
"grad_norm": 29.609702959008317,
|
|
"learning_rate": 2.1233595492659382e-07,
|
|
"logits/chosen": -1.0717233419418335,
|
|
"logits/rejected": -1.0747785568237305,
|
|
"logps/chosen": -56.3906135559082,
|
|
"logps/rejected": -58.85280990600586,
|
|
"loss": 0.1651,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.6519337892532349,
|
|
"rewards/margins": 3.8963236808776855,
|
|
"rewards/rejected": -3.244389772415161,
|
|
"step": 514
|
|
},
|
|
{
|
|
"epoch": 1.1909219130657775,
|
|
"grad_norm": 20.918336765308602,
|
|
"learning_rate": 2.1034379010432542e-07,
|
|
"logits/chosen": -1.1730085611343384,
|
|
"logits/rejected": -1.1727977991104126,
|
|
"logps/chosen": -44.0880126953125,
|
|
"logps/rejected": -56.85806655883789,
|
|
"loss": 0.1976,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.4944622218608856,
|
|
"rewards/margins": 3.6419970989227295,
|
|
"rewards/rejected": -3.1475343704223633,
|
|
"step": 516
|
|
},
|
|
{
|
|
"epoch": 1.1955378894730093,
|
|
"grad_norm": 25.288305102390122,
|
|
"learning_rate": 2.0835420512739957e-07,
|
|
"logits/chosen": -1.1438689231872559,
|
|
"logits/rejected": -1.1519646644592285,
|
|
"logps/chosen": -47.1004638671875,
|
|
"logps/rejected": -83.05133819580078,
|
|
"loss": 0.1839,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.2153804749250412,
|
|
"rewards/margins": 5.248907089233398,
|
|
"rewards/rejected": -5.03352689743042,
|
|
"step": 518
|
|
},
|
|
{
|
|
"epoch": 1.200153865880241,
|
|
"grad_norm": 19.59880042292574,
|
|
"learning_rate": 2.0636732942879917e-07,
|
|
"logits/chosen": -1.1264581680297852,
|
|
"logits/rejected": -1.1307095289230347,
|
|
"logps/chosen": -50.47024917602539,
|
|
"logps/rejected": -64.73220825195312,
|
|
"loss": 0.1619,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.3765060007572174,
|
|
"rewards/margins": 4.3557209968566895,
|
|
"rewards/rejected": -3.979214906692505,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 1.200153865880241,
|
|
"eval_logits/chosen": -1.1257847547531128,
|
|
"eval_logits/rejected": -1.1301593780517578,
|
|
"eval_logps/chosen": -48.62382507324219,
|
|
"eval_logps/rejected": -63.695838928222656,
|
|
"eval_loss": 0.24041977524757385,
|
|
"eval_rewards/accuracies": 0.8335253596305847,
|
|
"eval_rewards/chosen": 0.004803389776498079,
|
|
"eval_rewards/margins": 3.8061721324920654,
|
|
"eval_rewards/rejected": -3.8013687133789062,
|
|
"eval_runtime": 227.3215,
|
|
"eval_samples_per_second": 7.628,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 1.2047698422874729,
|
|
"grad_norm": 17.44277679097044,
|
|
"learning_rate": 2.0438329226525415e-07,
|
|
"logits/chosen": -1.144020915031433,
|
|
"logits/rejected": -1.1453847885131836,
|
|
"logps/chosen": -49.49299621582031,
|
|
"logps/rejected": -53.57789611816406,
|
|
"loss": 0.2025,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.42205584049224854,
|
|
"rewards/margins": 3.364236831665039,
|
|
"rewards/rejected": -2.942180871963501,
|
|
"step": 522
|
|
},
|
|
{
|
|
"epoch": 1.2093858186947044,
|
|
"grad_norm": 37.73486109040497,
|
|
"learning_rate": 2.0240222270883288e-07,
|
|
"logits/chosen": -1.1556731462478638,
|
|
"logits/rejected": -1.1702880859375,
|
|
"logps/chosen": -50.98395538330078,
|
|
"logps/rejected": -76.1040267944336,
|
|
"loss": 0.2225,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.07603338360786438,
|
|
"rewards/margins": 4.920933246612549,
|
|
"rewards/rejected": -4.9969658851623535,
|
|
"step": 524
|
|
},
|
|
{
|
|
"epoch": 1.2140017951019362,
|
|
"grad_norm": 25.377653028594498,
|
|
"learning_rate": 2.0042424963854542e-07,
|
|
"logits/chosen": -1.196961760520935,
|
|
"logits/rejected": -1.2145916223526,
|
|
"logps/chosen": -47.896732330322266,
|
|
"logps/rejected": -83.96469116210938,
|
|
"loss": 0.1463,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.26778745651245117,
|
|
"rewards/margins": 5.1378092765808105,
|
|
"rewards/rejected": -5.40559720993042,
|
|
"step": 526
|
|
},
|
|
{
|
|
"epoch": 1.2186177715091677,
|
|
"grad_norm": 21.037533939572622,
|
|
"learning_rate": 1.9844950173195883e-07,
|
|
"logits/chosen": -1.2031718492507935,
|
|
"logits/rejected": -1.207302212715149,
|
|
"logps/chosen": -48.16632843017578,
|
|
"logps/rejected": -63.03504943847656,
|
|
"loss": 0.1798,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.3718983829021454,
|
|
"rewards/margins": 3.580104351043701,
|
|
"rewards/rejected": -3.95200252532959,
|
|
"step": 528
|
|
},
|
|
{
|
|
"epoch": 1.2232337479163995,
|
|
"grad_norm": 22.283357312346677,
|
|
"learning_rate": 1.964781074568265e-07,
|
|
"logits/chosen": -1.2361119985580444,
|
|
"logits/rejected": -1.2355589866638184,
|
|
"logps/chosen": -48.786136627197266,
|
|
"logps/rejected": -53.23166275024414,
|
|
"loss": 0.1959,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.26044440269470215,
|
|
"rewards/margins": 3.046844482421875,
|
|
"rewards/rejected": -3.307288885116577,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 1.2278497243236313,
|
|
"grad_norm": 32.696983725912126,
|
|
"learning_rate": 1.9451019506273018e-07,
|
|
"logits/chosen": -1.1622250080108643,
|
|
"logits/rejected": -1.16159987449646,
|
|
"logps/chosen": -42.64177322387695,
|
|
"logps/rejected": -55.659339904785156,
|
|
"loss": 0.2379,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.17224609851837158,
|
|
"rewards/margins": 3.0260884761810303,
|
|
"rewards/rejected": -3.1983346939086914,
|
|
"step": 532
|
|
},
|
|
{
|
|
"epoch": 1.232465700730863,
|
|
"grad_norm": 31.81561570512381,
|
|
"learning_rate": 1.9254589257273712e-07,
|
|
"logits/chosen": -1.1568024158477783,
|
|
"logits/rejected": -1.1644660234451294,
|
|
"logps/chosen": -43.20817565917969,
|
|
"logps/rejected": -66.71856689453125,
|
|
"loss": 0.1648,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.03500910475850105,
|
|
"rewards/margins": 4.864539623260498,
|
|
"rewards/rejected": -4.899548530578613,
|
|
"step": 534
|
|
},
|
|
{
|
|
"epoch": 1.2370816771380946,
|
|
"grad_norm": 28.042577352949,
|
|
"learning_rate": 1.9058532777507141e-07,
|
|
"logits/chosen": -1.1810351610183716,
|
|
"logits/rejected": -1.186877727508545,
|
|
"logps/chosen": -46.86735534667969,
|
|
"logps/rejected": -58.00865173339844,
|
|
"loss": 0.1946,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.07706524431705475,
|
|
"rewards/margins": 3.6953649520874023,
|
|
"rewards/rejected": -3.772430896759033,
|
|
"step": 536
|
|
},
|
|
{
|
|
"epoch": 1.2416976535453264,
|
|
"grad_norm": 31.06776524909045,
|
|
"learning_rate": 1.886286282148002e-07,
|
|
"logits/chosen": -1.1419155597686768,
|
|
"logits/rejected": -1.1518969535827637,
|
|
"logps/chosen": -48.36051559448242,
|
|
"logps/rejected": -68.71621704101562,
|
|
"loss": 0.2628,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": -0.3396787643432617,
|
|
"rewards/margins": 4.181530952453613,
|
|
"rewards/rejected": -4.521209716796875,
|
|
"step": 538
|
|
},
|
|
{
|
|
"epoch": 1.246313629952558,
|
|
"grad_norm": 25.267305267108494,
|
|
"learning_rate": 1.8667592118553693e-07,
|
|
"logits/chosen": -1.2024831771850586,
|
|
"logits/rejected": -1.2076871395111084,
|
|
"logps/chosen": -52.623023986816406,
|
|
"logps/rejected": -61.89004898071289,
|
|
"loss": 0.2083,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": -0.21523982286453247,
|
|
"rewards/margins": 3.845079183578491,
|
|
"rewards/rejected": -4.060319423675537,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 1.246313629952558,
|
|
"eval_logits/chosen": -1.1086678504943848,
|
|
"eval_logits/rejected": -1.11427903175354,
|
|
"eval_logps/chosen": -48.652427673339844,
|
|
"eval_logps/rejected": -63.7818717956543,
|
|
"eval_loss": 0.2395094931125641,
|
|
"eval_rewards/accuracies": 0.8323732614517212,
|
|
"eval_rewards/chosen": -0.009500053711235523,
|
|
"eval_rewards/margins": 3.8348822593688965,
|
|
"eval_rewards/rejected": -3.8443822860717773,
|
|
"eval_runtime": 227.3478,
|
|
"eval_samples_per_second": 7.627,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 1.2509296063597897,
|
|
"grad_norm": 29.79858133121253,
|
|
"learning_rate": 1.8472733372115956e-07,
|
|
"logits/chosen": -1.2199275493621826,
|
|
"logits/rejected": -1.2279648780822754,
|
|
"logps/chosen": -50.78293228149414,
|
|
"logps/rejected": -72.13998413085938,
|
|
"loss": 0.2006,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.18516860902309418,
|
|
"rewards/margins": 4.692964553833008,
|
|
"rewards/rejected": -4.878133773803711,
|
|
"step": 542
|
|
},
|
|
{
|
|
"epoch": 1.2555455827670214,
|
|
"grad_norm": 18.335692774068683,
|
|
"learning_rate": 1.8278299258754692e-07,
|
|
"logits/chosen": -1.1382191181182861,
|
|
"logits/rejected": -1.1532717943191528,
|
|
"logps/chosen": -51.21404266357422,
|
|
"logps/rejected": -84.46864318847656,
|
|
"loss": 0.2507,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.002586497226729989,
|
|
"rewards/margins": 6.08302116394043,
|
|
"rewards/rejected": -6.085607528686523,
|
|
"step": 544
|
|
},
|
|
{
|
|
"epoch": 1.2601615591742532,
|
|
"grad_norm": 16.812848293352094,
|
|
"learning_rate": 1.808430242743316e-07,
|
|
"logits/chosen": -1.1635518074035645,
|
|
"logits/rejected": -1.1678366661071777,
|
|
"logps/chosen": -50.37785339355469,
|
|
"logps/rejected": -63.68332290649414,
|
|
"loss": 0.207,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.4088381230831146,
|
|
"rewards/margins": 4.470311641693115,
|
|
"rewards/rejected": -4.0614728927612305,
|
|
"step": 546
|
|
},
|
|
{
|
|
"epoch": 1.2647775355814848,
|
|
"grad_norm": 15.038796138409147,
|
|
"learning_rate": 1.7890755498667104e-07,
|
|
"logits/chosen": -1.1664639711380005,
|
|
"logits/rejected": -1.1719496250152588,
|
|
"logps/chosen": -42.69700622558594,
|
|
"logps/rejected": -64.51258850097656,
|
|
"loss": 0.1614,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.17392092943191528,
|
|
"rewards/margins": 4.139625549316406,
|
|
"rewards/rejected": -3.9657046794891357,
|
|
"step": 548
|
|
},
|
|
{
|
|
"epoch": 1.2693935119887165,
|
|
"grad_norm": 27.84348089734983,
|
|
"learning_rate": 1.7697671063703756e-07,
|
|
"logits/chosen": -1.1588454246520996,
|
|
"logits/rejected": -1.1661314964294434,
|
|
"logps/chosen": -44.9870719909668,
|
|
"logps/rejected": -65.78500366210938,
|
|
"loss": 0.2201,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.46526622772216797,
|
|
"rewards/margins": 4.418604373931885,
|
|
"rewards/rejected": -3.953338623046875,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 1.274009488395948,
|
|
"grad_norm": 44.28075638457142,
|
|
"learning_rate": 1.750506168370267e-07,
|
|
"logits/chosen": -1.1754214763641357,
|
|
"logits/rejected": -1.175520896911621,
|
|
"logps/chosen": -46.97793960571289,
|
|
"logps/rejected": -54.8778076171875,
|
|
"loss": 0.2392,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.9450171589851379,
|
|
"rewards/margins": 3.314897298812866,
|
|
"rewards/rejected": -2.369880199432373,
|
|
"step": 552
|
|
},
|
|
{
|
|
"epoch": 1.2786254648031798,
|
|
"grad_norm": 17.20774183339189,
|
|
"learning_rate": 1.7312939888918594e-07,
|
|
"logits/chosen": -1.1337149143218994,
|
|
"logits/rejected": -1.1420766115188599,
|
|
"logps/chosen": -49.699161529541016,
|
|
"logps/rejected": -73.22956085205078,
|
|
"loss": 0.1671,
|
|
"rewards/accuracies": 0.9444444179534912,
|
|
"rewards/chosen": 0.23295314610004425,
|
|
"rewards/margins": 4.598634243011475,
|
|
"rewards/rejected": -4.36568021774292,
|
|
"step": 554
|
|
},
|
|
{
|
|
"epoch": 1.2832414412104116,
|
|
"grad_norm": 12.879730723232214,
|
|
"learning_rate": 1.712131817788628e-07,
|
|
"logits/chosen": -1.1121582984924316,
|
|
"logits/rejected": -1.1107348203659058,
|
|
"logps/chosen": -46.21583938598633,
|
|
"logps/rejected": -56.827064514160156,
|
|
"loss": 0.2081,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.3201579451560974,
|
|
"rewards/margins": 3.4782567024230957,
|
|
"rewards/rejected": -3.1580984592437744,
|
|
"step": 556
|
|
},
|
|
{
|
|
"epoch": 1.2878574176176434,
|
|
"grad_norm": 18.379299257318767,
|
|
"learning_rate": 1.693020901660738e-07,
|
|
"logits/chosen": -1.1363815069198608,
|
|
"logits/rejected": -1.1399273872375488,
|
|
"logps/chosen": -53.33127212524414,
|
|
"logps/rejected": -65.93392181396484,
|
|
"loss": 0.1484,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.24407842755317688,
|
|
"rewards/margins": 4.509576320648193,
|
|
"rewards/rejected": -4.265497207641602,
|
|
"step": 558
|
|
},
|
|
{
|
|
"epoch": 1.292473394024875,
|
|
"grad_norm": 30.348807794181617,
|
|
"learning_rate": 1.6739624837739518e-07,
|
|
"logits/chosen": -1.1836738586425781,
|
|
"logits/rejected": -1.188474416732788,
|
|
"logps/chosen": -53.792484283447266,
|
|
"logps/rejected": -60.70852279663086,
|
|
"loss": 0.2063,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.07742541283369064,
|
|
"rewards/margins": 3.257985830307007,
|
|
"rewards/rejected": -3.335411310195923,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 1.292473394024875,
|
|
"eval_logits/chosen": -1.1171412467956543,
|
|
"eval_logits/rejected": -1.1220649480819702,
|
|
"eval_logps/chosen": -48.55030059814453,
|
|
"eval_logps/rejected": -63.84280014038086,
|
|
"eval_loss": 0.2384917438030243,
|
|
"eval_rewards/accuracies": 0.8329492807388306,
|
|
"eval_rewards/chosen": 0.04156512767076492,
|
|
"eval_rewards/margins": 3.9164135456085205,
|
|
"eval_rewards/rejected": -3.8748483657836914,
|
|
"eval_runtime": 227.4385,
|
|
"eval_samples_per_second": 7.624,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 1.2970893704321067,
|
|
"grad_norm": 16.156045882880065,
|
|
"learning_rate": 1.6549578039787434e-07,
|
|
"logits/chosen": -1.174346923828125,
|
|
"logits/rejected": -1.1770930290222168,
|
|
"logps/chosen": -50.916481018066406,
|
|
"logps/rejected": -73.8545913696289,
|
|
"loss": 0.2379,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.09411442279815674,
|
|
"rewards/margins": 4.1428680419921875,
|
|
"rewards/rejected": -4.04875373840332,
|
|
"step": 562
|
|
},
|
|
{
|
|
"epoch": 1.3017053468393383,
|
|
"grad_norm": 16.73959305434838,
|
|
"learning_rate": 1.6360080986296384e-07,
|
|
"logits/chosen": -1.14383065700531,
|
|
"logits/rejected": -1.1593358516693115,
|
|
"logps/chosen": -43.3416862487793,
|
|
"logps/rejected": -74.88224029541016,
|
|
"loss": 0.1743,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.07543455064296722,
|
|
"rewards/margins": 5.390232563018799,
|
|
"rewards/rejected": -5.465667247772217,
|
|
"step": 564
|
|
},
|
|
{
|
|
"epoch": 1.30632132324657,
|
|
"grad_norm": 19.01188883701719,
|
|
"learning_rate": 1.6171146005047894e-07,
|
|
"logits/chosen": -1.1023046970367432,
|
|
"logits/rejected": -1.1084368228912354,
|
|
"logps/chosen": -55.277671813964844,
|
|
"logps/rejected": -72.66205596923828,
|
|
"loss": 0.1773,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.12563864886760712,
|
|
"rewards/margins": 4.530452728271484,
|
|
"rewards/rejected": -4.404813766479492,
|
|
"step": 566
|
|
},
|
|
{
|
|
"epoch": 1.3109372996538018,
|
|
"grad_norm": 21.87767395078719,
|
|
"learning_rate": 1.5982785387257694e-07,
|
|
"logits/chosen": -1.1128134727478027,
|
|
"logits/rejected": -1.1113499402999878,
|
|
"logps/chosen": -49.72315979003906,
|
|
"logps/rejected": -56.22350311279297,
|
|
"loss": 0.1993,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.21243299543857574,
|
|
"rewards/margins": 3.1131999492645264,
|
|
"rewards/rejected": -3.3256328105926514,
|
|
"step": 568
|
|
},
|
|
{
|
|
"epoch": 1.3155532760610336,
|
|
"grad_norm": 23.959413620362366,
|
|
"learning_rate": 1.5795011386776159e-07,
|
|
"logits/chosen": -1.2445893287658691,
|
|
"logits/rejected": -1.2455319166183472,
|
|
"logps/chosen": -49.374000549316406,
|
|
"logps/rejected": -54.24154281616211,
|
|
"loss": 0.2022,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.03143635019659996,
|
|
"rewards/margins": 3.3321659564971924,
|
|
"rewards/rejected": -3.300729751586914,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 1.320169252468265,
|
|
"grad_norm": 16.120326025420926,
|
|
"learning_rate": 1.560783621929113e-07,
|
|
"logits/chosen": -1.216759204864502,
|
|
"logits/rejected": -1.2203327417373657,
|
|
"logps/chosen": -57.26462936401367,
|
|
"logps/rejected": -62.25014877319336,
|
|
"loss": 0.1895,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.36375537514686584,
|
|
"rewards/margins": 3.7947163581848145,
|
|
"rewards/rejected": -3.4309606552124023,
|
|
"step": 572
|
|
},
|
|
{
|
|
"epoch": 1.3247852288754969,
|
|
"grad_norm": 48.11745636864845,
|
|
"learning_rate": 1.5421272061533177e-07,
|
|
"logits/chosen": -1.1405613422393799,
|
|
"logits/rejected": -1.1527469158172607,
|
|
"logps/chosen": -43.10445785522461,
|
|
"logps/rejected": -70.5496826171875,
|
|
"loss": 0.3131,
|
|
"rewards/accuracies": 0.7638888955116272,
|
|
"rewards/chosen": 0.3498223125934601,
|
|
"rewards/margins": 4.530541896820068,
|
|
"rewards/rejected": -4.180719375610352,
|
|
"step": 574
|
|
},
|
|
{
|
|
"epoch": 1.3294012052827284,
|
|
"grad_norm": 36.29330680340726,
|
|
"learning_rate": 1.5235331050483513e-07,
|
|
"logits/chosen": -1.110296607017517,
|
|
"logits/rejected": -1.1132102012634277,
|
|
"logps/chosen": -50.03364944458008,
|
|
"logps/rejected": -65.54788208007812,
|
|
"loss": 0.2241,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.058246809989213943,
|
|
"rewards/margins": 3.8965775966644287,
|
|
"rewards/rejected": -3.954824209213257,
|
|
"step": 576
|
|
},
|
|
{
|
|
"epoch": 1.3340171816899602,
|
|
"grad_norm": 12.515890509319174,
|
|
"learning_rate": 1.5050025282584327e-07,
|
|
"logits/chosen": -1.1224780082702637,
|
|
"logits/rejected": -1.1311124563217163,
|
|
"logps/chosen": -56.89671325683594,
|
|
"logps/rejected": -74.29792785644531,
|
|
"loss": 0.1357,
|
|
"rewards/accuracies": 0.9444444179534912,
|
|
"rewards/chosen": 0.14432966709136963,
|
|
"rewards/margins": 4.722050666809082,
|
|
"rewards/rejected": -4.577720642089844,
|
|
"step": 578
|
|
},
|
|
{
|
|
"epoch": 1.338633158097192,
|
|
"grad_norm": 16.31324930298768,
|
|
"learning_rate": 1.4865366812951921e-07,
|
|
"logits/chosen": -1.0956053733825684,
|
|
"logits/rejected": -1.0948615074157715,
|
|
"logps/chosen": -44.66694641113281,
|
|
"logps/rejected": -53.35417175292969,
|
|
"loss": 0.186,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.01459770742803812,
|
|
"rewards/margins": 3.5722241401672363,
|
|
"rewards/rejected": -3.586822032928467,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 1.338633158097192,
|
|
"eval_logits/chosen": -1.1196925640106201,
|
|
"eval_logits/rejected": -1.1241984367370605,
|
|
"eval_logps/chosen": -49.20341873168945,
|
|
"eval_logps/rejected": -64.5632095336914,
|
|
"eval_loss": 0.23692870140075684,
|
|
"eval_rewards/accuracies": 0.8352534770965576,
|
|
"eval_rewards/chosen": -0.28499341011047363,
|
|
"eval_rewards/margins": 3.9500606060028076,
|
|
"eval_rewards/rejected": -4.2350544929504395,
|
|
"eval_runtime": 227.2284,
|
|
"eval_samples_per_second": 7.631,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 1.3432491345044237,
|
|
"grad_norm": 20.781517687408783,
|
|
"learning_rate": 1.4681367654592446e-07,
|
|
"logits/chosen": -1.1334997415542603,
|
|
"logits/rejected": -1.1324316263198853,
|
|
"logps/chosen": -51.77867889404297,
|
|
"logps/rejected": -59.6033935546875,
|
|
"loss": 0.1616,
|
|
"rewards/accuracies": 0.9444444179534912,
|
|
"rewards/chosen": -0.17863652110099792,
|
|
"rewards/margins": 3.280036449432373,
|
|
"rewards/rejected": -3.4586730003356934,
|
|
"step": 582
|
|
},
|
|
{
|
|
"epoch": 1.3478651109116553,
|
|
"grad_norm": 23.414987494186462,
|
|
"learning_rate": 1.4498039777620353e-07,
|
|
"logits/chosen": -1.1315072774887085,
|
|
"logits/rejected": -1.1402992010116577,
|
|
"logps/chosen": -56.817928314208984,
|
|
"logps/rejected": -78.69415283203125,
|
|
"loss": 0.1822,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.08201665431261063,
|
|
"rewards/margins": 4.948797702789307,
|
|
"rewards/rejected": -4.866781234741211,
|
|
"step": 584
|
|
},
|
|
{
|
|
"epoch": 1.352481087318887,
|
|
"grad_norm": 29.704044318217893,
|
|
"learning_rate": 1.4315395108479728e-07,
|
|
"logits/chosen": -1.170173168182373,
|
|
"logits/rejected": -1.1773362159729004,
|
|
"logps/chosen": -49.731021881103516,
|
|
"logps/rejected": -67.60116577148438,
|
|
"loss": 0.1818,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.5396385788917542,
|
|
"rewards/margins": 3.7138872146606445,
|
|
"rewards/rejected": -4.253525733947754,
|
|
"step": 586
|
|
},
|
|
{
|
|
"epoch": 1.3570970637261186,
|
|
"grad_norm": 26.692588714788616,
|
|
"learning_rate": 1.4133445529168365e-07,
|
|
"logits/chosen": -1.1388497352600098,
|
|
"logits/rejected": -1.1425156593322754,
|
|
"logps/chosen": -54.95615005493164,
|
|
"logps/rejected": -69.78382873535156,
|
|
"loss": 0.1626,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.5585896968841553,
|
|
"rewards/margins": 4.065035820007324,
|
|
"rewards/rejected": -4.623625755310059,
|
|
"step": 588
|
|
},
|
|
{
|
|
"epoch": 1.3617130401333504,
|
|
"grad_norm": 17.240579232471294,
|
|
"learning_rate": 1.395220287646483e-07,
|
|
"logits/chosen": -1.1489366292953491,
|
|
"logits/rejected": -1.1541228294372559,
|
|
"logps/chosen": -52.8204345703125,
|
|
"logps/rejected": -65.34224700927734,
|
|
"loss": 0.1726,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.6690115332603455,
|
|
"rewards/margins": 3.6602673530578613,
|
|
"rewards/rejected": -4.329278945922852,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 1.3663290165405821,
|
|
"grad_norm": 26.542897065871216,
|
|
"learning_rate": 1.377167894115837e-07,
|
|
"logits/chosen": -1.092912197113037,
|
|
"logits/rejected": -1.1030445098876953,
|
|
"logps/chosen": -45.3072509765625,
|
|
"logps/rejected": -80.26785278320312,
|
|
"loss": 0.1874,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.4750543236732483,
|
|
"rewards/margins": 4.889781475067139,
|
|
"rewards/rejected": -5.3648362159729,
|
|
"step": 592
|
|
},
|
|
{
|
|
"epoch": 1.370944992947814,
|
|
"grad_norm": 14.292615359684781,
|
|
"learning_rate": 1.3591885467281877e-07,
|
|
"logits/chosen": -1.2319241762161255,
|
|
"logits/rejected": -1.2370344400405884,
|
|
"logps/chosen": -47.54234313964844,
|
|
"logps/rejected": -69.16310119628906,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": -0.6050369739532471,
|
|
"rewards/margins": 4.743907451629639,
|
|
"rewards/rejected": -5.348944664001465,
|
|
"step": 594
|
|
},
|
|
{
|
|
"epoch": 1.3755609693550455,
|
|
"grad_norm": 24.44530683273813,
|
|
"learning_rate": 1.3412834151347896e-07,
|
|
"logits/chosen": -1.1558417081832886,
|
|
"logits/rejected": -1.1581149101257324,
|
|
"logps/chosen": -51.461524963378906,
|
|
"logps/rejected": -66.95028686523438,
|
|
"loss": 0.1852,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.5921568274497986,
|
|
"rewards/margins": 4.088617324829102,
|
|
"rewards/rejected": -4.680773735046387,
|
|
"step": 596
|
|
},
|
|
{
|
|
"epoch": 1.3801769457622772,
|
|
"grad_norm": 20.235949045232623,
|
|
"learning_rate": 1.323453664158769e-07,
|
|
"logits/chosen": -1.15675687789917,
|
|
"logits/rejected": -1.1713188886642456,
|
|
"logps/chosen": -47.177574157714844,
|
|
"logps/rejected": -76.56927490234375,
|
|
"loss": 0.2271,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": -0.8099576234817505,
|
|
"rewards/margins": 4.676289081573486,
|
|
"rewards/rejected": -5.486246585845947,
|
|
"step": 598
|
|
},
|
|
{
|
|
"epoch": 1.3847929221695088,
|
|
"grad_norm": 17.33444042750883,
|
|
"learning_rate": 1.3057004537193422e-07,
|
|
"logits/chosen": -1.178117036819458,
|
|
"logits/rejected": -1.1799274682998657,
|
|
"logps/chosen": -53.55020523071289,
|
|
"logps/rejected": -62.39503860473633,
|
|
"loss": 0.1845,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.5546784996986389,
|
|
"rewards/margins": 4.285789966583252,
|
|
"rewards/rejected": -4.840468883514404,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 1.3847929221695088,
|
|
"eval_logits/chosen": -1.1150095462799072,
|
|
"eval_logits/rejected": -1.1200028657913208,
|
|
"eval_logps/chosen": -49.0994873046875,
|
|
"eval_logps/rejected": -64.55924987792969,
|
|
"eval_loss": 0.23688365519046783,
|
|
"eval_rewards/accuracies": 0.8312212228775024,
|
|
"eval_rewards/chosen": -0.2330285757780075,
|
|
"eval_rewards/margins": 4.000042915344238,
|
|
"eval_rewards/rejected": -4.2330708503723145,
|
|
"eval_runtime": 227.4951,
|
|
"eval_samples_per_second": 7.622,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 1.3894088985767405,
|
|
"grad_norm": 37.7777346930926,
|
|
"learning_rate": 1.2880249387563662e-07,
|
|
"logits/chosen": -1.1187705993652344,
|
|
"logits/rejected": -1.1262003183364868,
|
|
"logps/chosen": -51.49440002441406,
|
|
"logps/rejected": -74.16474914550781,
|
|
"loss": 0.1865,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.5179623365402222,
|
|
"rewards/margins": 4.877338409423828,
|
|
"rewards/rejected": -5.395299911499023,
|
|
"step": 602
|
|
},
|
|
{
|
|
"epoch": 1.3940248749839723,
|
|
"grad_norm": 9.235672719669628,
|
|
"learning_rate": 1.2704282691551938e-07,
|
|
"logits/chosen": -1.116797924041748,
|
|
"logits/rejected": -1.1314423084259033,
|
|
"logps/chosen": -47.851707458496094,
|
|
"logps/rejected": -79.61589813232422,
|
|
"loss": 0.1526,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.13351619243621826,
|
|
"rewards/margins": 5.681245803833008,
|
|
"rewards/rejected": -5.5477294921875,
|
|
"step": 604
|
|
},
|
|
{
|
|
"epoch": 1.398640851391204,
|
|
"grad_norm": 22.69345823971204,
|
|
"learning_rate": 1.2529115896718714e-07,
|
|
"logits/chosen": -1.1550525426864624,
|
|
"logits/rejected": -1.158216118812561,
|
|
"logps/chosen": -52.753231048583984,
|
|
"logps/rejected": -61.50723648071289,
|
|
"loss": 0.1924,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": -0.3491640090942383,
|
|
"rewards/margins": 3.678175926208496,
|
|
"rewards/rejected": -4.027340412139893,
|
|
"step": 606
|
|
},
|
|
{
|
|
"epoch": 1.4032568277984356,
|
|
"grad_norm": 18.832219757797976,
|
|
"learning_rate": 1.2354760398586708e-07,
|
|
"logits/chosen": -1.0966627597808838,
|
|
"logits/rejected": -1.108534574508667,
|
|
"logps/chosen": -55.49364471435547,
|
|
"logps/rejected": -82.76998901367188,
|
|
"loss": 0.1459,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": -0.10722073167562485,
|
|
"rewards/margins": 5.6869659423828125,
|
|
"rewards/rejected": -5.794186592102051,
|
|
"step": 608
|
|
},
|
|
{
|
|
"epoch": 1.4078728042056674,
|
|
"grad_norm": 14.9525121487636,
|
|
"learning_rate": 1.2181227539899468e-07,
|
|
"logits/chosen": -1.1296883821487427,
|
|
"logits/rejected": -1.1345244646072388,
|
|
"logps/chosen": -52.32283020019531,
|
|
"logps/rejected": -67.40359497070312,
|
|
"loss": 0.1857,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.009007109329104424,
|
|
"rewards/margins": 4.094158172607422,
|
|
"rewards/rejected": -4.103165626525879,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 1.412488780612899,
|
|
"grad_norm": 14.336076086644082,
|
|
"learning_rate": 1.2008528609883557e-07,
|
|
"logits/chosen": -1.1148794889450073,
|
|
"logits/rejected": -1.125455379486084,
|
|
"logps/chosen": -54.79003143310547,
|
|
"logps/rejected": -75.44340515136719,
|
|
"loss": 0.1438,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.29885244369506836,
|
|
"rewards/margins": 5.630979537963867,
|
|
"rewards/rejected": -5.332127571105957,
|
|
"step": 612
|
|
},
|
|
{
|
|
"epoch": 1.4171047570201307,
|
|
"grad_norm": 19.871071765511616,
|
|
"learning_rate": 1.1836674843514042e-07,
|
|
"logits/chosen": -1.1470178365707397,
|
|
"logits/rejected": -1.1546311378479004,
|
|
"logps/chosen": -43.762474060058594,
|
|
"logps/rejected": -64.04544830322266,
|
|
"loss": 0.1737,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.18334037065505981,
|
|
"rewards/margins": 4.691874027252197,
|
|
"rewards/rejected": -4.508533477783203,
|
|
"step": 614
|
|
},
|
|
{
|
|
"epoch": 1.4217207334273625,
|
|
"grad_norm": 12.336389870346276,
|
|
"learning_rate": 1.1665677420783671e-07,
|
|
"logits/chosen": -1.1334505081176758,
|
|
"logits/rejected": -1.1354079246520996,
|
|
"logps/chosen": -48.85646057128906,
|
|
"logps/rejected": -58.98961639404297,
|
|
"loss": 0.1774,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.5762940645217896,
|
|
"rewards/margins": 4.42154598236084,
|
|
"rewards/rejected": -3.84525203704834,
|
|
"step": 616
|
|
},
|
|
{
|
|
"epoch": 1.4263367098345943,
|
|
"grad_norm": 26.99839545918975,
|
|
"learning_rate": 1.149554746597553e-07,
|
|
"logits/chosen": -1.153773546218872,
|
|
"logits/rejected": -1.1617780923843384,
|
|
"logps/chosen": -51.52302551269531,
|
|
"logps/rejected": -71.93586730957031,
|
|
"loss": 0.2209,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": -0.06947656720876694,
|
|
"rewards/margins": 4.659074783325195,
|
|
"rewards/rejected": -4.728551387786865,
|
|
"step": 618
|
|
},
|
|
{
|
|
"epoch": 1.4309526862418258,
|
|
"grad_norm": 23.120670434819147,
|
|
"learning_rate": 1.1326296046939333e-07,
|
|
"logits/chosen": -1.2297728061676025,
|
|
"logits/rejected": -1.2327196598052979,
|
|
"logps/chosen": -46.15282440185547,
|
|
"logps/rejected": -57.13591766357422,
|
|
"loss": 0.2511,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.22867631912231445,
|
|
"rewards/margins": 3.8101589679718018,
|
|
"rewards/rejected": -3.5814828872680664,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 1.4309526862418258,
|
|
"eval_logits/chosen": -1.1074106693267822,
|
|
"eval_logits/rejected": -1.1129966974258423,
|
|
"eval_logps/chosen": -48.4957389831543,
|
|
"eval_logps/rejected": -64.15252685546875,
|
|
"eval_loss": 0.23753519356250763,
|
|
"eval_rewards/accuracies": 0.8341013789176941,
|
|
"eval_rewards/chosen": 0.06884526461362839,
|
|
"eval_rewards/margins": 4.098559379577637,
|
|
"eval_rewards/rejected": -4.029714107513428,
|
|
"eval_runtime": 227.1817,
|
|
"eval_samples_per_second": 7.633,
|
|
"eval_steps_per_second": 1.91,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 1.4355686626490576,
|
|
"grad_norm": 19.372274896588006,
|
|
"learning_rate": 1.1157934174371413e-07,
|
|
"logits/chosen": -1.1408151388168335,
|
|
"logits/rejected": -1.15198814868927,
|
|
"logps/chosen": -50.5245361328125,
|
|
"logps/rejected": -73.73757934570312,
|
|
"loss": 0.1935,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.12286674976348877,
|
|
"rewards/margins": 4.926203727722168,
|
|
"rewards/rejected": -4.803337574005127,
|
|
"step": 622
|
|
},
|
|
{
|
|
"epoch": 1.4401846390562894,
|
|
"grad_norm": 17.679599039225316,
|
|
"learning_rate": 1.0990472801098419e-07,
|
|
"logits/chosen": -1.1729627847671509,
|
|
"logits/rejected": -1.1764029264450073,
|
|
"logps/chosen": -45.59742736816406,
|
|
"logps/rejected": -67.27173614501953,
|
|
"loss": 0.1504,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.4040929973125458,
|
|
"rewards/margins": 4.669921398162842,
|
|
"rewards/rejected": -4.265828609466553,
|
|
"step": 624
|
|
},
|
|
{
|
|
"epoch": 1.444800615463521,
|
|
"grad_norm": 21.626789388307476,
|
|
"learning_rate": 1.0823922821364795e-07,
|
|
"logits/chosen": -1.0937749147415161,
|
|
"logits/rejected": -1.0970505475997925,
|
|
"logps/chosen": -57.45598602294922,
|
|
"logps/rejected": -64.92636108398438,
|
|
"loss": 0.1922,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.20599400997161865,
|
|
"rewards/margins": 4.274693489074707,
|
|
"rewards/rejected": -4.068699836730957,
|
|
"step": 626
|
|
},
|
|
{
|
|
"epoch": 1.4494165918707527,
|
|
"grad_norm": 22.846453164687816,
|
|
"learning_rate": 1.0658295070124026e-07,
|
|
"logits/chosen": -1.1624855995178223,
|
|
"logits/rejected": -1.1643033027648926,
|
|
"logps/chosen": -54.79329299926758,
|
|
"logps/rejected": -61.6580810546875,
|
|
"loss": 0.2111,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.43274542689323425,
|
|
"rewards/margins": 4.024546146392822,
|
|
"rewards/rejected": -3.5918006896972656,
|
|
"step": 628
|
|
},
|
|
{
|
|
"epoch": 1.4540325682779844,
|
|
"grad_norm": 35.15148920852289,
|
|
"learning_rate": 1.0493600322333762e-07,
|
|
"logits/chosen": -1.1498773097991943,
|
|
"logits/rejected": -1.1634445190429688,
|
|
"logps/chosen": -50.85679626464844,
|
|
"logps/rejected": -84.39616394042969,
|
|
"loss": 0.167,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": -0.023858733475208282,
|
|
"rewards/margins": 6.123683929443359,
|
|
"rewards/rejected": -6.147542953491211,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 1.458648544685216,
|
|
"grad_norm": 30.044775332799237,
|
|
"learning_rate": 1.0329849292254883e-07,
|
|
"logits/chosen": -1.0466785430908203,
|
|
"logits/rejected": -1.0532869100570679,
|
|
"logps/chosen": -50.80934143066406,
|
|
"logps/rejected": -70.91539001464844,
|
|
"loss": 0.2205,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.08874227106571198,
|
|
"rewards/margins": 4.631314277648926,
|
|
"rewards/rejected": -4.542571544647217,
|
|
"step": 632
|
|
},
|
|
{
|
|
"epoch": 1.4632645210924478,
|
|
"grad_norm": 27.395602840016966,
|
|
"learning_rate": 1.0167052632754458e-07,
|
|
"logits/chosen": -1.1735262870788574,
|
|
"logits/rejected": -1.1718860864639282,
|
|
"logps/chosen": -46.975406646728516,
|
|
"logps/rejected": -56.96874237060547,
|
|
"loss": 0.2108,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.012903611175715923,
|
|
"rewards/margins": 3.0641860961914062,
|
|
"rewards/rejected": -3.0512828826904297,
|
|
"step": 634
|
|
},
|
|
{
|
|
"epoch": 1.4678804974996795,
|
|
"grad_norm": 19.703963008444365,
|
|
"learning_rate": 1.0005220934612713e-07,
|
|
"logits/chosen": -1.0792059898376465,
|
|
"logits/rejected": -1.0779961347579956,
|
|
"logps/chosen": -54.955718994140625,
|
|
"logps/rejected": -60.88299560546875,
|
|
"loss": 0.1733,
|
|
"rewards/accuracies": 0.9166666865348816,
|
|
"rewards/chosen": 0.10138123482465744,
|
|
"rewards/margins": 3.779219627380371,
|
|
"rewards/rejected": -3.6778385639190674,
|
|
"step": 636
|
|
},
|
|
{
|
|
"epoch": 1.472496473906911,
|
|
"grad_norm": 26.332520125277515,
|
|
"learning_rate": 9.844364725834056e-08,
|
|
"logits/chosen": -1.1881197690963745,
|
|
"logits/rejected": -1.201986312866211,
|
|
"logps/chosen": -51.59233093261719,
|
|
"logps/rejected": -88.54263305664062,
|
|
"loss": 0.1426,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.19230936467647552,
|
|
"rewards/margins": 6.799825668334961,
|
|
"rewards/rejected": -6.607515811920166,
|
|
"step": 638
|
|
},
|
|
{
|
|
"epoch": 1.4771124503141428,
|
|
"grad_norm": 28.45092893361144,
|
|
"learning_rate": 9.68449447096217e-08,
|
|
"logits/chosen": -1.2595468759536743,
|
|
"logits/rejected": -1.2635498046875,
|
|
"logps/chosen": -45.43921661376953,
|
|
"logps/rejected": -59.12704849243164,
|
|
"loss": 0.3573,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": 0.18244637548923492,
|
|
"rewards/margins": 3.4541211128234863,
|
|
"rewards/rejected": -3.271674394607544,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 1.4771124503141428,
|
|
"eval_logits/chosen": -1.1004152297973633,
|
|
"eval_logits/rejected": -1.1063543558120728,
|
|
"eval_logps/chosen": -48.19259262084961,
|
|
"eval_logps/rejected": -63.969642639160156,
|
|
"eval_loss": 0.23769782483577728,
|
|
"eval_rewards/accuracies": 0.8346773982048035,
|
|
"eval_rewards/chosen": 0.22041727602481842,
|
|
"eval_rewards/margins": 4.15868616104126,
|
|
"eval_rewards/rejected": -3.9382688999176025,
|
|
"eval_runtime": 227.6304,
|
|
"eval_samples_per_second": 7.618,
|
|
"eval_steps_per_second": 1.907,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 1.4817284267213746,
|
|
"grad_norm": 24.31958981085731,
|
|
"learning_rate": 9.525620570399259e-08,
|
|
"logits/chosen": -1.1648564338684082,
|
|
"logits/rejected": -1.1766667366027832,
|
|
"logps/chosen": -50.69769287109375,
|
|
"logps/rejected": -73.31265258789062,
|
|
"loss": 0.138,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.1818230152130127,
|
|
"rewards/margins": 4.770647048950195,
|
|
"rewards/rejected": -4.5888237953186035,
|
|
"step": 642
|
|
},
|
|
{
|
|
"epoch": 1.4863444031286062,
|
|
"grad_norm": 16.147422378148644,
|
|
"learning_rate": 9.36775335972943e-08,
|
|
"logits/chosen": -1.1746997833251953,
|
|
"logits/rejected": -1.2103776931762695,
|
|
"logps/chosen": -45.329200744628906,
|
|
"logps/rejected": -116.44515228271484,
|
|
"loss": 0.166,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.2069842666387558,
|
|
"rewards/margins": 9.259431838989258,
|
|
"rewards/rejected": -9.052447319030762,
|
|
"step": 644
|
|
},
|
|
{
|
|
"epoch": 1.490960379535838,
|
|
"grad_norm": 26.623831420383585,
|
|
"learning_rate": 9.210903109046284e-08,
|
|
"logits/chosen": -1.186785340309143,
|
|
"logits/rejected": -1.1978414058685303,
|
|
"logps/chosen": -49.64828872680664,
|
|
"logps/rejected": -75.10488891601562,
|
|
"loss": 0.1953,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": -0.27270856499671936,
|
|
"rewards/margins": 5.708976745605469,
|
|
"rewards/rejected": -5.981686115264893,
|
|
"step": 646
|
|
},
|
|
{
|
|
"epoch": 1.4955763559430697,
|
|
"grad_norm": 20.261642357856545,
|
|
"learning_rate": 9.05508002228485e-08,
|
|
"logits/chosen": -1.1485164165496826,
|
|
"logits/rejected": -1.1560351848602295,
|
|
"logps/chosen": -43.76812744140625,
|
|
"logps/rejected": -61.48163604736328,
|
|
"loss": 0.2165,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.30232900381088257,
|
|
"rewards/margins": 4.651912689208984,
|
|
"rewards/rejected": -4.349584579467773,
|
|
"step": 648
|
|
},
|
|
{
|
|
"epoch": 1.5001923323503012,
|
|
"grad_norm": 16.434029065803497,
|
|
"learning_rate": 8.900294236557707e-08,
|
|
"logits/chosen": -1.1660001277923584,
|
|
"logits/rejected": -1.1689536571502686,
|
|
"logps/chosen": -43.022518157958984,
|
|
"logps/rejected": -55.89129638671875,
|
|
"loss": 0.2104,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.252067506313324,
|
|
"rewards/margins": 3.607164144515991,
|
|
"rewards/rejected": -3.3550968170166016,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 1.504808308757533,
|
|
"grad_norm": 17.153074627752197,
|
|
"learning_rate": 8.746555821495561e-08,
|
|
"logits/chosen": -1.1337089538574219,
|
|
"logits/rejected": -1.149064540863037,
|
|
"logps/chosen": -49.65050506591797,
|
|
"logps/rejected": -73.29889678955078,
|
|
"loss": 0.168,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.3097386956214905,
|
|
"rewards/margins": 5.331449031829834,
|
|
"rewards/rejected": -5.0217108726501465,
|
|
"step": 652
|
|
},
|
|
{
|
|
"epoch": 1.5094242851647648,
|
|
"grad_norm": 23.56988440345321,
|
|
"learning_rate": 8.593874778592122e-08,
|
|
"logits/chosen": -1.1784387826919556,
|
|
"logits/rejected": -1.1783757209777832,
|
|
"logps/chosen": -43.02362823486328,
|
|
"logps/rejected": -56.035400390625,
|
|
"loss": 0.1716,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.2998148500919342,
|
|
"rewards/margins": 3.706061840057373,
|
|
"rewards/rejected": -3.4062466621398926,
|
|
"step": 654
|
|
},
|
|
{
|
|
"epoch": 1.5140402615719966,
|
|
"grad_norm": 24.44217268761521,
|
|
"learning_rate": 8.442261040553472e-08,
|
|
"logits/chosen": -1.1735872030258179,
|
|
"logits/rejected": -1.1768840551376343,
|
|
"logps/chosen": -50.59683609008789,
|
|
"logps/rejected": -56.11674880981445,
|
|
"loss": 0.1561,
|
|
"rewards/accuracies": 0.9444444179534912,
|
|
"rewards/chosen": 0.28288352489471436,
|
|
"rewards/margins": 3.881596088409424,
|
|
"rewards/rejected": -3.59871244430542,
|
|
"step": 656
|
|
},
|
|
{
|
|
"epoch": 1.518656237979228,
|
|
"grad_norm": 32.02127603424348,
|
|
"learning_rate": 8.291724470651903e-08,
|
|
"logits/chosen": -1.1492629051208496,
|
|
"logits/rejected": -1.1569753885269165,
|
|
"logps/chosen": -51.28215026855469,
|
|
"logps/rejected": -65.28805541992188,
|
|
"loss": 0.2696,
|
|
"rewards/accuracies": 0.7916666865348816,
|
|
"rewards/chosen": -0.23993118107318878,
|
|
"rewards/margins": 3.802645444869995,
|
|
"rewards/rejected": -4.042576313018799,
|
|
"step": 658
|
|
},
|
|
{
|
|
"epoch": 1.5232722143864597,
|
|
"grad_norm": 16.1030974632108,
|
|
"learning_rate": 8.14227486208423e-08,
|
|
"logits/chosen": -1.2377209663391113,
|
|
"logits/rejected": -1.2406808137893677,
|
|
"logps/chosen": -45.322593688964844,
|
|
"logps/rejected": -61.13237762451172,
|
|
"loss": 0.1727,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.44821304082870483,
|
|
"rewards/margins": 4.551301956176758,
|
|
"rewards/rejected": -4.103089332580566,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 1.5232722143864597,
|
|
"eval_logits/chosen": -1.106886386871338,
|
|
"eval_logits/rejected": -1.1123414039611816,
|
|
"eval_logps/chosen": -48.28040313720703,
|
|
"eval_logps/rejected": -64.126708984375,
|
|
"eval_loss": 0.23655745387077332,
|
|
"eval_rewards/accuracies": 0.8346773982048035,
|
|
"eval_rewards/chosen": 0.17651285231113434,
|
|
"eval_rewards/margins": 4.193314552307129,
|
|
"eval_rewards/rejected": -4.016801834106445,
|
|
"eval_runtime": 227.3008,
|
|
"eval_samples_per_second": 7.629,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 1.5278881907936914,
|
|
"grad_norm": 26.89012407667588,
|
|
"learning_rate": 7.993921937334716e-08,
|
|
"logits/chosen": -1.1547244787216187,
|
|
"logits/rejected": -1.1549021005630493,
|
|
"logps/chosen": -48.07485580444336,
|
|
"logps/rejected": -55.9273567199707,
|
|
"loss": 0.2437,
|
|
"rewards/accuracies": 0.8055555820465088,
|
|
"rewards/chosen": 0.16911821067333221,
|
|
"rewards/margins": 3.7287042140960693,
|
|
"rewards/rejected": -3.5595860481262207,
|
|
"step": 662
|
|
},
|
|
{
|
|
"epoch": 1.5325041672009232,
|
|
"grad_norm": 9.277145884274647,
|
|
"learning_rate": 7.846675347542578e-08,
|
|
"logits/chosen": -1.0987221002578735,
|
|
"logits/rejected": -1.0982441902160645,
|
|
"logps/chosen": -44.44068908691406,
|
|
"logps/rejected": -55.658538818359375,
|
|
"loss": 0.1239,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.7777649760246277,
|
|
"rewards/margins": 4.145383834838867,
|
|
"rewards/rejected": -3.3676185607910156,
|
|
"step": 664
|
|
},
|
|
{
|
|
"epoch": 1.537120143608155,
|
|
"grad_norm": 23.47472041668218,
|
|
"learning_rate": 7.700544671874079e-08,
|
|
"logits/chosen": -1.1225018501281738,
|
|
"logits/rejected": -1.1206145286560059,
|
|
"logps/chosen": -55.324520111083984,
|
|
"logps/rejected": -59.72848892211914,
|
|
"loss": 0.1971,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.16201752424240112,
|
|
"rewards/margins": 3.718522071838379,
|
|
"rewards/rejected": -3.556504487991333,
|
|
"step": 666
|
|
},
|
|
{
|
|
"epoch": 1.5417361200153867,
|
|
"grad_norm": 36.079809765281745,
|
|
"learning_rate": 7.555539416899437e-08,
|
|
"logits/chosen": -1.1977320909500122,
|
|
"logits/rejected": -1.2057994604110718,
|
|
"logps/chosen": -42.88581848144531,
|
|
"logps/rejected": -62.64973831176758,
|
|
"loss": 0.2275,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.011536385864019394,
|
|
"rewards/margins": 4.496410369873047,
|
|
"rewards/rejected": -4.484873294830322,
|
|
"step": 668
|
|
},
|
|
{
|
|
"epoch": 1.5463520964226183,
|
|
"grad_norm": 24.671849788858164,
|
|
"learning_rate": 7.41166901597429e-08,
|
|
"logits/chosen": -1.115236520767212,
|
|
"logits/rejected": -1.117641806602478,
|
|
"logps/chosen": -48.77861785888672,
|
|
"logps/rejected": -63.02436065673828,
|
|
"loss": 0.1757,
|
|
"rewards/accuracies": 0.9027777910232544,
|
|
"rewards/chosen": 0.31542646884918213,
|
|
"rewards/margins": 4.368900775909424,
|
|
"rewards/rejected": -4.053474426269531,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 1.5509680728298498,
|
|
"grad_norm": 22.124151348706462,
|
|
"learning_rate": 7.268942828626046e-08,
|
|
"logits/chosen": -1.201170802116394,
|
|
"logits/rejected": -1.2061718702316284,
|
|
"logps/chosen": -46.17606735229492,
|
|
"logps/rejected": -60.99459457397461,
|
|
"loss": 0.2017,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.4047514796257019,
|
|
"rewards/margins": 4.501495361328125,
|
|
"rewards/rejected": -4.096743583679199,
|
|
"step": 672
|
|
},
|
|
{
|
|
"epoch": 1.5555840492370816,
|
|
"grad_norm": 24.405076004524442,
|
|
"learning_rate": 7.127370139945018e-08,
|
|
"logits/chosen": -1.1625523567199707,
|
|
"logits/rejected": -1.1654243469238281,
|
|
"logps/chosen": -48.26844024658203,
|
|
"logps/rejected": -65.458740234375,
|
|
"loss": 0.1711,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.4592117667198181,
|
|
"rewards/margins": 4.522068500518799,
|
|
"rewards/rejected": -4.062856674194336,
|
|
"step": 674
|
|
},
|
|
{
|
|
"epoch": 1.5602000256443134,
|
|
"grad_norm": 18.612456705571446,
|
|
"learning_rate": 6.986960159980326e-08,
|
|
"logits/chosen": -1.1604725122451782,
|
|
"logits/rejected": -1.16167414188385,
|
|
"logps/chosen": -50.374324798583984,
|
|
"logps/rejected": -61.40808868408203,
|
|
"loss": 0.1842,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.5715973377227783,
|
|
"rewards/margins": 3.9924135208129883,
|
|
"rewards/rejected": -3.42081618309021,
|
|
"step": 676
|
|
},
|
|
{
|
|
"epoch": 1.5648160020515451,
|
|
"grad_norm": 43.48667538563028,
|
|
"learning_rate": 6.847722023140776e-08,
|
|
"logits/chosen": -1.2388062477111816,
|
|
"logits/rejected": -1.234206199645996,
|
|
"logps/chosen": -44.29112243652344,
|
|
"logps/rejected": -51.78242492675781,
|
|
"loss": 0.2001,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.6832877993583679,
|
|
"rewards/margins": 3.618730306625366,
|
|
"rewards/rejected": -2.9354422092437744,
|
|
"step": 678
|
|
},
|
|
{
|
|
"epoch": 1.569431978458777,
|
|
"grad_norm": 28.795198035373833,
|
|
"learning_rate": 6.709664787600616e-08,
|
|
"logits/chosen": -1.209147572517395,
|
|
"logits/rejected": -1.2093759775161743,
|
|
"logps/chosen": -42.70883560180664,
|
|
"logps/rejected": -50.379886627197266,
|
|
"loss": 0.2779,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.21883456408977509,
|
|
"rewards/margins": 2.795126438140869,
|
|
"rewards/rejected": -2.576291799545288,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 1.569431978458777,
|
|
"eval_logits/chosen": -1.118242621421814,
|
|
"eval_logits/rejected": -1.1229368448257446,
|
|
"eval_logps/chosen": -47.97200012207031,
|
|
"eval_logps/rejected": -63.929046630859375,
|
|
"eval_loss": 0.2367607206106186,
|
|
"eval_rewards/accuracies": 0.8312212228775024,
|
|
"eval_rewards/chosen": 0.33071503043174744,
|
|
"eval_rewards/margins": 4.248687744140625,
|
|
"eval_rewards/rejected": -3.9179720878601074,
|
|
"eval_runtime": 227.4731,
|
|
"eval_samples_per_second": 7.623,
|
|
"eval_steps_per_second": 1.908,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 1.5740479548660085,
|
|
"grad_norm": 22.618944291226985,
|
|
"learning_rate": 6.572797434710219e-08,
|
|
"logits/chosen": -1.2124552726745605,
|
|
"logits/rejected": -1.2253483533859253,
|
|
"logps/chosen": -45.591331481933594,
|
|
"logps/rejected": -77.16690063476562,
|
|
"loss": 0.1948,
|
|
"rewards/accuracies": 0.8611111044883728,
|
|
"rewards/chosen": 0.740224301815033,
|
|
"rewards/margins": 5.656050682067871,
|
|
"rewards/rejected": -4.91582727432251,
|
|
"step": 682
|
|
},
|
|
{
|
|
"epoch": 1.57866393127324,
|
|
"grad_norm": 17.206603905842275,
|
|
"learning_rate": 6.437128868411856e-08,
|
|
"logits/chosen": -1.1554473638534546,
|
|
"logits/rejected": -1.1552696228027344,
|
|
"logps/chosen": -45.49040222167969,
|
|
"logps/rejected": -53.40934753417969,
|
|
"loss": 0.2204,
|
|
"rewards/accuracies": 0.8194444179534912,
|
|
"rewards/chosen": 0.516470193862915,
|
|
"rewards/margins": 3.7008235454559326,
|
|
"rewards/rejected": -3.1843535900115967,
|
|
"step": 684
|
|
},
|
|
{
|
|
"epoch": 1.5832799076804718,
|
|
"grad_norm": 14.245504042199357,
|
|
"learning_rate": 6.302667914660384e-08,
|
|
"logits/chosen": -1.1610139608383179,
|
|
"logits/rejected": -1.1691855192184448,
|
|
"logps/chosen": -41.98570251464844,
|
|
"logps/rejected": -61.45694351196289,
|
|
"loss": 0.2166,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.34538733959198,
|
|
"rewards/margins": 4.076152324676514,
|
|
"rewards/rejected": -3.7307653427124023,
|
|
"step": 686
|
|
},
|
|
{
|
|
"epoch": 1.5878958840877035,
|
|
"grad_norm": 25.659568066978697,
|
|
"learning_rate": 6.169423320849112e-08,
|
|
"logits/chosen": -1.139572024345398,
|
|
"logits/rejected": -1.1325372457504272,
|
|
"logps/chosen": -52.09668731689453,
|
|
"logps/rejected": -52.88841247558594,
|
|
"loss": 0.1753,
|
|
"rewards/accuracies": 0.9583333134651184,
|
|
"rewards/chosen": 0.5496838092803955,
|
|
"rewards/margins": 3.8007943630218506,
|
|
"rewards/rejected": -3.251110553741455,
|
|
"step": 688
|
|
},
|
|
{
|
|
"epoch": 1.5925118604949353,
|
|
"grad_norm": 23.501661963559606,
|
|
"learning_rate": 6.037403755240748e-08,
|
|
"logits/chosen": -1.1798467636108398,
|
|
"logits/rejected": -1.1862027645111084,
|
|
"logps/chosen": -51.716487884521484,
|
|
"logps/rejected": -67.60077667236328,
|
|
"loss": 0.175,
|
|
"rewards/accuracies": 0.9305555820465088,
|
|
"rewards/chosen": 0.23453055322170258,
|
|
"rewards/margins": 4.55530309677124,
|
|
"rewards/rejected": -4.320772171020508,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 1.597127836902167,
|
|
"grad_norm": 17.275740083868094,
|
|
"learning_rate": 5.9066178064034326e-08,
|
|
"logits/chosen": -1.1528249979019165,
|
|
"logits/rejected": -1.1694761514663696,
|
|
"logps/chosen": -38.17667770385742,
|
|
"logps/rejected": -83.85364532470703,
|
|
"loss": 0.2264,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.28412505984306335,
|
|
"rewards/margins": 6.359869003295898,
|
|
"rewards/rejected": -6.075743675231934,
|
|
"step": 692
|
|
},
|
|
{
|
|
"epoch": 1.6017438133093986,
|
|
"grad_norm": 26.873806928959727,
|
|
"learning_rate": 5.777073982652064e-08,
|
|
"logits/chosen": -1.1307650804519653,
|
|
"logits/rejected": -1.132928490638733,
|
|
"logps/chosen": -41.0783576965332,
|
|
"logps/rejected": -60.499267578125,
|
|
"loss": 0.2291,
|
|
"rewards/accuracies": 0.8472222089767456,
|
|
"rewards/chosen": 0.24884945154190063,
|
|
"rewards/margins": 4.286365509033203,
|
|
"rewards/rejected": -4.037516117095947,
|
|
"step": 694
|
|
},
|
|
{
|
|
"epoch": 1.6063597897166302,
|
|
"grad_norm": 32.76563481878735,
|
|
"learning_rate": 5.6487807114947325e-08,
|
|
"logits/chosen": -1.133086085319519,
|
|
"logits/rejected": -1.149233102798462,
|
|
"logps/chosen": -49.21199035644531,
|
|
"logps/rejected": -82.91252899169922,
|
|
"loss": 0.1834,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.2540854215621948,
|
|
"rewards/margins": 5.426255226135254,
|
|
"rewards/rejected": -5.172169208526611,
|
|
"step": 696
|
|
},
|
|
{
|
|
"epoch": 1.610975766123862,
|
|
"grad_norm": 30.041135081079663,
|
|
"learning_rate": 5.521746339084532e-08,
|
|
"logits/chosen": -1.1215559244155884,
|
|
"logits/rejected": -1.130250334739685,
|
|
"logps/chosen": -54.02631759643555,
|
|
"logps/rejected": -67.99839782714844,
|
|
"loss": 0.2581,
|
|
"rewards/accuracies": 0.8333333134651184,
|
|
"rewards/chosen": 0.2139756679534912,
|
|
"rewards/margins": 4.321435928344727,
|
|
"rewards/rejected": -4.107460021972656,
|
|
"step": 698
|
|
},
|
|
{
|
|
"epoch": 1.6155917425310937,
|
|
"grad_norm": 21.854488700327504,
|
|
"learning_rate": 5.39597912967652e-08,
|
|
"logits/chosen": -1.131272315979004,
|
|
"logits/rejected": -1.1449000835418701,
|
|
"logps/chosen": -45.570159912109375,
|
|
"logps/rejected": -73.73873901367188,
|
|
"loss": 0.1902,
|
|
"rewards/accuracies": 0.8888888955116272,
|
|
"rewards/chosen": 0.329825758934021,
|
|
"rewards/margins": 5.084650039672852,
|
|
"rewards/rejected": -4.754825115203857,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 1.6155917425310937,
|
|
"eval_logits/chosen": -1.1056500673294067,
|
|
"eval_logits/rejected": -1.1112511157989502,
|
|
"eval_logps/chosen": -48.541831970214844,
|
|
"eval_logps/rejected": -64.42839050292969,
|
|
"eval_loss": 0.23584917187690735,
|
|
"eval_rewards/accuracies": 0.8323732614517212,
|
|
"eval_rewards/chosen": 0.04579799994826317,
|
|
"eval_rewards/margins": 4.213436603546143,
|
|
"eval_rewards/rejected": -4.167638778686523,
|
|
"eval_runtime": 227.3674,
|
|
"eval_samples_per_second": 7.626,
|
|
"eval_steps_per_second": 1.909,
|
|
"step": 700
|
|
}
|
|
],
|
|
"logging_steps": 2,
|
|
"max_steps": 866,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 20,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|