1499 lines
51 KiB
JSON
1499 lines
51 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.998691442030882,
|
|
"eval_steps": 400,
|
|
"global_step": 477,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.002093692750588851,
|
|
"grad_norm": 23.60069859438079,
|
|
"learning_rate": 1.0416666666666666e-08,
|
|
"logits/chosen": -2.8280014991760254,
|
|
"logits/rejected": -2.8466408252716064,
|
|
"logps/chosen": -1.1081000566482544,
|
|
"logps/rejected": -1.146370530128479,
|
|
"loss": 1.825,
|
|
"rewards/accuracies": 0.46875,
|
|
"rewards/chosen": -2.216200113296509,
|
|
"rewards/margins": 0.0765407383441925,
|
|
"rewards/rejected": -2.292741060256958,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.010468463752944255,
|
|
"grad_norm": 20.150796004914934,
|
|
"learning_rate": 5.208333333333333e-08,
|
|
"logits/chosen": -2.8811752796173096,
|
|
"logits/rejected": -2.847626209259033,
|
|
"logps/chosen": -1.0211418867111206,
|
|
"logps/rejected": -1.080704689025879,
|
|
"loss": 1.7105,
|
|
"rewards/accuracies": 0.5390625,
|
|
"rewards/chosen": -2.042283773422241,
|
|
"rewards/margins": 0.11912553012371063,
|
|
"rewards/rejected": -2.161409378051758,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.02093692750588851,
|
|
"grad_norm": 19.30396074137424,
|
|
"learning_rate": 1.0416666666666667e-07,
|
|
"logits/chosen": -2.992589235305786,
|
|
"logits/rejected": -2.9415202140808105,
|
|
"logps/chosen": -1.0235936641693115,
|
|
"logps/rejected": -1.1658079624176025,
|
|
"loss": 1.7413,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -2.047187328338623,
|
|
"rewards/margins": 0.2844284772872925,
|
|
"rewards/rejected": -2.331615924835205,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.031405391258832765,
|
|
"grad_norm": 22.37404641220351,
|
|
"learning_rate": 1.5624999999999999e-07,
|
|
"logits/chosen": -2.9186646938323975,
|
|
"logits/rejected": -2.905055522918701,
|
|
"logps/chosen": -1.0526732206344604,
|
|
"logps/rejected": -1.1032135486602783,
|
|
"loss": 1.7229,
|
|
"rewards/accuracies": 0.5375000238418579,
|
|
"rewards/chosen": -2.105346441268921,
|
|
"rewards/margins": 0.10108034312725067,
|
|
"rewards/rejected": -2.2064270973205566,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.04187385501177702,
|
|
"grad_norm": 18.24180275182366,
|
|
"learning_rate": 2.0833333333333333e-07,
|
|
"logits/chosen": -2.9416229724884033,
|
|
"logits/rejected": -2.890882968902588,
|
|
"logps/chosen": -1.0062425136566162,
|
|
"logps/rejected": -1.1030701398849487,
|
|
"loss": 1.6956,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": -2.0124850273132324,
|
|
"rewards/margins": 0.19365549087524414,
|
|
"rewards/rejected": -2.2061402797698975,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05234231876472128,
|
|
"grad_norm": 17.74156497951028,
|
|
"learning_rate": 2.604166666666667e-07,
|
|
"logits/chosen": -2.9460272789001465,
|
|
"logits/rejected": -2.882147789001465,
|
|
"logps/chosen": -0.9283784031867981,
|
|
"logps/rejected": -1.1168667078018188,
|
|
"loss": 1.664,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": -1.8567568063735962,
|
|
"rewards/margins": 0.37697672843933105,
|
|
"rewards/rejected": -2.2337334156036377,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.06281078251766553,
|
|
"grad_norm": 17.414699170293087,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"logits/chosen": -2.919449806213379,
|
|
"logits/rejected": -2.9000954627990723,
|
|
"logps/chosen": -0.9085705876350403,
|
|
"logps/rejected": -0.9413141012191772,
|
|
"loss": 1.6764,
|
|
"rewards/accuracies": 0.550000011920929,
|
|
"rewards/chosen": -1.8171411752700806,
|
|
"rewards/margins": 0.06548723578453064,
|
|
"rewards/rejected": -1.8826282024383545,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.07327924627060979,
|
|
"grad_norm": 18.306181256584207,
|
|
"learning_rate": 3.645833333333333e-07,
|
|
"logits/chosen": -2.929173469543457,
|
|
"logits/rejected": -2.8954787254333496,
|
|
"logps/chosen": -0.9194827079772949,
|
|
"logps/rejected": -1.1145708560943604,
|
|
"loss": 1.6503,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": -1.8389654159545898,
|
|
"rewards/margins": 0.39017611742019653,
|
|
"rewards/rejected": -2.2291417121887207,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.08374771002355404,
|
|
"grad_norm": 17.72310110786498,
|
|
"learning_rate": 4.1666666666666667e-07,
|
|
"logits/chosen": -2.920375347137451,
|
|
"logits/rejected": -2.878530979156494,
|
|
"logps/chosen": -0.9253458976745605,
|
|
"logps/rejected": -1.0473893880844116,
|
|
"loss": 1.6223,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -1.850691795349121,
|
|
"rewards/margins": 0.2440868616104126,
|
|
"rewards/rejected": -2.0947787761688232,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.0942161737764983,
|
|
"grad_norm": 16.885070690793345,
|
|
"learning_rate": 4.6874999999999996e-07,
|
|
"logits/chosen": -2.875908851623535,
|
|
"logits/rejected": -2.8810861110687256,
|
|
"logps/chosen": -0.903559684753418,
|
|
"logps/rejected": -1.0813857316970825,
|
|
"loss": 1.5727,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -1.807119369506836,
|
|
"rewards/margins": 0.3556520342826843,
|
|
"rewards/rejected": -2.162771463394165,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.10468463752944256,
|
|
"grad_norm": 23.820902499939343,
|
|
"learning_rate": 4.999731868769026e-07,
|
|
"logits/chosen": -2.820807933807373,
|
|
"logits/rejected": -2.832883358001709,
|
|
"logps/chosen": -0.9573640823364258,
|
|
"logps/rejected": -1.2130459547042847,
|
|
"loss": 1.6274,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -1.9147281646728516,
|
|
"rewards/margins": 0.5113638043403625,
|
|
"rewards/rejected": -2.4260919094085693,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.11515310128238682,
|
|
"grad_norm": 24.790862979912625,
|
|
"learning_rate": 4.996716052911017e-07,
|
|
"logits/chosen": -2.9123668670654297,
|
|
"logits/rejected": -2.9268505573272705,
|
|
"logps/chosen": -1.0739004611968994,
|
|
"logps/rejected": -1.1845686435699463,
|
|
"loss": 1.6259,
|
|
"rewards/accuracies": 0.574999988079071,
|
|
"rewards/chosen": -2.147800922393799,
|
|
"rewards/margins": 0.22133660316467285,
|
|
"rewards/rejected": -2.3691372871398926,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.12562156503533106,
|
|
"grad_norm": 30.216317311158747,
|
|
"learning_rate": 4.990353313429303e-07,
|
|
"logits/chosen": -2.847761869430542,
|
|
"logits/rejected": -2.8407883644104004,
|
|
"logps/chosen": -1.0353600978851318,
|
|
"logps/rejected": -1.207129716873169,
|
|
"loss": 1.5827,
|
|
"rewards/accuracies": 0.5625,
|
|
"rewards/chosen": -2.0707201957702637,
|
|
"rewards/margins": 0.3435389995574951,
|
|
"rewards/rejected": -2.414259433746338,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.1360900287882753,
|
|
"grad_norm": 25.74208520684934,
|
|
"learning_rate": 4.980652179769217e-07,
|
|
"logits/chosen": -2.8678090572357178,
|
|
"logits/rejected": -2.89147686958313,
|
|
"logps/chosen": -1.0218393802642822,
|
|
"logps/rejected": -1.196505069732666,
|
|
"loss": 1.5964,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": -2.0436787605285645,
|
|
"rewards/margins": 0.3493315577507019,
|
|
"rewards/rejected": -2.393010139465332,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.14655849254121958,
|
|
"grad_norm": 26.44073609906296,
|
|
"learning_rate": 4.967625656594781e-07,
|
|
"logits/chosen": -2.86854887008667,
|
|
"logits/rejected": -2.8683791160583496,
|
|
"logps/chosen": -1.1050409078598022,
|
|
"logps/rejected": -1.300787091255188,
|
|
"loss": 1.5446,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -2.2100818157196045,
|
|
"rewards/margins": 0.391492635011673,
|
|
"rewards/rejected": -2.601574182510376,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.15702695629416383,
|
|
"grad_norm": 32.621154621425745,
|
|
"learning_rate": 4.951291206355559e-07,
|
|
"logits/chosen": -2.816270351409912,
|
|
"logits/rejected": -2.7658441066741943,
|
|
"logps/chosen": -1.1289862394332886,
|
|
"logps/rejected": -1.42450749874115,
|
|
"loss": 1.5692,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -2.257972478866577,
|
|
"rewards/margins": 0.5910428762435913,
|
|
"rewards/rejected": -2.8490149974823,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.16749542004710807,
|
|
"grad_norm": 23.011756326022333,
|
|
"learning_rate": 4.93167072587771e-07,
|
|
"logits/chosen": -2.7463507652282715,
|
|
"logits/rejected": -2.741854667663574,
|
|
"logps/chosen": -1.118318796157837,
|
|
"logps/rejected": -1.486810564994812,
|
|
"loss": 1.5494,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -2.236637592315674,
|
|
"rewards/margins": 0.7369835376739502,
|
|
"rewards/rejected": -2.973621129989624,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.17796388380005235,
|
|
"grad_norm": 28.536903966310312,
|
|
"learning_rate": 4.908790517010636e-07,
|
|
"logits/chosen": -2.8005614280700684,
|
|
"logits/rejected": -2.7525715827941895,
|
|
"logps/chosen": -1.1057350635528564,
|
|
"logps/rejected": -1.4478759765625,
|
|
"loss": 1.5328,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -2.211470127105713,
|
|
"rewards/margins": 0.6842816472053528,
|
|
"rewards/rejected": -2.895751953125,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.1884323475529966,
|
|
"grad_norm": 21.41681785684056,
|
|
"learning_rate": 4.882681251368548e-07,
|
|
"logits/chosen": -2.7789416313171387,
|
|
"logits/rejected": -2.7269153594970703,
|
|
"logps/chosen": -1.1642733812332153,
|
|
"logps/rejected": -1.5922167301177979,
|
|
"loss": 1.5065,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -2.3285467624664307,
|
|
"rewards/margins": 0.8558871150016785,
|
|
"rewards/rejected": -3.1844334602355957,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.19890081130594087,
|
|
"grad_norm": 31.29059195928571,
|
|
"learning_rate": 4.853377929214243e-07,
|
|
"logits/chosen": -2.705059051513672,
|
|
"logits/rejected": -2.689885139465332,
|
|
"logps/chosen": -1.1885052919387817,
|
|
"logps/rejected": -1.503861665725708,
|
|
"loss": 1.5348,
|
|
"rewards/accuracies": 0.581250011920929,
|
|
"rewards/chosen": -2.3770105838775635,
|
|
"rewards/margins": 0.6307128667831421,
|
|
"rewards/rejected": -3.007723331451416,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.2093692750588851,
|
|
"grad_norm": 29.067491467613685,
|
|
"learning_rate": 4.820919832540181e-07,
|
|
"logits/chosen": -2.776214122772217,
|
|
"logits/rejected": -2.7497711181640625,
|
|
"logps/chosen": -1.157089352607727,
|
|
"logps/rejected": -1.3819860219955444,
|
|
"loss": 1.5864,
|
|
"rewards/accuracies": 0.5562499761581421,
|
|
"rewards/chosen": -2.314178705215454,
|
|
"rewards/margins": 0.44979315996170044,
|
|
"rewards/rejected": -2.763972043991089,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.21983773881182936,
|
|
"grad_norm": 23.067455098331433,
|
|
"learning_rate": 4.785350472409791e-07,
|
|
"logits/chosen": -2.7103278636932373,
|
|
"logits/rejected": -2.68127703666687,
|
|
"logps/chosen": -1.180944800376892,
|
|
"logps/rejected": -1.535935640335083,
|
|
"loss": 1.4346,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": -2.361889600753784,
|
|
"rewards/margins": 0.7099816203117371,
|
|
"rewards/rejected": -3.071871280670166,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.23030620256477363,
|
|
"grad_norm": 38.83165169666446,
|
|
"learning_rate": 4.7467175306295647e-07,
|
|
"logits/chosen": -2.6472716331481934,
|
|
"logits/rejected": -2.6239981651306152,
|
|
"logps/chosen": -1.1390531063079834,
|
|
"logps/rejected": -1.4835283756256104,
|
|
"loss": 1.4351,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -2.278106212615967,
|
|
"rewards/margins": 0.6889505386352539,
|
|
"rewards/rejected": -2.9670567512512207,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.24077466631771788,
|
|
"grad_norm": 28.07429755635963,
|
|
"learning_rate": 4.70507279583015e-07,
|
|
"logits/chosen": -2.6805942058563232,
|
|
"logits/rejected": -2.6706345081329346,
|
|
"logps/chosen": -1.1800041198730469,
|
|
"logps/rejected": -1.4017146825790405,
|
|
"loss": 1.527,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -2.3600082397460938,
|
|
"rewards/margins": 0.4434211254119873,
|
|
"rewards/rejected": -2.803429365158081,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.2512431300706621,
|
|
"grad_norm": 25.451878448448234,
|
|
"learning_rate": 4.6604720940421207e-07,
|
|
"logits/chosen": -2.695066213607788,
|
|
"logits/rejected": -2.6383955478668213,
|
|
"logps/chosen": -1.11952805519104,
|
|
"logps/rejected": -1.5505688190460205,
|
|
"loss": 1.5091,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -2.23905611038208,
|
|
"rewards/margins": 0.8620816469192505,
|
|
"rewards/rejected": -3.101137638092041,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.26171159382360637,
|
|
"grad_norm": 28.525020675058567,
|
|
"learning_rate": 4.612975213859487e-07,
|
|
"logits/chosen": -2.5875375270843506,
|
|
"logits/rejected": -2.5485615730285645,
|
|
"logps/chosen": -1.1817591190338135,
|
|
"logps/rejected": -1.588836669921875,
|
|
"loss": 1.5135,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -2.363518238067627,
|
|
"rewards/margins": 0.8141552209854126,
|
|
"rewards/rejected": -3.17767333984375,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.2721800575765506,
|
|
"grad_norm": 33.5709446027395,
|
|
"learning_rate": 4.5626458262912735e-07,
|
|
"logits/chosen": -2.6082234382629395,
|
|
"logits/rejected": -2.614490032196045,
|
|
"logps/chosen": -1.1771992444992065,
|
|
"logps/rejected": -1.5893863439559937,
|
|
"loss": 1.4422,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -2.354398488998413,
|
|
"rewards/margins": 0.8243740200996399,
|
|
"rewards/rejected": -3.1787726879119873,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.2826485213294949,
|
|
"grad_norm": 26.6001483024502,
|
|
"learning_rate": 4.5095513994085974e-07,
|
|
"logits/chosen": -2.54782772064209,
|
|
"logits/rejected": -2.51760196685791,
|
|
"logps/chosen": -1.0749025344848633,
|
|
"logps/rejected": -1.5542266368865967,
|
|
"loss": 1.4927,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.1498050689697266,
|
|
"rewards/margins": 0.9586483240127563,
|
|
"rewards/rejected": -3.1084532737731934,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.29311698508243916,
|
|
"grad_norm": 37.50133455368946,
|
|
"learning_rate": 4.453763107901675e-07,
|
|
"logits/chosen": -2.544795274734497,
|
|
"logits/rejected": -2.545607089996338,
|
|
"logps/chosen": -1.3422894477844238,
|
|
"logps/rejected": -1.5928326845169067,
|
|
"loss": 1.5489,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -2.6845788955688477,
|
|
"rewards/margins": 0.5010861158370972,
|
|
"rewards/rejected": -3.1856653690338135,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.3035854488353834,
|
|
"grad_norm": 30.749771656186557,
|
|
"learning_rate": 4.395355737667985e-07,
|
|
"logits/chosen": -2.5184309482574463,
|
|
"logits/rejected": -2.4989206790924072,
|
|
"logps/chosen": -1.229201078414917,
|
|
"logps/rejected": -1.7190883159637451,
|
|
"loss": 1.4308,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -2.458402156829834,
|
|
"rewards/margins": 0.9797741770744324,
|
|
"rewards/rejected": -3.4381766319274902,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.31405391258832765,
|
|
"grad_norm": 34.705616703882306,
|
|
"learning_rate": 4.3344075855595097e-07,
|
|
"logits/chosen": -2.5644986629486084,
|
|
"logits/rejected": -2.5165212154388428,
|
|
"logps/chosen": -1.2188150882720947,
|
|
"logps/rejected": -1.596199631690979,
|
|
"loss": 1.4579,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": -2.4376301765441895,
|
|
"rewards/margins": 0.7547694444656372,
|
|
"rewards/rejected": -3.192399263381958,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.3245223763412719,
|
|
"grad_norm": 52.49121820937414,
|
|
"learning_rate": 4.271000354423425e-07,
|
|
"logits/chosen": -2.475571393966675,
|
|
"logits/rejected": -2.447605609893799,
|
|
"logps/chosen": -1.2845265865325928,
|
|
"logps/rejected": -1.8401352167129517,
|
|
"loss": 1.4002,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -2.5690531730651855,
|
|
"rewards/margins": 1.1112172603607178,
|
|
"rewards/rejected": -3.6802704334259033,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.33499084009421615,
|
|
"grad_norm": 38.68489554173724,
|
|
"learning_rate": 4.2052190435769554e-07,
|
|
"logits/chosen": -2.445486545562744,
|
|
"logits/rejected": -2.4290173053741455,
|
|
"logps/chosen": -1.2640022039413452,
|
|
"logps/rejected": -1.90715754032135,
|
|
"loss": 1.4271,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -2.5280044078826904,
|
|
"rewards/margins": 1.2863105535507202,
|
|
"rewards/rejected": -3.8143150806427,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.34545930384716045,
|
|
"grad_norm": 47.847635567656944,
|
|
"learning_rate": 4.137151834863213e-07,
|
|
"logits/chosen": -2.5186421871185303,
|
|
"logits/rejected": -2.476241111755371,
|
|
"logps/chosen": -1.2429786920547485,
|
|
"logps/rejected": -1.6412330865859985,
|
|
"loss": 1.3755,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -2.485957384109497,
|
|
"rewards/margins": 0.7965084314346313,
|
|
"rewards/rejected": -3.282466173171997,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.3559277676001047,
|
|
"grad_norm": 39.55414229808935,
|
|
"learning_rate": 4.0668899744407567e-07,
|
|
"logits/chosen": -2.5068397521972656,
|
|
"logits/rejected": -2.458843231201172,
|
|
"logps/chosen": -1.2319917678833008,
|
|
"logps/rejected": -1.6991866827011108,
|
|
"loss": 1.3875,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.4639835357666016,
|
|
"rewards/margins": 0.9343900680541992,
|
|
"rewards/rejected": -3.3983733654022217,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.36639623135304894,
|
|
"grad_norm": 45.87476259725973,
|
|
"learning_rate": 3.994527650465352e-07,
|
|
"logits/chosen": -2.4231743812561035,
|
|
"logits/rejected": -2.4120521545410156,
|
|
"logps/chosen": -1.2813334465026855,
|
|
"logps/rejected": -1.8030840158462524,
|
|
"loss": 1.3979,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.562666893005371,
|
|
"rewards/margins": 1.0435012578964233,
|
|
"rewards/rejected": -3.606168031692505,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.3768646951059932,
|
|
"grad_norm": 52.555406414516895,
|
|
"learning_rate": 3.920161866827889e-07,
|
|
"logits/chosen": -2.438419818878174,
|
|
"logits/rejected": -2.4260263442993164,
|
|
"logps/chosen": -1.4097450971603394,
|
|
"logps/rejected": -1.9904677867889404,
|
|
"loss": 1.4005,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -2.8194901943206787,
|
|
"rewards/margins": 1.1614452600479126,
|
|
"rewards/rejected": -3.980935573577881,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.38733315885893743,
|
|
"grad_norm": 45.270627593903136,
|
|
"learning_rate": 3.8438923131177237e-07,
|
|
"logits/chosen": -2.390608787536621,
|
|
"logits/rejected": -2.4067492485046387,
|
|
"logps/chosen": -1.4339230060577393,
|
|
"logps/rejected": -2.0207736492156982,
|
|
"loss": 1.3374,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -2.8678460121154785,
|
|
"rewards/margins": 1.173701524734497,
|
|
"rewards/rejected": -4.0415472984313965,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.39780162261188173,
|
|
"grad_norm": 35.97699766429664,
|
|
"learning_rate": 3.765821230985757e-07,
|
|
"logits/chosen": -2.389159679412842,
|
|
"logits/rejected": -2.3969435691833496,
|
|
"logps/chosen": -1.4970250129699707,
|
|
"logps/rejected": -2.0378897190093994,
|
|
"loss": 1.3428,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -2.9940500259399414,
|
|
"rewards/margins": 1.081729531288147,
|
|
"rewards/rejected": -4.075779438018799,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.408270086364826,
|
|
"grad_norm": 53.06912771848456,
|
|
"learning_rate": 3.6860532770864005e-07,
|
|
"logits/chosen": -2.3946540355682373,
|
|
"logits/rejected": -2.3885226249694824,
|
|
"logps/chosen": -1.5624958276748657,
|
|
"logps/rejected": -2.2428553104400635,
|
|
"loss": 1.4342,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -3.1249916553497314,
|
|
"rewards/margins": 1.3607194423675537,
|
|
"rewards/rejected": -4.485710620880127,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.4187385501177702,
|
|
"grad_norm": 46.84209895719228,
|
|
"learning_rate": 3.604695382782159e-07,
|
|
"logits/chosen": -2.2939867973327637,
|
|
"logits/rejected": -2.276329755783081,
|
|
"logps/chosen": -1.6615016460418701,
|
|
"logps/rejected": -2.154167652130127,
|
|
"loss": 1.3573,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": -3.3230032920837402,
|
|
"rewards/margins": 0.9853318333625793,
|
|
"rewards/rejected": -4.308335304260254,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.42920701387071447,
|
|
"grad_norm": 63.174735320239826,
|
|
"learning_rate": 3.5218566107988867e-07,
|
|
"logits/chosen": -2.364863634109497,
|
|
"logits/rejected": -2.3571414947509766,
|
|
"logps/chosen": -1.7413444519042969,
|
|
"logps/rejected": -2.463257312774658,
|
|
"loss": 1.3484,
|
|
"rewards/accuracies": 0.65625,
|
|
"rewards/chosen": -3.4826889038085938,
|
|
"rewards/margins": 1.4438252449035645,
|
|
"rewards/rejected": -4.926514625549316,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.4396754776236587,
|
|
"grad_norm": 53.13105715426384,
|
|
"learning_rate": 3.4376480090239047e-07,
|
|
"logits/chosen": -2.216089963912964,
|
|
"logits/rejected": -2.2170519828796387,
|
|
"logps/chosen": -1.8861106634140015,
|
|
"logps/rejected": -2.334871768951416,
|
|
"loss": 1.4309,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -3.772221326828003,
|
|
"rewards/margins": 0.8975218534469604,
|
|
"rewards/rejected": -4.669743537902832,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.45014394137660296,
|
|
"grad_norm": 59.15829566376919,
|
|
"learning_rate": 3.3521824616429284e-07,
|
|
"logits/chosen": -2.225267171859741,
|
|
"logits/rejected": -2.250946521759033,
|
|
"logps/chosen": -1.865648627281189,
|
|
"logps/rejected": -2.464839220046997,
|
|
"loss": 1.3613,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -3.731297254562378,
|
|
"rewards/margins": 1.198380708694458,
|
|
"rewards/rejected": -4.929678440093994,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.46061240512954726,
|
|
"grad_norm": 60.027916344260326,
|
|
"learning_rate": 3.265574537815398e-07,
|
|
"logits/chosen": -2.2151072025299072,
|
|
"logits/rejected": -2.2236618995666504,
|
|
"logps/chosen": -1.9221923351287842,
|
|
"logps/rejected": -2.5661563873291016,
|
|
"loss": 1.2543,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -3.8443846702575684,
|
|
"rewards/margins": 1.2879277467727661,
|
|
"rewards/rejected": -5.132312774658203,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.4710808688824915,
|
|
"grad_norm": 58.043157481996744,
|
|
"learning_rate": 3.1779403380910425e-07,
|
|
"logits/chosen": -2.1862709522247314,
|
|
"logits/rejected": -2.1562201976776123,
|
|
"logps/chosen": -2.2186365127563477,
|
|
"logps/rejected": -2.7678346633911133,
|
|
"loss": 1.3579,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -4.437273025512695,
|
|
"rewards/margins": 1.0983966588974,
|
|
"rewards/rejected": -5.535669326782227,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.48154933263543576,
|
|
"grad_norm": 61.982027045427564,
|
|
"learning_rate": 3.0893973387735683e-07,
|
|
"logits/chosen": -2.1588408946990967,
|
|
"logits/rejected": -2.1733763217926025,
|
|
"logps/chosen": -2.2563939094543457,
|
|
"logps/rejected": -2.928832530975342,
|
|
"loss": 1.2783,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -4.512787818908691,
|
|
"rewards/margins": 1.3448779582977295,
|
|
"rewards/rejected": -5.857665061950684,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.49201779638838,
|
|
"grad_norm": 53.47910914358791,
|
|
"learning_rate": 3.000064234440111e-07,
|
|
"logits/chosen": -2.066746234893799,
|
|
"logits/rejected": -2.0809385776519775,
|
|
"logps/chosen": -2.1830739974975586,
|
|
"logps/rejected": -2.7093000411987305,
|
|
"loss": 1.3083,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -4.366147994995117,
|
|
"rewards/margins": 1.0524519681930542,
|
|
"rewards/rejected": -5.418600082397461,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.5024862601413242,
|
|
"grad_norm": 51.26667753004939,
|
|
"learning_rate": 2.910060778827554e-07,
|
|
"logits/chosen": -2.0583231449127197,
|
|
"logits/rejected": -2.0678017139434814,
|
|
"logps/chosen": -2.203321933746338,
|
|
"logps/rejected": -2.9744162559509277,
|
|
"loss": 1.3103,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.406643867492676,
|
|
"rewards/margins": 1.5421884059906006,
|
|
"rewards/rejected": -5.9488325119018555,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5129547238942685,
|
|
"grad_norm": 54.582002602834834,
|
|
"learning_rate": 2.8195076242990116e-07,
|
|
"logits/chosen": -2.076049327850342,
|
|
"logits/rejected": -2.105649948120117,
|
|
"logps/chosen": -2.2140307426452637,
|
|
"logps/rejected": -3.014204740524292,
|
|
"loss": 1.242,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -4.428061485290527,
|
|
"rewards/margins": 1.6003482341766357,
|
|
"rewards/rejected": -6.028409481048584,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.5234231876472127,
|
|
"grad_norm": 65.50612310325059,
|
|
"learning_rate": 2.7285261601056697e-07,
|
|
"logits/chosen": -1.9772818088531494,
|
|
"logits/rejected": -2.0134644508361816,
|
|
"logps/chosen": -2.4714977741241455,
|
|
"logps/rejected": -3.3275609016418457,
|
|
"loss": 1.3888,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -4.942995548248291,
|
|
"rewards/margins": 1.7121270895004272,
|
|
"rewards/rejected": -6.655121803283691,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.533891651400157,
|
|
"grad_norm": 67.4992864768084,
|
|
"learning_rate": 2.6372383496608186e-07,
|
|
"logits/chosen": -2.059023141860962,
|
|
"logits/rejected": -2.0484743118286133,
|
|
"logps/chosen": -2.5257086753845215,
|
|
"logps/rejected": -3.2463812828063965,
|
|
"loss": 1.3315,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -5.051417350769043,
|
|
"rewards/margins": 1.4413448572158813,
|
|
"rewards/rejected": -6.492762565612793,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.5443601151531012,
|
|
"grad_norm": 69.13744141717783,
|
|
"learning_rate": 2.5457665670441937e-07,
|
|
"logits/chosen": -2.006592273712158,
|
|
"logits/rejected": -2.014500141143799,
|
|
"logps/chosen": -2.560147762298584,
|
|
"logps/rejected": -3.3215479850769043,
|
|
"loss": 1.3082,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.120295524597168,
|
|
"rewards/margins": 1.5228006839752197,
|
|
"rewards/rejected": -6.643095970153809,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.5548285789060455,
|
|
"grad_norm": 58.072442212510744,
|
|
"learning_rate": 2.454233432955807e-07,
|
|
"logits/chosen": -1.9831031560897827,
|
|
"logits/rejected": -1.9949731826782227,
|
|
"logps/chosen": -2.81583833694458,
|
|
"logps/rejected": -3.404637575149536,
|
|
"loss": 1.3035,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -5.63167667388916,
|
|
"rewards/margins": 1.1775987148284912,
|
|
"rewards/rejected": -6.809275150299072,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.5652970426589898,
|
|
"grad_norm": 87.65400359085467,
|
|
"learning_rate": 2.3627616503391812e-07,
|
|
"logits/chosen": -2.0101704597473145,
|
|
"logits/rejected": -2.017418384552002,
|
|
"logps/chosen": -2.8499555587768555,
|
|
"logps/rejected": -3.4598793983459473,
|
|
"loss": 1.2368,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.699911117553711,
|
|
"rewards/margins": 1.219847559928894,
|
|
"rewards/rejected": -6.9197587966918945,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.575765506411934,
|
|
"grad_norm": 45.097233149719436,
|
|
"learning_rate": 2.2714738398943308e-07,
|
|
"logits/chosen": -1.9900894165039062,
|
|
"logits/rejected": -1.990896224975586,
|
|
"logps/chosen": -2.661165714263916,
|
|
"logps/rejected": -3.678067445755005,
|
|
"loss": 1.1497,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -5.322331428527832,
|
|
"rewards/margins": 2.033804416656494,
|
|
"rewards/rejected": -7.35613489151001,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.5862339701648783,
|
|
"grad_norm": 65.10418020833369,
|
|
"learning_rate": 2.1804923757009882e-07,
|
|
"logits/chosen": -2.0073654651641846,
|
|
"logits/rejected": -2.005659580230713,
|
|
"logps/chosen": -2.6216259002685547,
|
|
"logps/rejected": -3.4922943115234375,
|
|
"loss": 1.3131,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.243251800537109,
|
|
"rewards/margins": 1.7413362264633179,
|
|
"rewards/rejected": -6.984588623046875,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.5967024339178225,
|
|
"grad_norm": 61.639250570571264,
|
|
"learning_rate": 2.089939221172446e-07,
|
|
"logits/chosen": -1.9917762279510498,
|
|
"logits/rejected": -1.982318639755249,
|
|
"logps/chosen": -2.5954482555389404,
|
|
"logps/rejected": -3.323718309402466,
|
|
"loss": 1.2697,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.190896511077881,
|
|
"rewards/margins": 1.4565393924713135,
|
|
"rewards/rejected": -6.647436618804932,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.6071708976707668,
|
|
"grad_norm": 69.64758924542811,
|
|
"learning_rate": 1.9999357655598891e-07,
|
|
"logits/chosen": -1.9363685846328735,
|
|
"logits/rejected": -1.9144127368927002,
|
|
"logps/chosen": -2.655473232269287,
|
|
"logps/rejected": -3.300741195678711,
|
|
"loss": 1.2942,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.310946464538574,
|
|
"rewards/margins": 1.2905361652374268,
|
|
"rewards/rejected": -6.601482391357422,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.6176393614237111,
|
|
"grad_norm": 72.7122045465263,
|
|
"learning_rate": 1.9106026612264315e-07,
|
|
"logits/chosen": -1.9503307342529297,
|
|
"logits/rejected": -1.938823938369751,
|
|
"logps/chosen": -2.770573139190674,
|
|
"logps/rejected": -3.3855674266815186,
|
|
"loss": 1.2444,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -5.541146278381348,
|
|
"rewards/margins": 1.2299891710281372,
|
|
"rewards/rejected": -6.771134853363037,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.6281078251766553,
|
|
"grad_norm": 79.46704807847372,
|
|
"learning_rate": 1.8220596619089573e-07,
|
|
"logits/chosen": -1.8155113458633423,
|
|
"logits/rejected": -1.800158143043518,
|
|
"logps/chosen": -2.862525463104248,
|
|
"logps/rejected": -3.680985927581787,
|
|
"loss": 1.2954,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.725050926208496,
|
|
"rewards/margins": 1.6369216442108154,
|
|
"rewards/rejected": -7.361971855163574,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6385762889295996,
|
|
"grad_norm": 77.33985147922212,
|
|
"learning_rate": 1.7344254621846017e-07,
|
|
"logits/chosen": -1.9133965969085693,
|
|
"logits/rejected": -1.9274393320083618,
|
|
"logps/chosen": -2.733630657196045,
|
|
"logps/rejected": -3.405717134475708,
|
|
"loss": 1.2303,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.46726131439209,
|
|
"rewards/margins": 1.3441731929779053,
|
|
"rewards/rejected": -6.811434268951416,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.6490447526825438,
|
|
"grad_norm": 87.53372002667572,
|
|
"learning_rate": 1.647817538357072e-07,
|
|
"logits/chosen": -1.8388316631317139,
|
|
"logits/rejected": -1.8692519664764404,
|
|
"logps/chosen": -2.9406466484069824,
|
|
"logps/rejected": -3.684401035308838,
|
|
"loss": 1.2409,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.881293296813965,
|
|
"rewards/margins": 1.4875091314315796,
|
|
"rewards/rejected": -7.368802070617676,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.6595132164354881,
|
|
"grad_norm": 63.982977249408776,
|
|
"learning_rate": 1.562351990976095e-07,
|
|
"logits/chosen": -1.9070667028427124,
|
|
"logits/rejected": -1.91787850856781,
|
|
"logps/chosen": -2.804577350616455,
|
|
"logps/rejected": -3.586026668548584,
|
|
"loss": 1.2734,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -5.60915470123291,
|
|
"rewards/margins": 1.5628979206085205,
|
|
"rewards/rejected": -7.172053337097168,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.6699816801884323,
|
|
"grad_norm": 68.2071236663429,
|
|
"learning_rate": 1.478143389201113e-07,
|
|
"logits/chosen": -1.9229049682617188,
|
|
"logits/rejected": -1.9212894439697266,
|
|
"logps/chosen": -2.7598495483398438,
|
|
"logps/rejected": -3.6366069316864014,
|
|
"loss": 1.173,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -5.5196990966796875,
|
|
"rewards/margins": 1.7535146474838257,
|
|
"rewards/rejected": -7.273213863372803,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.6804501439413766,
|
|
"grad_norm": 82.89442348714914,
|
|
"learning_rate": 1.3953046172178413e-07,
|
|
"logits/chosen": -1.8585288524627686,
|
|
"logits/rejected": -1.8806421756744385,
|
|
"logps/chosen": -3.1847102642059326,
|
|
"logps/rejected": -4.028578758239746,
|
|
"loss": 1.319,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -6.369420528411865,
|
|
"rewards/margins": 1.6877357959747314,
|
|
"rewards/rejected": -8.057157516479492,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.6909186076943209,
|
|
"grad_norm": 63.652198082327615,
|
|
"learning_rate": 1.3139467229135998e-07,
|
|
"logits/chosen": -1.8519096374511719,
|
|
"logits/rejected": -1.8569042682647705,
|
|
"logps/chosen": -3.092832565307617,
|
|
"logps/rejected": -4.055886268615723,
|
|
"loss": 1.2226,
|
|
"rewards/accuracies": 0.737500011920929,
|
|
"rewards/chosen": -6.185665130615234,
|
|
"rewards/margins": 1.9261070489883423,
|
|
"rewards/rejected": -8.111772537231445,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.7013870714472651,
|
|
"grad_norm": 88.02707766997058,
|
|
"learning_rate": 1.2341787690142435e-07,
|
|
"logits/chosen": -1.8144184350967407,
|
|
"logits/rejected": -1.8429317474365234,
|
|
"logps/chosen": -3.2898623943328857,
|
|
"logps/rejected": -4.14624547958374,
|
|
"loss": 1.2802,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -6.5797247886657715,
|
|
"rewards/margins": 1.7127659320831299,
|
|
"rewards/rejected": -8.29249095916748,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.7118555352002094,
|
|
"grad_norm": 81.51547237456022,
|
|
"learning_rate": 1.1561076868822755e-07,
|
|
"logits/chosen": -1.8092453479766846,
|
|
"logits/rejected": -1.884722113609314,
|
|
"logps/chosen": -3.1887125968933105,
|
|
"logps/rejected": -4.263826370239258,
|
|
"loss": 1.2472,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -6.377425193786621,
|
|
"rewards/margins": 2.1502270698547363,
|
|
"rewards/rejected": -8.527652740478516,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7223239989531536,
|
|
"grad_norm": 61.57880406452509,
|
|
"learning_rate": 1.0798381331721107e-07,
|
|
"logits/chosen": -1.8666706085205078,
|
|
"logits/rejected": -1.8893043994903564,
|
|
"logps/chosen": -3.0370490550994873,
|
|
"logps/rejected": -3.9561619758605957,
|
|
"loss": 1.255,
|
|
"rewards/accuracies": 0.768750011920929,
|
|
"rewards/chosen": -6.074098110198975,
|
|
"rewards/margins": 1.838226079940796,
|
|
"rewards/rejected": -7.912323951721191,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.7327924627060979,
|
|
"grad_norm": 58.397329958567745,
|
|
"learning_rate": 1.0054723495346482e-07,
|
|
"logits/chosen": -1.895630121231079,
|
|
"logits/rejected": -1.9232200384140015,
|
|
"logps/chosen": -2.7903852462768555,
|
|
"logps/rejected": -3.5561904907226562,
|
|
"loss": 1.1821,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -5.580770492553711,
|
|
"rewards/margins": 1.5316107273101807,
|
|
"rewards/rejected": -7.1123809814453125,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.7432609264590422,
|
|
"grad_norm": 52.74691249746145,
|
|
"learning_rate": 9.331100255592436e-08,
|
|
"logits/chosen": -1.8785375356674194,
|
|
"logits/rejected": -1.9092729091644287,
|
|
"logps/chosen": -2.7013652324676514,
|
|
"logps/rejected": -3.447122573852539,
|
|
"loss": 1.3265,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -5.402730464935303,
|
|
"rewards/margins": 1.4915144443511963,
|
|
"rewards/rejected": -6.894245147705078,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.7537293902119864,
|
|
"grad_norm": 65.82463759506157,
|
|
"learning_rate": 8.628481651367875e-08,
|
|
"logits/chosen": -1.8855186700820923,
|
|
"logits/rejected": -1.9008718729019165,
|
|
"logps/chosen": -2.6092782020568848,
|
|
"logps/rejected": -3.4059150218963623,
|
|
"loss": 1.2498,
|
|
"rewards/accuracies": 0.762499988079071,
|
|
"rewards/chosen": -5.2185564041137695,
|
|
"rewards/margins": 1.5932741165161133,
|
|
"rewards/rejected": -6.811830043792725,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.7641978539649307,
|
|
"grad_norm": 62.941225991768896,
|
|
"learning_rate": 7.947809564230445e-08,
|
|
"logits/chosen": -1.9058167934417725,
|
|
"logits/rejected": -1.9186041355133057,
|
|
"logps/chosen": -2.606133222579956,
|
|
"logps/rejected": -3.5266499519348145,
|
|
"loss": 1.2337,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -5.212266445159912,
|
|
"rewards/margins": 1.8410332202911377,
|
|
"rewards/rejected": -7.053299903869629,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.7746663177178749,
|
|
"grad_norm": 80.15726866517387,
|
|
"learning_rate": 7.289996455765748e-08,
|
|
"logits/chosen": -1.8724944591522217,
|
|
"logits/rejected": -1.888501524925232,
|
|
"logps/chosen": -2.636202573776245,
|
|
"logps/rejected": -3.564650058746338,
|
|
"loss": 1.1151,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.27240514755249,
|
|
"rewards/margins": 1.8568944931030273,
|
|
"rewards/rejected": -7.129300117492676,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.7851347814708192,
|
|
"grad_norm": 85.74581196541867,
|
|
"learning_rate": 6.655924144404906e-08,
|
|
"logits/chosen": -1.8323078155517578,
|
|
"logits/rejected": -1.8531125783920288,
|
|
"logps/chosen": -2.757204532623291,
|
|
"logps/rejected": -3.6793594360351562,
|
|
"loss": 1.2422,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -5.514409065246582,
|
|
"rewards/margins": 1.8443094491958618,
|
|
"rewards/rejected": -7.3587188720703125,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.7956032452237635,
|
|
"grad_norm": 65.11322516172854,
|
|
"learning_rate": 6.046442623320145e-08,
|
|
"logits/chosen": -1.881985068321228,
|
|
"logits/rejected": -1.9223979711532593,
|
|
"logps/chosen": -2.7589480876922607,
|
|
"logps/rejected": -3.593773603439331,
|
|
"loss": 1.2412,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.5178961753845215,
|
|
"rewards/margins": 1.6696510314941406,
|
|
"rewards/rejected": -7.187547206878662,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.8060717089767077,
|
|
"grad_norm": 70.17153962504261,
|
|
"learning_rate": 5.4623689209832484e-08,
|
|
"logits/chosen": -1.8261827230453491,
|
|
"logits/rejected": -1.8423763513565063,
|
|
"logps/chosen": -2.850209951400757,
|
|
"logps/rejected": -3.670152187347412,
|
|
"loss": 1.2011,
|
|
"rewards/accuracies": 0.731249988079071,
|
|
"rewards/chosen": -5.700419902801514,
|
|
"rewards/margins": 1.639884352684021,
|
|
"rewards/rejected": -7.340304374694824,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.816540172729652,
|
|
"grad_norm": 68.38146040720316,
|
|
"learning_rate": 4.904486005914027e-08,
|
|
"logits/chosen": -1.8114173412322998,
|
|
"logits/rejected": -1.8267343044281006,
|
|
"logps/chosen": -3.0265450477600098,
|
|
"logps/rejected": -3.899582624435425,
|
|
"loss": 1.187,
|
|
"rewards/accuracies": 0.7250000238418579,
|
|
"rewards/chosen": -6.0530900955200195,
|
|
"rewards/margins": 1.7460750341415405,
|
|
"rewards/rejected": -7.79916524887085,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.8270086364825961,
|
|
"grad_norm": 77.09446527435051,
|
|
"learning_rate": 4.373541737087263e-08,
|
|
"logits/chosen": -1.7677643299102783,
|
|
"logits/rejected": -1.80731999874115,
|
|
"logps/chosen": -2.9306697845458984,
|
|
"logps/rejected": -3.8741061687469482,
|
|
"loss": 1.2481,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.861339569091797,
|
|
"rewards/margins": 1.8868728876113892,
|
|
"rewards/rejected": -7.7482123374938965,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.8374771002355405,
|
|
"grad_norm": 71.1284274145702,
|
|
"learning_rate": 3.8702478614051345e-08,
|
|
"logits/chosen": -1.7906978130340576,
|
|
"logits/rejected": -1.8041505813598633,
|
|
"logps/chosen": -2.976975202560425,
|
|
"logps/rejected": -3.7604854106903076,
|
|
"loss": 1.2816,
|
|
"rewards/accuracies": 0.75,
|
|
"rewards/chosen": -5.95395040512085,
|
|
"rewards/margins": 1.5670208930969238,
|
|
"rewards/rejected": -7.520970821380615,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.8374771002355405,
|
|
"eval_logits/chosen": -1.8679399490356445,
|
|
"eval_logits/rejected": -1.879980444908142,
|
|
"eval_logps/chosen": -2.9611897468566895,
|
|
"eval_logps/rejected": -3.90179443359375,
|
|
"eval_loss": 1.2016023397445679,
|
|
"eval_rewards/accuracies": 0.7379999756813049,
|
|
"eval_rewards/chosen": -5.922379493713379,
|
|
"eval_rewards/margins": 1.8812097311019897,
|
|
"eval_rewards/rejected": -7.8035888671875,
|
|
"eval_runtime": 171.6266,
|
|
"eval_samples_per_second": 11.653,
|
|
"eval_steps_per_second": 0.728,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.8479455639884846,
|
|
"grad_norm": 63.83271294085232,
|
|
"learning_rate": 3.3952790595787986e-08,
|
|
"logits/chosen": -1.8161646127700806,
|
|
"logits/rejected": -1.8377959728240967,
|
|
"logps/chosen": -3.039977788925171,
|
|
"logps/rejected": -3.8821442127227783,
|
|
"loss": 1.1734,
|
|
"rewards/accuracies": 0.71875,
|
|
"rewards/chosen": -6.079955577850342,
|
|
"rewards/margins": 1.6843324899673462,
|
|
"rewards/rejected": -7.764288425445557,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.8584140277414289,
|
|
"grad_norm": 58.30518411516352,
|
|
"learning_rate": 2.9492720416985e-08,
|
|
"logits/chosen": -1.7875831127166748,
|
|
"logits/rejected": -1.8193409442901611,
|
|
"logps/chosen": -3.0099644660949707,
|
|
"logps/rejected": -3.809281587600708,
|
|
"loss": 1.2211,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -6.019928932189941,
|
|
"rewards/margins": 1.5986337661743164,
|
|
"rewards/rejected": -7.618563175201416,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.8688824914943732,
|
|
"grad_norm": 75.30650643769822,
|
|
"learning_rate": 2.5328246937043525e-08,
|
|
"logits/chosen": -1.8199684619903564,
|
|
"logits/rejected": -1.8271119594573975,
|
|
"logps/chosen": -3.101931571960449,
|
|
"logps/rejected": -3.806241273880005,
|
|
"loss": 1.2569,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -6.203863143920898,
|
|
"rewards/margins": 1.4086189270019531,
|
|
"rewards/rejected": -7.61248254776001,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.8793509552473174,
|
|
"grad_norm": 78.83242409965422,
|
|
"learning_rate": 2.1464952759020856e-08,
|
|
"logits/chosen": -1.7683451175689697,
|
|
"logits/rejected": -1.7631126642227173,
|
|
"logps/chosen": -2.9747061729431152,
|
|
"logps/rejected": -4.003817558288574,
|
|
"loss": 1.1673,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -5.9494123458862305,
|
|
"rewards/margins": 2.0582237243652344,
|
|
"rewards/rejected": -8.007635116577148,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.8898194190002617,
|
|
"grad_norm": 78.91890630683983,
|
|
"learning_rate": 1.7908016745981856e-08,
|
|
"logits/chosen": -1.794054627418518,
|
|
"logits/rejected": -1.823462724685669,
|
|
"logps/chosen": -2.8728091716766357,
|
|
"logps/rejected": -3.8705430030822754,
|
|
"loss": 1.1418,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.7456183433532715,
|
|
"rewards/margins": 1.9954681396484375,
|
|
"rewards/rejected": -7.741086006164551,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.9002878827532059,
|
|
"grad_norm": 77.02994695164918,
|
|
"learning_rate": 1.4662207078575684e-08,
|
|
"logits/chosen": -1.777092695236206,
|
|
"logits/rejected": -1.8050721883773804,
|
|
"logps/chosen": -3.0531764030456543,
|
|
"logps/rejected": -3.9285216331481934,
|
|
"loss": 1.1231,
|
|
"rewards/accuracies": 0.706250011920929,
|
|
"rewards/chosen": -6.106352806091309,
|
|
"rewards/margins": 1.7506898641586304,
|
|
"rewards/rejected": -7.857043266296387,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.9107563465061502,
|
|
"grad_norm": 78.61899751414452,
|
|
"learning_rate": 1.1731874863145142e-08,
|
|
"logits/chosen": -1.783837914466858,
|
|
"logits/rejected": -1.7817462682724,
|
|
"logps/chosen": -2.908364772796631,
|
|
"logps/rejected": -3.625439405441284,
|
|
"loss": 1.1821,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -5.816729545593262,
|
|
"rewards/margins": 1.4341497421264648,
|
|
"rewards/rejected": -7.250878810882568,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.9212248102590945,
|
|
"grad_norm": 94.12554697658324,
|
|
"learning_rate": 9.12094829893642e-09,
|
|
"logits/chosen": -1.835268259048462,
|
|
"logits/rejected": -1.8342323303222656,
|
|
"logps/chosen": -3.0066416263580322,
|
|
"logps/rejected": -3.7469406127929688,
|
|
"loss": 1.209,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -6.0132832527160645,
|
|
"rewards/margins": 1.4805978536605835,
|
|
"rewards/rejected": -7.4938812255859375,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.9316932740120387,
|
|
"grad_norm": 78.08601822772991,
|
|
"learning_rate": 6.832927412229017e-09,
|
|
"logits/chosen": -1.7435308694839478,
|
|
"logits/rejected": -1.7644704580307007,
|
|
"logps/chosen": -3.0101089477539062,
|
|
"logps/rejected": -3.985473155975342,
|
|
"loss": 1.2211,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -6.0202178955078125,
|
|
"rewards/margins": 1.9507286548614502,
|
|
"rewards/rejected": -7.970946311950684,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.942161737764983,
|
|
"grad_norm": 93.57542246833127,
|
|
"learning_rate": 4.8708793644441086e-09,
|
|
"logits/chosen": -1.7990074157714844,
|
|
"logits/rejected": -1.8333011865615845,
|
|
"logps/chosen": -2.7810821533203125,
|
|
"logps/rejected": -3.82600474357605,
|
|
"loss": 1.1351,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": -5.562164306640625,
|
|
"rewards/margins": 2.08984375,
|
|
"rewards/rejected": -7.6520094871521,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.9526302015179272,
|
|
"grad_norm": 70.87010486644522,
|
|
"learning_rate": 3.2374343405217884e-09,
|
|
"logits/chosen": -1.7927166223526,
|
|
"logits/rejected": -1.7781349420547485,
|
|
"logps/chosen": -2.9593749046325684,
|
|
"logps/rejected": -3.826608180999756,
|
|
"loss": 1.2607,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -5.918749809265137,
|
|
"rewards/margins": 1.7344669103622437,
|
|
"rewards/rejected": -7.653216361999512,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.9630986652708715,
|
|
"grad_norm": 93.5003333433491,
|
|
"learning_rate": 1.9347820230782295e-09,
|
|
"logits/chosen": -1.7764625549316406,
|
|
"logits/rejected": -1.8235756158828735,
|
|
"logps/chosen": -2.923929214477539,
|
|
"logps/rejected": -3.9943060874938965,
|
|
"loss": 1.2149,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": -5.847858428955078,
|
|
"rewards/margins": 2.1407532691955566,
|
|
"rewards/rejected": -7.988612174987793,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.9735671290238157,
|
|
"grad_norm": 58.15978819844721,
|
|
"learning_rate": 9.64668657069706e-10,
|
|
"logits/chosen": -1.8141473531723022,
|
|
"logits/rejected": -1.805326223373413,
|
|
"logps/chosen": -2.911797046661377,
|
|
"logps/rejected": -4.0116353034973145,
|
|
"loss": 1.1457,
|
|
"rewards/accuracies": 0.7562500238418579,
|
|
"rewards/chosen": -5.823594093322754,
|
|
"rewards/margins": 2.1996755599975586,
|
|
"rewards/rejected": -8.023270606994629,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.98403559277676,
|
|
"grad_norm": 66.91626514842031,
|
|
"learning_rate": 3.2839470889836627e-10,
|
|
"logits/chosen": -1.7597990036010742,
|
|
"logits/rejected": -1.7440340518951416,
|
|
"logps/chosen": -2.7474265098571777,
|
|
"logps/rejected": -3.8517417907714844,
|
|
"loss": 1.1939,
|
|
"rewards/accuracies": 0.7437499761581421,
|
|
"rewards/chosen": -5.4948530197143555,
|
|
"rewards/margins": 2.2086310386657715,
|
|
"rewards/rejected": -7.703483581542969,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.9945040565297043,
|
|
"grad_norm": 95.9824717757533,
|
|
"learning_rate": 2.6813123097352287e-11,
|
|
"logits/chosen": -1.8236198425292969,
|
|
"logits/rejected": -1.8562953472137451,
|
|
"logps/chosen": -2.951925754547119,
|
|
"logps/rejected": -3.691092014312744,
|
|
"loss": 1.1781,
|
|
"rewards/accuracies": 0.6937500238418579,
|
|
"rewards/chosen": -5.903851509094238,
|
|
"rewards/margins": 1.4783318042755127,
|
|
"rewards/rejected": -7.382184028625488,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.998691442030882,
|
|
"step": 477,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.3635074789918944,
|
|
"train_runtime": 22192.2901,
|
|
"train_samples_per_second": 2.755,
|
|
"train_steps_per_second": 0.021
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 477,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|