1159 lines
40 KiB
JSON
1159 lines
40 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.134796238244514,
|
|
"eval_steps": 500,
|
|
"global_step": 2000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.047021943573667714,
|
|
"grad_norm": 1.1235875661288681,
|
|
"learning_rate": 9.999801114386507e-06,
|
|
"logits/chosen": 0.11724523454904556,
|
|
"logits/rejected": 0.2596210241317749,
|
|
"logps/chosen": -1.0975141525268555,
|
|
"logps/rejected": -0.9545950889587402,
|
|
"loss": 1.7296,
|
|
"odds_ratio_loss": 6.3209943771362305,
|
|
"rewards/accuracies": 0.24635416269302368,
|
|
"rewards/chosen": -0.10975142568349838,
|
|
"rewards/margins": -0.014291909523308277,
|
|
"rewards/rejected": -0.09545952081680298,
|
|
"sft_loss": 1.0975141525268555,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.09404388714733543,
|
|
"grad_norm": 1.086092490147137,
|
|
"learning_rate": 9.996265810285894e-06,
|
|
"logits/chosen": 0.06234961748123169,
|
|
"logits/rejected": 0.22102031111717224,
|
|
"logps/chosen": -0.9451744556427002,
|
|
"logps/rejected": -1.1230847835540771,
|
|
"loss": 1.4772,
|
|
"odds_ratio_loss": 5.320559501647949,
|
|
"rewards/accuracies": 0.7739583849906921,
|
|
"rewards/chosen": -0.09451745450496674,
|
|
"rewards/margins": 0.017791030928492546,
|
|
"rewards/rejected": -0.11230848729610443,
|
|
"sft_loss": 0.9451744556427002,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.14106583072100312,
|
|
"grad_norm": 1.1108478150041545,
|
|
"learning_rate": 9.988314422702716e-06,
|
|
"logits/chosen": 0.12033864855766296,
|
|
"logits/rejected": 0.11232076585292816,
|
|
"logps/chosen": -0.9296269416809082,
|
|
"logps/rejected": -5.353595733642578,
|
|
"loss": 1.4246,
|
|
"odds_ratio_loss": 4.9492692947387695,
|
|
"rewards/accuracies": 0.9562500715255737,
|
|
"rewards/chosen": -0.09296268969774246,
|
|
"rewards/margins": 0.4423968493938446,
|
|
"rewards/rejected": -0.5353595018386841,
|
|
"sft_loss": 0.9296269416809082,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.18808777429467086,
|
|
"grad_norm": 1.0395955342939103,
|
|
"learning_rate": 9.975953979684062e-06,
|
|
"logits/chosen": -0.059552229940891266,
|
|
"logits/rejected": -0.010672621428966522,
|
|
"logps/chosen": -0.9207684397697449,
|
|
"logps/rejected": -24.890125274658203,
|
|
"loss": 1.3819,
|
|
"odds_ratio_loss": 4.610863208770752,
|
|
"rewards/accuracies": 0.9979166984558105,
|
|
"rewards/chosen": -0.0920768529176712,
|
|
"rewards/margins": 2.3969357013702393,
|
|
"rewards/rejected": -2.4890124797821045,
|
|
"sft_loss": 0.9207684397697449,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.23510971786833856,
|
|
"grad_norm": 1.0746539201985001,
|
|
"learning_rate": 9.959195406338866e-06,
|
|
"logits/chosen": -0.24456781148910522,
|
|
"logits/rejected": -0.25657919049263,
|
|
"logps/chosen": -0.9097563028335571,
|
|
"logps/rejected": -33.72811508178711,
|
|
"loss": 1.3651,
|
|
"odds_ratio_loss": 4.553545951843262,
|
|
"rewards/accuracies": 0.9973958730697632,
|
|
"rewards/chosen": -0.09097563475370407,
|
|
"rewards/margins": 3.2818357944488525,
|
|
"rewards/rejected": -3.3728113174438477,
|
|
"sft_loss": 0.9097563028335571,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.28213166144200624,
|
|
"grad_norm": 1.0528834041448145,
|
|
"learning_rate": 9.938053515181465e-06,
|
|
"logits/chosen": -0.19424977898597717,
|
|
"logits/rejected": -0.6158410310745239,
|
|
"logps/chosen": -0.9048575162887573,
|
|
"logps/rejected": -39.44026184082031,
|
|
"loss": 1.3581,
|
|
"odds_ratio_loss": 4.532027721405029,
|
|
"rewards/accuracies": 0.9979166984558105,
|
|
"rewards/chosen": -0.09048575162887573,
|
|
"rewards/margins": 3.853541135787964,
|
|
"rewards/rejected": -3.9440269470214844,
|
|
"sft_loss": 0.9048575162887573,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.329153605015674,
|
|
"grad_norm": 1.0415419362116665,
|
|
"learning_rate": 9.912546993039157e-06,
|
|
"logits/chosen": -0.02191479504108429,
|
|
"logits/rejected": -0.8679313063621521,
|
|
"logps/chosen": -0.89907306432724,
|
|
"logps/rejected": -41.08076477050781,
|
|
"loss": 1.3498,
|
|
"odds_ratio_loss": 4.506892681121826,
|
|
"rewards/accuracies": 0.9937500357627869,
|
|
"rewards/chosen": -0.08990731090307236,
|
|
"rewards/margins": 4.018168926239014,
|
|
"rewards/rejected": -4.108077049255371,
|
|
"sft_loss": 0.89907306432724,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.3761755485893417,
|
|
"grad_norm": 1.105436137299816,
|
|
"learning_rate": 9.882698384535345e-06,
|
|
"logits/chosen": -0.02618134766817093,
|
|
"logits/rejected": -0.9593080878257751,
|
|
"logps/chosen": -0.8983286619186401,
|
|
"logps/rejected": -40.29463195800781,
|
|
"loss": 1.3482,
|
|
"odds_ratio_loss": 4.498676300048828,
|
|
"rewards/accuracies": 0.9968750476837158,
|
|
"rewards/chosen": -0.0898328572511673,
|
|
"rewards/margins": 3.9396305084228516,
|
|
"rewards/rejected": -4.029463291168213,
|
|
"sft_loss": 0.8983286619186401,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.4231974921630094,
|
|
"grad_norm": 1.0565241888624548,
|
|
"learning_rate": 9.848534072162876e-06,
|
|
"logits/chosen": -0.3394937217235565,
|
|
"logits/rejected": -0.8553600311279297,
|
|
"logps/chosen": -0.898851752281189,
|
|
"logps/rejected": -22.5789852142334,
|
|
"loss": 1.3715,
|
|
"odds_ratio_loss": 4.726236343383789,
|
|
"rewards/accuracies": 0.9609376192092896,
|
|
"rewards/chosen": -0.0898851826786995,
|
|
"rewards/margins": 2.168013334274292,
|
|
"rewards/rejected": -2.2578983306884766,
|
|
"sft_loss": 0.898851752281189,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.4702194357366771,
|
|
"grad_norm": 1.0100518691076887,
|
|
"learning_rate": 9.810084252965162e-06,
|
|
"logits/chosen": -0.5359396934509277,
|
|
"logits/rejected": -0.7189143896102905,
|
|
"logps/chosen": -0.8881340622901917,
|
|
"logps/rejected": -5.904677391052246,
|
|
"loss": 1.3596,
|
|
"odds_ratio_loss": 4.7141828536987305,
|
|
"rewards/accuracies": 0.9843750596046448,
|
|
"rewards/chosen": -0.0888134092092514,
|
|
"rewards/margins": 0.501654326915741,
|
|
"rewards/rejected": -0.5904676914215088,
|
|
"sft_loss": 0.8881340622901917,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.5172413793103449,
|
|
"grad_norm": 0.9690370862954015,
|
|
"learning_rate": 9.767382911845711e-06,
|
|
"logits/chosen": -0.6090168356895447,
|
|
"logits/rejected": -1.3381540775299072,
|
|
"logps/chosen": -0.8794980049133301,
|
|
"logps/rejected": -26.965978622436523,
|
|
"loss": 1.3202,
|
|
"odds_ratio_loss": 4.406744956970215,
|
|
"rewards/accuracies": 0.9963542222976685,
|
|
"rewards/chosen": -0.08794979751110077,
|
|
"rewards/margins": 2.6086480617523193,
|
|
"rewards/rejected": -2.6965980529785156,
|
|
"sft_loss": 0.8794980049133301,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.5642633228840125,
|
|
"grad_norm": 0.9969800996732315,
|
|
"learning_rate": 9.720467791529659e-06,
|
|
"logits/chosen": -0.5933035612106323,
|
|
"logits/rejected": -1.2798489332199097,
|
|
"logps/chosen": -0.877310037612915,
|
|
"logps/rejected": -29.602815628051758,
|
|
"loss": 1.3168,
|
|
"odds_ratio_loss": 4.394427299499512,
|
|
"rewards/accuracies": 0.9963542222976685,
|
|
"rewards/chosen": -0.0877310037612915,
|
|
"rewards/margins": 2.8725509643554688,
|
|
"rewards/rejected": -2.9602818489074707,
|
|
"sft_loss": 0.877310037612915,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.6112852664576802,
|
|
"grad_norm": 0.9419202894251142,
|
|
"learning_rate": 9.669380359203862e-06,
|
|
"logits/chosen": -0.7155650854110718,
|
|
"logits/rejected": -1.3223919868469238,
|
|
"logps/chosen": -0.8825143575668335,
|
|
"logps/rejected": -31.081226348876953,
|
|
"loss": 1.3245,
|
|
"odds_ratio_loss": 4.420156478881836,
|
|
"rewards/accuracies": 0.9968750476837158,
|
|
"rewards/chosen": -0.08825143426656723,
|
|
"rewards/margins": 3.0198709964752197,
|
|
"rewards/rejected": -3.1081225872039795,
|
|
"sft_loss": 0.8825143575668335,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.658307210031348,
|
|
"grad_norm": 1.0278725212806465,
|
|
"learning_rate": 9.614165769864993e-06,
|
|
"logits/chosen": -0.8998770713806152,
|
|
"logits/rejected": -1.4382452964782715,
|
|
"logps/chosen": -0.862067461013794,
|
|
"logps/rejected": -32.7113151550293,
|
|
"loss": 1.294,
|
|
"odds_ratio_loss": 4.319062232971191,
|
|
"rewards/accuracies": 0.9968750476837158,
|
|
"rewards/chosen": -0.08620674908161163,
|
|
"rewards/margins": 3.184924602508545,
|
|
"rewards/rejected": -3.2711315155029297,
|
|
"sft_loss": 0.862067461013794,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.7053291536050157,
|
|
"grad_norm": 0.9866461928420249,
|
|
"learning_rate": 9.554872826408102e-06,
|
|
"logits/chosen": -0.9998570084571838,
|
|
"logits/rejected": -1.401207685470581,
|
|
"logps/chosen": -0.8621182441711426,
|
|
"logps/rejected": -33.86608123779297,
|
|
"loss": 1.294,
|
|
"odds_ratio_loss": 4.3188796043396,
|
|
"rewards/accuracies": 0.9963542222976685,
|
|
"rewards/chosen": -0.08621183037757874,
|
|
"rewards/margins": 3.300396203994751,
|
|
"rewards/rejected": -3.386608123779297,
|
|
"sft_loss": 0.8621182441711426,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.7523510971786834,
|
|
"grad_norm": 0.9659912223798843,
|
|
"learning_rate": 9.49155393649086e-06,
|
|
"logits/chosen": -0.9582356810569763,
|
|
"logits/rejected": -1.3586877584457397,
|
|
"logps/chosen": -0.8651710748672485,
|
|
"logps/rejected": -34.725868225097656,
|
|
"loss": 1.2983,
|
|
"odds_ratio_loss": 4.3313446044921875,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.0865171030163765,
|
|
"rewards/margins": 3.3860700130462646,
|
|
"rewards/rejected": -3.4725871086120605,
|
|
"sft_loss": 0.8651710748672485,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.799373040752351,
|
|
"grad_norm": 1.023178923568536,
|
|
"learning_rate": 9.424265066211657e-06,
|
|
"logits/chosen": -0.979565441608429,
|
|
"logits/rejected": -1.3727631568908691,
|
|
"logps/chosen": -0.8631203770637512,
|
|
"logps/rejected": -35.165767669677734,
|
|
"loss": 1.2958,
|
|
"odds_ratio_loss": 4.326568603515625,
|
|
"rewards/accuracies": 0.9942708611488342,
|
|
"rewards/chosen": -0.08631204068660736,
|
|
"rewards/margins": 3.430264711380005,
|
|
"rewards/rejected": -3.5165767669677734,
|
|
"sft_loss": 0.8631203770637512,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.8463949843260188,
|
|
"grad_norm": 0.9266864933596661,
|
|
"learning_rate": 9.353065690642453e-06,
|
|
"logits/chosen": -0.9409717917442322,
|
|
"logits/rejected": -1.401993751525879,
|
|
"logps/chosen": -0.8574859499931335,
|
|
"logps/rejected": -35.339935302734375,
|
|
"loss": 1.2869,
|
|
"odds_ratio_loss": 4.294265270233154,
|
|
"rewards/accuracies": 0.9968750476837158,
|
|
"rewards/chosen": -0.085748590528965,
|
|
"rewards/margins": 3.448244571685791,
|
|
"rewards/rejected": -3.5339932441711426,
|
|
"sft_loss": 0.8574859499931335,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.8934169278996865,
|
|
"grad_norm": 0.991116661066985,
|
|
"learning_rate": 9.278018741260144e-06,
|
|
"logits/chosen": -0.9096575379371643,
|
|
"logits/rejected": -1.3825011253356934,
|
|
"logps/chosen": -0.8583310842514038,
|
|
"logps/rejected": -36.20825958251953,
|
|
"loss": 1.288,
|
|
"odds_ratio_loss": 4.296856880187988,
|
|
"rewards/accuracies": 0.9968750476837158,
|
|
"rewards/chosen": -0.0858331024646759,
|
|
"rewards/margins": 3.5349931716918945,
|
|
"rewards/rejected": -3.620826244354248,
|
|
"sft_loss": 0.8583310842514038,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.9404388714733543,
|
|
"grad_norm": 0.9532041283621853,
|
|
"learning_rate": 9.199190550322895e-06,
|
|
"logits/chosen": -1.046135425567627,
|
|
"logits/rejected": -1.4427109956741333,
|
|
"logps/chosen": -0.8530829548835754,
|
|
"logps/rejected": -36.00242614746094,
|
|
"loss": 1.2804,
|
|
"odds_ratio_loss": 4.273621082305908,
|
|
"rewards/accuracies": 0.9973958730697632,
|
|
"rewards/chosen": -0.08530830591917038,
|
|
"rewards/margins": 3.5149343013763428,
|
|
"rewards/rejected": -3.600242853164673,
|
|
"sft_loss": 0.8530829548835754,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.987460815047022,
|
|
"grad_norm": 0.9736159250338984,
|
|
"learning_rate": 9.116650792240592e-06,
|
|
"logits/chosen": -1.136152982711792,
|
|
"logits/rejected": -1.4546109437942505,
|
|
"logps/chosen": -0.8512221574783325,
|
|
"logps/rejected": -36.04446029663086,
|
|
"loss": 1.2774,
|
|
"odds_ratio_loss": 4.2616400718688965,
|
|
"rewards/accuracies": 0.9979166984558105,
|
|
"rewards/chosen": -0.08512221276760101,
|
|
"rewards/margins": 3.5193233489990234,
|
|
"rewards/rejected": -3.604445695877075,
|
|
"sft_loss": 0.8512221574783325,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 1.0344827586206897,
|
|
"grad_norm": 0.9519135160773134,
|
|
"learning_rate": 9.030472421991243e-06,
|
|
"logits/chosen": -1.4093230962753296,
|
|
"logits/rejected": -1.5503480434417725,
|
|
"logps/chosen": -0.67490553855896,
|
|
"logps/rejected": -37.34769058227539,
|
|
"loss": 1.0127,
|
|
"odds_ratio_loss": 3.377951145172119,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06749054789543152,
|
|
"rewards/margins": 3.66727876663208,
|
|
"rewards/rejected": -3.734768867492676,
|
|
"sft_loss": 0.67490553855896,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 1.0815047021943573,
|
|
"grad_norm": 1.0315297815628177,
|
|
"learning_rate": 8.940731610637772e-06,
|
|
"logits/chosen": -1.5676097869873047,
|
|
"logits/rejected": -1.540395975112915,
|
|
"logps/chosen": -0.6025470495223999,
|
|
"logps/rejected": -37.542171478271484,
|
|
"loss": 0.9043,
|
|
"odds_ratio_loss": 3.0179269313812256,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.06025470420718193,
|
|
"rewards/margins": 3.693962574005127,
|
|
"rewards/rejected": -3.7542171478271484,
|
|
"sft_loss": 0.6025470495223999,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 1.1285266457680252,
|
|
"grad_norm": 1.0247278224991563,
|
|
"learning_rate": 8.847507678002177e-06,
|
|
"logits/chosen": -1.4729366302490234,
|
|
"logits/rejected": -1.5617899894714355,
|
|
"logps/chosen": -0.6058939695358276,
|
|
"logps/rejected": -38.69839096069336,
|
|
"loss": 0.9094,
|
|
"odds_ratio_loss": 3.0350539684295654,
|
|
"rewards/accuracies": 0.9979166984558105,
|
|
"rewards/chosen": -0.060589395463466644,
|
|
"rewards/margins": 3.8092496395111084,
|
|
"rewards/rejected": -3.8698391914367676,
|
|
"sft_loss": 0.6058939695358276,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 1.1755485893416928,
|
|
"grad_norm": 1.0213178135656136,
|
|
"learning_rate": 8.75088302255658e-06,
|
|
"logits/chosen": -1.4907186031341553,
|
|
"logits/rejected": -1.5485866069793701,
|
|
"logps/chosen": -0.6068739295005798,
|
|
"logps/rejected": -38.33498001098633,
|
|
"loss": 0.9111,
|
|
"odds_ratio_loss": 3.041877508163452,
|
|
"rewards/accuracies": 0.9963542222976685,
|
|
"rewards/chosen": -0.06068739295005798,
|
|
"rewards/margins": 3.772810697555542,
|
|
"rewards/rejected": -3.8334977626800537,
|
|
"sft_loss": 0.6068739295005798,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 1.2225705329153604,
|
|
"grad_norm": 1.0161893086023928,
|
|
"learning_rate": 8.650943048593135e-06,
|
|
"logits/chosen": -1.5544003248214722,
|
|
"logits/rejected": -1.540315866470337,
|
|
"logps/chosen": -0.6121038794517517,
|
|
"logps/rejected": -38.208885192871094,
|
|
"loss": 0.9187,
|
|
"odds_ratio_loss": 3.065685749053955,
|
|
"rewards/accuracies": 0.9979166984558105,
|
|
"rewards/chosen": -0.06121038645505905,
|
|
"rewards/margins": 3.759678363800049,
|
|
"rewards/rejected": -3.8208887577056885,
|
|
"sft_loss": 0.6121038794517517,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.2695924764890283,
|
|
"grad_norm": 1.0772783309503091,
|
|
"learning_rate": 8.547776090737143e-06,
|
|
"logits/chosen": -1.4951478242874146,
|
|
"logits/rejected": -1.542786955833435,
|
|
"logps/chosen": -0.6139963269233704,
|
|
"logps/rejected": -38.818702697753906,
|
|
"loss": 0.9213,
|
|
"odds_ratio_loss": 3.073441505432129,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06139963120222092,
|
|
"rewards/margins": 3.8204703330993652,
|
|
"rewards/rejected": -3.8818702697753906,
|
|
"sft_loss": 0.6139963269233704,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 1.316614420062696,
|
|
"grad_norm": 0.9900333449019929,
|
|
"learning_rate": 8.441473335870134e-06,
|
|
"logits/chosen": -1.4873510599136353,
|
|
"logits/rejected": -1.5388774871826172,
|
|
"logps/chosen": -0.6097849607467651,
|
|
"logps/rejected": -39.01841354370117,
|
|
"loss": 0.915,
|
|
"odds_ratio_loss": 3.0520551204681396,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.060978494584560394,
|
|
"rewards/margins": 3.840862989425659,
|
|
"rewards/rejected": -3.901841402053833,
|
|
"sft_loss": 0.6097849607467651,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.3636363636363638,
|
|
"grad_norm": 1.0446900620996404,
|
|
"learning_rate": 8.332128742531895e-06,
|
|
"logits/chosen": -1.5400224924087524,
|
|
"logits/rejected": -1.5519053936004639,
|
|
"logps/chosen": -0.6121001243591309,
|
|
"logps/rejected": -39.485267639160156,
|
|
"loss": 0.9186,
|
|
"odds_ratio_loss": 3.0645432472229004,
|
|
"rewards/accuracies": 0.9984375238418579,
|
|
"rewards/chosen": -0.061210013926029205,
|
|
"rewards/margins": 3.887317180633545,
|
|
"rewards/rejected": -3.9485275745391846,
|
|
"sft_loss": 0.6121001243591309,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 1.4106583072100314,
|
|
"grad_norm": 1.0584585285332069,
|
|
"learning_rate": 8.219838957872695e-06,
|
|
"logits/chosen": -1.5601458549499512,
|
|
"logits/rejected": -1.5774872303009033,
|
|
"logps/chosen": -0.612558126449585,
|
|
"logps/rejected": -39.039276123046875,
|
|
"loss": 0.9193,
|
|
"odds_ratio_loss": 3.0671072006225586,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.061255816370248795,
|
|
"rewards/margins": 3.842672109603882,
|
|
"rewards/rejected": -3.903927803039551,
|
|
"sft_loss": 0.612558126449585,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.457680250783699,
|
|
"grad_norm": 1.03154453551069,
|
|
"learning_rate": 8.10470323222911e-06,
|
|
"logits/chosen": -1.5531983375549316,
|
|
"logits/rejected": -1.5607452392578125,
|
|
"logps/chosen": -0.6180831789970398,
|
|
"logps/rejected": -39.3043212890625,
|
|
"loss": 0.9278,
|
|
"odds_ratio_loss": 3.0967118740081787,
|
|
"rewards/accuracies": 0.9979166984558105,
|
|
"rewards/chosen": -0.061808325350284576,
|
|
"rewards/margins": 3.868623733520508,
|
|
"rewards/rejected": -3.9304323196411133,
|
|
"sft_loss": 0.6180831789970398,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 1.5047021943573666,
|
|
"grad_norm": 1.1180953361782457,
|
|
"learning_rate": 7.98682333139895e-06,
|
|
"logits/chosen": -1.55546236038208,
|
|
"logits/rejected": -1.5939733982086182,
|
|
"logps/chosen": -0.6205478310585022,
|
|
"logps/rejected": -39.821533203125,
|
|
"loss": 0.9313,
|
|
"odds_ratio_loss": 3.1071197986602783,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.06205478310585022,
|
|
"rewards/margins": 3.9200985431671143,
|
|
"rewards/rejected": -3.9821529388427734,
|
|
"sft_loss": 0.6205478310585022,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.5517241379310345,
|
|
"grad_norm": 1.0650244622896785,
|
|
"learning_rate": 7.866303446692838e-06,
|
|
"logits/chosen": -1.5789411067962646,
|
|
"logits/rejected": -1.6112656593322754,
|
|
"logps/chosen": -0.6237287521362305,
|
|
"logps/rejected": -39.89510726928711,
|
|
"loss": 0.9359,
|
|
"odds_ratio_loss": 3.1217641830444336,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06237287074327469,
|
|
"rewards/margins": 3.927137851715088,
|
|
"rewards/rejected": -3.989511013031006,
|
|
"sft_loss": 0.6237287521362305,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 1.5987460815047023,
|
|
"grad_norm": 1.0533917228283542,
|
|
"learning_rate": 7.74325010284192e-06,
|
|
"logits/chosen": -1.448456883430481,
|
|
"logits/rejected": -1.5946437120437622,
|
|
"logps/chosen": -0.6208869218826294,
|
|
"logps/rejected": -40.297550201416016,
|
|
"loss": 0.9319,
|
|
"odds_ratio_loss": 3.1104769706726074,
|
|
"rewards/accuracies": 0.9984375238418579,
|
|
"rewards/chosen": -0.06208869069814682,
|
|
"rewards/margins": 3.9676666259765625,
|
|
"rewards/rejected": -4.029755115509033,
|
|
"sft_loss": 0.6208869218826294,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.64576802507837,
|
|
"grad_norm": 1.0981490419572077,
|
|
"learning_rate": 7.617772063843125e-06,
|
|
"logits/chosen": -1.3358079195022583,
|
|
"logits/rejected": -1.6043446063995361,
|
|
"logps/chosen": -0.6198667883872986,
|
|
"logps/rejected": -40.07209014892578,
|
|
"loss": 0.9302,
|
|
"odds_ratio_loss": 3.103623867034912,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06198667734861374,
|
|
"rewards/margins": 3.9452221393585205,
|
|
"rewards/rejected": -4.007209300994873,
|
|
"sft_loss": 0.6198667883872986,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 1.6927899686520376,
|
|
"grad_norm": 1.002702342649321,
|
|
"learning_rate": 7.489980236825194e-06,
|
|
"logits/chosen": -1.4426370859146118,
|
|
"logits/rejected": -1.573171615600586,
|
|
"logps/chosen": -0.6225786805152893,
|
|
"logps/rejected": -40.08259582519531,
|
|
"loss": 0.9343,
|
|
"odds_ratio_loss": 3.1171209812164307,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06225787475705147,
|
|
"rewards/margins": 3.9460020065307617,
|
|
"rewards/rejected": -4.0082597732543945,
|
|
"sft_loss": 0.6225786805152893,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.7398119122257052,
|
|
"grad_norm": 1.045644355148547,
|
|
"learning_rate": 7.3599875740204405e-06,
|
|
"logits/chosen": -1.3436530828475952,
|
|
"logits/rejected": -1.578587532043457,
|
|
"logps/chosen": -0.6230885982513428,
|
|
"logps/rejected": -40.49763870239258,
|
|
"loss": 0.935,
|
|
"odds_ratio_loss": 3.1195812225341797,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.062308862805366516,
|
|
"rewards/margins": 3.987455129623413,
|
|
"rewards/rejected": -4.0497636795043945,
|
|
"sft_loss": 0.6230885982513428,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 1.786833855799373,
|
|
"grad_norm": 1.0927200855751504,
|
|
"learning_rate": 7.227908972928892e-06,
|
|
"logits/chosen": -1.3714853525161743,
|
|
"logits/rejected": -1.6073870658874512,
|
|
"logps/chosen": -0.6151573657989502,
|
|
"logps/rejected": -39.90351867675781,
|
|
"loss": 0.9232,
|
|
"odds_ratio_loss": 3.079930543899536,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06151573359966278,
|
|
"rewards/margins": 3.928835868835449,
|
|
"rewards/rejected": -3.990351676940918,
|
|
"sft_loss": 0.6151573657989502,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.8338557993730409,
|
|
"grad_norm": 1.042619184198925,
|
|
"learning_rate": 7.0938611747630496e-06,
|
|
"logits/chosen": -1.4685415029525757,
|
|
"logits/rejected": -1.5880926847457886,
|
|
"logps/chosen": -0.6156089305877686,
|
|
"logps/rejected": -40.116119384765625,
|
|
"loss": 0.9239,
|
|
"odds_ratio_loss": 3.082674741744995,
|
|
"rewards/accuracies": 0.9984375238418579,
|
|
"rewards/chosen": -0.061560891568660736,
|
|
"rewards/margins": 3.9500508308410645,
|
|
"rewards/rejected": -4.0116119384765625,
|
|
"sft_loss": 0.6156089305877686,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 1.8808777429467085,
|
|
"grad_norm": 1.0318644739056106,
|
|
"learning_rate": 6.957962661263043e-06,
|
|
"logits/chosen": -1.4960691928863525,
|
|
"logits/rejected": -1.6382880210876465,
|
|
"logps/chosen": -0.6132216453552246,
|
|
"logps/rejected": -40.39379119873047,
|
|
"loss": 0.9202,
|
|
"odds_ratio_loss": 3.0698792934417725,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.0613221675157547,
|
|
"rewards/margins": 3.9780571460723877,
|
|
"rewards/rejected": -4.039379119873047,
|
|
"sft_loss": 0.6132216453552246,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.9278996865203761,
|
|
"grad_norm": 0.9587286583895697,
|
|
"learning_rate": 6.820333549973367e-06,
|
|
"logits/chosen": -1.4510760307312012,
|
|
"logits/rejected": -1.6273317337036133,
|
|
"logps/chosen": -0.6141430139541626,
|
|
"logps/rejected": -40.484169006347656,
|
|
"loss": 0.9216,
|
|
"odds_ratio_loss": 3.074673652648926,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06141430139541626,
|
|
"rewards/margins": 3.9870028495788574,
|
|
"rewards/rejected": -4.048417091369629,
|
|
"sft_loss": 0.6141430139541626,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 1.9749216300940438,
|
|
"grad_norm": 0.9879114442885194,
|
|
"learning_rate": 6.68109548807377e-06,
|
|
"logits/chosen": -1.4274954795837402,
|
|
"logits/rejected": -1.612345576286316,
|
|
"logps/chosen": -0.6152713894844055,
|
|
"logps/rejected": -38.77534484863281,
|
|
"loss": 0.9233,
|
|
"odds_ratio_loss": 3.0799922943115234,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.06152713671326637,
|
|
"rewards/margins": 3.816007375717163,
|
|
"rewards/rejected": -3.8775346279144287,
|
|
"sft_loss": 0.6152713894844055,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 2.0219435736677114,
|
|
"grad_norm": 1.0845364591851332,
|
|
"learning_rate": 6.5403715448581284e-06,
|
|
"logits/chosen": -1.7770729064941406,
|
|
"logits/rejected": -1.7219361066818237,
|
|
"logps/chosen": -0.5043900012969971,
|
|
"logps/rejected": -38.080379486083984,
|
|
"loss": 0.7567,
|
|
"odds_ratio_loss": 2.52349853515625,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.05043900012969971,
|
|
"rewards/margins": 3.757598876953125,
|
|
"rewards/rejected": -3.808037757873535,
|
|
"sft_loss": 0.5043900012969971,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 2.0689655172413794,
|
|
"grad_norm": 1.0611444492838484,
|
|
"learning_rate": 6.398286102956354e-06,
|
|
"logits/chosen": -2.0708460807800293,
|
|
"logits/rejected": -1.8045943975448608,
|
|
"logps/chosen": -0.36996203660964966,
|
|
"logps/rejected": -39.32792282104492,
|
|
"loss": 0.5552,
|
|
"odds_ratio_loss": 1.8524706363677979,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.03699620068073273,
|
|
"rewards/margins": 3.895796298980713,
|
|
"rewards/rejected": -3.9327926635742188,
|
|
"sft_loss": 0.36996203660964966,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 2.115987460815047,
|
|
"grad_norm": 1.0826753914388125,
|
|
"learning_rate": 6.254964748395482e-06,
|
|
"logits/chosen": -1.8300830125808716,
|
|
"logits/rejected": -1.7398159503936768,
|
|
"logps/chosen": -0.3730930685997009,
|
|
"logps/rejected": -40.005470275878906,
|
|
"loss": 0.5598,
|
|
"odds_ratio_loss": 1.866748571395874,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03730930760502815,
|
|
"rewards/margins": 3.963238000869751,
|
|
"rewards/rejected": -4.000547409057617,
|
|
"sft_loss": 0.3730930685997009,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 2.1630094043887147,
|
|
"grad_norm": 1.1611530323505992,
|
|
"learning_rate": 6.110534159597075e-06,
|
|
"logits/chosen": -1.4871913194656372,
|
|
"logits/rejected": -1.6748030185699463,
|
|
"logps/chosen": -0.37595564126968384,
|
|
"logps/rejected": -40.809329986572266,
|
|
"loss": 0.5642,
|
|
"odds_ratio_loss": 1.882346749305725,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.03759556636214256,
|
|
"rewards/margins": 4.043337821960449,
|
|
"rewards/rejected": -4.080933094024658,
|
|
"sft_loss": 0.37595564126968384,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 2.2100313479623823,
|
|
"grad_norm": 1.0518829938359235,
|
|
"learning_rate": 5.965121995409121e-06,
|
|
"logits/chosen": -1.8022518157958984,
|
|
"logits/rejected": -1.7283188104629517,
|
|
"logps/chosen": -0.37588319182395935,
|
|
"logps/rejected": -40.30139923095703,
|
|
"loss": 0.5641,
|
|
"odds_ratio_loss": 1.8821475505828857,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.037588316947221756,
|
|
"rewards/margins": 3.992551326751709,
|
|
"rewards/rejected": -4.030139923095703,
|
|
"sft_loss": 0.37588319182395935,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 2.2570532915360504,
|
|
"grad_norm": 1.078581003449218,
|
|
"learning_rate": 5.818856782271325e-06,
|
|
"logits/chosen": -1.7889518737792969,
|
|
"logits/rejected": -1.7148984670639038,
|
|
"logps/chosen": -0.3744273781776428,
|
|
"logps/rejected": -41.171321868896484,
|
|
"loss": 0.5618,
|
|
"odds_ratio_loss": 1.8740653991699219,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.03744274005293846,
|
|
"rewards/margins": 4.0796895027160645,
|
|
"rewards/rejected": -4.117132186889648,
|
|
"sft_loss": 0.3744273781776428,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 2.304075235109718,
|
|
"grad_norm": 1.1352714624377223,
|
|
"learning_rate": 5.671867800613582e-06,
|
|
"logits/chosen": -1.654802680015564,
|
|
"logits/rejected": -1.7102715969085693,
|
|
"logps/chosen": -0.37380197644233704,
|
|
"logps/rejected": -40.510955810546875,
|
|
"loss": 0.5609,
|
|
"odds_ratio_loss": 1.871164321899414,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03738019987940788,
|
|
"rewards/margins": 4.0137152671813965,
|
|
"rewards/rejected": -4.051095962524414,
|
|
"sft_loss": 0.37380197644233704,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 2.3510971786833856,
|
|
"grad_norm": 1.0544550616593722,
|
|
"learning_rate": 5.524284970587997e-06,
|
|
"logits/chosen": -1.6725318431854248,
|
|
"logits/rejected": -1.6666339635849,
|
|
"logps/chosen": -0.37714874744415283,
|
|
"logps/rejected": -41.287841796875,
|
|
"loss": 0.5661,
|
|
"odds_ratio_loss": 1.8892247676849365,
|
|
"rewards/accuracies": 0.9984375238418579,
|
|
"rewards/chosen": -0.0377148762345314,
|
|
"rewards/margins": 4.09106969833374,
|
|
"rewards/rejected": -4.128784656524658,
|
|
"sft_loss": 0.37714874744415283,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 2.3981191222570533,
|
|
"grad_norm": 1.0647146692947609,
|
|
"learning_rate": 5.376238737235469e-06,
|
|
"logits/chosen": -1.7326326370239258,
|
|
"logits/rejected": -1.692516803741455,
|
|
"logps/chosen": -0.37957924604415894,
|
|
"logps/rejected": -40.64752197265625,
|
|
"loss": 0.5696,
|
|
"odds_ratio_loss": 1.8999431133270264,
|
|
"rewards/accuracies": 0.9989583492279053,
|
|
"rewards/chosen": -0.03795792534947395,
|
|
"rewards/margins": 4.026793956756592,
|
|
"rewards/rejected": -4.064752101898193,
|
|
"sft_loss": 0.37957924604415894,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 2.445141065830721,
|
|
"grad_norm": 1.0668568570109873,
|
|
"learning_rate": 5.227859955188355e-06,
|
|
"logits/chosen": -1.7023158073425293,
|
|
"logits/rejected": -1.704237937927246,
|
|
"logps/chosen": -0.37669476866722107,
|
|
"logps/rejected": -41.38389587402344,
|
|
"loss": 0.5652,
|
|
"odds_ratio_loss": 1.8851966857910156,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03766947612166405,
|
|
"rewards/margins": 4.100719928741455,
|
|
"rewards/rejected": -4.138389587402344,
|
|
"sft_loss": 0.37669476866722107,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.492163009404389,
|
|
"grad_norm": 1.048799823874522,
|
|
"learning_rate": 5.079279773011095e-06,
|
|
"logits/chosen": -1.5848591327667236,
|
|
"logits/rejected": -1.663787603378296,
|
|
"logps/chosen": -0.38333389163017273,
|
|
"logps/rejected": -42.06602096557617,
|
|
"loss": 0.5752,
|
|
"odds_ratio_loss": 1.9189646244049072,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.03833338990807533,
|
|
"rewards/margins": 4.168269157409668,
|
|
"rewards/rejected": -4.206602096557617,
|
|
"sft_loss": 0.38333389163017273,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 2.5391849529780566,
|
|
"grad_norm": 1.0924219426439041,
|
|
"learning_rate": 4.930629517281028e-06,
|
|
"logits/chosen": -1.6451200246810913,
|
|
"logits/rejected": -1.6708714962005615,
|
|
"logps/chosen": -0.37655967473983765,
|
|
"logps/rejected": -40.94800567626953,
|
|
"loss": 0.5651,
|
|
"odds_ratio_loss": 1.8852598667144775,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03765597194433212,
|
|
"rewards/margins": 4.057145118713379,
|
|
"rewards/rejected": -4.09480094909668,
|
|
"sft_loss": 0.37655967473983765,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.586206896551724,
|
|
"grad_norm": 1.0780617647053894,
|
|
"learning_rate": 4.782040576511881e-06,
|
|
"logits/chosen": -1.6605018377304077,
|
|
"logits/rejected": -1.7122681140899658,
|
|
"logps/chosen": -0.37766554951667786,
|
|
"logps/rejected": -41.40230178833008,
|
|
"loss": 0.5668,
|
|
"odds_ratio_loss": 1.8910880088806152,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.037766553461551666,
|
|
"rewards/margins": 4.102463722229004,
|
|
"rewards/rejected": -4.140230655670166,
|
|
"sft_loss": 0.37766554951667786,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 2.633228840125392,
|
|
"grad_norm": 1.0665283784627981,
|
|
"learning_rate": 4.633644285022514e-06,
|
|
"logits/chosen": -1.4954478740692139,
|
|
"logits/rejected": -1.6388036012649536,
|
|
"logps/chosen": -0.37818458676338196,
|
|
"logps/rejected": -41.43574905395508,
|
|
"loss": 0.5676,
|
|
"odds_ratio_loss": 1.8939425945281982,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.037818457931280136,
|
|
"rewards/margins": 4.105756759643555,
|
|
"rewards/rejected": -4.1435747146606445,
|
|
"sft_loss": 0.37818458676338196,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.6802507836990594,
|
|
"grad_norm": 1.117909376672666,
|
|
"learning_rate": 4.485571806853553e-06,
|
|
"logits/chosen": -1.4043054580688477,
|
|
"logits/rejected": -1.6599198579788208,
|
|
"logps/chosen": -0.3815458118915558,
|
|
"logps/rejected": -42.49534606933594,
|
|
"loss": 0.5726,
|
|
"odds_ratio_loss": 1.9101753234863281,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03815457969903946,
|
|
"rewards/margins": 4.2113800048828125,
|
|
"rewards/rejected": -4.249535083770752,
|
|
"sft_loss": 0.3815458118915558,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 2.7272727272727275,
|
|
"grad_norm": 0.9957664276621001,
|
|
"learning_rate": 4.337954019834537e-06,
|
|
"logits/chosen": -1.382237195968628,
|
|
"logits/rejected": -1.6407064199447632,
|
|
"logps/chosen": -0.37606796622276306,
|
|
"logps/rejected": -41.618125915527344,
|
|
"loss": 0.5642,
|
|
"odds_ratio_loss": 1.8813918828964233,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.037606798112392426,
|
|
"rewards/margins": 4.124205589294434,
|
|
"rewards/rejected": -4.161812782287598,
|
|
"sft_loss": 0.37606796622276306,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.774294670846395,
|
|
"grad_norm": 1.0842791833086312,
|
|
"learning_rate": 4.190921399904048e-06,
|
|
"logits/chosen": -1.4750745296478271,
|
|
"logits/rejected": -1.6820265054702759,
|
|
"logps/chosen": -0.37413859367370605,
|
|
"logps/rejected": -41.810272216796875,
|
|
"loss": 0.5615,
|
|
"odds_ratio_loss": 1.873866319656372,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.037413861602544785,
|
|
"rewards/margins": 4.143613815307617,
|
|
"rewards/rejected": -4.181027412414551,
|
|
"sft_loss": 0.37413859367370605,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 2.8213166144200628,
|
|
"grad_norm": 1.0015237419368779,
|
|
"learning_rate": 4.044603905785044e-06,
|
|
"logits/chosen": -1.3445394039154053,
|
|
"logits/rejected": -1.6244174242019653,
|
|
"logps/chosen": -0.3810182809829712,
|
|
"logps/rejected": -42.303462982177734,
|
|
"loss": 0.5717,
|
|
"odds_ratio_loss": 1.9063372611999512,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03810183331370354,
|
|
"rewards/margins": 4.192244529724121,
|
|
"rewards/rejected": -4.2303466796875,
|
|
"sft_loss": 0.3810182809829712,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.8683385579937304,
|
|
"grad_norm": 1.1765923790147168,
|
|
"learning_rate": 3.899130864117366e-06,
|
|
"logits/chosen": -1.3384199142456055,
|
|
"logits/rejected": -1.626448154449463,
|
|
"logps/chosen": -0.382070928812027,
|
|
"logps/rejected": -42.68085479736328,
|
|
"loss": 0.5733,
|
|
"odds_ratio_loss": 1.9123166799545288,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.03820709139108658,
|
|
"rewards/margins": 4.229878902435303,
|
|
"rewards/rejected": -4.268085479736328,
|
|
"sft_loss": 0.382070928812027,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 2.915360501567398,
|
|
"grad_norm": 1.090679323980196,
|
|
"learning_rate": 3.754630855148914e-06,
|
|
"logits/chosen": -1.3685989379882812,
|
|
"logits/rejected": -1.5960211753845215,
|
|
"logps/chosen": -0.3773561120033264,
|
|
"logps/rejected": -42.771400451660156,
|
|
"loss": 0.5662,
|
|
"odds_ratio_loss": 1.8884392976760864,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03773561120033264,
|
|
"rewards/margins": 4.239404678344727,
|
|
"rewards/rejected": -4.277140140533447,
|
|
"sft_loss": 0.3773561120033264,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.962382445141066,
|
|
"grad_norm": 1.0970942641955401,
|
|
"learning_rate": 3.6112315990865543e-06,
|
|
"logits/chosen": -1.268080472946167,
|
|
"logits/rejected": -1.5860413312911987,
|
|
"logps/chosen": -0.3762770891189575,
|
|
"logps/rejected": -42.43409729003906,
|
|
"loss": 0.5647,
|
|
"odds_ratio_loss": 1.8838173151016235,
|
|
"rewards/accuracies": 0.9994791746139526,
|
|
"rewards/chosen": -0.03762771189212799,
|
|
"rewards/margins": 4.205781936645508,
|
|
"rewards/rejected": -4.243409633636475,
|
|
"sft_loss": 0.3762770891189575,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 3.0094043887147337,
|
|
"grad_norm": 1.2095236484058132,
|
|
"learning_rate": 3.4690598432071793e-06,
|
|
"logits/chosen": -1.1365634202957153,
|
|
"logits/rejected": -1.5553410053253174,
|
|
"logps/chosen": -0.3466207981109619,
|
|
"logps/rejected": -42.42127227783203,
|
|
"loss": 0.5201,
|
|
"odds_ratio_loss": 1.7351850271224976,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.03466207906603813,
|
|
"rewards/margins": 4.207465171813965,
|
|
"rewards/rejected": -4.242127418518066,
|
|
"sft_loss": 0.3466207981109619,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 3.0564263322884013,
|
|
"grad_norm": 1.024347525310305,
|
|
"learning_rate": 3.328241249828747e-06,
|
|
"logits/chosen": -1.4228293895721436,
|
|
"logits/rejected": -1.614565134048462,
|
|
"logps/chosen": -0.21424469351768494,
|
|
"logps/rejected": -41.821380615234375,
|
|
"loss": 0.3216,
|
|
"odds_ratio_loss": 1.073145866394043,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.021424470469355583,
|
|
"rewards/margins": 4.160713195800781,
|
|
"rewards/rejected": -4.182137966156006,
|
|
"sft_loss": 0.21424469351768494,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 3.103448275862069,
|
|
"grad_norm": 0.9863432232044451,
|
|
"learning_rate": 3.1889002852402505e-06,
|
|
"logits/chosen": -1.3458492755889893,
|
|
"logits/rejected": -1.5922932624816895,
|
|
"logps/chosen": -0.21549250185489655,
|
|
"logps/rejected": -42.468841552734375,
|
|
"loss": 0.3234,
|
|
"odds_ratio_loss": 1.078763723373413,
|
|
"rewards/accuracies": 1.0,
|
|
"rewards/chosen": -0.021549250930547714,
|
|
"rewards/margins": 4.225335121154785,
|
|
"rewards/rejected": -4.246884346008301,
|
|
"sft_loss": 0.21549250185489655,
|
|
"step": 1980
|
|
}
|
|
],
|
|
"log_save_evaluate_time": 457.3154537677765,
|
|
"logging_steps": 30,
|
|
"max_steps": 3190,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 5,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 680295014072320.0,
|
|
"total_tokens": 0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|