27acccf3d4
Model: W-61/qwen3-8b-base-margin-dpo-ultrafeedback-4xh200-batch-128-20260423-040315 Source: Original Platform
7231 lines
255 KiB
JSON
7231 lines
255 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9989528795811519,
|
|
"eval_steps": 200,
|
|
"global_step": 477,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0020942408376963353,
|
|
"grad_norm": 14.617609977722168,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": 2.203179359436035,
|
|
"logits/rejected": 2.035616397857666,
|
|
"logps/chosen": -257.4821472167969,
|
|
"logps/ref_chosen": -257.55841064453125,
|
|
"logps/ref_rejected": -199.84764099121094,
|
|
"logps/rejected": -199.93338012695312,
|
|
"loss": 5.5446,
|
|
"margin_dpo/margin_mean": 0.16199058294296265,
|
|
"margin_dpo/margin_std": 0.6907856464385986,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.004188481675392671,
|
|
"grad_norm": 15.140374183654785,
|
|
"learning_rate": 1.0416666666666666e-08,
|
|
"logits/chosen": 2.1704792976379395,
|
|
"logits/rejected": 2.0754430294036865,
|
|
"logps/chosen": -224.03538513183594,
|
|
"logps/ref_chosen": -224.12454223632812,
|
|
"logps/ref_rejected": -182.62721252441406,
|
|
"logps/rejected": -182.67271423339844,
|
|
"loss": 5.5417,
|
|
"margin_dpo/margin_mean": 0.13464844226837158,
|
|
"margin_dpo/margin_std": 0.5429617166519165,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.0062827225130890054,
|
|
"grad_norm": 14.625223159790039,
|
|
"learning_rate": 2.083333333333333e-08,
|
|
"logits/chosen": 2.4683523178100586,
|
|
"logits/rejected": 2.463977098464966,
|
|
"logps/chosen": -312.9666748046875,
|
|
"logps/ref_chosen": -312.8153991699219,
|
|
"logps/ref_rejected": -291.1138916015625,
|
|
"logps/rejected": -291.2332763671875,
|
|
"loss": 5.5426,
|
|
"margin_dpo/margin_mean": -0.03191244602203369,
|
|
"margin_dpo/margin_std": 0.6326964497566223,
|
|
"step": 3
|
|
},
|
|
{
|
|
"epoch": 0.008376963350785341,
|
|
"grad_norm": 15.790285110473633,
|
|
"learning_rate": 3.125e-08,
|
|
"logits/chosen": 1.5894497632980347,
|
|
"logits/rejected": 1.4774465560913086,
|
|
"logps/chosen": -310.7625427246094,
|
|
"logps/ref_chosen": -310.8699645996094,
|
|
"logps/ref_rejected": -323.95556640625,
|
|
"logps/rejected": -323.9718933105469,
|
|
"loss": 5.5437,
|
|
"margin_dpo/margin_mean": 0.12377279996871948,
|
|
"margin_dpo/margin_std": 0.9771984815597534,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.010471204188481676,
|
|
"grad_norm": 15.793586730957031,
|
|
"learning_rate": 4.166666666666666e-08,
|
|
"logits/chosen": 1.5695815086364746,
|
|
"logits/rejected": 1.5709682703018188,
|
|
"logps/chosen": -303.8356628417969,
|
|
"logps/ref_chosen": -303.7280578613281,
|
|
"logps/ref_rejected": -262.055419921875,
|
|
"logps/rejected": -261.8935546875,
|
|
"loss": 5.548,
|
|
"margin_dpo/margin_mean": -0.26944446563720703,
|
|
"margin_dpo/margin_std": 0.66167151927948,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.012565445026178011,
|
|
"grad_norm": 15.511699676513672,
|
|
"learning_rate": 5.208333333333333e-08,
|
|
"logits/chosen": 2.0192410945892334,
|
|
"logits/rejected": 1.9741183519363403,
|
|
"logps/chosen": -252.2058563232422,
|
|
"logps/ref_chosen": -252.3014373779297,
|
|
"logps/ref_rejected": -214.40451049804688,
|
|
"logps/rejected": -214.4804229736328,
|
|
"loss": 5.5507,
|
|
"margin_dpo/margin_mean": 0.1714714765548706,
|
|
"margin_dpo/margin_std": 0.6865968108177185,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.014659685863874346,
|
|
"grad_norm": 15.63283634185791,
|
|
"learning_rate": 6.25e-08,
|
|
"logits/chosen": 2.191936492919922,
|
|
"logits/rejected": 2.0201575756073,
|
|
"logps/chosen": -248.16464233398438,
|
|
"logps/ref_chosen": -248.10345458984375,
|
|
"logps/ref_rejected": -204.55133056640625,
|
|
"logps/rejected": -204.63514709472656,
|
|
"loss": 5.5465,
|
|
"margin_dpo/margin_mean": 0.022650957107543945,
|
|
"margin_dpo/margin_std": 0.7195451855659485,
|
|
"step": 7
|
|
},
|
|
{
|
|
"epoch": 0.016753926701570682,
|
|
"grad_norm": 15.911747932434082,
|
|
"learning_rate": 7.291666666666667e-08,
|
|
"logits/chosen": 2.4633631706237793,
|
|
"logits/rejected": 2.229030132293701,
|
|
"logps/chosen": -446.24395751953125,
|
|
"logps/ref_chosen": -446.1068115234375,
|
|
"logps/ref_rejected": -316.3032531738281,
|
|
"logps/rejected": -316.33001708984375,
|
|
"loss": 5.5447,
|
|
"margin_dpo/margin_mean": -0.11035525798797607,
|
|
"margin_dpo/margin_std": 0.8465025424957275,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.018848167539267015,
|
|
"grad_norm": 14.066997528076172,
|
|
"learning_rate": 8.333333333333333e-08,
|
|
"logits/chosen": 1.9973905086517334,
|
|
"logits/rejected": 1.8876209259033203,
|
|
"logps/chosen": -291.28857421875,
|
|
"logps/ref_chosen": -291.0896911621094,
|
|
"logps/ref_rejected": -298.3818054199219,
|
|
"logps/rejected": -298.3582763671875,
|
|
"loss": 5.5483,
|
|
"margin_dpo/margin_mean": -0.22240149974822998,
|
|
"margin_dpo/margin_std": 0.7139020562171936,
|
|
"step": 9
|
|
},
|
|
{
|
|
"epoch": 0.020942408376963352,
|
|
"grad_norm": 14.026876449584961,
|
|
"learning_rate": 9.375e-08,
|
|
"logits/chosen": 1.6050350666046143,
|
|
"logits/rejected": 1.755211591720581,
|
|
"logps/chosen": -221.44143676757812,
|
|
"logps/ref_chosen": -221.42408752441406,
|
|
"logps/ref_rejected": -210.35684204101562,
|
|
"logps/rejected": -210.39434814453125,
|
|
"loss": 5.544,
|
|
"margin_dpo/margin_mean": 0.02016240358352661,
|
|
"margin_dpo/margin_std": 0.5195479989051819,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.023036649214659685,
|
|
"grad_norm": 15.404158592224121,
|
|
"learning_rate": 1.0416666666666667e-07,
|
|
"logits/chosen": 1.8669978380203247,
|
|
"logits/rejected": 1.7889609336853027,
|
|
"logps/chosen": -307.2198181152344,
|
|
"logps/ref_chosen": -307.2149658203125,
|
|
"logps/ref_rejected": -264.55902099609375,
|
|
"logps/rejected": -264.7065734863281,
|
|
"loss": 5.5427,
|
|
"margin_dpo/margin_mean": 0.1427026391029358,
|
|
"margin_dpo/margin_std": 0.9485504627227783,
|
|
"step": 11
|
|
},
|
|
{
|
|
"epoch": 0.025130890052356022,
|
|
"grad_norm": 14.81792163848877,
|
|
"learning_rate": 1.1458333333333332e-07,
|
|
"logits/chosen": 1.494691014289856,
|
|
"logits/rejected": 1.6338729858398438,
|
|
"logps/chosen": -273.935302734375,
|
|
"logps/ref_chosen": -273.97259521484375,
|
|
"logps/ref_rejected": -312.4557189941406,
|
|
"logps/rejected": -312.26611328125,
|
|
"loss": 5.5513,
|
|
"margin_dpo/margin_mean": -0.15232467651367188,
|
|
"margin_dpo/margin_std": 0.7628190517425537,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.027225130890052355,
|
|
"grad_norm": 14.786741256713867,
|
|
"learning_rate": 1.25e-07,
|
|
"logits/chosen": 1.8189257383346558,
|
|
"logits/rejected": 1.8658004999160767,
|
|
"logps/chosen": -264.774658203125,
|
|
"logps/ref_chosen": -264.722412109375,
|
|
"logps/ref_rejected": -264.62823486328125,
|
|
"logps/rejected": -264.7838134765625,
|
|
"loss": 5.5457,
|
|
"margin_dpo/margin_mean": 0.10335606336593628,
|
|
"margin_dpo/margin_std": 0.7768966555595398,
|
|
"step": 13
|
|
},
|
|
{
|
|
"epoch": 0.02931937172774869,
|
|
"grad_norm": 15.321511268615723,
|
|
"learning_rate": 1.3541666666666666e-07,
|
|
"logits/chosen": 1.8423357009887695,
|
|
"logits/rejected": 1.6009153127670288,
|
|
"logps/chosen": -357.5430603027344,
|
|
"logps/ref_chosen": -357.3697509765625,
|
|
"logps/ref_rejected": -231.3351287841797,
|
|
"logps/rejected": -231.34188842773438,
|
|
"loss": 5.5436,
|
|
"margin_dpo/margin_mean": -0.16655707359313965,
|
|
"margin_dpo/margin_std": 0.6755635738372803,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.031413612565445025,
|
|
"grad_norm": 16.096477508544922,
|
|
"learning_rate": 1.4583333333333335e-07,
|
|
"logits/chosen": 2.050579071044922,
|
|
"logits/rejected": 1.9528357982635498,
|
|
"logps/chosen": -282.3099670410156,
|
|
"logps/ref_chosen": -282.4208984375,
|
|
"logps/ref_rejected": -193.90872192382812,
|
|
"logps/rejected": -193.78834533691406,
|
|
"loss": 5.5457,
|
|
"margin_dpo/margin_mean": -0.009424567222595215,
|
|
"margin_dpo/margin_std": 0.5681266784667969,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.033507853403141365,
|
|
"grad_norm": 16.60857391357422,
|
|
"learning_rate": 1.5624999999999999e-07,
|
|
"logits/chosen": 2.2264082431793213,
|
|
"logits/rejected": 1.9722710847854614,
|
|
"logps/chosen": -291.3759460449219,
|
|
"logps/ref_chosen": -291.56591796875,
|
|
"logps/ref_rejected": -252.4170684814453,
|
|
"logps/rejected": -252.54373168945312,
|
|
"loss": 5.54,
|
|
"margin_dpo/margin_mean": 0.31664133071899414,
|
|
"margin_dpo/margin_std": 0.7804574370384216,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.0356020942408377,
|
|
"grad_norm": 15.15626049041748,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"logits/chosen": 1.9703552722930908,
|
|
"logits/rejected": 1.9993352890014648,
|
|
"logps/chosen": -343.3455505371094,
|
|
"logps/ref_chosen": -343.4768981933594,
|
|
"logps/ref_rejected": -338.89654541015625,
|
|
"logps/rejected": -338.8592224121094,
|
|
"loss": 5.5409,
|
|
"margin_dpo/margin_mean": 0.09399676322937012,
|
|
"margin_dpo/margin_std": 0.5367782115936279,
|
|
"step": 17
|
|
},
|
|
{
|
|
"epoch": 0.03769633507853403,
|
|
"grad_norm": 15.167213439941406,
|
|
"learning_rate": 1.7708333333333334e-07,
|
|
"logits/chosen": 1.8425214290618896,
|
|
"logits/rejected": 1.8331950902938843,
|
|
"logps/chosen": -213.01934814453125,
|
|
"logps/ref_chosen": -213.05694580078125,
|
|
"logps/ref_rejected": -211.70962524414062,
|
|
"logps/rejected": -211.76414489746094,
|
|
"loss": 5.5491,
|
|
"margin_dpo/margin_mean": 0.09212470054626465,
|
|
"margin_dpo/margin_std": 0.6411672234535217,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.039790575916230364,
|
|
"grad_norm": 14.854358673095703,
|
|
"learning_rate": 1.875e-07,
|
|
"logits/chosen": 2.0766916275024414,
|
|
"logits/rejected": 2.0941522121429443,
|
|
"logps/chosen": -240.00901794433594,
|
|
"logps/ref_chosen": -240.0670928955078,
|
|
"logps/ref_rejected": -246.15377807617188,
|
|
"logps/rejected": -246.24050903320312,
|
|
"loss": 5.5489,
|
|
"margin_dpo/margin_mean": 0.14478152990341187,
|
|
"margin_dpo/margin_std": 0.584217369556427,
|
|
"step": 19
|
|
},
|
|
{
|
|
"epoch": 0.041884816753926704,
|
|
"grad_norm": 15.4912748336792,
|
|
"learning_rate": 1.9791666666666664e-07,
|
|
"logits/chosen": 2.1966586112976074,
|
|
"logits/rejected": 1.9358861446380615,
|
|
"logps/chosen": -315.5570983886719,
|
|
"logps/ref_chosen": -315.71331787109375,
|
|
"logps/ref_rejected": -230.0822296142578,
|
|
"logps/rejected": -230.0750732421875,
|
|
"loss": 5.5455,
|
|
"margin_dpo/margin_mean": 0.14912045001983643,
|
|
"margin_dpo/margin_std": 0.5315914750099182,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.04397905759162304,
|
|
"grad_norm": 15.429500579833984,
|
|
"learning_rate": 2.0833333333333333e-07,
|
|
"logits/chosen": 2.09773588180542,
|
|
"logits/rejected": 2.0702552795410156,
|
|
"logps/chosen": -279.3077697753906,
|
|
"logps/ref_chosen": -279.2261657714844,
|
|
"logps/ref_rejected": -300.1985168457031,
|
|
"logps/rejected": -300.22119140625,
|
|
"loss": 5.5468,
|
|
"margin_dpo/margin_mean": -0.05891883373260498,
|
|
"margin_dpo/margin_std": 0.8325001001358032,
|
|
"step": 21
|
|
},
|
|
{
|
|
"epoch": 0.04607329842931937,
|
|
"grad_norm": 13.734630584716797,
|
|
"learning_rate": 2.1875e-07,
|
|
"logits/chosen": 1.8216187953948975,
|
|
"logits/rejected": 1.9799120426177979,
|
|
"logps/chosen": -225.4229736328125,
|
|
"logps/ref_chosen": -225.4801788330078,
|
|
"logps/ref_rejected": -236.63134765625,
|
|
"logps/rejected": -236.60411071777344,
|
|
"loss": 5.5409,
|
|
"margin_dpo/margin_mean": 0.029949307441711426,
|
|
"margin_dpo/margin_std": 0.5145200490951538,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.048167539267015703,
|
|
"grad_norm": 15.402115821838379,
|
|
"learning_rate": 2.2916666666666663e-07,
|
|
"logits/chosen": 1.9867033958435059,
|
|
"logits/rejected": 1.8609161376953125,
|
|
"logps/chosen": -340.4596862792969,
|
|
"logps/ref_chosen": -340.510986328125,
|
|
"logps/ref_rejected": -273.1431579589844,
|
|
"logps/rejected": -273.184814453125,
|
|
"loss": 5.5456,
|
|
"margin_dpo/margin_mean": 0.09300780296325684,
|
|
"margin_dpo/margin_std": 0.5188795924186707,
|
|
"step": 23
|
|
},
|
|
{
|
|
"epoch": 0.050261780104712044,
|
|
"grad_norm": 16.485750198364258,
|
|
"learning_rate": 2.3958333333333335e-07,
|
|
"logits/chosen": 1.7313284873962402,
|
|
"logits/rejected": 1.6817138195037842,
|
|
"logps/chosen": -274.0079040527344,
|
|
"logps/ref_chosen": -273.9709777832031,
|
|
"logps/ref_rejected": -269.8603210449219,
|
|
"logps/rejected": -269.9830017089844,
|
|
"loss": 5.5462,
|
|
"margin_dpo/margin_mean": 0.08572280406951904,
|
|
"margin_dpo/margin_std": 0.4962030053138733,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.05235602094240838,
|
|
"grad_norm": 14.515819549560547,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": 1.7567241191864014,
|
|
"logits/rejected": 1.772882342338562,
|
|
"logps/chosen": -245.420654296875,
|
|
"logps/ref_chosen": -245.38388061523438,
|
|
"logps/ref_rejected": -251.77703857421875,
|
|
"logps/rejected": -251.8808135986328,
|
|
"loss": 5.5402,
|
|
"margin_dpo/margin_mean": 0.06698936223983765,
|
|
"margin_dpo/margin_std": 0.7465457916259766,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.05445026178010471,
|
|
"grad_norm": 15.561816215515137,
|
|
"learning_rate": 2.604166666666667e-07,
|
|
"logits/chosen": 1.6602405309677124,
|
|
"logits/rejected": 1.611204743385315,
|
|
"logps/chosen": -245.07839965820312,
|
|
"logps/ref_chosen": -245.162109375,
|
|
"logps/ref_rejected": -167.06671142578125,
|
|
"logps/rejected": -166.95631408691406,
|
|
"loss": 5.5441,
|
|
"margin_dpo/margin_mean": -0.026699483394622803,
|
|
"margin_dpo/margin_std": 0.7909866571426392,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.05654450261780105,
|
|
"grad_norm": 15.185941696166992,
|
|
"learning_rate": 2.708333333333333e-07,
|
|
"logits/chosen": 2.148705244064331,
|
|
"logits/rejected": 1.9048577547073364,
|
|
"logps/chosen": -309.2626037597656,
|
|
"logps/ref_chosen": -309.4706115722656,
|
|
"logps/ref_rejected": -200.16006469726562,
|
|
"logps/rejected": -200.23269653320312,
|
|
"loss": 5.5469,
|
|
"margin_dpo/margin_mean": 0.2806363105773926,
|
|
"margin_dpo/margin_std": 0.6260923147201538,
|
|
"step": 27
|
|
},
|
|
{
|
|
"epoch": 0.05863874345549738,
|
|
"grad_norm": 15.434507369995117,
|
|
"learning_rate": 2.8125e-07,
|
|
"logits/chosen": 1.9996970891952515,
|
|
"logits/rejected": 2.1089255809783936,
|
|
"logps/chosen": -203.73443603515625,
|
|
"logps/ref_chosen": -203.72039794921875,
|
|
"logps/ref_rejected": -228.1062469482422,
|
|
"logps/rejected": -228.02944946289062,
|
|
"loss": 5.5409,
|
|
"margin_dpo/margin_mean": -0.09086447954177856,
|
|
"margin_dpo/margin_std": 0.3806726932525635,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.060732984293193716,
|
|
"grad_norm": 14.699873924255371,
|
|
"learning_rate": 2.916666666666667e-07,
|
|
"logits/chosen": 2.243607997894287,
|
|
"logits/rejected": 1.9699711799621582,
|
|
"logps/chosen": -341.47991943359375,
|
|
"logps/ref_chosen": -341.7933349609375,
|
|
"logps/ref_rejected": -323.7848815917969,
|
|
"logps/rejected": -323.83416748046875,
|
|
"loss": 5.5414,
|
|
"margin_dpo/margin_mean": 0.3627087473869324,
|
|
"margin_dpo/margin_std": 0.9482086896896362,
|
|
"step": 29
|
|
},
|
|
{
|
|
"epoch": 0.06282722513089005,
|
|
"grad_norm": 14.436098098754883,
|
|
"learning_rate": 3.020833333333333e-07,
|
|
"logits/chosen": 1.4743300676345825,
|
|
"logits/rejected": 1.4441381692886353,
|
|
"logps/chosen": -239.34152221679688,
|
|
"logps/ref_chosen": -239.4767303466797,
|
|
"logps/ref_rejected": -228.0832977294922,
|
|
"logps/rejected": -228.0165252685547,
|
|
"loss": 5.5418,
|
|
"margin_dpo/margin_mean": 0.06841355562210083,
|
|
"margin_dpo/margin_std": 0.7110106348991394,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.06492146596858639,
|
|
"grad_norm": 13.857452392578125,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"logits/chosen": 1.6719400882720947,
|
|
"logits/rejected": 1.52069091796875,
|
|
"logps/chosen": -268.8196105957031,
|
|
"logps/ref_chosen": -268.9744567871094,
|
|
"logps/ref_rejected": -221.5098114013672,
|
|
"logps/rejected": -221.68231201171875,
|
|
"loss": 5.5392,
|
|
"margin_dpo/margin_mean": 0.3273264765739441,
|
|
"margin_dpo/margin_std": 0.8021472692489624,
|
|
"step": 31
|
|
},
|
|
{
|
|
"epoch": 0.06701570680628273,
|
|
"grad_norm": 15.621495246887207,
|
|
"learning_rate": 3.2291666666666666e-07,
|
|
"logits/chosen": 1.6164491176605225,
|
|
"logits/rejected": 1.4590275287628174,
|
|
"logps/chosen": -236.6236572265625,
|
|
"logps/ref_chosen": -236.76123046875,
|
|
"logps/ref_rejected": -191.0041046142578,
|
|
"logps/rejected": -190.91786193847656,
|
|
"loss": 5.5383,
|
|
"margin_dpo/margin_mean": 0.051319420337677,
|
|
"margin_dpo/margin_std": 0.562064528465271,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.06910994764397906,
|
|
"grad_norm": 14.935791015625,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"logits/chosen": 1.937072515487671,
|
|
"logits/rejected": 1.866725206375122,
|
|
"logps/chosen": -258.4335021972656,
|
|
"logps/ref_chosen": -258.6623840332031,
|
|
"logps/ref_rejected": -233.15805053710938,
|
|
"logps/rejected": -233.19522094726562,
|
|
"loss": 5.5401,
|
|
"margin_dpo/margin_mean": 0.26607221364974976,
|
|
"margin_dpo/margin_std": 0.9210672974586487,
|
|
"step": 33
|
|
},
|
|
{
|
|
"epoch": 0.0712041884816754,
|
|
"grad_norm": 17.70219612121582,
|
|
"learning_rate": 3.4375e-07,
|
|
"logits/chosen": 2.076815128326416,
|
|
"logits/rejected": 2.0185177326202393,
|
|
"logps/chosen": -380.03729248046875,
|
|
"logps/ref_chosen": -380.25201416015625,
|
|
"logps/ref_rejected": -315.8236389160156,
|
|
"logps/rejected": -315.7915954589844,
|
|
"loss": 5.5395,
|
|
"margin_dpo/margin_mean": 0.18271714448928833,
|
|
"margin_dpo/margin_std": 0.7701175212860107,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.07329842931937172,
|
|
"grad_norm": 13.645162582397461,
|
|
"learning_rate": 3.541666666666667e-07,
|
|
"logits/chosen": 1.5646406412124634,
|
|
"logits/rejected": 1.7504596710205078,
|
|
"logps/chosen": -245.80335998535156,
|
|
"logps/ref_chosen": -246.0772705078125,
|
|
"logps/ref_rejected": -317.1019592285156,
|
|
"logps/rejected": -317.00274658203125,
|
|
"loss": 5.54,
|
|
"margin_dpo/margin_mean": 0.17473018169403076,
|
|
"margin_dpo/margin_std": 0.7614114284515381,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.07539267015706806,
|
|
"grad_norm": 17.520965576171875,
|
|
"learning_rate": 3.645833333333333e-07,
|
|
"logits/chosen": 1.7731884717941284,
|
|
"logits/rejected": 1.8305914402008057,
|
|
"logps/chosen": -343.9805908203125,
|
|
"logps/ref_chosen": -344.1368408203125,
|
|
"logps/ref_rejected": -343.6894836425781,
|
|
"logps/rejected": -343.47882080078125,
|
|
"loss": 5.5342,
|
|
"margin_dpo/margin_mean": -0.05438530445098877,
|
|
"margin_dpo/margin_std": 0.7112289071083069,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.0774869109947644,
|
|
"grad_norm": 15.14476203918457,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": 1.9591785669326782,
|
|
"logits/rejected": 1.9149752855300903,
|
|
"logps/chosen": -310.9266357421875,
|
|
"logps/ref_chosen": -311.3376770019531,
|
|
"logps/ref_rejected": -278.5052185058594,
|
|
"logps/rejected": -278.489990234375,
|
|
"loss": 5.5375,
|
|
"margin_dpo/margin_mean": 0.3958609700202942,
|
|
"margin_dpo/margin_std": 0.6456325054168701,
|
|
"step": 37
|
|
},
|
|
{
|
|
"epoch": 0.07958115183246073,
|
|
"grad_norm": 15.079659461975098,
|
|
"learning_rate": 3.8541666666666665e-07,
|
|
"logits/chosen": 2.1111977100372314,
|
|
"logits/rejected": 2.3584368228912354,
|
|
"logps/chosen": -193.07827758789062,
|
|
"logps/ref_chosen": -193.3851318359375,
|
|
"logps/ref_rejected": -234.6280975341797,
|
|
"logps/rejected": -234.42193603515625,
|
|
"loss": 5.5401,
|
|
"margin_dpo/margin_mean": 0.10068202018737793,
|
|
"margin_dpo/margin_std": 0.5997118353843689,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.08167539267015707,
|
|
"grad_norm": 15.749566078186035,
|
|
"learning_rate": 3.958333333333333e-07,
|
|
"logits/chosen": 1.7943568229675293,
|
|
"logits/rejected": 1.8780990839004517,
|
|
"logps/chosen": -290.79742431640625,
|
|
"logps/ref_chosen": -291.5687255859375,
|
|
"logps/ref_rejected": -317.7392578125,
|
|
"logps/rejected": -317.748779296875,
|
|
"loss": 5.5255,
|
|
"margin_dpo/margin_mean": 0.7807860374450684,
|
|
"margin_dpo/margin_std": 1.0324419736862183,
|
|
"step": 39
|
|
},
|
|
{
|
|
"epoch": 0.08376963350785341,
|
|
"grad_norm": 15.053966522216797,
|
|
"learning_rate": 4.0625e-07,
|
|
"logits/chosen": 1.7152007818222046,
|
|
"logits/rejected": 1.685285210609436,
|
|
"logps/chosen": -211.45947265625,
|
|
"logps/ref_chosen": -211.951904296875,
|
|
"logps/ref_rejected": -166.82864379882812,
|
|
"logps/rejected": -166.58428955078125,
|
|
"loss": 5.528,
|
|
"margin_dpo/margin_mean": 0.24808716773986816,
|
|
"margin_dpo/margin_std": 0.9657536745071411,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.08586387434554973,
|
|
"grad_norm": 15.568541526794434,
|
|
"learning_rate": 4.1666666666666667e-07,
|
|
"logits/chosen": 1.968687653541565,
|
|
"logits/rejected": 1.8429195880889893,
|
|
"logps/chosen": -300.13665771484375,
|
|
"logps/ref_chosen": -300.6400146484375,
|
|
"logps/ref_rejected": -224.77317810058594,
|
|
"logps/rejected": -224.72613525390625,
|
|
"loss": 5.535,
|
|
"margin_dpo/margin_mean": 0.45627307891845703,
|
|
"margin_dpo/margin_std": 0.7364793419837952,
|
|
"step": 41
|
|
},
|
|
{
|
|
"epoch": 0.08795811518324607,
|
|
"grad_norm": 14.581147193908691,
|
|
"learning_rate": 4.270833333333333e-07,
|
|
"logits/chosen": 2.1621668338775635,
|
|
"logits/rejected": 2.195481061935425,
|
|
"logps/chosen": -291.0929870605469,
|
|
"logps/ref_chosen": -291.4709167480469,
|
|
"logps/ref_rejected": -285.62982177734375,
|
|
"logps/rejected": -285.6851501464844,
|
|
"loss": 5.5294,
|
|
"margin_dpo/margin_mean": 0.43321943283081055,
|
|
"margin_dpo/margin_std": 1.0551120042800903,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.09005235602094241,
|
|
"grad_norm": 15.62813663482666,
|
|
"learning_rate": 4.375e-07,
|
|
"logits/chosen": 1.9382034540176392,
|
|
"logits/rejected": 1.9245309829711914,
|
|
"logps/chosen": -313.7782897949219,
|
|
"logps/ref_chosen": -314.3768615722656,
|
|
"logps/ref_rejected": -246.80313110351562,
|
|
"logps/rejected": -246.7808380126953,
|
|
"loss": 5.5273,
|
|
"margin_dpo/margin_mean": 0.5762431621551514,
|
|
"margin_dpo/margin_std": 0.9067457914352417,
|
|
"step": 43
|
|
},
|
|
{
|
|
"epoch": 0.09214659685863874,
|
|
"grad_norm": 15.793681144714355,
|
|
"learning_rate": 4.479166666666667e-07,
|
|
"logits/chosen": 1.7762880325317383,
|
|
"logits/rejected": 1.7065317630767822,
|
|
"logps/chosen": -209.00802612304688,
|
|
"logps/ref_chosen": -209.8181915283203,
|
|
"logps/ref_rejected": -246.21340942382812,
|
|
"logps/rejected": -246.0368194580078,
|
|
"loss": 5.5223,
|
|
"margin_dpo/margin_mean": 0.633570671081543,
|
|
"margin_dpo/margin_std": 1.1882718801498413,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.09424083769633508,
|
|
"grad_norm": 16.929059982299805,
|
|
"learning_rate": 4.5833333333333327e-07,
|
|
"logits/chosen": 1.7606732845306396,
|
|
"logits/rejected": 1.5792968273162842,
|
|
"logps/chosen": -308.1605224609375,
|
|
"logps/ref_chosen": -309.0930480957031,
|
|
"logps/ref_rejected": -269.3559265136719,
|
|
"logps/rejected": -268.9593200683594,
|
|
"loss": 5.5261,
|
|
"margin_dpo/margin_mean": 0.535961389541626,
|
|
"margin_dpo/margin_std": 1.0746005773544312,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.09633507853403141,
|
|
"grad_norm": 16.489280700683594,
|
|
"learning_rate": 4.6874999999999996e-07,
|
|
"logits/chosen": 1.9223171472549438,
|
|
"logits/rejected": 1.9758403301239014,
|
|
"logps/chosen": -298.0412292480469,
|
|
"logps/ref_chosen": -298.72467041015625,
|
|
"logps/ref_rejected": -309.87786865234375,
|
|
"logps/rejected": -309.4717712402344,
|
|
"loss": 5.5236,
|
|
"margin_dpo/margin_mean": 0.277274489402771,
|
|
"margin_dpo/margin_std": 0.9340643882751465,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.09842931937172775,
|
|
"grad_norm": 13.506661415100098,
|
|
"learning_rate": 4.791666666666667e-07,
|
|
"logits/chosen": 1.6691988706588745,
|
|
"logits/rejected": 2.0380465984344482,
|
|
"logps/chosen": -215.84332275390625,
|
|
"logps/ref_chosen": -216.43553161621094,
|
|
"logps/ref_rejected": -292.6329345703125,
|
|
"logps/rejected": -291.96148681640625,
|
|
"loss": 5.5293,
|
|
"margin_dpo/margin_mean": -0.0792464017868042,
|
|
"margin_dpo/margin_std": 1.1294535398483276,
|
|
"step": 47
|
|
},
|
|
{
|
|
"epoch": 0.10052356020942409,
|
|
"grad_norm": 14.86147403717041,
|
|
"learning_rate": 4.895833333333333e-07,
|
|
"logits/chosen": 2.211613178253174,
|
|
"logits/rejected": 2.186110496520996,
|
|
"logps/chosen": -234.05947875976562,
|
|
"logps/ref_chosen": -234.77496337890625,
|
|
"logps/ref_rejected": -240.41433715820312,
|
|
"logps/rejected": -240.24525451660156,
|
|
"loss": 5.5203,
|
|
"margin_dpo/margin_mean": 0.5463833212852478,
|
|
"margin_dpo/margin_std": 1.3220971822738647,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.10261780104712041,
|
|
"grad_norm": 15.43526840209961,
|
|
"learning_rate": 5e-07,
|
|
"logits/chosen": 1.7962108850479126,
|
|
"logits/rejected": 1.9277849197387695,
|
|
"logps/chosen": -245.73326110839844,
|
|
"logps/ref_chosen": -246.7688446044922,
|
|
"logps/ref_rejected": -253.47378540039062,
|
|
"logps/rejected": -253.3438720703125,
|
|
"loss": 5.5203,
|
|
"margin_dpo/margin_mean": 0.9057276248931885,
|
|
"margin_dpo/margin_std": 1.4221221208572388,
|
|
"step": 49
|
|
},
|
|
{
|
|
"epoch": 0.10471204188481675,
|
|
"grad_norm": 15.111068725585938,
|
|
"learning_rate": 4.999932966293553e-07,
|
|
"logits/chosen": 2.2119665145874023,
|
|
"logits/rejected": 2.335810422897339,
|
|
"logps/chosen": -281.3116760253906,
|
|
"logps/ref_chosen": -282.61981201171875,
|
|
"logps/ref_rejected": -340.8515625,
|
|
"logps/rejected": -340.31781005859375,
|
|
"loss": 5.5284,
|
|
"margin_dpo/margin_mean": 0.7743173837661743,
|
|
"margin_dpo/margin_std": 1.4886585474014282,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.1068062827225131,
|
|
"grad_norm": 14.817649841308594,
|
|
"learning_rate": 4.999731868769026e-07,
|
|
"logits/chosen": 1.637377381324768,
|
|
"logits/rejected": 1.7862030267715454,
|
|
"logps/chosen": -244.794677734375,
|
|
"logps/ref_chosen": -245.87562561035156,
|
|
"logps/ref_rejected": -309.7420654296875,
|
|
"logps/rejected": -309.6230773925781,
|
|
"loss": 5.5202,
|
|
"margin_dpo/margin_mean": 0.961925208568573,
|
|
"margin_dpo/margin_std": 1.9343087673187256,
|
|
"step": 51
|
|
},
|
|
{
|
|
"epoch": 0.10890052356020942,
|
|
"grad_norm": 17.035507202148438,
|
|
"learning_rate": 4.99939671821067e-07,
|
|
"logits/chosen": 1.8847734928131104,
|
|
"logits/rejected": 2.039155960083008,
|
|
"logps/chosen": -276.9980163574219,
|
|
"logps/ref_chosen": -278.3123474121094,
|
|
"logps/ref_rejected": -320.58203125,
|
|
"logps/rejected": -319.9336853027344,
|
|
"loss": 5.5067,
|
|
"margin_dpo/margin_mean": 0.6659917235374451,
|
|
"margin_dpo/margin_std": 1.508874773979187,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.11099476439790576,
|
|
"grad_norm": 15.945631980895996,
|
|
"learning_rate": 4.998927532591591e-07,
|
|
"logits/chosen": 2.085860013961792,
|
|
"logits/rejected": 2.0801711082458496,
|
|
"logps/chosen": -331.2710266113281,
|
|
"logps/ref_chosen": -332.776123046875,
|
|
"logps/ref_rejected": -325.1794128417969,
|
|
"logps/rejected": -324.69622802734375,
|
|
"loss": 5.5144,
|
|
"margin_dpo/margin_mean": 1.0218517780303955,
|
|
"margin_dpo/margin_std": 1.590319037437439,
|
|
"step": 53
|
|
},
|
|
{
|
|
"epoch": 0.1130890052356021,
|
|
"grad_norm": 14.57484245300293,
|
|
"learning_rate": 4.998324337072792e-07,
|
|
"logits/chosen": 1.3913365602493286,
|
|
"logits/rejected": 1.4456019401550293,
|
|
"logps/chosen": -294.7577819824219,
|
|
"logps/ref_chosen": -296.2243347167969,
|
|
"logps/ref_rejected": -267.64251708984375,
|
|
"logps/rejected": -267.3682861328125,
|
|
"loss": 5.5131,
|
|
"margin_dpo/margin_mean": 1.1922770738601685,
|
|
"margin_dpo/margin_std": 1.47157621383667,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.11518324607329843,
|
|
"grad_norm": 12.808218955993652,
|
|
"learning_rate": 4.997587164001815e-07,
|
|
"logits/chosen": 1.8794647455215454,
|
|
"logits/rejected": 1.8777508735656738,
|
|
"logps/chosen": -197.05091857910156,
|
|
"logps/ref_chosen": -198.1138916015625,
|
|
"logps/ref_rejected": -185.93772888183594,
|
|
"logps/rejected": -185.5297088623047,
|
|
"loss": 5.522,
|
|
"margin_dpo/margin_mean": 0.6549429893493652,
|
|
"margin_dpo/margin_std": 1.0751301050186157,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.11727748691099477,
|
|
"grad_norm": 14.403154373168945,
|
|
"learning_rate": 4.996716052911017e-07,
|
|
"logits/chosen": 2.004265785217285,
|
|
"logits/rejected": 1.965603232383728,
|
|
"logps/chosen": -267.2569580078125,
|
|
"logps/ref_chosen": -268.8618469238281,
|
|
"logps/ref_rejected": -245.21348571777344,
|
|
"logps/rejected": -244.97555541992188,
|
|
"loss": 5.5085,
|
|
"margin_dpo/margin_mean": 1.3669579029083252,
|
|
"margin_dpo/margin_std": 1.768923282623291,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.1193717277486911,
|
|
"grad_norm": 17.60223960876465,
|
|
"learning_rate": 4.99571105051544e-07,
|
|
"logits/chosen": 2.1416828632354736,
|
|
"logits/rejected": 1.8643951416015625,
|
|
"logps/chosen": -286.81622314453125,
|
|
"logps/ref_chosen": -288.4784851074219,
|
|
"logps/ref_rejected": -239.400146484375,
|
|
"logps/rejected": -238.46566772460938,
|
|
"loss": 5.4919,
|
|
"margin_dpo/margin_mean": 0.7277634739875793,
|
|
"margin_dpo/margin_std": 1.487809658050537,
|
|
"step": 57
|
|
},
|
|
{
|
|
"epoch": 0.12146596858638743,
|
|
"grad_norm": 14.885680198669434,
|
|
"learning_rate": 4.994572210710314e-07,
|
|
"logits/chosen": 1.8542314767837524,
|
|
"logits/rejected": 1.8795952796936035,
|
|
"logps/chosen": -276.6270446777344,
|
|
"logps/ref_chosen": -278.2837219238281,
|
|
"logps/ref_rejected": -262.5280456542969,
|
|
"logps/rejected": -262.4251708984375,
|
|
"loss": 5.5076,
|
|
"margin_dpo/margin_mean": 1.5538146495819092,
|
|
"margin_dpo/margin_std": 1.7451914548873901,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.12356020942408377,
|
|
"grad_norm": 15.256675720214844,
|
|
"learning_rate": 4.993299594568162e-07,
|
|
"logits/chosen": 1.695129632949829,
|
|
"logits/rejected": 1.76768159866333,
|
|
"logps/chosen": -231.92245483398438,
|
|
"logps/ref_chosen": -232.77662658691406,
|
|
"logps/ref_rejected": -226.2711181640625,
|
|
"logps/rejected": -225.69354248046875,
|
|
"loss": 5.5103,
|
|
"margin_dpo/margin_mean": 0.27659308910369873,
|
|
"margin_dpo/margin_std": 1.5459599494934082,
|
|
"step": 59
|
|
},
|
|
{
|
|
"epoch": 0.1256544502617801,
|
|
"grad_norm": 14.66773509979248,
|
|
"learning_rate": 4.991893270335525e-07,
|
|
"logits/chosen": 1.8791348934173584,
|
|
"logits/rejected": 1.587320327758789,
|
|
"logps/chosen": -314.26800537109375,
|
|
"logps/ref_chosen": -315.6903991699219,
|
|
"logps/ref_rejected": -190.40899658203125,
|
|
"logps/rejected": -189.97193908691406,
|
|
"loss": 5.4951,
|
|
"margin_dpo/margin_mean": 0.9853799939155579,
|
|
"margin_dpo/margin_std": 1.7359843254089355,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.12774869109947645,
|
|
"grad_norm": 15.011839866638184,
|
|
"learning_rate": 4.990353313429303e-07,
|
|
"logits/chosen": 2.034388542175293,
|
|
"logits/rejected": 2.065513849258423,
|
|
"logps/chosen": -249.50173950195312,
|
|
"logps/ref_chosen": -251.527099609375,
|
|
"logps/ref_rejected": -261.0340270996094,
|
|
"logps/rejected": -259.8365173339844,
|
|
"loss": 5.503,
|
|
"margin_dpo/margin_mean": 0.8278936147689819,
|
|
"margin_dpo/margin_std": 1.6765094995498657,
|
|
"step": 61
|
|
},
|
|
{
|
|
"epoch": 0.12984293193717278,
|
|
"grad_norm": 14.675267219543457,
|
|
"learning_rate": 4.988679806432711e-07,
|
|
"logits/chosen": 1.8862786293029785,
|
|
"logits/rejected": 1.8161289691925049,
|
|
"logps/chosen": -255.54144287109375,
|
|
"logps/ref_chosen": -257.3919982910156,
|
|
"logps/ref_rejected": -282.1814880371094,
|
|
"logps/rejected": -281.52008056640625,
|
|
"loss": 5.4967,
|
|
"margin_dpo/margin_mean": 1.189134955406189,
|
|
"margin_dpo/margin_std": 2.0515801906585693,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.1319371727748691,
|
|
"grad_norm": 14.802937507629395,
|
|
"learning_rate": 4.986872839090852e-07,
|
|
"logits/chosen": 2.1043763160705566,
|
|
"logits/rejected": 2.2130091190338135,
|
|
"logps/chosen": -320.39398193359375,
|
|
"logps/ref_chosen": -322.24725341796875,
|
|
"logps/ref_rejected": -327.70892333984375,
|
|
"logps/rejected": -326.7320251464844,
|
|
"loss": 5.4938,
|
|
"margin_dpo/margin_mean": 0.876427948474884,
|
|
"margin_dpo/margin_std": 2.8320491313934326,
|
|
"step": 63
|
|
},
|
|
{
|
|
"epoch": 0.13403141361256546,
|
|
"grad_norm": 15.675727844238281,
|
|
"learning_rate": 4.9849325083059e-07,
|
|
"logits/chosen": 1.840427279472351,
|
|
"logits/rejected": 2.098795175552368,
|
|
"logps/chosen": -333.1174621582031,
|
|
"logps/ref_chosen": -335.7379455566406,
|
|
"logps/ref_rejected": -337.8742980957031,
|
|
"logps/rejected": -337.04913330078125,
|
|
"loss": 5.4794,
|
|
"margin_dpo/margin_mean": 1.795319676399231,
|
|
"margin_dpo/margin_std": 3.2046210765838623,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.13612565445026178,
|
|
"grad_norm": 14.844298362731934,
|
|
"learning_rate": 4.982858918131906e-07,
|
|
"logits/chosen": 1.8770238161087036,
|
|
"logits/rejected": 1.9210056066513062,
|
|
"logps/chosen": -309.9880676269531,
|
|
"logps/ref_chosen": -312.36358642578125,
|
|
"logps/ref_rejected": -300.220947265625,
|
|
"logps/rejected": -298.6002502441406,
|
|
"loss": 5.5002,
|
|
"margin_dpo/margin_mean": 0.754831075668335,
|
|
"margin_dpo/margin_std": 2.1830434799194336,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.1382198952879581,
|
|
"grad_norm": 14.79240894317627,
|
|
"learning_rate": 4.980652179769217e-07,
|
|
"logits/chosen": 1.8671329021453857,
|
|
"logits/rejected": 2.058912515640259,
|
|
"logps/chosen": -195.7026824951172,
|
|
"logps/ref_chosen": -198.186767578125,
|
|
"logps/ref_rejected": -248.18748474121094,
|
|
"logps/rejected": -247.15882873535156,
|
|
"loss": 5.4918,
|
|
"margin_dpo/margin_mean": 1.4554123878479004,
|
|
"margin_dpo/margin_std": 2.3160204887390137,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.14031413612565444,
|
|
"grad_norm": 14.211318969726562,
|
|
"learning_rate": 4.978312411558517e-07,
|
|
"logits/chosen": 2.104246139526367,
|
|
"logits/rejected": 2.138582706451416,
|
|
"logps/chosen": -289.1455078125,
|
|
"logps/ref_chosen": -291.9940490722656,
|
|
"logps/ref_rejected": -269.945068359375,
|
|
"logps/rejected": -268.6217346191406,
|
|
"loss": 5.4991,
|
|
"margin_dpo/margin_mean": 1.5252022743225098,
|
|
"margin_dpo/margin_std": 2.973376989364624,
|
|
"step": 67
|
|
},
|
|
{
|
|
"epoch": 0.1424083769633508,
|
|
"grad_norm": 14.742173194885254,
|
|
"learning_rate": 4.975839738974473e-07,
|
|
"logits/chosen": 1.597095012664795,
|
|
"logits/rejected": 1.4467945098876953,
|
|
"logps/chosen": -287.52056884765625,
|
|
"logps/ref_chosen": -289.9323425292969,
|
|
"logps/ref_rejected": -225.7897491455078,
|
|
"logps/rejected": -225.21658325195312,
|
|
"loss": 5.4771,
|
|
"margin_dpo/margin_mean": 1.8386340141296387,
|
|
"margin_dpo/margin_std": 3.168964385986328,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.14450261780104712,
|
|
"grad_norm": 14.997187614440918,
|
|
"learning_rate": 4.97323429461901e-07,
|
|
"logits/chosen": 2.1530022621154785,
|
|
"logits/rejected": 2.05501651763916,
|
|
"logps/chosen": -263.3707580566406,
|
|
"logps/ref_chosen": -266.7104797363281,
|
|
"logps/ref_rejected": -229.5946502685547,
|
|
"logps/rejected": -228.25326538085938,
|
|
"loss": 5.4597,
|
|
"margin_dpo/margin_mean": 1.9983257055282593,
|
|
"margin_dpo/margin_std": 3.47263503074646,
|
|
"step": 69
|
|
},
|
|
{
|
|
"epoch": 0.14659685863874344,
|
|
"grad_norm": 15.571727752685547,
|
|
"learning_rate": 4.970496218214204e-07,
|
|
"logits/chosen": 2.235600471496582,
|
|
"logits/rejected": 2.3245067596435547,
|
|
"logps/chosen": -265.7595520019531,
|
|
"logps/ref_chosen": -268.6711120605469,
|
|
"logps/ref_rejected": -261.61273193359375,
|
|
"logps/rejected": -260.5652160644531,
|
|
"loss": 5.4636,
|
|
"margin_dpo/margin_mean": 1.864060878753662,
|
|
"margin_dpo/margin_std": 2.7824549674987793,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.1486910994764398,
|
|
"grad_norm": 14.973682403564453,
|
|
"learning_rate": 4.967625656594781e-07,
|
|
"logits/chosen": 1.9091215133666992,
|
|
"logits/rejected": 1.9661266803741455,
|
|
"logps/chosen": -241.69586181640625,
|
|
"logps/ref_chosen": -244.97821044921875,
|
|
"logps/ref_rejected": -263.7174377441406,
|
|
"logps/rejected": -262.50848388671875,
|
|
"loss": 5.469,
|
|
"margin_dpo/margin_mean": 2.0734434127807617,
|
|
"margin_dpo/margin_std": 4.373683929443359,
|
|
"step": 71
|
|
},
|
|
{
|
|
"epoch": 0.15078534031413612,
|
|
"grad_norm": 14.649141311645508,
|
|
"learning_rate": 4.964622763700252e-07,
|
|
"logits/chosen": 1.8277561664581299,
|
|
"logits/rejected": 1.8917427062988281,
|
|
"logps/chosen": -276.90264892578125,
|
|
"logps/ref_chosen": -280.0353698730469,
|
|
"logps/ref_rejected": -291.1289367675781,
|
|
"logps/rejected": -289.7200927734375,
|
|
"loss": 5.4668,
|
|
"margin_dpo/margin_mean": 1.7238655090332031,
|
|
"margin_dpo/margin_std": 3.216670274734497,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.15287958115183245,
|
|
"grad_norm": 14.681085586547852,
|
|
"learning_rate": 4.961487700566646e-07,
|
|
"logits/chosen": 2.040257453918457,
|
|
"logits/rejected": 2.0101261138916016,
|
|
"logps/chosen": -237.6248321533203,
|
|
"logps/ref_chosen": -241.37384033203125,
|
|
"logps/ref_rejected": -227.28871154785156,
|
|
"logps/rejected": -224.7103271484375,
|
|
"loss": 5.4684,
|
|
"margin_dpo/margin_mean": 1.1706353425979614,
|
|
"margin_dpo/margin_std": 2.684138774871826,
|
|
"step": 73
|
|
},
|
|
{
|
|
"epoch": 0.1549738219895288,
|
|
"grad_norm": 16.057296752929688,
|
|
"learning_rate": 4.958220635317885e-07,
|
|
"logits/chosen": 1.7149076461791992,
|
|
"logits/rejected": 1.616335153579712,
|
|
"logps/chosen": -427.9046630859375,
|
|
"logps/ref_chosen": -432.6361389160156,
|
|
"logps/ref_rejected": -408.990478515625,
|
|
"logps/rejected": -406.4610595703125,
|
|
"loss": 5.4703,
|
|
"margin_dpo/margin_mean": 2.2020528316497803,
|
|
"margin_dpo/margin_std": 3.4136807918548584,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.15706806282722513,
|
|
"grad_norm": 15.3256254196167,
|
|
"learning_rate": 4.954821743156767e-07,
|
|
"logits/chosen": 1.8307483196258545,
|
|
"logits/rejected": 1.8694071769714355,
|
|
"logps/chosen": -277.4227294921875,
|
|
"logps/ref_chosen": -282.2913513183594,
|
|
"logps/ref_rejected": -227.30093383789062,
|
|
"logps/rejected": -225.89971923828125,
|
|
"loss": 5.4384,
|
|
"margin_dpo/margin_mean": 3.4673845767974854,
|
|
"margin_dpo/margin_std": 3.8357293605804443,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.15916230366492146,
|
|
"grad_norm": 16.74871253967285,
|
|
"learning_rate": 4.951291206355559e-07,
|
|
"logits/chosen": 1.9061857461929321,
|
|
"logits/rejected": 1.6594858169555664,
|
|
"logps/chosen": -272.63018798828125,
|
|
"logps/ref_chosen": -277.90081787109375,
|
|
"logps/ref_rejected": -214.1353302001953,
|
|
"logps/rejected": -211.89590454101562,
|
|
"loss": 5.431,
|
|
"margin_dpo/margin_mean": 3.031224250793457,
|
|
"margin_dpo/margin_std": 3.183443546295166,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.1612565445026178,
|
|
"grad_norm": 18.260398864746094,
|
|
"learning_rate": 4.947629214246236e-07,
|
|
"logits/chosen": 2.142491102218628,
|
|
"logits/rejected": 2.1160454750061035,
|
|
"logps/chosen": -278.9680480957031,
|
|
"logps/ref_chosen": -283.3741455078125,
|
|
"logps/ref_rejected": -239.51246643066406,
|
|
"logps/rejected": -237.45001220703125,
|
|
"loss": 5.4527,
|
|
"margin_dpo/margin_mean": 2.3436222076416016,
|
|
"margin_dpo/margin_std": 3.3090319633483887,
|
|
"step": 77
|
|
},
|
|
{
|
|
"epoch": 0.16335078534031414,
|
|
"grad_norm": 14.633062362670898,
|
|
"learning_rate": 4.943835963210323e-07,
|
|
"logits/chosen": 1.6990811824798584,
|
|
"logits/rejected": 1.6937521696090698,
|
|
"logps/chosen": -202.76388549804688,
|
|
"logps/ref_chosen": -207.1702423095703,
|
|
"logps/ref_rejected": -196.26866149902344,
|
|
"logps/rejected": -194.35032653808594,
|
|
"loss": 5.4294,
|
|
"margin_dpo/margin_mean": 2.4880149364471436,
|
|
"margin_dpo/margin_std": 3.424355983734131,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.16544502617801046,
|
|
"grad_norm": 16.774738311767578,
|
|
"learning_rate": 4.939911656668361e-07,
|
|
"logits/chosen": 1.9445700645446777,
|
|
"logits/rejected": 2.229759454727173,
|
|
"logps/chosen": -208.6917724609375,
|
|
"logps/ref_chosen": -212.90396118164062,
|
|
"logps/ref_rejected": -242.32528686523438,
|
|
"logps/rejected": -239.5742950439453,
|
|
"loss": 5.4268,
|
|
"margin_dpo/margin_mean": 1.461201786994934,
|
|
"margin_dpo/margin_std": 4.005527973175049,
|
|
"step": 79
|
|
},
|
|
{
|
|
"epoch": 0.16753926701570682,
|
|
"grad_norm": 15.637154579162598,
|
|
"learning_rate": 4.935856505068998e-07,
|
|
"logits/chosen": 1.3914299011230469,
|
|
"logits/rejected": 1.5492628812789917,
|
|
"logps/chosen": -251.85031127929688,
|
|
"logps/ref_chosen": -257.9057312011719,
|
|
"logps/ref_rejected": -246.391845703125,
|
|
"logps/rejected": -243.01177978515625,
|
|
"loss": 5.4504,
|
|
"margin_dpo/margin_mean": 2.6753690242767334,
|
|
"margin_dpo/margin_std": 3.9176571369171143,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.16963350785340314,
|
|
"grad_norm": 14.57850456237793,
|
|
"learning_rate": 4.93167072587771e-07,
|
|
"logits/chosen": 2.009546995162964,
|
|
"logits/rejected": 2.2237491607666016,
|
|
"logps/chosen": -220.03717041015625,
|
|
"logps/ref_chosen": -226.68576049804688,
|
|
"logps/ref_rejected": -215.2713623046875,
|
|
"logps/rejected": -212.5276336669922,
|
|
"loss": 5.4326,
|
|
"margin_dpo/margin_mean": 3.9048891067504883,
|
|
"margin_dpo/margin_std": 3.982060432434082,
|
|
"step": 81
|
|
},
|
|
{
|
|
"epoch": 0.17172774869109947,
|
|
"grad_norm": 15.640838623046875,
|
|
"learning_rate": 4.92735454356513e-07,
|
|
"logits/chosen": 1.8449329137802124,
|
|
"logits/rejected": 1.773772954940796,
|
|
"logps/chosen": -290.084228515625,
|
|
"logps/ref_chosen": -296.12799072265625,
|
|
"logps/ref_rejected": -261.3748474121094,
|
|
"logps/rejected": -258.4228515625,
|
|
"loss": 5.4297,
|
|
"margin_dpo/margin_mean": 3.091761589050293,
|
|
"margin_dpo/margin_std": 4.757362365722656,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.17382198952879582,
|
|
"grad_norm": 15.583915710449219,
|
|
"learning_rate": 4.922908189595017e-07,
|
|
"logits/chosen": 1.8198847770690918,
|
|
"logits/rejected": 1.8020501136779785,
|
|
"logps/chosen": -255.5862274169922,
|
|
"logps/ref_chosen": -261.39862060546875,
|
|
"logps/ref_rejected": -279.9942626953125,
|
|
"logps/rejected": -276.3531799316406,
|
|
"loss": 5.4115,
|
|
"margin_dpo/margin_mean": 2.1713221073150635,
|
|
"margin_dpo/margin_std": 4.908409118652344,
|
|
"step": 83
|
|
},
|
|
{
|
|
"epoch": 0.17591623036649215,
|
|
"grad_norm": 15.165980339050293,
|
|
"learning_rate": 4.918331902411841e-07,
|
|
"logits/chosen": 2.0596227645874023,
|
|
"logits/rejected": 1.9471057653427124,
|
|
"logps/chosen": -385.02862548828125,
|
|
"logps/ref_chosen": -392.54547119140625,
|
|
"logps/ref_rejected": -342.066162109375,
|
|
"logps/rejected": -336.90234375,
|
|
"loss": 5.4311,
|
|
"margin_dpo/margin_mean": 2.353001356124878,
|
|
"margin_dpo/margin_std": 5.491085052490234,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.17801047120418848,
|
|
"grad_norm": 13.898218154907227,
|
|
"learning_rate": 4.913625927427995e-07,
|
|
"logits/chosen": 1.505142331123352,
|
|
"logits/rejected": 1.6710948944091797,
|
|
"logps/chosen": -186.115478515625,
|
|
"logps/ref_chosen": -192.9306640625,
|
|
"logps/ref_rejected": -231.5825653076172,
|
|
"logps/rejected": -227.84988403320312,
|
|
"loss": 5.4719,
|
|
"margin_dpo/margin_mean": 3.082519054412842,
|
|
"margin_dpo/margin_std": 4.139708995819092,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.18010471204188483,
|
|
"grad_norm": 16.21003532409668,
|
|
"learning_rate": 4.908790517010636e-07,
|
|
"logits/chosen": 1.800257682800293,
|
|
"logits/rejected": 1.8198274374008179,
|
|
"logps/chosen": -306.5592346191406,
|
|
"logps/ref_chosen": -313.5525207519531,
|
|
"logps/ref_rejected": -285.59228515625,
|
|
"logps/rejected": -280.493896484375,
|
|
"loss": 5.4098,
|
|
"margin_dpo/margin_mean": 1.8949062824249268,
|
|
"margin_dpo/margin_std": 6.387627601623535,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.18219895287958116,
|
|
"grad_norm": 15.313575744628906,
|
|
"learning_rate": 4.903825930468148e-07,
|
|
"logits/chosen": 1.5017904043197632,
|
|
"logits/rejected": 1.4683315753936768,
|
|
"logps/chosen": -227.59046936035156,
|
|
"logps/ref_chosen": -236.03445434570312,
|
|
"logps/ref_rejected": -225.67410278320312,
|
|
"logps/rejected": -221.80938720703125,
|
|
"loss": 5.3899,
|
|
"margin_dpo/margin_mean": 4.5792436599731445,
|
|
"margin_dpo/margin_std": 6.2218732833862305,
|
|
"step": 87
|
|
},
|
|
{
|
|
"epoch": 0.18429319371727748,
|
|
"grad_norm": 14.063505172729492,
|
|
"learning_rate": 4.898732434036243e-07,
|
|
"logits/chosen": 1.7088102102279663,
|
|
"logits/rejected": 1.6172915697097778,
|
|
"logps/chosen": -273.8514709472656,
|
|
"logps/ref_chosen": -280.1703186035156,
|
|
"logps/ref_rejected": -219.1881103515625,
|
|
"logps/rejected": -216.62831115722656,
|
|
"loss": 5.4232,
|
|
"margin_dpo/margin_mean": 3.7590301036834717,
|
|
"margin_dpo/margin_std": 6.66114616394043,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.18638743455497384,
|
|
"grad_norm": 15.837873458862305,
|
|
"learning_rate": 4.893510300863676e-07,
|
|
"logits/chosen": 2.1236486434936523,
|
|
"logits/rejected": 2.1029891967773438,
|
|
"logps/chosen": -202.62815856933594,
|
|
"logps/ref_chosen": -211.3966827392578,
|
|
"logps/ref_rejected": -171.04954528808594,
|
|
"logps/rejected": -165.4285430908203,
|
|
"loss": 5.3997,
|
|
"margin_dpo/margin_mean": 3.1475319862365723,
|
|
"margin_dpo/margin_std": 4.286096572875977,
|
|
"step": 89
|
|
},
|
|
{
|
|
"epoch": 0.18848167539267016,
|
|
"grad_norm": 15.2637357711792,
|
|
"learning_rate": 4.8881598109976e-07,
|
|
"logits/chosen": 2.190295696258545,
|
|
"logits/rejected": 2.0822367668151855,
|
|
"logps/chosen": -271.2816467285156,
|
|
"logps/ref_chosen": -280.9217834472656,
|
|
"logps/ref_rejected": -245.75814819335938,
|
|
"logps/rejected": -239.31825256347656,
|
|
"loss": 5.4166,
|
|
"margin_dpo/margin_mean": 3.2002599239349365,
|
|
"margin_dpo/margin_std": 5.397155284881592,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.1905759162303665,
|
|
"grad_norm": 14.667741775512695,
|
|
"learning_rate": 4.882681251368548e-07,
|
|
"logits/chosen": 1.3757317066192627,
|
|
"logits/rejected": 1.691314697265625,
|
|
"logps/chosen": -121.55278778076172,
|
|
"logps/ref_chosen": -130.23472595214844,
|
|
"logps/ref_rejected": -177.76895141601562,
|
|
"logps/rejected": -172.3560028076172,
|
|
"loss": 5.4142,
|
|
"margin_dpo/margin_mean": 3.2689881324768066,
|
|
"margin_dpo/margin_std": 4.278058052062988,
|
|
"step": 91
|
|
},
|
|
{
|
|
"epoch": 0.19267015706806281,
|
|
"grad_norm": 15.648965835571289,
|
|
"learning_rate": 4.877074915775048e-07,
|
|
"logits/chosen": 1.6639858484268188,
|
|
"logits/rejected": 1.4799772500991821,
|
|
"logps/chosen": -334.3116455078125,
|
|
"logps/ref_chosen": -344.4306335449219,
|
|
"logps/ref_rejected": -276.291748046875,
|
|
"logps/rejected": -270.984375,
|
|
"loss": 5.4004,
|
|
"margin_dpo/margin_mean": 4.811601161956787,
|
|
"margin_dpo/margin_std": 5.4291887283325195,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.19476439790575917,
|
|
"grad_norm": 14.095191955566406,
|
|
"learning_rate": 4.871341104867864e-07,
|
|
"logits/chosen": 1.9756680727005005,
|
|
"logits/rejected": 1.923811674118042,
|
|
"logps/chosen": -196.8525390625,
|
|
"logps/ref_chosen": -206.1533660888672,
|
|
"logps/ref_rejected": -231.759033203125,
|
|
"logps/rejected": -227.26364135742188,
|
|
"loss": 5.408,
|
|
"margin_dpo/margin_mean": 4.805446147918701,
|
|
"margin_dpo/margin_std": 6.056692600250244,
|
|
"step": 93
|
|
},
|
|
{
|
|
"epoch": 0.1968586387434555,
|
|
"grad_norm": 15.55691909790039,
|
|
"learning_rate": 4.865480126133871e-07,
|
|
"logits/chosen": 1.7521295547485352,
|
|
"logits/rejected": 1.8241287469863892,
|
|
"logps/chosen": -250.36639404296875,
|
|
"logps/ref_chosen": -261.2528381347656,
|
|
"logps/ref_rejected": -269.2928771972656,
|
|
"logps/rejected": -263.65771484375,
|
|
"loss": 5.3919,
|
|
"margin_dpo/margin_mean": 5.2512712478637695,
|
|
"margin_dpo/margin_std": 7.996842384338379,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.19895287958115182,
|
|
"grad_norm": 16.091550827026367,
|
|
"learning_rate": 4.859492293879573e-07,
|
|
"logits/chosen": 1.884320855140686,
|
|
"logits/rejected": 1.6460635662078857,
|
|
"logps/chosen": -334.9806823730469,
|
|
"logps/ref_chosen": -345.480224609375,
|
|
"logps/ref_rejected": -294.0064697265625,
|
|
"logps/rejected": -288.2855529785156,
|
|
"loss": 5.3719,
|
|
"margin_dpo/margin_mean": 4.778585433959961,
|
|
"margin_dpo/margin_std": 8.979715347290039,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.20104712041884817,
|
|
"grad_norm": 15.211227416992188,
|
|
"learning_rate": 4.853377929214243e-07,
|
|
"logits/chosen": 1.442068099975586,
|
|
"logits/rejected": 1.3402963876724243,
|
|
"logps/chosen": -239.22625732421875,
|
|
"logps/ref_chosen": -249.85205078125,
|
|
"logps/ref_rejected": -274.1024169921875,
|
|
"logps/rejected": -266.5990295410156,
|
|
"loss": 5.3763,
|
|
"margin_dpo/margin_mean": 3.1224074363708496,
|
|
"margin_dpo/margin_std": 6.072546482086182,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.2031413612565445,
|
|
"grad_norm": 15.3523588180542,
|
|
"learning_rate": 4.847137360032699e-07,
|
|
"logits/chosen": 1.682770013809204,
|
|
"logits/rejected": 1.789080262184143,
|
|
"logps/chosen": -224.3016815185547,
|
|
"logps/ref_chosen": -233.62025451660156,
|
|
"logps/ref_rejected": -258.32647705078125,
|
|
"logps/rejected": -253.4158935546875,
|
|
"loss": 5.3694,
|
|
"margin_dpo/margin_mean": 4.407979965209961,
|
|
"margin_dpo/margin_std": 6.528227806091309,
|
|
"step": 97
|
|
},
|
|
{
|
|
"epoch": 0.20523560209424083,
|
|
"grad_norm": 15.678691864013672,
|
|
"learning_rate": 4.84077092099773e-07,
|
|
"logits/chosen": 1.9161100387573242,
|
|
"logits/rejected": 2.1308605670928955,
|
|
"logps/chosen": -256.5081787109375,
|
|
"logps/ref_chosen": -267.27911376953125,
|
|
"logps/ref_rejected": -335.98284912109375,
|
|
"logps/rejected": -327.7537841796875,
|
|
"loss": 5.3668,
|
|
"margin_dpo/margin_mean": 2.5418522357940674,
|
|
"margin_dpo/margin_std": 7.70307731628418,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.20732984293193718,
|
|
"grad_norm": 15.061153411865234,
|
|
"learning_rate": 4.834278953522137e-07,
|
|
"logits/chosen": 1.8618698120117188,
|
|
"logits/rejected": 1.8229163885116577,
|
|
"logps/chosen": -275.3375244140625,
|
|
"logps/ref_chosen": -285.90435791015625,
|
|
"logps/ref_rejected": -278.0072021484375,
|
|
"logps/rejected": -271.364013671875,
|
|
"loss": 5.3595,
|
|
"margin_dpo/margin_mean": 3.9236538410186768,
|
|
"margin_dpo/margin_std": 9.717151641845703,
|
|
"step": 99
|
|
},
|
|
{
|
|
"epoch": 0.2094240837696335,
|
|
"grad_norm": 15.537282943725586,
|
|
"learning_rate": 4.827661805750437e-07,
|
|
"logits/chosen": 1.6382941007614136,
|
|
"logits/rejected": 1.5340853929519653,
|
|
"logps/chosen": -327.0155944824219,
|
|
"logps/ref_chosen": -335.2471008300781,
|
|
"logps/ref_rejected": -304.7597351074219,
|
|
"logps/rejected": -300.1502380371094,
|
|
"loss": 5.3785,
|
|
"margin_dpo/margin_mean": 3.622096061706543,
|
|
"margin_dpo/margin_std": 7.071560859680176,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.21151832460732983,
|
|
"grad_norm": 15.370798110961914,
|
|
"learning_rate": 4.820919832540181e-07,
|
|
"logits/chosen": 1.507110357284546,
|
|
"logits/rejected": 1.7560882568359375,
|
|
"logps/chosen": -262.4298400878906,
|
|
"logps/ref_chosen": -272.9364318847656,
|
|
"logps/ref_rejected": -271.82366943359375,
|
|
"logps/rejected": -268.6051330566406,
|
|
"loss": 5.3734,
|
|
"margin_dpo/margin_mean": 7.288076400756836,
|
|
"margin_dpo/margin_std": 8.229599952697754,
|
|
"step": 101
|
|
},
|
|
{
|
|
"epoch": 0.2136125654450262,
|
|
"grad_norm": 15.161721229553223,
|
|
"learning_rate": 4.814053395442932e-07,
|
|
"logits/chosen": 1.76149582862854,
|
|
"logits/rejected": 1.8778889179229736,
|
|
"logps/chosen": -151.35018920898438,
|
|
"logps/ref_chosen": -159.15536499023438,
|
|
"logps/ref_rejected": -191.47312927246094,
|
|
"logps/rejected": -188.49746704101562,
|
|
"loss": 5.305,
|
|
"margin_dpo/margin_mean": 4.829489707946777,
|
|
"margin_dpo/margin_std": 7.208276271820068,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.2157068062827225,
|
|
"grad_norm": 15.669957160949707,
|
|
"learning_rate": 4.807062862684873e-07,
|
|
"logits/chosen": 2.092226505279541,
|
|
"logits/rejected": 2.202396869659424,
|
|
"logps/chosen": -291.3448486328125,
|
|
"logps/ref_chosen": -301.0699768066406,
|
|
"logps/ref_rejected": -306.12469482421875,
|
|
"logps/rejected": -298.96844482421875,
|
|
"loss": 5.3684,
|
|
"margin_dpo/margin_mean": 2.5688788890838623,
|
|
"margin_dpo/margin_std": 8.519815444946289,
|
|
"step": 103
|
|
},
|
|
{
|
|
"epoch": 0.21780104712041884,
|
|
"grad_norm": 14.514609336853027,
|
|
"learning_rate": 4.799948609147061e-07,
|
|
"logits/chosen": 1.875953197479248,
|
|
"logits/rejected": 1.7595632076263428,
|
|
"logps/chosen": -308.81158447265625,
|
|
"logps/ref_chosen": -316.44036865234375,
|
|
"logps/ref_rejected": -245.41790771484375,
|
|
"logps/rejected": -242.4790496826172,
|
|
"loss": 5.3826,
|
|
"margin_dpo/margin_mean": 4.689910411834717,
|
|
"margin_dpo/margin_std": 10.412727355957031,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.2198952879581152,
|
|
"grad_norm": 17.934940338134766,
|
|
"learning_rate": 4.792711016345321e-07,
|
|
"logits/chosen": 1.8088258504867554,
|
|
"logits/rejected": 1.6915315389633179,
|
|
"logps/chosen": -253.7894744873047,
|
|
"logps/ref_chosen": -264.70599365234375,
|
|
"logps/ref_rejected": -232.14236450195312,
|
|
"logps/rejected": -229.86798095703125,
|
|
"loss": 5.241,
|
|
"margin_dpo/margin_mean": 8.642158508300781,
|
|
"margin_dpo/margin_std": 9.996341705322266,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.22198952879581152,
|
|
"grad_norm": 16.461326599121094,
|
|
"learning_rate": 4.785350472409791e-07,
|
|
"logits/chosen": 1.8355200290679932,
|
|
"logits/rejected": 2.0365209579467773,
|
|
"logps/chosen": -274.2940673828125,
|
|
"logps/ref_chosen": -280.6784973144531,
|
|
"logps/ref_rejected": -353.0090026855469,
|
|
"logps/rejected": -352.0072021484375,
|
|
"loss": 5.3413,
|
|
"margin_dpo/margin_mean": 5.382654666900635,
|
|
"margin_dpo/margin_std": 9.296738624572754,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.22408376963350785,
|
|
"grad_norm": 16.5199031829834,
|
|
"learning_rate": 4.777867372064105e-07,
|
|
"logits/chosen": 1.6165478229522705,
|
|
"logits/rejected": 1.5343804359436035,
|
|
"logps/chosen": -327.95794677734375,
|
|
"logps/ref_chosen": -336.91058349609375,
|
|
"logps/ref_rejected": -280.02325439453125,
|
|
"logps/rejected": -277.4742431640625,
|
|
"loss": 5.2697,
|
|
"margin_dpo/margin_mean": 6.403599262237549,
|
|
"margin_dpo/margin_std": 9.143040657043457,
|
|
"step": 107
|
|
},
|
|
{
|
|
"epoch": 0.2261780104712042,
|
|
"grad_norm": 16.170669555664062,
|
|
"learning_rate": 4.770262116604223e-07,
|
|
"logits/chosen": 1.8304221630096436,
|
|
"logits/rejected": 2.0288898944854736,
|
|
"logps/chosen": -224.6934356689453,
|
|
"logps/ref_chosen": -232.04891967773438,
|
|
"logps/ref_rejected": -248.3793487548828,
|
|
"logps/rejected": -246.96351623535156,
|
|
"loss": 5.2351,
|
|
"margin_dpo/margin_mean": 5.9396281242370605,
|
|
"margin_dpo/margin_std": 9.726874351501465,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.22827225130890053,
|
|
"grad_norm": 17.59023094177246,
|
|
"learning_rate": 4.7625351138769166e-07,
|
|
"logits/chosen": 1.8960250616073608,
|
|
"logits/rejected": 1.919461965560913,
|
|
"logps/chosen": -236.6331329345703,
|
|
"logps/ref_chosen": -243.42401123046875,
|
|
"logps/ref_rejected": -276.1861877441406,
|
|
"logps/rejected": -274.469482421875,
|
|
"loss": 5.2323,
|
|
"margin_dpo/margin_mean": 5.074185371398926,
|
|
"margin_dpo/margin_std": 8.536012649536133,
|
|
"step": 109
|
|
},
|
|
{
|
|
"epoch": 0.23036649214659685,
|
|
"grad_norm": 15.948025703430176,
|
|
"learning_rate": 4.75468677825789e-07,
|
|
"logits/chosen": 1.6093004941940308,
|
|
"logits/rejected": 1.6397433280944824,
|
|
"logps/chosen": -234.94406127929688,
|
|
"logps/ref_chosen": -242.5493621826172,
|
|
"logps/ref_rejected": -195.59750366210938,
|
|
"logps/rejected": -193.1940155029297,
|
|
"loss": 5.246,
|
|
"margin_dpo/margin_mean": 5.201825141906738,
|
|
"margin_dpo/margin_std": 10.898710250854492,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.2324607329842932,
|
|
"grad_norm": 18.407573699951172,
|
|
"learning_rate": 4.7467175306295647e-07,
|
|
"logits/chosen": 1.6897281408309937,
|
|
"logits/rejected": 1.7771556377410889,
|
|
"logps/chosen": -272.2618408203125,
|
|
"logps/ref_chosen": -279.930908203125,
|
|
"logps/ref_rejected": -281.9147644042969,
|
|
"logps/rejected": -282.97882080078125,
|
|
"loss": 5.223,
|
|
"margin_dpo/margin_mean": 8.733101844787598,
|
|
"margin_dpo/margin_std": 11.250627517700195,
|
|
"step": 111
|
|
},
|
|
{
|
|
"epoch": 0.23455497382198953,
|
|
"grad_norm": 14.606575012207031,
|
|
"learning_rate": 4.7386277983585053e-07,
|
|
"logits/chosen": 1.776769757270813,
|
|
"logits/rejected": 1.8782975673675537,
|
|
"logps/chosen": -243.624755859375,
|
|
"logps/ref_chosen": -246.89129638671875,
|
|
"logps/ref_rejected": -266.50506591796875,
|
|
"logps/rejected": -265.4157409667969,
|
|
"loss": 5.395,
|
|
"margin_dpo/margin_mean": 2.177186965942383,
|
|
"margin_dpo/margin_std": 10.55683422088623,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.23664921465968586,
|
|
"grad_norm": 16.843202590942383,
|
|
"learning_rate": 4.7304180152725024e-07,
|
|
"logits/chosen": 1.4505597352981567,
|
|
"logits/rejected": 1.5904918909072876,
|
|
"logps/chosen": -269.72711181640625,
|
|
"logps/ref_chosen": -276.4613342285156,
|
|
"logps/ref_rejected": -341.7659912109375,
|
|
"logps/rejected": -342.48956298828125,
|
|
"loss": 5.1716,
|
|
"margin_dpo/margin_mean": 7.457816123962402,
|
|
"margin_dpo/margin_std": 12.05935287475586,
|
|
"step": 113
|
|
},
|
|
{
|
|
"epoch": 0.2387434554973822,
|
|
"grad_norm": 16.080974578857422,
|
|
"learning_rate": 4.7220886216373085e-07,
|
|
"logits/chosen": 1.6391416788101196,
|
|
"logits/rejected": 1.5624871253967285,
|
|
"logps/chosen": -247.58502197265625,
|
|
"logps/ref_chosen": -251.4463653564453,
|
|
"logps/ref_rejected": -210.03152465820312,
|
|
"logps/rejected": -213.17721557617188,
|
|
"loss": 5.3559,
|
|
"margin_dpo/margin_mean": 7.00706672668457,
|
|
"margin_dpo/margin_std": 8.853893280029297,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.24083769633507854,
|
|
"grad_norm": 15.944089889526367,
|
|
"learning_rate": 4.7136400641330245e-07,
|
|
"logits/chosen": 1.8735270500183105,
|
|
"logits/rejected": 1.6029636859893799,
|
|
"logps/chosen": -253.3223876953125,
|
|
"logps/ref_chosen": -257.82574462890625,
|
|
"logps/ref_rejected": -192.41648864746094,
|
|
"logps/rejected": -191.51156616210938,
|
|
"loss": 5.3403,
|
|
"margin_dpo/margin_mean": 3.5983924865722656,
|
|
"margin_dpo/margin_std": 8.953197479248047,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.24293193717277486,
|
|
"grad_norm": 16.291976928710938,
|
|
"learning_rate": 4.70507279583015e-07,
|
|
"logits/chosen": 1.695469856262207,
|
|
"logits/rejected": 1.8081481456756592,
|
|
"logps/chosen": -242.69943237304688,
|
|
"logps/ref_chosen": -248.17518615722656,
|
|
"logps/ref_rejected": -274.10870361328125,
|
|
"logps/rejected": -276.470703125,
|
|
"loss": 5.2459,
|
|
"margin_dpo/margin_mean": 7.837741851806641,
|
|
"margin_dpo/margin_std": 9.762773513793945,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.2450261780104712,
|
|
"grad_norm": 16.642024993896484,
|
|
"learning_rate": 4.6963872761652834e-07,
|
|
"logits/chosen": 1.6590253114700317,
|
|
"logits/rejected": 1.4430992603302002,
|
|
"logps/chosen": -229.59909057617188,
|
|
"logps/ref_chosen": -235.29620361328125,
|
|
"logps/ref_rejected": -190.87095642089844,
|
|
"logps/rejected": -194.7079620361328,
|
|
"loss": 5.2344,
|
|
"margin_dpo/margin_mean": 9.53414535522461,
|
|
"margin_dpo/margin_std": 9.101675033569336,
|
|
"step": 117
|
|
},
|
|
{
|
|
"epoch": 0.24712041884816754,
|
|
"grad_norm": 20.776172637939453,
|
|
"learning_rate": 4.687583970916486e-07,
|
|
"logits/chosen": 1.6007872819900513,
|
|
"logits/rejected": 1.6555330753326416,
|
|
"logps/chosen": -256.0022277832031,
|
|
"logps/ref_chosen": -260.44781494140625,
|
|
"logps/ref_rejected": -308.3326416015625,
|
|
"logps/rejected": -313.2330627441406,
|
|
"loss": 5.2168,
|
|
"margin_dpo/margin_mean": 9.346000671386719,
|
|
"margin_dpo/margin_std": 13.443426132202148,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.24921465968586387,
|
|
"grad_norm": 15.843025207519531,
|
|
"learning_rate": 4.6786633521783005e-07,
|
|
"logits/chosen": 1.9080662727355957,
|
|
"logits/rejected": 2.017760753631592,
|
|
"logps/chosen": -282.1200866699219,
|
|
"logps/ref_chosen": -286.9692687988281,
|
|
"logps/ref_rejected": -331.7510986328125,
|
|
"logps/rejected": -331.0477600097656,
|
|
"loss": 5.2789,
|
|
"margin_dpo/margin_mean": 4.145843982696533,
|
|
"margin_dpo/margin_std": 13.910889625549316,
|
|
"step": 119
|
|
},
|
|
{
|
|
"epoch": 0.2513089005235602,
|
|
"grad_norm": 16.17180061340332,
|
|
"learning_rate": 4.669625898336438e-07,
|
|
"logits/chosen": 1.9562854766845703,
|
|
"logits/rejected": 1.8660322427749634,
|
|
"logps/chosen": -278.0622253417969,
|
|
"logps/ref_chosen": -281.98077392578125,
|
|
"logps/ref_rejected": -283.52679443359375,
|
|
"logps/rejected": -288.8341369628906,
|
|
"loss": 5.2221,
|
|
"margin_dpo/margin_mean": 9.225922584533691,
|
|
"margin_dpo/margin_std": 13.011504173278809,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.2534031413612565,
|
|
"grad_norm": 14.78394889831543,
|
|
"learning_rate": 4.6604720940421207e-07,
|
|
"logits/chosen": 1.1911287307739258,
|
|
"logits/rejected": 1.5024229288101196,
|
|
"logps/chosen": -144.0179443359375,
|
|
"logps/ref_chosen": -145.69662475585938,
|
|
"logps/ref_rejected": -195.21612548828125,
|
|
"logps/rejected": -199.57223510742188,
|
|
"loss": 5.3557,
|
|
"margin_dpo/margin_mean": 6.034780025482178,
|
|
"margin_dpo/margin_std": 9.371333122253418,
|
|
"step": 121
|
|
},
|
|
{
|
|
"epoch": 0.2554973821989529,
|
|
"grad_norm": 15.84109115600586,
|
|
"learning_rate": 4.651202430186092e-07,
|
|
"logits/chosen": 1.7703770399093628,
|
|
"logits/rejected": 2.0983457565307617,
|
|
"logps/chosen": -245.3528289794922,
|
|
"logps/ref_chosen": -252.1569366455078,
|
|
"logps/ref_rejected": -309.68548583984375,
|
|
"logps/rejected": -306.59942626953125,
|
|
"loss": 5.2869,
|
|
"margin_dpo/margin_mean": 3.718092441558838,
|
|
"margin_dpo/margin_std": 17.503427505493164,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.25759162303664923,
|
|
"grad_norm": 18.21240997314453,
|
|
"learning_rate": 4.6418174038722924e-07,
|
|
"logits/chosen": 1.6439917087554932,
|
|
"logits/rejected": 1.4977948665618896,
|
|
"logps/chosen": -358.5697326660156,
|
|
"logps/ref_chosen": -366.5253601074219,
|
|
"logps/ref_rejected": -285.2503662109375,
|
|
"logps/rejected": -286.8184814453125,
|
|
"loss": 5.1686,
|
|
"margin_dpo/margin_mean": 9.523737907409668,
|
|
"margin_dpo/margin_std": 12.473346710205078,
|
|
"step": 123
|
|
},
|
|
{
|
|
"epoch": 0.25968586387434556,
|
|
"grad_norm": 16.816696166992188,
|
|
"learning_rate": 4.6323175183912023e-07,
|
|
"logits/chosen": 1.4895159006118774,
|
|
"logits/rejected": 1.6259382963180542,
|
|
"logps/chosen": -244.55775451660156,
|
|
"logps/ref_chosen": -251.4420623779297,
|
|
"logps/ref_rejected": -231.0302734375,
|
|
"logps/rejected": -230.74337768554688,
|
|
"loss": 5.2019,
|
|
"margin_dpo/margin_mean": 6.597394943237305,
|
|
"margin_dpo/margin_std": 15.111127853393555,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.2617801047120419,
|
|
"grad_norm": 16.386260986328125,
|
|
"learning_rate": 4.6227032831928483e-07,
|
|
"logits/chosen": 1.6032323837280273,
|
|
"logits/rejected": 1.598075032234192,
|
|
"logps/chosen": -242.79583740234375,
|
|
"logps/ref_chosen": -248.3984375,
|
|
"logps/ref_rejected": -307.77557373046875,
|
|
"logps/rejected": -308.05059814453125,
|
|
"loss": 5.2189,
|
|
"margin_dpo/margin_mean": 5.877676010131836,
|
|
"margin_dpo/margin_std": 13.811307907104492,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.2638743455497382,
|
|
"grad_norm": 16.69825553894043,
|
|
"learning_rate": 4.612975213859487e-07,
|
|
"logits/chosen": 1.7347309589385986,
|
|
"logits/rejected": 1.9158421754837036,
|
|
"logps/chosen": -291.75244140625,
|
|
"logps/ref_chosen": -295.82366943359375,
|
|
"logps/ref_rejected": -295.2666931152344,
|
|
"logps/rejected": -299.0240783691406,
|
|
"loss": 5.1658,
|
|
"margin_dpo/margin_mean": 7.8285651206970215,
|
|
"margin_dpo/margin_std": 13.573448181152344,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.26596858638743454,
|
|
"grad_norm": 16.867650985717773,
|
|
"learning_rate": 4.603133832077953e-07,
|
|
"logits/chosen": 1.1577448844909668,
|
|
"logits/rejected": 1.106504201889038,
|
|
"logps/chosen": -273.8604736328125,
|
|
"logps/ref_chosen": -279.496337890625,
|
|
"logps/ref_rejected": -278.802978515625,
|
|
"logps/rejected": -282.5022277832031,
|
|
"loss": 5.1019,
|
|
"margin_dpo/margin_mean": 9.335136413574219,
|
|
"margin_dpo/margin_std": 13.274786949157715,
|
|
"step": 127
|
|
},
|
|
{
|
|
"epoch": 0.2680628272251309,
|
|
"grad_norm": 16.078149795532227,
|
|
"learning_rate": 4.5931796656116837e-07,
|
|
"logits/chosen": 1.4048773050308228,
|
|
"logits/rejected": 1.4003376960754395,
|
|
"logps/chosen": -258.86883544921875,
|
|
"logps/ref_chosen": -264.52252197265625,
|
|
"logps/ref_rejected": -239.76937866210938,
|
|
"logps/rejected": -247.31756591796875,
|
|
"loss": 5.057,
|
|
"margin_dpo/margin_mean": 13.201865196228027,
|
|
"margin_dpo/margin_std": 13.145772933959961,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.27015706806282724,
|
|
"grad_norm": 16.511791229248047,
|
|
"learning_rate": 4.5831132482724193e-07,
|
|
"logits/chosen": 1.5169470310211182,
|
|
"logits/rejected": 1.6699875593185425,
|
|
"logps/chosen": -290.08258056640625,
|
|
"logps/ref_chosen": -296.95233154296875,
|
|
"logps/ref_rejected": -260.0984802246094,
|
|
"logps/rejected": -267.3520812988281,
|
|
"loss": 5.1034,
|
|
"margin_dpo/margin_mean": 14.12340259552002,
|
|
"margin_dpo/margin_std": 17.277435302734375,
|
|
"step": 129
|
|
},
|
|
{
|
|
"epoch": 0.27225130890052357,
|
|
"grad_norm": 21.19081687927246,
|
|
"learning_rate": 4.5729351198915705e-07,
|
|
"logits/chosen": 1.5894699096679688,
|
|
"logits/rejected": 1.8434865474700928,
|
|
"logps/chosen": -263.253173828125,
|
|
"logps/ref_chosen": -274.7286682128906,
|
|
"logps/ref_rejected": -325.187255859375,
|
|
"logps/rejected": -327.59503173828125,
|
|
"loss": 5.1333,
|
|
"margin_dpo/margin_mean": 13.883302688598633,
|
|
"margin_dpo/margin_std": 18.286108016967773,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.2743455497382199,
|
|
"grad_norm": 17.56028175354004,
|
|
"learning_rate": 4.5626458262912735e-07,
|
|
"logits/chosen": 1.44374680519104,
|
|
"logits/rejected": 1.400553822517395,
|
|
"logps/chosen": -270.84796142578125,
|
|
"logps/ref_chosen": -279.3233642578125,
|
|
"logps/ref_rejected": -299.2681884765625,
|
|
"logps/rejected": -304.762451171875,
|
|
"loss": 5.2149,
|
|
"margin_dpo/margin_mean": 13.969644546508789,
|
|
"margin_dpo/margin_std": 20.493703842163086,
|
|
"step": 131
|
|
},
|
|
{
|
|
"epoch": 0.2764397905759162,
|
|
"grad_norm": 17.945659637451172,
|
|
"learning_rate": 4.5522459192551166e-07,
|
|
"logits/chosen": 1.5821537971496582,
|
|
"logits/rejected": 1.6266090869903564,
|
|
"logps/chosen": -281.5635681152344,
|
|
"logps/ref_chosen": -291.3346862792969,
|
|
"logps/ref_rejected": -283.13311767578125,
|
|
"logps/rejected": -291.1026306152344,
|
|
"loss": 5.1007,
|
|
"margin_dpo/margin_mean": 17.740650177001953,
|
|
"margin_dpo/margin_std": 17.511295318603516,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.27853403141361255,
|
|
"grad_norm": 16.447092056274414,
|
|
"learning_rate": 4.541735956498554e-07,
|
|
"logits/chosen": 1.6146799325942993,
|
|
"logits/rejected": 1.5560858249664307,
|
|
"logps/chosen": -223.23194885253906,
|
|
"logps/ref_chosen": -233.71875,
|
|
"logps/ref_rejected": -216.53781127929688,
|
|
"logps/rejected": -223.18417358398438,
|
|
"loss": 5.0683,
|
|
"margin_dpo/margin_mean": 17.133150100708008,
|
|
"margin_dpo/margin_std": 14.197755813598633,
|
|
"step": 133
|
|
},
|
|
{
|
|
"epoch": 0.2806282722513089,
|
|
"grad_norm": 20.631309509277344,
|
|
"learning_rate": 4.5311165016389914e-07,
|
|
"logits/chosen": 1.920145869255066,
|
|
"logits/rejected": 1.981586217880249,
|
|
"logps/chosen": -348.9212951660156,
|
|
"logps/ref_chosen": -348.29547119140625,
|
|
"logps/ref_rejected": -343.04510498046875,
|
|
"logps/rejected": -351.0985412597656,
|
|
"loss": 5.22,
|
|
"margin_dpo/margin_mean": 7.427634239196777,
|
|
"margin_dpo/margin_std": 17.99872589111328,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.28272251308900526,
|
|
"grad_norm": 17.117826461791992,
|
|
"learning_rate": 4.520388124165564e-07,
|
|
"logits/chosen": 1.1408627033233643,
|
|
"logits/rejected": 0.9298585057258606,
|
|
"logps/chosen": -226.50486755371094,
|
|
"logps/ref_chosen": -232.59129333496094,
|
|
"logps/ref_rejected": -175.74066162109375,
|
|
"logps/rejected": -181.21482849121094,
|
|
"loss": 5.0762,
|
|
"margin_dpo/margin_mean": 11.560598373413086,
|
|
"margin_dpo/margin_std": 14.716351509094238,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.2848167539267016,
|
|
"grad_norm": 19.100990295410156,
|
|
"learning_rate": 4.5095513994085974e-07,
|
|
"logits/chosen": 1.0842311382293701,
|
|
"logits/rejected": 1.3095265626907349,
|
|
"logps/chosen": -183.53028869628906,
|
|
"logps/ref_chosen": -189.21795654296875,
|
|
"logps/ref_rejected": -191.75979614257812,
|
|
"logps/rejected": -200.99093627929688,
|
|
"loss": 5.1055,
|
|
"margin_dpo/margin_mean": 14.91882038116455,
|
|
"margin_dpo/margin_std": 16.662824630737305,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.2869109947643979,
|
|
"grad_norm": 17.85831642150879,
|
|
"learning_rate": 4.498606908508753e-07,
|
|
"logits/chosen": 1.86328125,
|
|
"logits/rejected": 1.6945784091949463,
|
|
"logps/chosen": -356.1871032714844,
|
|
"logps/ref_chosen": -358.9820861816406,
|
|
"logps/ref_rejected": -277.2926330566406,
|
|
"logps/rejected": -286.5790710449219,
|
|
"loss": 5.1522,
|
|
"margin_dpo/margin_mean": 12.081487655639648,
|
|
"margin_dpo/margin_std": 15.93864631652832,
|
|
"step": 137
|
|
},
|
|
{
|
|
"epoch": 0.28900523560209423,
|
|
"grad_norm": 16.747636795043945,
|
|
"learning_rate": 4.487555238385862e-07,
|
|
"logits/chosen": 1.8756850957870483,
|
|
"logits/rejected": 1.9149004220962524,
|
|
"logps/chosen": -284.18572998046875,
|
|
"logps/ref_chosen": -283.7969055175781,
|
|
"logps/ref_rejected": -269.28643798828125,
|
|
"logps/rejected": -280.577880859375,
|
|
"loss": 5.1753,
|
|
"margin_dpo/margin_mean": 10.902585983276367,
|
|
"margin_dpo/margin_std": 22.474666595458984,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.29109947643979056,
|
|
"grad_norm": 16.213245391845703,
|
|
"learning_rate": 4.476396981707453e-07,
|
|
"logits/chosen": 1.4325203895568848,
|
|
"logits/rejected": 1.5791009664535522,
|
|
"logps/chosen": -218.7724609375,
|
|
"logps/ref_chosen": -221.46124267578125,
|
|
"logps/ref_rejected": -234.3295440673828,
|
|
"logps/rejected": -236.3891143798828,
|
|
"loss": 5.2792,
|
|
"margin_dpo/margin_mean": 4.748367786407471,
|
|
"margin_dpo/margin_std": 21.647659301757812,
|
|
"step": 139
|
|
},
|
|
{
|
|
"epoch": 0.2931937172774869,
|
|
"grad_norm": 25.28333854675293,
|
|
"learning_rate": 4.4651327368569684e-07,
|
|
"logits/chosen": 1.5151917934417725,
|
|
"logits/rejected": 1.5757999420166016,
|
|
"logps/chosen": -237.8951873779297,
|
|
"logps/ref_chosen": -246.27151489257812,
|
|
"logps/ref_rejected": -255.00428771972656,
|
|
"logps/rejected": -261.1990661621094,
|
|
"loss": 5.0124,
|
|
"margin_dpo/margin_mean": 14.571114540100098,
|
|
"margin_dpo/margin_std": 15.991363525390625,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.29528795811518327,
|
|
"grad_norm": 23.41806411743164,
|
|
"learning_rate": 4.453763107901675e-07,
|
|
"logits/chosen": 1.5907336473464966,
|
|
"logits/rejected": 1.6988296508789062,
|
|
"logps/chosen": -264.4815979003906,
|
|
"logps/ref_chosen": -267.79345703125,
|
|
"logps/ref_rejected": -295.4119873046875,
|
|
"logps/rejected": -308.6583557128906,
|
|
"loss": 5.0535,
|
|
"margin_dpo/margin_mean": 16.5582332611084,
|
|
"margin_dpo/margin_std": 21.787641525268555,
|
|
"step": 141
|
|
},
|
|
{
|
|
"epoch": 0.2973821989528796,
|
|
"grad_norm": 17.388578414916992,
|
|
"learning_rate": 4.4422887045602674e-07,
|
|
"logits/chosen": 1.8900976181030273,
|
|
"logits/rejected": 1.6236388683319092,
|
|
"logps/chosen": -341.6228942871094,
|
|
"logps/ref_chosen": -352.8658752441406,
|
|
"logps/ref_rejected": -219.5095672607422,
|
|
"logps/rejected": -223.31808471679688,
|
|
"loss": 5.0641,
|
|
"margin_dpo/margin_mean": 15.05146598815918,
|
|
"margin_dpo/margin_std": 16.49203109741211,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.2994764397905759,
|
|
"grad_norm": 18.831209182739258,
|
|
"learning_rate": 4.4307101421701755e-07,
|
|
"logits/chosen": 1.4374781847000122,
|
|
"logits/rejected": 1.3258070945739746,
|
|
"logps/chosen": -327.2297058105469,
|
|
"logps/ref_chosen": -336.38482666015625,
|
|
"logps/ref_rejected": -213.85707092285156,
|
|
"logps/rejected": -229.42831420898438,
|
|
"loss": 4.9898,
|
|
"margin_dpo/margin_mean": 24.726341247558594,
|
|
"margin_dpo/margin_std": 23.126943588256836,
|
|
"step": 143
|
|
},
|
|
{
|
|
"epoch": 0.30157068062827225,
|
|
"grad_norm": 19.49022674560547,
|
|
"learning_rate": 4.419028041654559e-07,
|
|
"logits/chosen": 1.4996888637542725,
|
|
"logits/rejected": 1.4556653499603271,
|
|
"logps/chosen": -264.51727294921875,
|
|
"logps/ref_chosen": -274.0345458984375,
|
|
"logps/ref_rejected": -274.5603942871094,
|
|
"logps/rejected": -273.80316162109375,
|
|
"loss": 5.0793,
|
|
"margin_dpo/margin_mean": 8.760041236877441,
|
|
"margin_dpo/margin_std": 19.655494689941406,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.3036649214659686,
|
|
"grad_norm": 16.73856544494629,
|
|
"learning_rate": 4.4072430294890166e-07,
|
|
"logits/chosen": 1.6811779737472534,
|
|
"logits/rejected": 1.7291994094848633,
|
|
"logps/chosen": -269.10491943359375,
|
|
"logps/ref_chosen": -274.1513366699219,
|
|
"logps/ref_rejected": -226.63064575195312,
|
|
"logps/rejected": -239.5631866455078,
|
|
"loss": 4.9405,
|
|
"margin_dpo/margin_mean": 17.978939056396484,
|
|
"margin_dpo/margin_std": 23.945186614990234,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.3057591623036649,
|
|
"grad_norm": 29.0201358795166,
|
|
"learning_rate": 4.395355737667985e-07,
|
|
"logits/chosen": 1.4950841665267944,
|
|
"logits/rejected": 1.7295074462890625,
|
|
"logps/chosen": -227.69259643554688,
|
|
"logps/ref_chosen": -229.48269653320312,
|
|
"logps/ref_rejected": -249.7940216064453,
|
|
"logps/rejected": -259.9455261230469,
|
|
"loss": 5.0665,
|
|
"margin_dpo/margin_mean": 11.941591262817383,
|
|
"margin_dpo/margin_std": 17.751564025878906,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.3078534031413613,
|
|
"grad_norm": 16.784557342529297,
|
|
"learning_rate": 4.3833668036708483e-07,
|
|
"logits/chosen": 1.4946480989456177,
|
|
"logits/rejected": 1.4648349285125732,
|
|
"logps/chosen": -284.0057067871094,
|
|
"logps/ref_chosen": -290.8128356933594,
|
|
"logps/ref_rejected": -218.97787475585938,
|
|
"logps/rejected": -229.5756072998047,
|
|
"loss": 5.1681,
|
|
"margin_dpo/margin_mean": 17.40483856201172,
|
|
"margin_dpo/margin_std": 24.006677627563477,
|
|
"step": 147
|
|
},
|
|
{
|
|
"epoch": 0.3099476439790576,
|
|
"grad_norm": 16.890840530395508,
|
|
"learning_rate": 4.3712768704277524e-07,
|
|
"logits/chosen": 1.5196683406829834,
|
|
"logits/rejected": 1.509922742843628,
|
|
"logps/chosen": -261.50762939453125,
|
|
"logps/ref_chosen": -263.70001220703125,
|
|
"logps/ref_rejected": -262.095703125,
|
|
"logps/rejected": -272.2943115234375,
|
|
"loss": 5.1353,
|
|
"margin_dpo/margin_mean": 12.390965461730957,
|
|
"margin_dpo/margin_std": 21.307296752929688,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.31204188481675393,
|
|
"grad_norm": 18.739173889160156,
|
|
"learning_rate": 4.3590865862851263e-07,
|
|
"logits/chosen": 1.9116061925888062,
|
|
"logits/rejected": 1.7204910516738892,
|
|
"logps/chosen": -344.4569396972656,
|
|
"logps/ref_chosen": -350.6168518066406,
|
|
"logps/ref_rejected": -277.2320251464844,
|
|
"logps/rejected": -288.07904052734375,
|
|
"loss": 5.0166,
|
|
"margin_dpo/margin_mean": 17.006885528564453,
|
|
"margin_dpo/margin_std": 17.385805130004883,
|
|
"step": 149
|
|
},
|
|
{
|
|
"epoch": 0.31413612565445026,
|
|
"grad_norm": 17.36412811279297,
|
|
"learning_rate": 4.346796604970912e-07,
|
|
"logits/chosen": 1.934645652770996,
|
|
"logits/rejected": 1.848956823348999,
|
|
"logps/chosen": -261.2005920410156,
|
|
"logps/ref_chosen": -264.05096435546875,
|
|
"logps/ref_rejected": -286.02313232421875,
|
|
"logps/rejected": -298.2835693359375,
|
|
"loss": 5.0369,
|
|
"margin_dpo/margin_mean": 15.110857963562012,
|
|
"margin_dpo/margin_std": 18.34941291809082,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.3162303664921466,
|
|
"grad_norm": 20.5943546295166,
|
|
"learning_rate": 4.3344075855595097e-07,
|
|
"logits/chosen": 1.3573246002197266,
|
|
"logits/rejected": 1.373565673828125,
|
|
"logps/chosen": -254.92498779296875,
|
|
"logps/ref_chosen": -257.74664306640625,
|
|
"logps/ref_rejected": -256.2339172363281,
|
|
"logps/rejected": -267.41278076171875,
|
|
"loss": 4.7848,
|
|
"margin_dpo/margin_mean": 14.000543594360352,
|
|
"margin_dpo/margin_std": 23.76491928100586,
|
|
"step": 151
|
|
},
|
|
{
|
|
"epoch": 0.3183246073298429,
|
|
"grad_norm": 21.099018096923828,
|
|
"learning_rate": 4.3219201924364323e-07,
|
|
"logits/chosen": 1.4018583297729492,
|
|
"logits/rejected": 1.803174376487732,
|
|
"logps/chosen": -245.9750213623047,
|
|
"logps/ref_chosen": -250.47512817382812,
|
|
"logps/ref_rejected": -322.36474609375,
|
|
"logps/rejected": -333.2466735839844,
|
|
"loss": 4.9817,
|
|
"margin_dpo/margin_mean": 15.381957054138184,
|
|
"margin_dpo/margin_std": 22.273956298828125,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.3204188481675393,
|
|
"grad_norm": 22.535673141479492,
|
|
"learning_rate": 4.309335095262675e-07,
|
|
"logits/chosen": 1.5490094423294067,
|
|
"logits/rejected": 1.5208497047424316,
|
|
"logps/chosen": -235.2023162841797,
|
|
"logps/ref_chosen": -238.36544799804688,
|
|
"logps/ref_rejected": -215.78970336914062,
|
|
"logps/rejected": -236.40200805664062,
|
|
"loss": 4.6931,
|
|
"margin_dpo/margin_mean": 23.775440216064453,
|
|
"margin_dpo/margin_std": 23.90810775756836,
|
|
"step": 153
|
|
},
|
|
{
|
|
"epoch": 0.3225130890052356,
|
|
"grad_norm": 19.634624481201172,
|
|
"learning_rate": 4.2966529689388064e-07,
|
|
"logits/chosen": 1.213348627090454,
|
|
"logits/rejected": 1.2180352210998535,
|
|
"logps/chosen": -264.2608337402344,
|
|
"logps/ref_chosen": -259.7012939453125,
|
|
"logps/ref_rejected": -255.74172973632812,
|
|
"logps/rejected": -272.3033142089844,
|
|
"loss": 4.9699,
|
|
"margin_dpo/margin_mean": 12.002017974853516,
|
|
"margin_dpo/margin_std": 28.510345458984375,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.32460732984293195,
|
|
"grad_norm": 19.311044692993164,
|
|
"learning_rate": 4.2838744935687716e-07,
|
|
"logits/chosen": 1.4307262897491455,
|
|
"logits/rejected": 1.4176700115203857,
|
|
"logps/chosen": -324.7783203125,
|
|
"logps/ref_chosen": -325.11517333984375,
|
|
"logps/ref_rejected": -288.08380126953125,
|
|
"logps/rejected": -307.0673828125,
|
|
"loss": 5.1244,
|
|
"margin_dpo/margin_mean": 19.320411682128906,
|
|
"margin_dpo/margin_std": 28.002426147460938,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.3267015706806283,
|
|
"grad_norm": 19.045074462890625,
|
|
"learning_rate": 4.271000354423425e-07,
|
|
"logits/chosen": 1.6414060592651367,
|
|
"logits/rejected": 1.486697793006897,
|
|
"logps/chosen": -260.87078857421875,
|
|
"logps/ref_chosen": -263.62353515625,
|
|
"logps/ref_rejected": -183.94119262695312,
|
|
"logps/rejected": -202.97857666015625,
|
|
"loss": 4.8187,
|
|
"margin_dpo/margin_mean": 21.790143966674805,
|
|
"margin_dpo/margin_std": 19.9763240814209,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.3287958115183246,
|
|
"grad_norm": 24.014020919799805,
|
|
"learning_rate": 4.258031241903777e-07,
|
|
"logits/chosen": 1.4358762502670288,
|
|
"logits/rejected": 1.5518109798431396,
|
|
"logps/chosen": -248.9981231689453,
|
|
"logps/ref_chosen": -237.6883087158203,
|
|
"logps/ref_rejected": -232.87484741210938,
|
|
"logps/rejected": -254.34902954101562,
|
|
"loss": 5.0507,
|
|
"margin_dpo/margin_mean": 10.164348602294922,
|
|
"margin_dpo/margin_std": 22.118553161621094,
|
|
"step": 157
|
|
},
|
|
{
|
|
"epoch": 0.3308900523560209,
|
|
"grad_norm": 19.902008056640625,
|
|
"learning_rate": 4.2449678515039743e-07,
|
|
"logits/chosen": 1.7699273824691772,
|
|
"logits/rejected": 1.8686857223510742,
|
|
"logps/chosen": -284.7595520019531,
|
|
"logps/ref_chosen": -279.62335205078125,
|
|
"logps/ref_rejected": -267.80615234375,
|
|
"logps/rejected": -285.53924560546875,
|
|
"loss": 4.9867,
|
|
"margin_dpo/margin_mean": 12.59688663482666,
|
|
"margin_dpo/margin_std": 24.365018844604492,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.33298429319371725,
|
|
"grad_norm": 22.825597763061523,
|
|
"learning_rate": 4.2318108837739986e-07,
|
|
"logits/chosen": 1.553140640258789,
|
|
"logits/rejected": 1.439896583557129,
|
|
"logps/chosen": -303.68487548828125,
|
|
"logps/ref_chosen": -301.5324401855469,
|
|
"logps/ref_rejected": -263.529541015625,
|
|
"logps/rejected": -274.5115966796875,
|
|
"loss": 5.1446,
|
|
"margin_dpo/margin_mean": 8.82960319519043,
|
|
"margin_dpo/margin_std": 30.41036605834961,
|
|
"step": 159
|
|
},
|
|
{
|
|
"epoch": 0.33507853403141363,
|
|
"grad_norm": 20.597837448120117,
|
|
"learning_rate": 4.218561044282098e-07,
|
|
"logits/chosen": 1.9710590839385986,
|
|
"logits/rejected": 1.699224829673767,
|
|
"logps/chosen": -311.9967041015625,
|
|
"logps/ref_chosen": -314.1754455566406,
|
|
"logps/ref_rejected": -241.1903076171875,
|
|
"logps/rejected": -267.9695129394531,
|
|
"loss": 4.8583,
|
|
"margin_dpo/margin_mean": 28.957944869995117,
|
|
"margin_dpo/margin_std": 29.692768096923828,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.33717277486910996,
|
|
"grad_norm": 25.702106475830078,
|
|
"learning_rate": 4.2052190435769554e-07,
|
|
"logits/chosen": 1.345297932624817,
|
|
"logits/rejected": 1.2094160318374634,
|
|
"logps/chosen": -268.9353942871094,
|
|
"logps/ref_chosen": -271.0775451660156,
|
|
"logps/ref_rejected": -212.71853637695312,
|
|
"logps/rejected": -228.42201232910156,
|
|
"loss": 4.8254,
|
|
"margin_dpo/margin_mean": 17.845624923706055,
|
|
"margin_dpo/margin_std": 25.553813934326172,
|
|
"step": 161
|
|
},
|
|
{
|
|
"epoch": 0.3392670157068063,
|
|
"grad_norm": 26.607454299926758,
|
|
"learning_rate": 4.1917855971495763e-07,
|
|
"logits/chosen": 1.6022093296051025,
|
|
"logits/rejected": 1.4934312105178833,
|
|
"logps/chosen": -293.98974609375,
|
|
"logps/ref_chosen": -296.7241516113281,
|
|
"logps/ref_rejected": -222.9241485595703,
|
|
"logps/rejected": -236.45635986328125,
|
|
"loss": 4.8946,
|
|
"margin_dpo/margin_mean": 16.266626358032227,
|
|
"margin_dpo/margin_std": 23.54033660888672,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.3413612565445026,
|
|
"grad_norm": 30.976999282836914,
|
|
"learning_rate": 4.1782614253949255e-07,
|
|
"logits/chosen": 1.7216789722442627,
|
|
"logits/rejected": 1.7484244108200073,
|
|
"logps/chosen": -246.4569549560547,
|
|
"logps/ref_chosen": -249.64366149902344,
|
|
"logps/ref_rejected": -244.58258056640625,
|
|
"logps/rejected": -260.55218505859375,
|
|
"loss": 4.7557,
|
|
"margin_dpo/margin_mean": 19.156299591064453,
|
|
"margin_dpo/margin_std": 21.998430252075195,
|
|
"step": 163
|
|
},
|
|
{
|
|
"epoch": 0.34345549738219894,
|
|
"grad_norm": 22.384260177612305,
|
|
"learning_rate": 4.164647253573289e-07,
|
|
"logits/chosen": 1.4122521877288818,
|
|
"logits/rejected": 1.5924245119094849,
|
|
"logps/chosen": -214.8105926513672,
|
|
"logps/ref_chosen": -203.6176300048828,
|
|
"logps/ref_rejected": -216.5535888671875,
|
|
"logps/rejected": -240.29396057128906,
|
|
"loss": 4.8891,
|
|
"margin_dpo/margin_mean": 12.547422409057617,
|
|
"margin_dpo/margin_std": 23.612470626831055,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.34554973821989526,
|
|
"grad_norm": 28.401796340942383,
|
|
"learning_rate": 4.1509438117713863e-07,
|
|
"logits/chosen": 2.1252005100250244,
|
|
"logits/rejected": 2.1613521575927734,
|
|
"logps/chosen": -350.52978515625,
|
|
"logps/ref_chosen": -344.1730651855469,
|
|
"logps/ref_rejected": -304.00128173828125,
|
|
"logps/rejected": -327.9066467285156,
|
|
"loss": 4.9931,
|
|
"margin_dpo/margin_mean": 17.548656463623047,
|
|
"margin_dpo/margin_std": 27.731534957885742,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.34764397905759165,
|
|
"grad_norm": 19.331459045410156,
|
|
"learning_rate": 4.137151834863213e-07,
|
|
"logits/chosen": 1.646728277206421,
|
|
"logits/rejected": 1.640990138053894,
|
|
"logps/chosen": -242.68841552734375,
|
|
"logps/ref_chosen": -233.72891235351562,
|
|
"logps/ref_rejected": -208.29397583007812,
|
|
"logps/rejected": -224.4010467529297,
|
|
"loss": 5.0854,
|
|
"margin_dpo/margin_mean": 7.147580146789551,
|
|
"margin_dpo/margin_std": 26.665321350097656,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.34973821989528797,
|
|
"grad_norm": 19.852638244628906,
|
|
"learning_rate": 4.123272062470633e-07,
|
|
"logits/chosen": 1.6361035108566284,
|
|
"logits/rejected": 1.411129117012024,
|
|
"logps/chosen": -327.1979064941406,
|
|
"logps/ref_chosen": -326.10198974609375,
|
|
"logps/ref_rejected": -232.0992889404297,
|
|
"logps/rejected": -256.6214294433594,
|
|
"loss": 4.9473,
|
|
"margin_dpo/margin_mean": 23.426191329956055,
|
|
"margin_dpo/margin_std": 28.530874252319336,
|
|
"step": 167
|
|
},
|
|
{
|
|
"epoch": 0.3518324607329843,
|
|
"grad_norm": 21.564373016357422,
|
|
"learning_rate": 4.1093052389237174e-07,
|
|
"logits/chosen": 1.3292641639709473,
|
|
"logits/rejected": 1.2153077125549316,
|
|
"logps/chosen": -246.62283325195312,
|
|
"logps/ref_chosen": -247.4376983642578,
|
|
"logps/ref_rejected": -216.68064880371094,
|
|
"logps/rejected": -241.31011962890625,
|
|
"loss": 4.7589,
|
|
"margin_dpo/margin_mean": 25.444313049316406,
|
|
"margin_dpo/margin_std": 17.723526000976562,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.3539267015706806,
|
|
"grad_norm": 19.15829086303711,
|
|
"learning_rate": 4.0952521132208267e-07,
|
|
"logits/chosen": 1.6247047185897827,
|
|
"logits/rejected": 1.7833751440048218,
|
|
"logps/chosen": -281.30078125,
|
|
"logps/ref_chosen": -285.1272277832031,
|
|
"logps/ref_rejected": -279.10943603515625,
|
|
"logps/rejected": -302.5621032714844,
|
|
"loss": 4.632,
|
|
"margin_dpo/margin_mean": 27.279136657714844,
|
|
"margin_dpo/margin_std": 23.28731346130371,
|
|
"step": 169
|
|
},
|
|
{
|
|
"epoch": 0.35602094240837695,
|
|
"grad_norm": 24.02274513244629,
|
|
"learning_rate": 4.081113438988443e-07,
|
|
"logits/chosen": 1.5731761455535889,
|
|
"logits/rejected": 1.4810683727264404,
|
|
"logps/chosen": -357.42327880859375,
|
|
"logps/ref_chosen": -358.3712463378906,
|
|
"logps/ref_rejected": -245.13316345214844,
|
|
"logps/rejected": -264.8816223144531,
|
|
"loss": 4.7427,
|
|
"margin_dpo/margin_mean": 20.696434020996094,
|
|
"margin_dpo/margin_std": 30.21214485168457,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.3581151832460733,
|
|
"grad_norm": 23.137300491333008,
|
|
"learning_rate": 4.0668899744407567e-07,
|
|
"logits/chosen": 1.5857964754104614,
|
|
"logits/rejected": 1.465027093887329,
|
|
"logps/chosen": -269.0282897949219,
|
|
"logps/ref_chosen": -273.9371337890625,
|
|
"logps/ref_rejected": -241.6103515625,
|
|
"logps/rejected": -259.7757263183594,
|
|
"loss": 4.7322,
|
|
"margin_dpo/margin_mean": 23.074222564697266,
|
|
"margin_dpo/margin_std": 30.494476318359375,
|
|
"step": 171
|
|
},
|
|
{
|
|
"epoch": 0.36020942408376966,
|
|
"grad_norm": 22.266416549682617,
|
|
"learning_rate": 4.0525824823390043e-07,
|
|
"logits/chosen": 1.6551828384399414,
|
|
"logits/rejected": 1.8315401077270508,
|
|
"logps/chosen": -254.328369140625,
|
|
"logps/ref_chosen": -255.1793975830078,
|
|
"logps/ref_rejected": -279.3556213378906,
|
|
"logps/rejected": -293.8561706542969,
|
|
"loss": 5.039,
|
|
"margin_dpo/margin_mean": 15.351570129394531,
|
|
"margin_dpo/margin_std": 22.528553009033203,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.362303664921466,
|
|
"grad_norm": 24.233097076416016,
|
|
"learning_rate": 4.0381917299505686e-07,
|
|
"logits/chosen": 1.626520037651062,
|
|
"logits/rejected": 1.3303242921829224,
|
|
"logps/chosen": -338.2034606933594,
|
|
"logps/ref_chosen": -333.66375732421875,
|
|
"logps/ref_rejected": -275.1485290527344,
|
|
"logps/rejected": -300.3768310546875,
|
|
"loss": 4.8793,
|
|
"margin_dpo/margin_mean": 20.68860626220703,
|
|
"margin_dpo/margin_std": 28.740659713745117,
|
|
"step": 173
|
|
},
|
|
{
|
|
"epoch": 0.3643979057591623,
|
|
"grad_norm": 22.030344009399414,
|
|
"learning_rate": 4.0237184890078243e-07,
|
|
"logits/chosen": 1.9243297576904297,
|
|
"logits/rejected": 1.6874244213104248,
|
|
"logps/chosen": -354.771484375,
|
|
"logps/ref_chosen": -362.5843505859375,
|
|
"logps/ref_rejected": -250.0384521484375,
|
|
"logps/rejected": -277.5650939941406,
|
|
"loss": 4.6927,
|
|
"margin_dpo/margin_mean": 35.33952713012695,
|
|
"margin_dpo/margin_std": 33.1160774230957,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.36649214659685864,
|
|
"grad_norm": 35.58210754394531,
|
|
"learning_rate": 4.00916353566676e-07,
|
|
"logits/chosen": 1.5620782375335693,
|
|
"logits/rejected": 1.598710536956787,
|
|
"logps/chosen": -242.1133270263672,
|
|
"logps/ref_chosen": -231.65187072753906,
|
|
"logps/ref_rejected": -264.08526611328125,
|
|
"logps/rejected": -294.81854248046875,
|
|
"loss": 4.8994,
|
|
"margin_dpo/margin_mean": 20.271865844726562,
|
|
"margin_dpo/margin_std": 31.47553062438965,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.36858638743455496,
|
|
"grad_norm": 23.747568130493164,
|
|
"learning_rate": 3.994527650465352e-07,
|
|
"logits/chosen": 1.3475306034088135,
|
|
"logits/rejected": 1.4316266775131226,
|
|
"logps/chosen": -278.8919372558594,
|
|
"logps/ref_chosen": -271.37152099609375,
|
|
"logps/ref_rejected": -281.20074462890625,
|
|
"logps/rejected": -299.3853759765625,
|
|
"loss": 5.067,
|
|
"margin_dpo/margin_mean": 10.664226531982422,
|
|
"margin_dpo/margin_std": 33.30470657348633,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.3706806282722513,
|
|
"grad_norm": 21.642898559570312,
|
|
"learning_rate": 3.979811618281705e-07,
|
|
"logits/chosen": 1.6163108348846436,
|
|
"logits/rejected": 1.4132215976715088,
|
|
"logps/chosen": -270.68548583984375,
|
|
"logps/ref_chosen": -266.7376403808594,
|
|
"logps/ref_rejected": -217.114990234375,
|
|
"logps/rejected": -240.8184356689453,
|
|
"loss": 5.1559,
|
|
"margin_dpo/margin_mean": 19.755634307861328,
|
|
"margin_dpo/margin_std": 33.71812438964844,
|
|
"step": 177
|
|
},
|
|
{
|
|
"epoch": 0.37277486910994767,
|
|
"grad_norm": 22.430517196655273,
|
|
"learning_rate": 3.9650162282919654e-07,
|
|
"logits/chosen": 1.463651180267334,
|
|
"logits/rejected": 1.5219404697418213,
|
|
"logps/chosen": -230.6317138671875,
|
|
"logps/ref_chosen": -230.67471313476562,
|
|
"logps/ref_rejected": -185.40577697753906,
|
|
"logps/rejected": -219.8003387451172,
|
|
"loss": 4.7678,
|
|
"margin_dpo/margin_mean": 34.437557220458984,
|
|
"margin_dpo/margin_std": 34.62123107910156,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.374869109947644,
|
|
"grad_norm": 30.32175064086914,
|
|
"learning_rate": 3.9501422739279953e-07,
|
|
"logits/chosen": 1.3042542934417725,
|
|
"logits/rejected": 1.3105361461639404,
|
|
"logps/chosen": -272.2032470703125,
|
|
"logps/ref_chosen": -267.849853515625,
|
|
"logps/ref_rejected": -270.5272521972656,
|
|
"logps/rejected": -286.4674987792969,
|
|
"loss": 4.9431,
|
|
"margin_dpo/margin_mean": 11.586915969848633,
|
|
"margin_dpo/margin_std": 27.10576629638672,
|
|
"step": 179
|
|
},
|
|
{
|
|
"epoch": 0.3769633507853403,
|
|
"grad_norm": 41.640106201171875,
|
|
"learning_rate": 3.935190552834828e-07,
|
|
"logits/chosen": 1.7063815593719482,
|
|
"logits/rejected": 1.6488580703735352,
|
|
"logps/chosen": -302.65380859375,
|
|
"logps/ref_chosen": -296.4002685546875,
|
|
"logps/ref_rejected": -224.35203552246094,
|
|
"logps/rejected": -253.70095825195312,
|
|
"loss": 4.7457,
|
|
"margin_dpo/margin_mean": 23.0954647064209,
|
|
"margin_dpo/margin_std": 34.00359344482422,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.37905759162303665,
|
|
"grad_norm": 30.104999542236328,
|
|
"learning_rate": 3.920161866827889e-07,
|
|
"logits/chosen": 1.1796238422393799,
|
|
"logits/rejected": 1.132272481918335,
|
|
"logps/chosen": -241.59747314453125,
|
|
"logps/ref_chosen": -243.10891723632812,
|
|
"logps/ref_rejected": -231.96902465820312,
|
|
"logps/rejected": -256.28497314453125,
|
|
"loss": 4.8516,
|
|
"margin_dpo/margin_mean": 25.827394485473633,
|
|
"margin_dpo/margin_std": 33.010704040527344,
|
|
"step": 181
|
|
},
|
|
{
|
|
"epoch": 0.381151832460733,
|
|
"grad_norm": 28.976200103759766,
|
|
"learning_rate": 3.90505702185e-07,
|
|
"logits/chosen": 1.584215521812439,
|
|
"logits/rejected": 1.6100966930389404,
|
|
"logps/chosen": -264.54351806640625,
|
|
"logps/ref_chosen": -263.5075988769531,
|
|
"logps/ref_rejected": -254.4083709716797,
|
|
"logps/rejected": -296.13641357421875,
|
|
"loss": 4.5773,
|
|
"margin_dpo/margin_mean": 40.692108154296875,
|
|
"margin_dpo/margin_std": 22.797809600830078,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.3832460732984293,
|
|
"grad_norm": 36.45281982421875,
|
|
"learning_rate": 3.889876827928156e-07,
|
|
"logits/chosen": 1.0563912391662598,
|
|
"logits/rejected": 1.159271001815796,
|
|
"logps/chosen": -233.31439208984375,
|
|
"logps/ref_chosen": -220.9555206298828,
|
|
"logps/ref_rejected": -224.3114471435547,
|
|
"logps/rejected": -247.57742309570312,
|
|
"loss": 4.7918,
|
|
"margin_dpo/margin_mean": 10.90709114074707,
|
|
"margin_dpo/margin_std": 35.502681732177734,
|
|
"step": 183
|
|
},
|
|
{
|
|
"epoch": 0.38534031413612563,
|
|
"grad_norm": 24.624616622924805,
|
|
"learning_rate": 3.874622099130087e-07,
|
|
"logits/chosen": 1.666105031967163,
|
|
"logits/rejected": 1.6945122480392456,
|
|
"logps/chosen": -290.854736328125,
|
|
"logps/ref_chosen": -285.35125732421875,
|
|
"logps/ref_rejected": -282.2647705078125,
|
|
"logps/rejected": -324.21044921875,
|
|
"loss": 4.4107,
|
|
"margin_dpo/margin_mean": 36.442237854003906,
|
|
"margin_dpo/margin_std": 38.43600082397461,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.387434554973822,
|
|
"grad_norm": 29.312538146972656,
|
|
"learning_rate": 3.859293653520604e-07,
|
|
"logits/chosen": 1.6671961545944214,
|
|
"logits/rejected": 1.7346235513687134,
|
|
"logps/chosen": -326.77490234375,
|
|
"logps/ref_chosen": -324.6773986816406,
|
|
"logps/ref_rejected": -275.9365539550781,
|
|
"logps/rejected": -308.9313659667969,
|
|
"loss": 4.8262,
|
|
"margin_dpo/margin_mean": 30.89735221862793,
|
|
"margin_dpo/margin_std": 34.63751220703125,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.38952879581151834,
|
|
"grad_norm": 34.26539611816406,
|
|
"learning_rate": 3.8438923131177237e-07,
|
|
"logits/chosen": 1.594333291053772,
|
|
"logits/rejected": 1.5077753067016602,
|
|
"logps/chosen": -304.6080017089844,
|
|
"logps/ref_chosen": -287.4004211425781,
|
|
"logps/ref_rejected": -222.46803283691406,
|
|
"logps/rejected": -260.6602783203125,
|
|
"loss": 4.8356,
|
|
"margin_dpo/margin_mean": 20.984676361083984,
|
|
"margin_dpo/margin_std": 20.14511489868164,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.39162303664921466,
|
|
"grad_norm": 25.851015090942383,
|
|
"learning_rate": 3.828418903848593e-07,
|
|
"logits/chosen": 1.441859245300293,
|
|
"logits/rejected": 1.5745567083358765,
|
|
"logps/chosen": -401.31182861328125,
|
|
"logps/ref_chosen": -378.8255310058594,
|
|
"logps/ref_rejected": -319.38116455078125,
|
|
"logps/rejected": -365.0159912109375,
|
|
"loss": 4.8863,
|
|
"margin_dpo/margin_mean": 23.14852523803711,
|
|
"margin_dpo/margin_std": 45.19081115722656,
|
|
"step": 187
|
|
},
|
|
{
|
|
"epoch": 0.393717277486911,
|
|
"grad_norm": 41.554141998291016,
|
|
"learning_rate": 3.812874255505191e-07,
|
|
"logits/chosen": 1.360278844833374,
|
|
"logits/rejected": 1.1752986907958984,
|
|
"logps/chosen": -250.5333251953125,
|
|
"logps/ref_chosen": -246.3994903564453,
|
|
"logps/ref_rejected": -204.85589599609375,
|
|
"logps/rejected": -239.05686950683594,
|
|
"loss": 4.8302,
|
|
"margin_dpo/margin_mean": 30.06714630126953,
|
|
"margin_dpo/margin_std": 34.51936340332031,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.3958115183246073,
|
|
"grad_norm": 38.48931884765625,
|
|
"learning_rate": 3.797259201699833e-07,
|
|
"logits/chosen": 1.4543706178665161,
|
|
"logits/rejected": 1.5096098184585571,
|
|
"logps/chosen": -264.8511047363281,
|
|
"logps/ref_chosen": -264.7483825683594,
|
|
"logps/ref_rejected": -292.3799743652344,
|
|
"logps/rejected": -328.85107421875,
|
|
"loss": 4.6022,
|
|
"margin_dpo/margin_mean": 36.36838912963867,
|
|
"margin_dpo/margin_std": 26.913986206054688,
|
|
"step": 189
|
|
},
|
|
{
|
|
"epoch": 0.39790575916230364,
|
|
"grad_norm": 24.164396286010742,
|
|
"learning_rate": 3.781574579820464e-07,
|
|
"logits/chosen": 0.8813581466674805,
|
|
"logits/rejected": 0.9559296369552612,
|
|
"logps/chosen": -223.26422119140625,
|
|
"logps/ref_chosen": -211.2392120361328,
|
|
"logps/ref_rejected": -204.55384826660156,
|
|
"logps/rejected": -233.70541381835938,
|
|
"loss": 4.6669,
|
|
"margin_dpo/margin_mean": 17.12653350830078,
|
|
"margin_dpo/margin_std": 40.82097625732422,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.4,
|
|
"grad_norm": 28.65985107421875,
|
|
"learning_rate": 3.765821230985757e-07,
|
|
"logits/chosen": 1.188876748085022,
|
|
"logits/rejected": 1.3144832849502563,
|
|
"logps/chosen": -177.3275604248047,
|
|
"logps/ref_chosen": -175.97952270507812,
|
|
"logps/ref_rejected": -206.85325622558594,
|
|
"logps/rejected": -228.22000122070312,
|
|
"loss": 4.7686,
|
|
"margin_dpo/margin_mean": 20.018733978271484,
|
|
"margin_dpo/margin_std": 31.200864791870117,
|
|
"step": 191
|
|
},
|
|
{
|
|
"epoch": 0.40209424083769635,
|
|
"grad_norm": 30.982559204101562,
|
|
"learning_rate": 3.75e-07,
|
|
"logits/chosen": 1.6706230640411377,
|
|
"logits/rejected": 1.854614496231079,
|
|
"logps/chosen": -253.131103515625,
|
|
"logps/ref_chosen": -241.5125732421875,
|
|
"logps/ref_rejected": -285.0710144042969,
|
|
"logps/rejected": -313.1866149902344,
|
|
"loss": 4.972,
|
|
"margin_dpo/margin_mean": 16.49706268310547,
|
|
"margin_dpo/margin_std": 54.19124221801758,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.4041884816753927,
|
|
"grad_norm": 28.91083526611328,
|
|
"learning_rate": 3.734111735307796e-07,
|
|
"logits/chosen": 1.7183902263641357,
|
|
"logits/rejected": 1.5575065612792969,
|
|
"logps/chosen": -255.6170196533203,
|
|
"logps/ref_chosen": -247.06581115722656,
|
|
"logps/ref_rejected": -221.4132537841797,
|
|
"logps/rejected": -248.817138671875,
|
|
"loss": 4.8338,
|
|
"margin_dpo/margin_mean": 18.852684020996094,
|
|
"margin_dpo/margin_std": 29.74808120727539,
|
|
"step": 193
|
|
},
|
|
{
|
|
"epoch": 0.406282722513089,
|
|
"grad_norm": 35.922122955322266,
|
|
"learning_rate": 3.7181572889485623e-07,
|
|
"logits/chosen": 1.3995440006256104,
|
|
"logits/rejected": 1.4992262125015259,
|
|
"logps/chosen": -216.14686584472656,
|
|
"logps/ref_chosen": -208.60263061523438,
|
|
"logps/ref_rejected": -189.4849090576172,
|
|
"logps/rejected": -212.74192810058594,
|
|
"loss": 5.0447,
|
|
"margin_dpo/margin_mean": 15.712799072265625,
|
|
"margin_dpo/margin_std": 31.20469093322754,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.4083769633507853,
|
|
"grad_norm": 31.717365264892578,
|
|
"learning_rate": 3.7021375165108377e-07,
|
|
"logits/chosen": 1.464687466621399,
|
|
"logits/rejected": 1.4596346616744995,
|
|
"logps/chosen": -287.36065673828125,
|
|
"logps/ref_chosen": -278.51275634765625,
|
|
"logps/ref_rejected": -298.09185791015625,
|
|
"logps/rejected": -318.23797607421875,
|
|
"loss": 5.0759,
|
|
"margin_dpo/margin_mean": 11.298222541809082,
|
|
"margin_dpo/margin_std": 27.541015625,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.41047120418848165,
|
|
"grad_norm": 25.350812911987305,
|
|
"learning_rate": 3.6860532770864005e-07,
|
|
"logits/chosen": 1.1140937805175781,
|
|
"logits/rejected": 1.2928128242492676,
|
|
"logps/chosen": -213.8653564453125,
|
|
"logps/ref_chosen": -213.48568725585938,
|
|
"logps/ref_rejected": -216.8994903564453,
|
|
"logps/rejected": -244.5856170654297,
|
|
"loss": 4.6056,
|
|
"margin_dpo/margin_mean": 27.30645751953125,
|
|
"margin_dpo/margin_std": 27.342227935791016,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.41256544502617803,
|
|
"grad_norm": 26.07918357849121,
|
|
"learning_rate": 3.6699054332241985e-07,
|
|
"logits/chosen": 1.4508588314056396,
|
|
"logits/rejected": 1.4278172254562378,
|
|
"logps/chosen": -255.36036682128906,
|
|
"logps/ref_chosen": -256.396728515625,
|
|
"logps/ref_rejected": -185.2763671875,
|
|
"logps/rejected": -232.59405517578125,
|
|
"loss": 4.3517,
|
|
"margin_dpo/margin_mean": 48.35406494140625,
|
|
"margin_dpo/margin_std": 30.625558853149414,
|
|
"step": 197
|
|
},
|
|
{
|
|
"epoch": 0.41465968586387436,
|
|
"grad_norm": 29.263551712036133,
|
|
"learning_rate": 3.653694850884091e-07,
|
|
"logits/chosen": 1.842124342918396,
|
|
"logits/rejected": 1.9477362632751465,
|
|
"logps/chosen": -362.33245849609375,
|
|
"logps/ref_chosen": -366.5196838378906,
|
|
"logps/ref_rejected": -361.7866516113281,
|
|
"logps/rejected": -392.13189697265625,
|
|
"loss": 4.6993,
|
|
"margin_dpo/margin_mean": 34.532501220703125,
|
|
"margin_dpo/margin_std": 40.5759162902832,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.4167539267015707,
|
|
"grad_norm": 32.77777862548828,
|
|
"learning_rate": 3.6374223993904124e-07,
|
|
"logits/chosen": 0.9073523879051208,
|
|
"logits/rejected": 0.9206515550613403,
|
|
"logps/chosen": -210.94586181640625,
|
|
"logps/ref_chosen": -207.86968994140625,
|
|
"logps/ref_rejected": -184.52076721191406,
|
|
"logps/rejected": -226.15948486328125,
|
|
"loss": 4.7308,
|
|
"margin_dpo/margin_mean": 38.562538146972656,
|
|
"margin_dpo/margin_std": 28.093774795532227,
|
|
"step": 199
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"grad_norm": 28.612226486206055,
|
|
"learning_rate": 3.621088951385353e-07,
|
|
"logits/chosen": 1.3658336400985718,
|
|
"logits/rejected": 1.3859562873840332,
|
|
"logps/chosen": -281.68792724609375,
|
|
"logps/ref_chosen": -276.4098205566406,
|
|
"logps/ref_rejected": -252.23086547851562,
|
|
"logps/rejected": -272.3934326171875,
|
|
"loss": 4.9383,
|
|
"margin_dpo/margin_mean": 14.884419441223145,
|
|
"margin_dpo/margin_std": 53.33942794799805,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"eval_logits/chosen": 1.4300137758255005,
|
|
"eval_logits/rejected": 1.4696903228759766,
|
|
"eval_logps/chosen": -287.022705078125,
|
|
"eval_logps/ref_chosen": -281.4588928222656,
|
|
"eval_logps/ref_rejected": -261.84954833984375,
|
|
"eval_logps/rejected": -295.6717529296875,
|
|
"eval_loss": 0.597048819065094,
|
|
"eval_margin_dpo/margin_mean": 28.258426666259766,
|
|
"eval_margin_dpo/margin_std": 39.02444076538086,
|
|
"eval_runtime": 93.548,
|
|
"eval_samples_per_second": 21.379,
|
|
"eval_steps_per_second": 1.336,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.42094240837696334,
|
|
"grad_norm": 32.25981903076172,
|
|
"learning_rate": 3.604695382782159e-07,
|
|
"logits/chosen": 1.2966924905776978,
|
|
"logits/rejected": 1.4700032472610474,
|
|
"logps/chosen": -266.32220458984375,
|
|
"logps/ref_chosen": -265.32904052734375,
|
|
"logps/ref_rejected": -255.19529724121094,
|
|
"logps/rejected": -297.01544189453125,
|
|
"loss": 4.6702,
|
|
"margin_dpo/margin_mean": 40.82699203491211,
|
|
"margin_dpo/margin_std": 30.352630615234375,
|
|
"step": 201
|
|
},
|
|
{
|
|
"epoch": 0.42303664921465967,
|
|
"grad_norm": 35.75096130371094,
|
|
"learning_rate": 3.588242572718162e-07,
|
|
"logits/chosen": 1.6444792747497559,
|
|
"logits/rejected": 1.5493888854980469,
|
|
"logps/chosen": -278.0890197753906,
|
|
"logps/ref_chosen": -274.6075439453125,
|
|
"logps/ref_rejected": -219.9969940185547,
|
|
"logps/rejected": -251.83348083496094,
|
|
"loss": 4.7312,
|
|
"margin_dpo/margin_mean": 28.354969024658203,
|
|
"margin_dpo/margin_std": 44.327239990234375,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.42513089005235605,
|
|
"grad_norm": 34.300376892089844,
|
|
"learning_rate": 3.571731403507635e-07,
|
|
"logits/chosen": 1.4360129833221436,
|
|
"logits/rejected": 1.3600637912750244,
|
|
"logps/chosen": -302.18707275390625,
|
|
"logps/ref_chosen": -295.6935119628906,
|
|
"logps/ref_rejected": -241.4007568359375,
|
|
"logps/rejected": -266.06201171875,
|
|
"loss": 4.8634,
|
|
"margin_dpo/margin_mean": 18.167736053466797,
|
|
"margin_dpo/margin_std": 24.508058547973633,
|
|
"step": 203
|
|
},
|
|
{
|
|
"epoch": 0.4272251308900524,
|
|
"grad_norm": 28.439380645751953,
|
|
"learning_rate": 3.5551627605944746e-07,
|
|
"logits/chosen": 2.044978141784668,
|
|
"logits/rejected": 1.9398431777954102,
|
|
"logps/chosen": -398.93328857421875,
|
|
"logps/ref_chosen": -392.3414611816406,
|
|
"logps/ref_rejected": -291.4375915527344,
|
|
"logps/rejected": -327.0179443359375,
|
|
"loss": 4.5824,
|
|
"margin_dpo/margin_mean": 28.98847007751465,
|
|
"margin_dpo/margin_std": 38.196693420410156,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.4293193717277487,
|
|
"grad_norm": 27.83588981628418,
|
|
"learning_rate": 3.5385375325047163e-07,
|
|
"logits/chosen": 1.3867862224578857,
|
|
"logits/rejected": 1.657343864440918,
|
|
"logps/chosen": -191.40664672851562,
|
|
"logps/ref_chosen": -190.1780242919922,
|
|
"logps/ref_rejected": -275.8878479003906,
|
|
"logps/rejected": -311.2901306152344,
|
|
"loss": 4.6413,
|
|
"margin_dpo/margin_mean": 34.17367935180664,
|
|
"margin_dpo/margin_std": 32.90446090698242,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.431413612565445,
|
|
"grad_norm": 30.838573455810547,
|
|
"learning_rate": 3.5218566107988867e-07,
|
|
"logits/chosen": 0.9244284629821777,
|
|
"logits/rejected": 1.1862902641296387,
|
|
"logps/chosen": -277.1954040527344,
|
|
"logps/ref_chosen": -278.95977783203125,
|
|
"logps/ref_rejected": -296.458984375,
|
|
"logps/rejected": -318.1335144042969,
|
|
"loss": 4.6764,
|
|
"margin_dpo/margin_mean": 23.43887710571289,
|
|
"margin_dpo/margin_std": 38.043148040771484,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.43350785340314135,
|
|
"grad_norm": 31.335426330566406,
|
|
"learning_rate": 3.505120890024195e-07,
|
|
"logits/chosen": 1.658402681350708,
|
|
"logits/rejected": 1.8528728485107422,
|
|
"logps/chosen": -222.63587951660156,
|
|
"logps/ref_chosen": -219.367919921875,
|
|
"logps/ref_rejected": -231.6876678466797,
|
|
"logps/rejected": -260.1963195800781,
|
|
"loss": 4.9355,
|
|
"margin_dpo/margin_mean": 25.240697860717773,
|
|
"margin_dpo/margin_std": 46.75006866455078,
|
|
"step": 207
|
|
},
|
|
{
|
|
"epoch": 0.4356020942408377,
|
|
"grad_norm": 34.799400329589844,
|
|
"learning_rate": 3.4883312676665534e-07,
|
|
"logits/chosen": 1.4422191381454468,
|
|
"logits/rejected": 1.4274728298187256,
|
|
"logps/chosen": -308.7106628417969,
|
|
"logps/ref_chosen": -303.848388671875,
|
|
"logps/ref_rejected": -252.1853485107422,
|
|
"logps/rejected": -288.1015625,
|
|
"loss": 4.7313,
|
|
"margin_dpo/margin_mean": 31.05390739440918,
|
|
"margin_dpo/margin_std": 41.717647552490234,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.437696335078534,
|
|
"grad_norm": 32.08354949951172,
|
|
"learning_rate": 3.4714886441024573e-07,
|
|
"logits/chosen": 1.48392915725708,
|
|
"logits/rejected": 1.2658922672271729,
|
|
"logps/chosen": -353.7491760253906,
|
|
"logps/ref_chosen": -347.6343688964844,
|
|
"logps/ref_rejected": -240.31988525390625,
|
|
"logps/rejected": -270.4415283203125,
|
|
"loss": 4.891,
|
|
"margin_dpo/margin_mean": 24.006847381591797,
|
|
"margin_dpo/margin_std": 38.43374252319336,
|
|
"step": 209
|
|
},
|
|
{
|
|
"epoch": 0.4397905759162304,
|
|
"grad_norm": 41.964515686035156,
|
|
"learning_rate": 3.454593922550693e-07,
|
|
"logits/chosen": 1.7486904859542847,
|
|
"logits/rejected": 1.899224042892456,
|
|
"logps/chosen": -230.97503662109375,
|
|
"logps/ref_chosen": -236.3311767578125,
|
|
"logps/ref_rejected": -289.6016845703125,
|
|
"logps/rejected": -317.5093688964844,
|
|
"loss": 4.7272,
|
|
"margin_dpo/margin_mean": 33.26382064819336,
|
|
"margin_dpo/margin_std": 33.677276611328125,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.4418848167539267,
|
|
"grad_norm": 31.20326805114746,
|
|
"learning_rate": 3.4376480090239047e-07,
|
|
"logits/chosen": 1.3021446466445923,
|
|
"logits/rejected": 1.3574562072753906,
|
|
"logps/chosen": -205.39637756347656,
|
|
"logps/ref_chosen": -204.38107299804688,
|
|
"logps/ref_rejected": -212.449462890625,
|
|
"logps/rejected": -242.7283172607422,
|
|
"loss": 4.3731,
|
|
"margin_dpo/margin_mean": 29.263545989990234,
|
|
"margin_dpo/margin_std": 35.2406005859375,
|
|
"step": 211
|
|
},
|
|
{
|
|
"epoch": 0.44397905759162304,
|
|
"grad_norm": 35.141807556152344,
|
|
"learning_rate": 3.4206518122800055e-07,
|
|
"logits/chosen": 1.1048368215560913,
|
|
"logits/rejected": 1.1881780624389648,
|
|
"logps/chosen": -241.2005615234375,
|
|
"logps/ref_chosen": -231.28570556640625,
|
|
"logps/ref_rejected": -222.65725708007812,
|
|
"logps/rejected": -248.41322326660156,
|
|
"loss": 4.7702,
|
|
"margin_dpo/margin_mean": 15.841072082519531,
|
|
"margin_dpo/margin_std": 39.27621841430664,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.44607329842931936,
|
|
"grad_norm": 27.873577117919922,
|
|
"learning_rate": 3.403606243773448e-07,
|
|
"logits/chosen": 1.5394573211669922,
|
|
"logits/rejected": 1.6554535627365112,
|
|
"logps/chosen": -336.94354248046875,
|
|
"logps/ref_chosen": -332.35968017578125,
|
|
"logps/ref_rejected": -329.94830322265625,
|
|
"logps/rejected": -365.0585021972656,
|
|
"loss": 4.7341,
|
|
"margin_dpo/margin_mean": 30.526296615600586,
|
|
"margin_dpo/margin_std": 37.60173034667969,
|
|
"step": 213
|
|
},
|
|
{
|
|
"epoch": 0.4481675392670157,
|
|
"grad_norm": 33.36276626586914,
|
|
"learning_rate": 3.3865122176063385e-07,
|
|
"logits/chosen": 1.8208783864974976,
|
|
"logits/rejected": 1.9227497577667236,
|
|
"logps/chosen": -320.77886962890625,
|
|
"logps/ref_chosen": -303.07257080078125,
|
|
"logps/ref_rejected": -310.52001953125,
|
|
"logps/rejected": -347.1795654296875,
|
|
"loss": 4.8278,
|
|
"margin_dpo/margin_mean": 18.953208923339844,
|
|
"margin_dpo/margin_std": 34.65628433227539,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.450261780104712,
|
|
"grad_norm": 31.66336441040039,
|
|
"learning_rate": 3.3693706504794243e-07,
|
|
"logits/chosen": 2.0376367568969727,
|
|
"logits/rejected": 2.075817584991455,
|
|
"logps/chosen": -283.66595458984375,
|
|
"logps/ref_chosen": -286.654296875,
|
|
"logps/ref_rejected": -272.1281433105469,
|
|
"logps/rejected": -317.6420593261719,
|
|
"loss": 4.8219,
|
|
"margin_dpo/margin_mean": 48.50217819213867,
|
|
"margin_dpo/margin_std": 41.356727600097656,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.4523560209424084,
|
|
"grad_norm": 51.738441467285156,
|
|
"learning_rate": 3.3521824616429284e-07,
|
|
"logits/chosen": 1.378481149673462,
|
|
"logits/rejected": 1.2707972526550293,
|
|
"logps/chosen": -364.43603515625,
|
|
"logps/ref_chosen": -351.34417724609375,
|
|
"logps/ref_rejected": -290.5171813964844,
|
|
"logps/rejected": -327.94488525390625,
|
|
"loss": 4.7106,
|
|
"margin_dpo/margin_mean": 24.335878372192383,
|
|
"margin_dpo/margin_std": 42.64997100830078,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.4544502617801047,
|
|
"grad_norm": 35.788352966308594,
|
|
"learning_rate": 3.334948572847253e-07,
|
|
"logits/chosen": 1.5968372821807861,
|
|
"logits/rejected": 1.7124537229537964,
|
|
"logps/chosen": -279.4749755859375,
|
|
"logps/ref_chosen": -273.76788330078125,
|
|
"logps/ref_rejected": -286.2580261230469,
|
|
"logps/rejected": -343.501953125,
|
|
"loss": 4.4028,
|
|
"margin_dpo/margin_mean": 51.53682327270508,
|
|
"margin_dpo/margin_std": 38.95985412597656,
|
|
"step": 217
|
|
},
|
|
{
|
|
"epoch": 0.45654450261780105,
|
|
"grad_norm": 41.920623779296875,
|
|
"learning_rate": 3.317669908293554e-07,
|
|
"logits/chosen": 1.5484070777893066,
|
|
"logits/rejected": 1.8093584775924683,
|
|
"logps/chosen": -235.32321166992188,
|
|
"logps/ref_chosen": -219.74948120117188,
|
|
"logps/ref_rejected": -308.801025390625,
|
|
"logps/rejected": -354.8564147949219,
|
|
"loss": 4.5987,
|
|
"margin_dpo/margin_mean": 30.481698989868164,
|
|
"margin_dpo/margin_std": 39.370521545410156,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.4586387434554974,
|
|
"grad_norm": 30.652027130126953,
|
|
"learning_rate": 3.300347394584172e-07,
|
|
"logits/chosen": 1.3850374221801758,
|
|
"logits/rejected": 1.4719693660736084,
|
|
"logps/chosen": -282.8400573730469,
|
|
"logps/ref_chosen": -264.65374755859375,
|
|
"logps/ref_rejected": -233.9711151123047,
|
|
"logps/rejected": -276.5810546875,
|
|
"loss": 4.6628,
|
|
"margin_dpo/margin_mean": 24.423654556274414,
|
|
"margin_dpo/margin_std": 40.1912841796875,
|
|
"step": 219
|
|
},
|
|
{
|
|
"epoch": 0.4607329842931937,
|
|
"grad_norm": 45.71303939819336,
|
|
"learning_rate": 3.2829819606729477e-07,
|
|
"logits/chosen": 1.9091517925262451,
|
|
"logits/rejected": 1.744594931602478,
|
|
"logps/chosen": -315.3508605957031,
|
|
"logps/ref_chosen": -295.8961486816406,
|
|
"logps/ref_rejected": -219.56228637695312,
|
|
"logps/rejected": -273.97418212890625,
|
|
"loss": 4.5451,
|
|
"margin_dpo/margin_mean": 34.957191467285156,
|
|
"margin_dpo/margin_std": 39.46710968017578,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.46282722513089003,
|
|
"grad_norm": 29.89345932006836,
|
|
"learning_rate": 3.265574537815398e-07,
|
|
"logits/chosen": 1.0531387329101562,
|
|
"logits/rejected": 1.2520170211791992,
|
|
"logps/chosen": -302.5050048828125,
|
|
"logps/ref_chosen": -284.9080810546875,
|
|
"logps/ref_rejected": -310.0538330078125,
|
|
"logps/rejected": -356.16510009765625,
|
|
"loss": 4.8325,
|
|
"margin_dpo/margin_mean": 28.514326095581055,
|
|
"margin_dpo/margin_std": 32.273284912109375,
|
|
"step": 221
|
|
},
|
|
{
|
|
"epoch": 0.4649214659685864,
|
|
"grad_norm": 47.45641326904297,
|
|
"learning_rate": 3.248126059518784e-07,
|
|
"logits/chosen": 1.3722844123840332,
|
|
"logits/rejected": 1.2933783531188965,
|
|
"logps/chosen": -329.0756530761719,
|
|
"logps/ref_chosen": -308.44622802734375,
|
|
"logps/ref_rejected": -254.99667358398438,
|
|
"logps/rejected": -303.69677734375,
|
|
"loss": 4.5312,
|
|
"margin_dpo/margin_mean": 28.07069969177246,
|
|
"margin_dpo/margin_std": 30.664283752441406,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.46701570680628274,
|
|
"grad_norm": 36.27118682861328,
|
|
"learning_rate": 3.230637461492043e-07,
|
|
"logits/chosen": 1.214386224746704,
|
|
"logits/rejected": 1.193519115447998,
|
|
"logps/chosen": -283.5272521972656,
|
|
"logps/ref_chosen": -258.5130310058594,
|
|
"logps/ref_rejected": -231.13885498046875,
|
|
"logps/rejected": -290.85296630859375,
|
|
"loss": 4.4404,
|
|
"margin_dpo/margin_mean": 34.699886322021484,
|
|
"margin_dpo/margin_std": 42.52618408203125,
|
|
"step": 223
|
|
},
|
|
{
|
|
"epoch": 0.46910994764397906,
|
|
"grad_norm": 36.553733825683594,
|
|
"learning_rate": 3.213109681595612e-07,
|
|
"logits/chosen": 1.2857964038848877,
|
|
"logits/rejected": 1.433445692062378,
|
|
"logps/chosen": -248.49815368652344,
|
|
"logps/ref_chosen": -234.55177307128906,
|
|
"logps/ref_rejected": -208.4610595703125,
|
|
"logps/rejected": -271.1744689941406,
|
|
"loss": 4.3795,
|
|
"margin_dpo/margin_mean": 48.76702117919922,
|
|
"margin_dpo/margin_std": 35.48724365234375,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.4712041884816754,
|
|
"grad_norm": 40.446937561035156,
|
|
"learning_rate": 3.1955436597911315e-07,
|
|
"logits/chosen": 1.6013773679733276,
|
|
"logits/rejected": 1.722807765007019,
|
|
"logps/chosen": -360.8241882324219,
|
|
"logps/ref_chosen": -339.7688903808594,
|
|
"logps/ref_rejected": -347.96112060546875,
|
|
"logps/rejected": -397.2833251953125,
|
|
"loss": 4.9012,
|
|
"margin_dpo/margin_mean": 28.266937255859375,
|
|
"margin_dpo/margin_std": 49.008575439453125,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.4732984293193717,
|
|
"grad_norm": 37.459991455078125,
|
|
"learning_rate": 3.1779403380910425e-07,
|
|
"logits/chosen": 0.7877386808395386,
|
|
"logits/rejected": 1.0014675855636597,
|
|
"logps/chosen": -225.65472412109375,
|
|
"logps/ref_chosen": -209.56515502929688,
|
|
"logps/ref_rejected": -207.83871459960938,
|
|
"logps/rejected": -261.1890563964844,
|
|
"loss": 4.8962,
|
|
"margin_dpo/margin_mean": 37.260780334472656,
|
|
"margin_dpo/margin_std": 42.05327606201172,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.47539267015706804,
|
|
"grad_norm": 29.621501922607422,
|
|
"learning_rate": 3.160300660508064e-07,
|
|
"logits/chosen": 1.4312937259674072,
|
|
"logits/rejected": 1.644526481628418,
|
|
"logps/chosen": -278.422607421875,
|
|
"logps/ref_chosen": -252.69004821777344,
|
|
"logps/ref_rejected": -252.89427185058594,
|
|
"logps/rejected": -317.8539733886719,
|
|
"loss": 4.4608,
|
|
"margin_dpo/margin_mean": 39.22712707519531,
|
|
"margin_dpo/margin_std": 55.3231086730957,
|
|
"step": 227
|
|
},
|
|
{
|
|
"epoch": 0.4774869109947644,
|
|
"grad_norm": 46.8765869140625,
|
|
"learning_rate": 3.1426255730045695e-07,
|
|
"logits/chosen": 1.5183122158050537,
|
|
"logits/rejected": 1.6018824577331543,
|
|
"logps/chosen": -235.6670684814453,
|
|
"logps/ref_chosen": -210.62913513183594,
|
|
"logps/ref_rejected": -174.08975219726562,
|
|
"logps/rejected": -226.82781982421875,
|
|
"loss": 4.4722,
|
|
"margin_dpo/margin_mean": 27.70014762878418,
|
|
"margin_dpo/margin_std": 42.50754165649414,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.47958115183246075,
|
|
"grad_norm": 39.29153823852539,
|
|
"learning_rate": 3.1249160234418644e-07,
|
|
"logits/chosen": 1.3761322498321533,
|
|
"logits/rejected": 1.3572083711624146,
|
|
"logps/chosen": -336.7579345703125,
|
|
"logps/ref_chosen": -315.1896057128906,
|
|
"logps/ref_rejected": -265.8664855957031,
|
|
"logps/rejected": -330.8331298828125,
|
|
"loss": 4.1991,
|
|
"margin_dpo/margin_mean": 43.39836883544922,
|
|
"margin_dpo/margin_std": 46.784549713134766,
|
|
"step": 229
|
|
},
|
|
{
|
|
"epoch": 0.4816753926701571,
|
|
"grad_norm": 39.0309944152832,
|
|
"learning_rate": 3.1071729615293424e-07,
|
|
"logits/chosen": 0.9634809494018555,
|
|
"logits/rejected": 0.9632886648178101,
|
|
"logps/chosen": -258.40020751953125,
|
|
"logps/ref_chosen": -240.54244995117188,
|
|
"logps/ref_rejected": -262.5657043457031,
|
|
"logps/rejected": -319.0147705078125,
|
|
"loss": 4.5649,
|
|
"margin_dpo/margin_mean": 38.59132385253906,
|
|
"margin_dpo/margin_std": 50.162498474121094,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.4837696335078534,
|
|
"grad_norm": 55.88585662841797,
|
|
"learning_rate": 3.0893973387735683e-07,
|
|
"logits/chosen": 1.1088343858718872,
|
|
"logits/rejected": 1.1820017099380493,
|
|
"logps/chosen": -326.6155700683594,
|
|
"logps/ref_chosen": -290.8667907714844,
|
|
"logps/ref_rejected": -277.01739501953125,
|
|
"logps/rejected": -330.43194580078125,
|
|
"loss": 4.7561,
|
|
"margin_dpo/margin_mean": 17.66571807861328,
|
|
"margin_dpo/margin_std": 42.80667495727539,
|
|
"step": 231
|
|
},
|
|
{
|
|
"epoch": 0.48586387434554973,
|
|
"grad_norm": 43.03524398803711,
|
|
"learning_rate": 3.071590108427243e-07,
|
|
"logits/chosen": 1.3966903686523438,
|
|
"logits/rejected": 1.5778224468231201,
|
|
"logps/chosen": -285.2840270996094,
|
|
"logps/ref_chosen": -260.0438232421875,
|
|
"logps/ref_rejected": -261.63507080078125,
|
|
"logps/rejected": -320.8078308105469,
|
|
"loss": 4.5309,
|
|
"margin_dpo/margin_mean": 33.93254089355469,
|
|
"margin_dpo/margin_std": 37.87663269042969,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.48795811518324606,
|
|
"grad_norm": 53.92399597167969,
|
|
"learning_rate": 3.05375222543809e-07,
|
|
"logits/chosen": 0.838955283164978,
|
|
"logits/rejected": 0.9410269856452942,
|
|
"logps/chosen": -240.427734375,
|
|
"logps/ref_chosen": -221.6608123779297,
|
|
"logps/ref_rejected": -261.16839599609375,
|
|
"logps/rejected": -328.1685485839844,
|
|
"loss": 4.5981,
|
|
"margin_dpo/margin_mean": 48.23321533203125,
|
|
"margin_dpo/margin_std": 38.929080963134766,
|
|
"step": 233
|
|
},
|
|
{
|
|
"epoch": 0.4900523560209424,
|
|
"grad_norm": 33.82901382446289,
|
|
"learning_rate": 3.035884646397637e-07,
|
|
"logits/chosen": 1.1958811283111572,
|
|
"logits/rejected": 1.237013816833496,
|
|
"logps/chosen": -297.06036376953125,
|
|
"logps/ref_chosen": -281.4861145019531,
|
|
"logps/ref_rejected": -276.58441162109375,
|
|
"logps/rejected": -340.1748046875,
|
|
"loss": 4.5243,
|
|
"margin_dpo/margin_mean": 48.01616668701172,
|
|
"margin_dpo/margin_std": 41.946895599365234,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.49214659685863876,
|
|
"grad_norm": 44.3351936340332,
|
|
"learning_rate": 3.017988329489923e-07,
|
|
"logits/chosen": 1.6281356811523438,
|
|
"logits/rejected": 1.5668871402740479,
|
|
"logps/chosen": -301.54229736328125,
|
|
"logps/ref_chosen": -300.5598449707031,
|
|
"logps/ref_rejected": -259.905029296875,
|
|
"logps/rejected": -302.8565368652344,
|
|
"loss": 4.6976,
|
|
"margin_dpo/margin_mean": 41.96904754638672,
|
|
"margin_dpo/margin_std": 42.275962829589844,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.4942408376963351,
|
|
"grad_norm": 32.59733963012695,
|
|
"learning_rate": 3.000064234440111e-07,
|
|
"logits/chosen": 1.2152283191680908,
|
|
"logits/rejected": 1.2443571090698242,
|
|
"logps/chosen": -282.0902099609375,
|
|
"logps/ref_chosen": -270.4844665527344,
|
|
"logps/ref_rejected": -231.67613220214844,
|
|
"logps/rejected": -282.9165954589844,
|
|
"loss": 4.4441,
|
|
"margin_dpo/margin_mean": 39.63469696044922,
|
|
"margin_dpo/margin_std": 43.977577209472656,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.4963350785340314,
|
|
"grad_norm": 47.38420486450195,
|
|
"learning_rate": 2.9821133224630223e-07,
|
|
"logits/chosen": 1.266021490097046,
|
|
"logits/rejected": 1.4988112449645996,
|
|
"logps/chosen": -219.76870727539062,
|
|
"logps/ref_chosen": -194.99342346191406,
|
|
"logps/ref_rejected": -243.12779235839844,
|
|
"logps/rejected": -310.861083984375,
|
|
"loss": 4.5863,
|
|
"margin_dpo/margin_mean": 42.9580078125,
|
|
"margin_dpo/margin_std": 39.53506851196289,
|
|
"step": 237
|
|
},
|
|
{
|
|
"epoch": 0.49842931937172774,
|
|
"grad_norm": 39.1376838684082,
|
|
"learning_rate": 2.964136556211588e-07,
|
|
"logits/chosen": 1.1324211359024048,
|
|
"logits/rejected": 1.0826090574264526,
|
|
"logps/chosen": -261.50543212890625,
|
|
"logps/ref_chosen": -240.9060516357422,
|
|
"logps/ref_rejected": -205.97012329101562,
|
|
"logps/rejected": -254.07977294921875,
|
|
"loss": 4.446,
|
|
"margin_dpo/margin_mean": 27.510299682617188,
|
|
"margin_dpo/margin_std": 46.14052200317383,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.5005235602094241,
|
|
"grad_norm": 40.104766845703125,
|
|
"learning_rate": 2.946134899725226e-07,
|
|
"logits/chosen": 1.3846328258514404,
|
|
"logits/rejected": 1.5582243204116821,
|
|
"logps/chosen": -303.406005859375,
|
|
"logps/ref_chosen": -277.0447998046875,
|
|
"logps/ref_rejected": -284.7602233886719,
|
|
"logps/rejected": -329.855712890625,
|
|
"loss": 4.701,
|
|
"margin_dpo/margin_mean": 18.734315872192383,
|
|
"margin_dpo/margin_std": 59.868492126464844,
|
|
"step": 239
|
|
},
|
|
{
|
|
"epoch": 0.5026178010471204,
|
|
"grad_norm": 38.7380485534668,
|
|
"learning_rate": 2.9281093183781403e-07,
|
|
"logits/chosen": 1.1157575845718384,
|
|
"logits/rejected": 1.0506106615066528,
|
|
"logps/chosen": -290.3915710449219,
|
|
"logps/ref_chosen": -285.29144287109375,
|
|
"logps/ref_rejected": -212.11915588378906,
|
|
"logps/rejected": -265.2477111816406,
|
|
"loss": 4.3702,
|
|
"margin_dpo/margin_mean": 48.02838134765625,
|
|
"margin_dpo/margin_std": 42.991214752197266,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5047120418848168,
|
|
"grad_norm": 41.1712646484375,
|
|
"learning_rate": 2.910060778827554e-07,
|
|
"logits/chosen": 1.3538631200790405,
|
|
"logits/rejected": 1.4909546375274658,
|
|
"logps/chosen": -260.87939453125,
|
|
"logps/ref_chosen": -254.9442901611328,
|
|
"logps/ref_rejected": -278.5121154785156,
|
|
"logps/rejected": -332.14483642578125,
|
|
"loss": 4.7755,
|
|
"margin_dpo/margin_mean": 47.697601318359375,
|
|
"margin_dpo/margin_std": 46.98115539550781,
|
|
"step": 241
|
|
},
|
|
{
|
|
"epoch": 0.506806282722513,
|
|
"grad_norm": 33.75428771972656,
|
|
"learning_rate": 2.891990248961871e-07,
|
|
"logits/chosen": 2.038219690322876,
|
|
"logits/rejected": 1.9361552000045776,
|
|
"logps/chosen": -274.6824645996094,
|
|
"logps/ref_chosen": -264.16876220703125,
|
|
"logps/ref_rejected": -215.627197265625,
|
|
"logps/rejected": -269.078125,
|
|
"loss": 4.4651,
|
|
"margin_dpo/margin_mean": 42.937225341796875,
|
|
"margin_dpo/margin_std": 52.65818786621094,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 0.5089005235602094,
|
|
"grad_norm": 45.30524826049805,
|
|
"learning_rate": 2.873898697848762e-07,
|
|
"logits/chosen": 1.3483995199203491,
|
|
"logits/rejected": 1.3744844198226929,
|
|
"logps/chosen": -322.53204345703125,
|
|
"logps/ref_chosen": -313.7347106933594,
|
|
"logps/ref_rejected": -357.50054931640625,
|
|
"logps/rejected": -403.6512756347656,
|
|
"loss": 4.1712,
|
|
"margin_dpo/margin_mean": 37.353302001953125,
|
|
"margin_dpo/margin_std": 46.82036590576172,
|
|
"step": 243
|
|
},
|
|
{
|
|
"epoch": 0.5109947643979058,
|
|
"grad_norm": 35.50245666503906,
|
|
"learning_rate": 2.8557870956832133e-07,
|
|
"logits/chosen": 1.1742347478866577,
|
|
"logits/rejected": 0.9833606481552124,
|
|
"logps/chosen": -291.64044189453125,
|
|
"logps/ref_chosen": -265.0720520019531,
|
|
"logps/ref_rejected": -235.29541015625,
|
|
"logps/rejected": -293.6773376464844,
|
|
"loss": 4.2395,
|
|
"margin_dpo/margin_mean": 31.813528060913086,
|
|
"margin_dpo/margin_std": 38.89430236816406,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 0.5130890052356021,
|
|
"grad_norm": 55.53532791137695,
|
|
"learning_rate": 2.837656413735479e-07,
|
|
"logits/chosen": 1.9007817506790161,
|
|
"logits/rejected": 1.6215357780456543,
|
|
"logps/chosen": -346.5862731933594,
|
|
"logps/ref_chosen": -338.6529235839844,
|
|
"logps/ref_rejected": -259.6473693847656,
|
|
"logps/rejected": -305.2576904296875,
|
|
"loss": 4.2236,
|
|
"margin_dpo/margin_mean": 37.676998138427734,
|
|
"margin_dpo/margin_std": 33.821868896484375,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.5151832460732985,
|
|
"grad_norm": 36.23422622680664,
|
|
"learning_rate": 2.8195076242990116e-07,
|
|
"logits/chosen": 1.154848337173462,
|
|
"logits/rejected": 1.1089671850204468,
|
|
"logps/chosen": -273.8831787109375,
|
|
"logps/ref_chosen": -254.98756408691406,
|
|
"logps/ref_rejected": -201.20782470703125,
|
|
"logps/rejected": -253.29745483398438,
|
|
"loss": 4.8429,
|
|
"margin_dpo/margin_mean": 33.19401550292969,
|
|
"margin_dpo/margin_std": 46.2290153503418,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 0.5172774869109947,
|
|
"grad_norm": 41.20656967163086,
|
|
"learning_rate": 2.801341700638307e-07,
|
|
"logits/chosen": 1.237385630607605,
|
|
"logits/rejected": 1.1006180047988892,
|
|
"logps/chosen": -284.4879455566406,
|
|
"logps/ref_chosen": -276.70361328125,
|
|
"logps/ref_rejected": -209.83523559570312,
|
|
"logps/rejected": -266.49542236328125,
|
|
"loss": 4.4147,
|
|
"margin_dpo/margin_mean": 48.87584686279297,
|
|
"margin_dpo/margin_std": 42.81962966918945,
|
|
"step": 247
|
|
},
|
|
{
|
|
"epoch": 0.5193717277486911,
|
|
"grad_norm": 57.703697204589844,
|
|
"learning_rate": 2.7831596169367227e-07,
|
|
"logits/chosen": 1.0914226770401,
|
|
"logits/rejected": 1.1984620094299316,
|
|
"logps/chosen": -258.7278747558594,
|
|
"logps/ref_chosen": -249.7368621826172,
|
|
"logps/ref_rejected": -230.7808837890625,
|
|
"logps/rejected": -274.591552734375,
|
|
"loss": 4.7502,
|
|
"margin_dpo/margin_mean": 34.819644927978516,
|
|
"margin_dpo/margin_std": 40.67426300048828,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 0.5214659685863874,
|
|
"grad_norm": 47.22350311279297,
|
|
"learning_rate": 2.7649623482442274e-07,
|
|
"logits/chosen": 1.0606677532196045,
|
|
"logits/rejected": 1.1147487163543701,
|
|
"logps/chosen": -266.5928649902344,
|
|
"logps/ref_chosen": -229.43399047851562,
|
|
"logps/ref_rejected": -242.59182739257812,
|
|
"logps/rejected": -302.368896484375,
|
|
"loss": 4.5617,
|
|
"margin_dpo/margin_mean": 22.618181228637695,
|
|
"margin_dpo/margin_std": 44.011497497558594,
|
|
"step": 249
|
|
},
|
|
{
|
|
"epoch": 0.5235602094240838,
|
|
"grad_norm": 34.5158576965332,
|
|
"learning_rate": 2.7467508704251135e-07,
|
|
"logits/chosen": 1.6137490272521973,
|
|
"logits/rejected": 1.7355223894119263,
|
|
"logps/chosen": -386.4211120605469,
|
|
"logps/ref_chosen": -374.47015380859375,
|
|
"logps/ref_rejected": -397.1805114746094,
|
|
"logps/rejected": -455.82952880859375,
|
|
"loss": 4.6106,
|
|
"margin_dpo/margin_mean": 46.698097229003906,
|
|
"margin_dpo/margin_std": 53.3001823425293,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.5256544502617801,
|
|
"grad_norm": 44.91852569580078,
|
|
"learning_rate": 2.7285261601056697e-07,
|
|
"logits/chosen": 1.0645577907562256,
|
|
"logits/rejected": 0.8425718545913696,
|
|
"logps/chosen": -355.9337463378906,
|
|
"logps/ref_chosen": -340.28240966796875,
|
|
"logps/ref_rejected": -255.56735229492188,
|
|
"logps/rejected": -305.7314147949219,
|
|
"loss": 4.5346,
|
|
"margin_dpo/margin_mean": 34.51277160644531,
|
|
"margin_dpo/margin_std": 47.6010627746582,
|
|
"step": 251
|
|
},
|
|
{
|
|
"epoch": 0.5277486910994764,
|
|
"grad_norm": 30.532474517822266,
|
|
"learning_rate": 2.7102891946217994e-07,
|
|
"logits/chosen": 1.4391117095947266,
|
|
"logits/rejected": 1.4691420793533325,
|
|
"logps/chosen": -215.19662475585938,
|
|
"logps/ref_chosen": -198.7939453125,
|
|
"logps/ref_rejected": -212.86849975585938,
|
|
"logps/rejected": -271.3706359863281,
|
|
"loss": 4.5578,
|
|
"margin_dpo/margin_mean": 42.09947967529297,
|
|
"margin_dpo/margin_std": 46.31159973144531,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 0.5298429319371728,
|
|
"grad_norm": 45.59535598754883,
|
|
"learning_rate": 2.692040951966617e-07,
|
|
"logits/chosen": 1.448297142982483,
|
|
"logits/rejected": 1.3689329624176025,
|
|
"logps/chosen": -370.5470275878906,
|
|
"logps/ref_chosen": -343.3220520019531,
|
|
"logps/ref_rejected": -258.52044677734375,
|
|
"logps/rejected": -316.4342041015625,
|
|
"loss": 4.7989,
|
|
"margin_dpo/margin_mean": 30.688785552978516,
|
|
"margin_dpo/margin_std": 51.24434280395508,
|
|
"step": 253
|
|
},
|
|
{
|
|
"epoch": 0.5319371727748691,
|
|
"grad_norm": 37.76780700683594,
|
|
"learning_rate": 2.6737824107379947e-07,
|
|
"logits/chosen": 1.4605791568756104,
|
|
"logits/rejected": 1.3956043720245361,
|
|
"logps/chosen": -326.6246337890625,
|
|
"logps/ref_chosen": -300.8880310058594,
|
|
"logps/ref_rejected": -288.5895690917969,
|
|
"logps/rejected": -342.62518310546875,
|
|
"loss": 4.4466,
|
|
"margin_dpo/margin_mean": 28.299026489257812,
|
|
"margin_dpo/margin_std": 43.362815856933594,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 0.5340314136125655,
|
|
"grad_norm": 38.82050323486328,
|
|
"learning_rate": 2.655514550086086e-07,
|
|
"logits/chosen": 1.3760805130004883,
|
|
"logits/rejected": 1.3785066604614258,
|
|
"logps/chosen": -309.2912902832031,
|
|
"logps/ref_chosen": -283.4182434082031,
|
|
"logps/ref_rejected": -317.677978515625,
|
|
"logps/rejected": -381.75701904296875,
|
|
"loss": 4.3792,
|
|
"margin_dpo/margin_mean": 38.20598220825195,
|
|
"margin_dpo/margin_std": 58.638553619384766,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.5361256544502618,
|
|
"grad_norm": 37.75017166137695,
|
|
"learning_rate": 2.6372383496608186e-07,
|
|
"logits/chosen": 1.3811287879943848,
|
|
"logits/rejected": 1.417975902557373,
|
|
"logps/chosen": -352.6160583496094,
|
|
"logps/ref_chosen": -333.6951599121094,
|
|
"logps/ref_rejected": -302.9135437011719,
|
|
"logps/rejected": -374.9257507324219,
|
|
"loss": 4.4641,
|
|
"margin_dpo/margin_mean": 53.091312408447266,
|
|
"margin_dpo/margin_std": 62.08375930786133,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 0.5382198952879581,
|
|
"grad_norm": 39.30986022949219,
|
|
"learning_rate": 2.618954789559356e-07,
|
|
"logits/chosen": 1.3786240816116333,
|
|
"logits/rejected": 1.5010215044021606,
|
|
"logps/chosen": -297.0694885253906,
|
|
"logps/ref_chosen": -269.2105712890625,
|
|
"logps/ref_rejected": -282.474365234375,
|
|
"logps/rejected": -354.8011474609375,
|
|
"loss": 4.3795,
|
|
"margin_dpo/margin_mean": 44.46790313720703,
|
|
"margin_dpo/margin_std": 52.83469772338867,
|
|
"step": 257
|
|
},
|
|
{
|
|
"epoch": 0.5403141361256545,
|
|
"grad_norm": 64.80396270751953,
|
|
"learning_rate": 2.600664850273538e-07,
|
|
"logits/chosen": 1.1780776977539062,
|
|
"logits/rejected": 1.358864665031433,
|
|
"logps/chosen": -304.0466613769531,
|
|
"logps/ref_chosen": -274.53314208984375,
|
|
"logps/ref_rejected": -284.3149108886719,
|
|
"logps/rejected": -365.9967346191406,
|
|
"loss": 4.2167,
|
|
"margin_dpo/margin_mean": 52.16826248168945,
|
|
"margin_dpo/margin_std": 58.13621139526367,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 0.5424083769633508,
|
|
"grad_norm": 57.088375091552734,
|
|
"learning_rate": 2.582369512637302e-07,
|
|
"logits/chosen": 1.1932607889175415,
|
|
"logits/rejected": 1.141600489616394,
|
|
"logps/chosen": -255.16656494140625,
|
|
"logps/ref_chosen": -235.41139221191406,
|
|
"logps/ref_rejected": -217.746826171875,
|
|
"logps/rejected": -282.0968933105469,
|
|
"loss": 4.5131,
|
|
"margin_dpo/margin_mean": 44.59490966796875,
|
|
"margin_dpo/margin_std": 53.4549674987793,
|
|
"step": 259
|
|
},
|
|
{
|
|
"epoch": 0.5445026178010471,
|
|
"grad_norm": 65.07582092285156,
|
|
"learning_rate": 2.5640697577740815e-07,
|
|
"logits/chosen": 0.8414401412010193,
|
|
"logits/rejected": 0.9391928911209106,
|
|
"logps/chosen": -242.9241943359375,
|
|
"logps/ref_chosen": -224.4993133544922,
|
|
"logps/ref_rejected": -215.19839477539062,
|
|
"logps/rejected": -268.3463439941406,
|
|
"loss": 5.2773,
|
|
"margin_dpo/margin_mean": 34.72306442260742,
|
|
"margin_dpo/margin_std": 62.74570083618164,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.5465968586387434,
|
|
"grad_norm": 62.26913070678711,
|
|
"learning_rate": 2.5457665670441937e-07,
|
|
"logits/chosen": 0.8039923906326294,
|
|
"logits/rejected": 0.6551789045333862,
|
|
"logps/chosen": -289.4314270019531,
|
|
"logps/ref_chosen": -251.2598114013672,
|
|
"logps/ref_rejected": -205.53323364257812,
|
|
"logps/rejected": -263.32464599609375,
|
|
"loss": 4.7681,
|
|
"margin_dpo/margin_mean": 19.61980438232422,
|
|
"margin_dpo/margin_std": 53.51505661010742,
|
|
"step": 261
|
|
},
|
|
{
|
|
"epoch": 0.5486910994764398,
|
|
"grad_norm": 57.36088562011719,
|
|
"learning_rate": 2.527460921992209e-07,
|
|
"logits/chosen": 1.5565768480300903,
|
|
"logits/rejected": 1.5727018117904663,
|
|
"logps/chosen": -370.4512634277344,
|
|
"logps/ref_chosen": -347.8548889160156,
|
|
"logps/ref_rejected": -309.43011474609375,
|
|
"logps/rejected": -387.045166015625,
|
|
"loss": 4.4059,
|
|
"margin_dpo/margin_mean": 55.01873016357422,
|
|
"margin_dpo/margin_std": 52.59794235229492,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 0.5507853403141362,
|
|
"grad_norm": 80.06723022460938,
|
|
"learning_rate": 2.509153804294318e-07,
|
|
"logits/chosen": 1.2121027708053589,
|
|
"logits/rejected": 1.3596720695495605,
|
|
"logps/chosen": -301.9384765625,
|
|
"logps/ref_chosen": -261.0179443359375,
|
|
"logps/ref_rejected": -295.4287109375,
|
|
"logps/rejected": -357.9337463378906,
|
|
"loss": 4.7515,
|
|
"margin_dpo/margin_mean": 21.584484100341797,
|
|
"margin_dpo/margin_std": 49.02084732055664,
|
|
"step": 263
|
|
},
|
|
{
|
|
"epoch": 0.5528795811518324,
|
|
"grad_norm": 65.33440399169922,
|
|
"learning_rate": 2.4908461957056825e-07,
|
|
"logits/chosen": 1.4055628776550293,
|
|
"logits/rejected": 1.2200889587402344,
|
|
"logps/chosen": -321.2078857421875,
|
|
"logps/ref_chosen": -297.6844482421875,
|
|
"logps/ref_rejected": -205.72137451171875,
|
|
"logps/rejected": -284.78070068359375,
|
|
"loss": 4.2315,
|
|
"margin_dpo/margin_mean": 55.53590393066406,
|
|
"margin_dpo/margin_std": 46.41938018798828,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 0.5549738219895288,
|
|
"grad_norm": 51.22663879394531,
|
|
"learning_rate": 2.4725390780077905e-07,
|
|
"logits/chosen": 1.3676010370254517,
|
|
"logits/rejected": 1.380630612373352,
|
|
"logps/chosen": -306.2537536621094,
|
|
"logps/ref_chosen": -285.8244323730469,
|
|
"logps/ref_rejected": -275.6885681152344,
|
|
"logps/rejected": -362.411865234375,
|
|
"loss": 4.4685,
|
|
"margin_dpo/margin_mean": 66.2939682006836,
|
|
"margin_dpo/margin_std": 51.13595199584961,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.5570680628272251,
|
|
"grad_norm": 56.91117477416992,
|
|
"learning_rate": 2.454233432955807e-07,
|
|
"logits/chosen": 1.253815770149231,
|
|
"logits/rejected": 1.3310532569885254,
|
|
"logps/chosen": -280.5023193359375,
|
|
"logps/ref_chosen": -273.0467834472656,
|
|
"logps/ref_rejected": -291.18133544921875,
|
|
"logps/rejected": -342.8387145996094,
|
|
"loss": 4.314,
|
|
"margin_dpo/margin_mean": 44.201820373535156,
|
|
"margin_dpo/margin_std": 46.703495025634766,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 0.5591623036649215,
|
|
"grad_norm": 44.81831741333008,
|
|
"learning_rate": 2.435930242225919e-07,
|
|
"logits/chosen": 1.1857373714447021,
|
|
"logits/rejected": 1.3162403106689453,
|
|
"logps/chosen": -294.0497131347656,
|
|
"logps/ref_chosen": -272.337890625,
|
|
"logps/ref_rejected": -279.97076416015625,
|
|
"logps/rejected": -351.3126525878906,
|
|
"loss": 4.6021,
|
|
"margin_dpo/margin_mean": 49.630027770996094,
|
|
"margin_dpo/margin_std": 56.707557678222656,
|
|
"step": 267
|
|
},
|
|
{
|
|
"epoch": 0.5612565445026177,
|
|
"grad_norm": 65.94684600830078,
|
|
"learning_rate": 2.4176304873626984e-07,
|
|
"logits/chosen": 1.1858967542648315,
|
|
"logits/rejected": 1.2295567989349365,
|
|
"logps/chosen": -257.2110900878906,
|
|
"logps/ref_chosen": -235.03692626953125,
|
|
"logps/ref_rejected": -245.3459014892578,
|
|
"logps/rejected": -307.6155090332031,
|
|
"loss": 4.4782,
|
|
"margin_dpo/margin_mean": 40.09546661376953,
|
|
"margin_dpo/margin_std": 52.132469177246094,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 0.5633507853403141,
|
|
"grad_norm": 40.701568603515625,
|
|
"learning_rate": 2.399335149726463e-07,
|
|
"logits/chosen": 1.1352908611297607,
|
|
"logits/rejected": 1.3305590152740479,
|
|
"logps/chosen": -262.03607177734375,
|
|
"logps/ref_chosen": -240.3035430908203,
|
|
"logps/ref_rejected": -233.82675170898438,
|
|
"logps/rejected": -302.03057861328125,
|
|
"loss": 4.6077,
|
|
"margin_dpo/margin_mean": 46.471290588378906,
|
|
"margin_dpo/margin_std": 58.428775787353516,
|
|
"step": 269
|
|
},
|
|
{
|
|
"epoch": 0.5654450261780105,
|
|
"grad_norm": 73.9042739868164,
|
|
"learning_rate": 2.381045210440644e-07,
|
|
"logits/chosen": 1.550492286682129,
|
|
"logits/rejected": 1.8568247556686401,
|
|
"logps/chosen": -273.93243408203125,
|
|
"logps/ref_chosen": -249.420166015625,
|
|
"logps/ref_rejected": -279.5133972167969,
|
|
"logps/rejected": -334.2721862792969,
|
|
"loss": 4.5035,
|
|
"margin_dpo/margin_mean": 30.24651336669922,
|
|
"margin_dpo/margin_std": 66.87718200683594,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.5675392670157068,
|
|
"grad_norm": 64.23003387451172,
|
|
"learning_rate": 2.3627616503391812e-07,
|
|
"logits/chosen": 1.003787636756897,
|
|
"logits/rejected": 1.0502243041992188,
|
|
"logps/chosen": -243.3548126220703,
|
|
"logps/ref_chosen": -227.45108032226562,
|
|
"logps/ref_rejected": -183.29275512695312,
|
|
"logps/rejected": -236.02288818359375,
|
|
"loss": 4.2933,
|
|
"margin_dpo/margin_mean": 36.826351165771484,
|
|
"margin_dpo/margin_std": 40.91915512084961,
|
|
"step": 271
|
|
},
|
|
{
|
|
"epoch": 0.5696335078534032,
|
|
"grad_norm": 54.780860900878906,
|
|
"learning_rate": 2.344485449913914e-07,
|
|
"logits/chosen": 1.5306094884872437,
|
|
"logits/rejected": 1.417227029800415,
|
|
"logps/chosen": -370.7244873046875,
|
|
"logps/ref_chosen": -360.17462158203125,
|
|
"logps/ref_rejected": -241.59568786621094,
|
|
"logps/rejected": -302.3287658691406,
|
|
"loss": 4.4468,
|
|
"margin_dpo/margin_mean": 50.18324661254883,
|
|
"margin_dpo/margin_std": 52.38308334350586,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 0.5717277486910994,
|
|
"grad_norm": 76.68623352050781,
|
|
"learning_rate": 2.3262175892620062e-07,
|
|
"logits/chosen": 1.463561773300171,
|
|
"logits/rejected": 1.513543725013733,
|
|
"logps/chosen": -323.9863586425781,
|
|
"logps/ref_chosen": -309.366455078125,
|
|
"logps/ref_rejected": -271.2337951660156,
|
|
"logps/rejected": -335.74420166015625,
|
|
"loss": 4.3448,
|
|
"margin_dpo/margin_mean": 49.89052963256836,
|
|
"margin_dpo/margin_std": 69.81361389160156,
|
|
"step": 273
|
|
},
|
|
{
|
|
"epoch": 0.5738219895287958,
|
|
"grad_norm": 40.13818359375,
|
|
"learning_rate": 2.3079590480333827e-07,
|
|
"logits/chosen": 1.596007227897644,
|
|
"logits/rejected": 1.7439165115356445,
|
|
"logps/chosen": -304.8982238769531,
|
|
"logps/ref_chosen": -295.56866455078125,
|
|
"logps/ref_rejected": -253.984130859375,
|
|
"logps/rejected": -311.5342102050781,
|
|
"loss": 4.2593,
|
|
"margin_dpo/margin_mean": 48.22050476074219,
|
|
"margin_dpo/margin_std": 47.02198791503906,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 0.5759162303664922,
|
|
"grad_norm": 51.986778259277344,
|
|
"learning_rate": 2.2897108053782e-07,
|
|
"logits/chosen": 0.9961601495742798,
|
|
"logits/rejected": 1.0950078964233398,
|
|
"logps/chosen": -251.74990844726562,
|
|
"logps/ref_chosen": -235.93154907226562,
|
|
"logps/ref_rejected": -230.19454956054688,
|
|
"logps/rejected": -288.5270080566406,
|
|
"loss": 4.061,
|
|
"margin_dpo/margin_mean": 42.51408386230469,
|
|
"margin_dpo/margin_std": 56.936180114746094,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.5780104712041885,
|
|
"grad_norm": 51.089576721191406,
|
|
"learning_rate": 2.2714738398943308e-07,
|
|
"logits/chosen": 1.7104884386062622,
|
|
"logits/rejected": 1.6105390787124634,
|
|
"logps/chosen": -365.57635498046875,
|
|
"logps/ref_chosen": -357.3829650878906,
|
|
"logps/ref_rejected": -273.025146484375,
|
|
"logps/rejected": -322.5356140136719,
|
|
"loss": 4.1898,
|
|
"margin_dpo/margin_mean": 41.317039489746094,
|
|
"margin_dpo/margin_std": 46.224205017089844,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 0.5801047120418849,
|
|
"grad_norm": 59.21977233886719,
|
|
"learning_rate": 2.2532491295748865e-07,
|
|
"logits/chosen": 1.0496208667755127,
|
|
"logits/rejected": 1.255394697189331,
|
|
"logps/chosen": -316.8267822265625,
|
|
"logps/ref_chosen": -289.98040771484375,
|
|
"logps/ref_rejected": -310.3972473144531,
|
|
"logps/rejected": -371.3232727050781,
|
|
"loss": 4.7638,
|
|
"margin_dpo/margin_mean": 34.079654693603516,
|
|
"margin_dpo/margin_std": 57.57683563232422,
|
|
"step": 277
|
|
},
|
|
{
|
|
"epoch": 0.5821989528795811,
|
|
"grad_norm": 52.5463981628418,
|
|
"learning_rate": 2.2350376517557726e-07,
|
|
"logits/chosen": 0.8676056861877441,
|
|
"logits/rejected": 0.8314589262008667,
|
|
"logps/chosen": -256.13165283203125,
|
|
"logps/ref_chosen": -237.13531494140625,
|
|
"logps/ref_rejected": -232.33502197265625,
|
|
"logps/rejected": -290.1609802246094,
|
|
"loss": 4.9549,
|
|
"margin_dpo/margin_mean": 38.829627990722656,
|
|
"margin_dpo/margin_std": 55.37847900390625,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 0.5842931937172775,
|
|
"grad_norm": 45.49783706665039,
|
|
"learning_rate": 2.2168403830632769e-07,
|
|
"logits/chosen": 1.3541253805160522,
|
|
"logits/rejected": 1.4431573152542114,
|
|
"logps/chosen": -361.9897766113281,
|
|
"logps/ref_chosen": -354.13311767578125,
|
|
"logps/ref_rejected": -305.6336975097656,
|
|
"logps/rejected": -358.853271484375,
|
|
"loss": 4.1861,
|
|
"margin_dpo/margin_mean": 45.362972259521484,
|
|
"margin_dpo/margin_std": 40.74034118652344,
|
|
"step": 279
|
|
},
|
|
{
|
|
"epoch": 0.5863874345549738,
|
|
"grad_norm": 45.03684997558594,
|
|
"learning_rate": 2.1986582993616925e-07,
|
|
"logits/chosen": 1.337092638015747,
|
|
"logits/rejected": 1.3559750318527222,
|
|
"logps/chosen": -274.98260498046875,
|
|
"logps/ref_chosen": -268.2659912109375,
|
|
"logps/ref_rejected": -232.44114685058594,
|
|
"logps/rejected": -289.997314453125,
|
|
"loss": 4.4116,
|
|
"margin_dpo/margin_mean": 50.83951950073242,
|
|
"margin_dpo/margin_std": 63.36719512939453,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.5884816753926702,
|
|
"grad_norm": 40.450538635253906,
|
|
"learning_rate": 2.1804923757009882e-07,
|
|
"logits/chosen": 1.3821660280227661,
|
|
"logits/rejected": 1.3343546390533447,
|
|
"logps/chosen": -287.6787414550781,
|
|
"logps/ref_chosen": -257.0721740722656,
|
|
"logps/ref_rejected": -248.18264770507812,
|
|
"logps/rejected": -319.9836730957031,
|
|
"loss": 4.3178,
|
|
"margin_dpo/margin_mean": 41.19449234008789,
|
|
"margin_dpo/margin_std": 56.606285095214844,
|
|
"step": 281
|
|
},
|
|
{
|
|
"epoch": 0.5905759162303665,
|
|
"grad_norm": 57.53801345825195,
|
|
"learning_rate": 2.1623435862645205e-07,
|
|
"logits/chosen": 1.5275428295135498,
|
|
"logits/rejected": 1.6021305322647095,
|
|
"logps/chosen": -293.01007080078125,
|
|
"logps/ref_chosen": -269.2411804199219,
|
|
"logps/ref_rejected": -323.8949279785156,
|
|
"logps/rejected": -384.69775390625,
|
|
"loss": 4.58,
|
|
"margin_dpo/margin_mean": 37.03391647338867,
|
|
"margin_dpo/margin_std": 60.21036148071289,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 0.5926701570680628,
|
|
"grad_norm": 48.43301773071289,
|
|
"learning_rate": 2.1442129043167873e-07,
|
|
"logits/chosen": 1.0364420413970947,
|
|
"logits/rejected": 1.2966769933700562,
|
|
"logps/chosen": -279.5700378417969,
|
|
"logps/ref_chosen": -257.61688232421875,
|
|
"logps/ref_rejected": -234.8463134765625,
|
|
"logps/rejected": -297.4212646484375,
|
|
"loss": 4.4572,
|
|
"margin_dpo/margin_mean": 40.621826171875,
|
|
"margin_dpo/margin_std": 61.95793533325195,
|
|
"step": 283
|
|
},
|
|
{
|
|
"epoch": 0.5947643979057592,
|
|
"grad_norm": 77.20549011230469,
|
|
"learning_rate": 2.1261013021512378e-07,
|
|
"logits/chosen": 1.3976938724517822,
|
|
"logits/rejected": 1.3549877405166626,
|
|
"logps/chosen": -252.79287719726562,
|
|
"logps/ref_chosen": -228.94891357421875,
|
|
"logps/ref_rejected": -288.43804931640625,
|
|
"logps/rejected": -346.0372009277344,
|
|
"loss": 4.6112,
|
|
"margin_dpo/margin_mean": 33.755226135253906,
|
|
"margin_dpo/margin_std": 55.99586486816406,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 0.5968586387434555,
|
|
"grad_norm": 57.3791389465332,
|
|
"learning_rate": 2.1080097510381294e-07,
|
|
"logits/chosen": 1.5715055465698242,
|
|
"logits/rejected": 1.4658746719360352,
|
|
"logps/chosen": -386.9960632324219,
|
|
"logps/ref_chosen": -364.84332275390625,
|
|
"logps/ref_rejected": -306.4946594238281,
|
|
"logps/rejected": -359.02520751953125,
|
|
"loss": 4.7905,
|
|
"margin_dpo/margin_mean": 30.377866744995117,
|
|
"margin_dpo/margin_std": 52.86358642578125,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.5989528795811518,
|
|
"grad_norm": 37.026641845703125,
|
|
"learning_rate": 2.089939221172446e-07,
|
|
"logits/chosen": 1.36098051071167,
|
|
"logits/rejected": 1.4245069026947021,
|
|
"logps/chosen": -299.2890625,
|
|
"logps/ref_chosen": -269.2027893066406,
|
|
"logps/ref_rejected": -286.9102478027344,
|
|
"logps/rejected": -346.26043701171875,
|
|
"loss": 4.6962,
|
|
"margin_dpo/margin_mean": 29.26395034790039,
|
|
"margin_dpo/margin_std": 46.75328063964844,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 0.6010471204188481,
|
|
"grad_norm": 58.61341094970703,
|
|
"learning_rate": 2.0718906816218595e-07,
|
|
"logits/chosen": 1.219170093536377,
|
|
"logits/rejected": 1.3217523097991943,
|
|
"logps/chosen": -259.4914855957031,
|
|
"logps/ref_chosen": -233.5873565673828,
|
|
"logps/ref_rejected": -230.03646850585938,
|
|
"logps/rejected": -291.0602722167969,
|
|
"loss": 4.594,
|
|
"margin_dpo/margin_mean": 35.11963653564453,
|
|
"margin_dpo/margin_std": 54.2941780090332,
|
|
"step": 287
|
|
},
|
|
{
|
|
"epoch": 0.6031413612565445,
|
|
"grad_norm": 46.509056091308594,
|
|
"learning_rate": 2.053865100274774e-07,
|
|
"logits/chosen": 1.5584362745285034,
|
|
"logits/rejected": 1.3865753412246704,
|
|
"logps/chosen": -412.9979553222656,
|
|
"logps/ref_chosen": -378.4530029296875,
|
|
"logps/ref_rejected": -302.7226257324219,
|
|
"logps/rejected": -366.6625061035156,
|
|
"loss": 4.5476,
|
|
"margin_dpo/margin_mean": 29.394969940185547,
|
|
"margin_dpo/margin_std": 42.88922119140625,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 0.6052356020942409,
|
|
"grad_norm": 42.84352493286133,
|
|
"learning_rate": 2.035863443788411e-07,
|
|
"logits/chosen": 1.5689976215362549,
|
|
"logits/rejected": 1.5198795795440674,
|
|
"logps/chosen": -373.4714660644531,
|
|
"logps/ref_chosen": -342.27532958984375,
|
|
"logps/ref_rejected": -317.79638671875,
|
|
"logps/rejected": -370.5872802734375,
|
|
"loss": 4.944,
|
|
"margin_dpo/margin_mean": 21.594791412353516,
|
|
"margin_dpo/margin_std": 44.652915954589844,
|
|
"step": 289
|
|
},
|
|
{
|
|
"epoch": 0.6073298429319371,
|
|
"grad_norm": 58.85520935058594,
|
|
"learning_rate": 2.0178866775369774e-07,
|
|
"logits/chosen": 1.3218892812728882,
|
|
"logits/rejected": 1.2929930686950684,
|
|
"logps/chosen": -374.0101623535156,
|
|
"logps/ref_chosen": -348.39788818359375,
|
|
"logps/ref_rejected": -349.3028564453125,
|
|
"logps/rejected": -415.2039489746094,
|
|
"loss": 4.6543,
|
|
"margin_dpo/margin_mean": 40.28877258300781,
|
|
"margin_dpo/margin_std": 68.06195831298828,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.6094240837696335,
|
|
"grad_norm": 39.78348922729492,
|
|
"learning_rate": 1.9999357655598891e-07,
|
|
"logits/chosen": 1.0092355012893677,
|
|
"logits/rejected": 1.146081805229187,
|
|
"logps/chosen": -268.5143737792969,
|
|
"logps/ref_chosen": -250.70835876464844,
|
|
"logps/ref_rejected": -240.2347869873047,
|
|
"logps/rejected": -312.80255126953125,
|
|
"loss": 4.2499,
|
|
"margin_dpo/margin_mean": 54.76176834106445,
|
|
"margin_dpo/margin_std": 55.031982421875,
|
|
"step": 291
|
|
},
|
|
{
|
|
"epoch": 0.6115183246073298,
|
|
"grad_norm": 56.82017135620117,
|
|
"learning_rate": 1.9820116705100775e-07,
|
|
"logits/chosen": 1.0279195308685303,
|
|
"logits/rejected": 1.0469276905059814,
|
|
"logps/chosen": -285.55364990234375,
|
|
"logps/ref_chosen": -277.9742431640625,
|
|
"logps/ref_rejected": -260.510986328125,
|
|
"logps/rejected": -321.3108825683594,
|
|
"loss": 4.4704,
|
|
"margin_dpo/margin_mean": 53.22050094604492,
|
|
"margin_dpo/margin_std": 38.52600860595703,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 0.6136125654450262,
|
|
"grad_norm": 82.68972778320312,
|
|
"learning_rate": 1.9641153536023642e-07,
|
|
"logits/chosen": 1.8253206014633179,
|
|
"logits/rejected": 1.6558302640914917,
|
|
"logps/chosen": -322.6553039550781,
|
|
"logps/ref_chosen": -300.9186096191406,
|
|
"logps/ref_rejected": -257.6700439453125,
|
|
"logps/rejected": -320.5356750488281,
|
|
"loss": 4.4861,
|
|
"margin_dpo/margin_mean": 41.12899398803711,
|
|
"margin_dpo/margin_std": 47.92676544189453,
|
|
"step": 293
|
|
},
|
|
{
|
|
"epoch": 0.6157068062827226,
|
|
"grad_norm": 59.99162673950195,
|
|
"learning_rate": 1.9462477745619106e-07,
|
|
"logits/chosen": 1.151703953742981,
|
|
"logits/rejected": 1.2953401803970337,
|
|
"logps/chosen": -282.59271240234375,
|
|
"logps/ref_chosen": -266.8080139160156,
|
|
"logps/ref_rejected": -283.26959228515625,
|
|
"logps/rejected": -355.72576904296875,
|
|
"loss": 4.5619,
|
|
"margin_dpo/margin_mean": 56.671478271484375,
|
|
"margin_dpo/margin_std": 48.01198959350586,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 0.6178010471204188,
|
|
"grad_norm": 43.26309585571289,
|
|
"learning_rate": 1.928409891572757e-07,
|
|
"logits/chosen": 1.2075080871582031,
|
|
"logits/rejected": 1.226868748664856,
|
|
"logps/chosen": -282.1271667480469,
|
|
"logps/ref_chosen": -240.19598388671875,
|
|
"logps/ref_rejected": -214.87818908691406,
|
|
"logps/rejected": -267.48272705078125,
|
|
"loss": 4.5008,
|
|
"margin_dpo/margin_mean": 10.673352241516113,
|
|
"margin_dpo/margin_std": 68.71654510498047,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.6198952879581152,
|
|
"grad_norm": 41.60967254638672,
|
|
"learning_rate": 1.9106026612264315e-07,
|
|
"logits/chosen": 1.4175227880477905,
|
|
"logits/rejected": 1.595857858657837,
|
|
"logps/chosen": -236.48651123046875,
|
|
"logps/ref_chosen": -227.85513305664062,
|
|
"logps/ref_rejected": -256.476318359375,
|
|
"logps/rejected": -316.8334655761719,
|
|
"loss": 4.0503,
|
|
"margin_dpo/margin_mean": 51.72578430175781,
|
|
"margin_dpo/margin_std": 55.78533935546875,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 0.6219895287958115,
|
|
"grad_norm": 49.98090744018555,
|
|
"learning_rate": 1.8928270384706582e-07,
|
|
"logits/chosen": 1.33966863155365,
|
|
"logits/rejected": 1.5027199983596802,
|
|
"logps/chosen": -248.77769470214844,
|
|
"logps/ref_chosen": -220.73609924316406,
|
|
"logps/ref_rejected": -272.24017333984375,
|
|
"logps/rejected": -329.5039367675781,
|
|
"loss": 4.359,
|
|
"margin_dpo/margin_mean": 29.222198486328125,
|
|
"margin_dpo/margin_std": 50.361183166503906,
|
|
"step": 297
|
|
},
|
|
{
|
|
"epoch": 0.6240837696335079,
|
|
"grad_norm": 70.29344940185547,
|
|
"learning_rate": 1.875083976558136e-07,
|
|
"logits/chosen": 1.4000345468521118,
|
|
"logits/rejected": 1.3036651611328125,
|
|
"logps/chosen": -363.42401123046875,
|
|
"logps/ref_chosen": -346.2327880859375,
|
|
"logps/ref_rejected": -285.7917785644531,
|
|
"logps/rejected": -350.0803527832031,
|
|
"loss": 4.2786,
|
|
"margin_dpo/margin_mean": 47.0973014831543,
|
|
"margin_dpo/margin_std": 58.554161071777344,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 0.6261780104712041,
|
|
"grad_norm": 68.31159973144531,
|
|
"learning_rate": 1.8573744269954297e-07,
|
|
"logits/chosen": 1.3761045932769775,
|
|
"logits/rejected": 1.3730204105377197,
|
|
"logps/chosen": -297.95330810546875,
|
|
"logps/ref_chosen": -266.99658203125,
|
|
"logps/ref_rejected": -262.5125427246094,
|
|
"logps/rejected": -327.41265869140625,
|
|
"loss": 4.5174,
|
|
"margin_dpo/margin_mean": 33.94337463378906,
|
|
"margin_dpo/margin_std": 67.46819305419922,
|
|
"step": 299
|
|
},
|
|
{
|
|
"epoch": 0.6282722513089005,
|
|
"grad_norm": 41.25592803955078,
|
|
"learning_rate": 1.839699339491937e-07,
|
|
"logits/chosen": 1.0598005056381226,
|
|
"logits/rejected": 1.1422343254089355,
|
|
"logps/chosen": -306.2508239746094,
|
|
"logps/ref_chosen": -281.19525146484375,
|
|
"logps/ref_rejected": -288.6803894042969,
|
|
"logps/rejected": -364.2799377441406,
|
|
"loss": 4.5063,
|
|
"margin_dpo/margin_mean": 50.54399490356445,
|
|
"margin_dpo/margin_std": 57.26484298706055,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6303664921465969,
|
|
"grad_norm": 80.2366943359375,
|
|
"learning_rate": 1.8220596619089573e-07,
|
|
"logits/chosen": 1.5710808038711548,
|
|
"logits/rejected": 1.6003844738006592,
|
|
"logps/chosen": -304.22662353515625,
|
|
"logps/ref_chosen": -289.8253173828125,
|
|
"logps/ref_rejected": -327.8699645996094,
|
|
"logps/rejected": -389.2873840332031,
|
|
"loss": 4.5827,
|
|
"margin_dpo/margin_mean": 47.01613235473633,
|
|
"margin_dpo/margin_std": 53.770626068115234,
|
|
"step": 301
|
|
},
|
|
{
|
|
"epoch": 0.6324607329842932,
|
|
"grad_norm": 45.32533645629883,
|
|
"learning_rate": 1.8044563402088682e-07,
|
|
"logits/chosen": 1.390702486038208,
|
|
"logits/rejected": 1.5861480236053467,
|
|
"logps/chosen": -341.7214660644531,
|
|
"logps/ref_chosen": -307.1119079589844,
|
|
"logps/ref_rejected": -296.61785888671875,
|
|
"logps/rejected": -385.1257019042969,
|
|
"loss": 4.3462,
|
|
"margin_dpo/margin_mean": 53.89823532104492,
|
|
"margin_dpo/margin_std": 60.43596267700195,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 0.6345549738219896,
|
|
"grad_norm": 35.210235595703125,
|
|
"learning_rate": 1.7868903184043885e-07,
|
|
"logits/chosen": 1.0414403676986694,
|
|
"logits/rejected": 1.2032232284545898,
|
|
"logps/chosen": -287.9827575683594,
|
|
"logps/ref_chosen": -261.281982421875,
|
|
"logps/ref_rejected": -287.9131164550781,
|
|
"logps/rejected": -370.0382385253906,
|
|
"loss": 4.4312,
|
|
"margin_dpo/margin_mean": 55.42430114746094,
|
|
"margin_dpo/margin_std": 58.83437728881836,
|
|
"step": 303
|
|
},
|
|
{
|
|
"epoch": 0.6366492146596858,
|
|
"grad_norm": 53.23714065551758,
|
|
"learning_rate": 1.7693625385079574e-07,
|
|
"logits/chosen": 1.1299974918365479,
|
|
"logits/rejected": 1.1754674911499023,
|
|
"logps/chosen": -317.24932861328125,
|
|
"logps/ref_chosen": -276.4831848144531,
|
|
"logps/ref_rejected": -257.2686462402344,
|
|
"logps/rejected": -332.5347595214844,
|
|
"loss": 4.6016,
|
|
"margin_dpo/margin_mean": 34.49999237060547,
|
|
"margin_dpo/margin_std": 42.753028869628906,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 0.6387434554973822,
|
|
"grad_norm": 50.1977653503418,
|
|
"learning_rate": 1.7518739404812155e-07,
|
|
"logits/chosen": 1.1471208333969116,
|
|
"logits/rejected": 1.1764111518859863,
|
|
"logps/chosen": -272.23565673828125,
|
|
"logps/ref_chosen": -253.3165283203125,
|
|
"logps/ref_rejected": -225.20468139648438,
|
|
"logps/rejected": -278.5679931640625,
|
|
"loss": 4.0448,
|
|
"margin_dpo/margin_mean": 34.444190979003906,
|
|
"margin_dpo/margin_std": 46.56166076660156,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.6408376963350786,
|
|
"grad_norm": 51.97341537475586,
|
|
"learning_rate": 1.7344254621846017e-07,
|
|
"logits/chosen": 1.2266101837158203,
|
|
"logits/rejected": 1.1241114139556885,
|
|
"logps/chosen": -338.9609069824219,
|
|
"logps/ref_chosen": -324.57122802734375,
|
|
"logps/ref_rejected": -299.1585693359375,
|
|
"logps/rejected": -352.9222412109375,
|
|
"loss": 4.306,
|
|
"margin_dpo/margin_mean": 39.374000549316406,
|
|
"margin_dpo/margin_std": 74.34258270263672,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 0.6429319371727749,
|
|
"grad_norm": 45.46051025390625,
|
|
"learning_rate": 1.717018039327053e-07,
|
|
"logits/chosen": 1.1175193786621094,
|
|
"logits/rejected": 1.2578641176223755,
|
|
"logps/chosen": -320.50177001953125,
|
|
"logps/ref_chosen": -289.5794372558594,
|
|
"logps/ref_rejected": -262.92510986328125,
|
|
"logps/rejected": -347.1852722167969,
|
|
"loss": 4.1751,
|
|
"margin_dpo/margin_mean": 53.3377685546875,
|
|
"margin_dpo/margin_std": 65.59990692138672,
|
|
"step": 307
|
|
},
|
|
{
|
|
"epoch": 0.6450261780104712,
|
|
"grad_norm": 34.76543045043945,
|
|
"learning_rate": 1.699652605415828e-07,
|
|
"logits/chosen": 1.338348388671875,
|
|
"logits/rejected": 1.3202842473983765,
|
|
"logps/chosen": -348.8934020996094,
|
|
"logps/ref_chosen": -305.04351806640625,
|
|
"logps/ref_rejected": -305.0120849609375,
|
|
"logps/rejected": -384.90301513671875,
|
|
"loss": 4.6399,
|
|
"margin_dpo/margin_mean": 36.04100036621094,
|
|
"margin_dpo/margin_std": 60.608455657958984,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 0.6471204188481675,
|
|
"grad_norm": 66.8311996459961,
|
|
"learning_rate": 1.6823300917064458e-07,
|
|
"logits/chosen": 1.619686484336853,
|
|
"logits/rejected": 1.3856267929077148,
|
|
"logps/chosen": -354.4423522949219,
|
|
"logps/ref_chosen": -316.80303955078125,
|
|
"logps/ref_rejected": -240.09307861328125,
|
|
"logps/rejected": -317.1116027832031,
|
|
"loss": 4.284,
|
|
"margin_dpo/margin_mean": 39.3791389465332,
|
|
"margin_dpo/margin_std": 59.34083938598633,
|
|
"step": 309
|
|
},
|
|
{
|
|
"epoch": 0.6492146596858639,
|
|
"grad_norm": 61.75398254394531,
|
|
"learning_rate": 1.6650514271527465e-07,
|
|
"logits/chosen": 1.2451605796813965,
|
|
"logits/rejected": 1.5067654848098755,
|
|
"logps/chosen": -289.0413818359375,
|
|
"logps/ref_chosen": -240.17652893066406,
|
|
"logps/ref_rejected": -242.7730712890625,
|
|
"logps/rejected": -332.27880859375,
|
|
"loss": 4.4389,
|
|
"margin_dpo/margin_mean": 40.6408576965332,
|
|
"margin_dpo/margin_std": 47.202674865722656,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.6513089005235602,
|
|
"grad_norm": 44.972686767578125,
|
|
"learning_rate": 1.647817538357072e-07,
|
|
"logits/chosen": 1.1516824960708618,
|
|
"logits/rejected": 1.3081568479537964,
|
|
"logps/chosen": -300.0003662109375,
|
|
"logps/ref_chosen": -257.53515625,
|
|
"logps/ref_rejected": -249.1999053955078,
|
|
"logps/rejected": -334.6231689453125,
|
|
"loss": 4.4418,
|
|
"margin_dpo/margin_mean": 42.95802307128906,
|
|
"margin_dpo/margin_std": 56.176063537597656,
|
|
"step": 311
|
|
},
|
|
{
|
|
"epoch": 0.6534031413612565,
|
|
"grad_norm": 70.0779037475586,
|
|
"learning_rate": 1.6306293495205755e-07,
|
|
"logits/chosen": 1.374745488166809,
|
|
"logits/rejected": 1.4253482818603516,
|
|
"logps/chosen": -301.5932922363281,
|
|
"logps/ref_chosen": -261.98828125,
|
|
"logps/ref_rejected": -238.6123504638672,
|
|
"logps/rejected": -317.7529296875,
|
|
"loss": 4.3747,
|
|
"margin_dpo/margin_mean": 39.53556442260742,
|
|
"margin_dpo/margin_std": 68.29449462890625,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 0.6554973821989529,
|
|
"grad_norm": 56.74006271362305,
|
|
"learning_rate": 1.6134877823936607e-07,
|
|
"logits/chosen": 1.480233073234558,
|
|
"logits/rejected": 1.6010148525238037,
|
|
"logps/chosen": -417.88751220703125,
|
|
"logps/ref_chosen": -380.5164794921875,
|
|
"logps/ref_rejected": -340.59722900390625,
|
|
"logps/rejected": -436.11846923828125,
|
|
"loss": 4.5172,
|
|
"margin_dpo/margin_mean": 58.15019607543945,
|
|
"margin_dpo/margin_std": 64.69535064697266,
|
|
"step": 313
|
|
},
|
|
{
|
|
"epoch": 0.6575916230366492,
|
|
"grad_norm": 52.93495559692383,
|
|
"learning_rate": 1.5963937562265522e-07,
|
|
"logits/chosen": 1.3546419143676758,
|
|
"logits/rejected": 1.3760360479354858,
|
|
"logps/chosen": -288.9587707519531,
|
|
"logps/ref_chosen": -254.8392791748047,
|
|
"logps/ref_rejected": -233.38494873046875,
|
|
"logps/rejected": -312.0257263183594,
|
|
"loss": 4.4718,
|
|
"margin_dpo/margin_mean": 44.52123260498047,
|
|
"margin_dpo/margin_std": 63.62104415893555,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 0.6596858638743456,
|
|
"grad_norm": 41.35818862915039,
|
|
"learning_rate": 1.5793481877199943e-07,
|
|
"logits/chosen": 1.7810715436935425,
|
|
"logits/rejected": 1.7476561069488525,
|
|
"logps/chosen": -315.27471923828125,
|
|
"logps/ref_chosen": -287.1436767578125,
|
|
"logps/ref_rejected": -245.744873046875,
|
|
"logps/rejected": -311.196044921875,
|
|
"loss": 4.1818,
|
|
"margin_dpo/margin_mean": 37.32012939453125,
|
|
"margin_dpo/margin_std": 46.346778869628906,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.6617801047120419,
|
|
"grad_norm": 60.255733489990234,
|
|
"learning_rate": 1.562351990976095e-07,
|
|
"logits/chosen": 0.9633012413978577,
|
|
"logits/rejected": 1.0967074632644653,
|
|
"logps/chosen": -310.6409912109375,
|
|
"logps/ref_chosen": -278.97003173828125,
|
|
"logps/ref_rejected": -268.5596618652344,
|
|
"logps/rejected": -364.9547119140625,
|
|
"loss": 4.1869,
|
|
"margin_dpo/margin_mean": 64.72406005859375,
|
|
"margin_dpo/margin_std": 64.02919006347656,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 0.6638743455497382,
|
|
"grad_norm": 63.166786193847656,
|
|
"learning_rate": 1.5454060774493065e-07,
|
|
"logits/chosen": 1.2942625284194946,
|
|
"logits/rejected": 1.294532060623169,
|
|
"logps/chosen": -277.548095703125,
|
|
"logps/ref_chosen": -252.86656188964844,
|
|
"logps/ref_rejected": -236.70155334472656,
|
|
"logps/rejected": -304.314208984375,
|
|
"loss": 4.3267,
|
|
"margin_dpo/margin_mean": 42.931148529052734,
|
|
"margin_dpo/margin_std": 60.30817413330078,
|
|
"step": 317
|
|
},
|
|
{
|
|
"epoch": 0.6659685863874345,
|
|
"grad_norm": 59.5412712097168,
|
|
"learning_rate": 1.5285113558975427e-07,
|
|
"logits/chosen": 1.2442307472229004,
|
|
"logits/rejected": 1.4497402906417847,
|
|
"logps/chosen": -252.0952606201172,
|
|
"logps/ref_chosen": -217.34515380859375,
|
|
"logps/ref_rejected": -243.4803009033203,
|
|
"logps/rejected": -328.711669921875,
|
|
"loss": 4.2442,
|
|
"margin_dpo/margin_mean": 50.481266021728516,
|
|
"margin_dpo/margin_std": 49.45831298828125,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 0.6680628272251309,
|
|
"grad_norm": 34.23768615722656,
|
|
"learning_rate": 1.5116687323334464e-07,
|
|
"logits/chosen": 1.0364283323287964,
|
|
"logits/rejected": 1.2878518104553223,
|
|
"logps/chosen": -290.0143737792969,
|
|
"logps/ref_chosen": -268.8816833496094,
|
|
"logps/ref_rejected": -275.4843444824219,
|
|
"logps/rejected": -347.4952697753906,
|
|
"loss": 4.0379,
|
|
"margin_dpo/margin_mean": 50.878273010253906,
|
|
"margin_dpo/margin_std": 42.63667678833008,
|
|
"step": 319
|
|
},
|
|
{
|
|
"epoch": 0.6701570680628273,
|
|
"grad_norm": 55.01826477050781,
|
|
"learning_rate": 1.4948791099758052e-07,
|
|
"logits/chosen": 1.7030011415481567,
|
|
"logits/rejected": 1.6658614873886108,
|
|
"logps/chosen": -328.2850341796875,
|
|
"logps/ref_chosen": -307.4996337890625,
|
|
"logps/ref_rejected": -251.08456420898438,
|
|
"logps/rejected": -320.5891418457031,
|
|
"loss": 4.3706,
|
|
"margin_dpo/margin_mean": 48.71923065185547,
|
|
"margin_dpo/margin_std": 54.48359680175781,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.6722513089005235,
|
|
"grad_norm": 36.16436004638672,
|
|
"learning_rate": 1.478143389201113e-07,
|
|
"logits/chosen": 1.5252739191055298,
|
|
"logits/rejected": 1.3168452978134155,
|
|
"logps/chosen": -343.71514892578125,
|
|
"logps/ref_chosen": -309.8309631347656,
|
|
"logps/ref_rejected": -248.75213623046875,
|
|
"logps/rejected": -326.5061340332031,
|
|
"loss": 4.4452,
|
|
"margin_dpo/margin_mean": 43.869834899902344,
|
|
"margin_dpo/margin_std": 63.80504608154297,
|
|
"step": 321
|
|
},
|
|
{
|
|
"epoch": 0.6743455497382199,
|
|
"grad_norm": 43.79127502441406,
|
|
"learning_rate": 1.461462467495284e-07,
|
|
"logits/chosen": 0.9715927243232727,
|
|
"logits/rejected": 1.0002247095108032,
|
|
"logps/chosen": -323.4966735839844,
|
|
"logps/ref_chosen": -291.58843994140625,
|
|
"logps/ref_rejected": -265.43023681640625,
|
|
"logps/rejected": -339.85174560546875,
|
|
"loss": 4.1918,
|
|
"margin_dpo/margin_mean": 42.5133056640625,
|
|
"margin_dpo/margin_std": 48.27851486206055,
|
|
"step": 322
|
|
},
|
|
{
|
|
"epoch": 0.6764397905759162,
|
|
"grad_norm": 47.61728286743164,
|
|
"learning_rate": 1.4448372394055246e-07,
|
|
"logits/chosen": 1.0764468908309937,
|
|
"logits/rejected": 0.8316705822944641,
|
|
"logps/chosen": -385.0590515136719,
|
|
"logps/ref_chosen": -343.968017578125,
|
|
"logps/ref_rejected": -254.12161254882812,
|
|
"logps/rejected": -329.58868408203125,
|
|
"loss": 4.879,
|
|
"margin_dpo/margin_mean": 34.37602996826172,
|
|
"margin_dpo/margin_std": 67.8411636352539,
|
|
"step": 323
|
|
},
|
|
{
|
|
"epoch": 0.6785340314136126,
|
|
"grad_norm": 35.46821975708008,
|
|
"learning_rate": 1.428268596492364e-07,
|
|
"logits/chosen": 1.5448215007781982,
|
|
"logits/rejected": 1.5194947719573975,
|
|
"logps/chosen": -213.710693359375,
|
|
"logps/ref_chosen": -206.94500732421875,
|
|
"logps/ref_rejected": -262.6962890625,
|
|
"logps/rejected": -316.6614990234375,
|
|
"loss": 3.9753,
|
|
"margin_dpo/margin_mean": 47.19957733154297,
|
|
"margin_dpo/margin_std": 56.4971923828125,
|
|
"step": 324
|
|
},
|
|
{
|
|
"epoch": 0.680628272251309,
|
|
"grad_norm": 40.15785598754883,
|
|
"learning_rate": 1.4117574272818386e-07,
|
|
"logits/chosen": 1.4091088771820068,
|
|
"logits/rejected": 1.5492061376571655,
|
|
"logps/chosen": -311.66400146484375,
|
|
"logps/ref_chosen": -301.9862060546875,
|
|
"logps/ref_rejected": -333.42236328125,
|
|
"logps/rejected": -399.2699890136719,
|
|
"loss": 4.5302,
|
|
"margin_dpo/margin_mean": 56.1698112487793,
|
|
"margin_dpo/margin_std": 54.43414306640625,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.6827225130890052,
|
|
"grad_norm": 52.91168975830078,
|
|
"learning_rate": 1.3953046172178413e-07,
|
|
"logits/chosen": 0.951869785785675,
|
|
"logits/rejected": 1.228202223777771,
|
|
"logps/chosen": -177.7034912109375,
|
|
"logps/ref_chosen": -164.46109008789062,
|
|
"logps/ref_rejected": -249.89413452148438,
|
|
"logps/rejected": -324.3243713378906,
|
|
"loss": 4.3905,
|
|
"margin_dpo/margin_mean": 61.18782043457031,
|
|
"margin_dpo/margin_std": 53.454124450683594,
|
|
"step": 326
|
|
},
|
|
{
|
|
"epoch": 0.6848167539267016,
|
|
"grad_norm": 42.79719543457031,
|
|
"learning_rate": 1.3789110486146468e-07,
|
|
"logits/chosen": 1.5188686847686768,
|
|
"logits/rejected": 1.4478236436843872,
|
|
"logps/chosen": -259.4933166503906,
|
|
"logps/ref_chosen": -246.3433837890625,
|
|
"logps/ref_rejected": -229.85508728027344,
|
|
"logps/rejected": -297.393798828125,
|
|
"loss": 4.1578,
|
|
"margin_dpo/margin_mean": 54.388763427734375,
|
|
"margin_dpo/margin_std": 67.07605743408203,
|
|
"step": 327
|
|
},
|
|
{
|
|
"epoch": 0.6869109947643979,
|
|
"grad_norm": 62.9756965637207,
|
|
"learning_rate": 1.362577600609588e-07,
|
|
"logits/chosen": 0.8666256666183472,
|
|
"logits/rejected": 0.9310898780822754,
|
|
"logps/chosen": -325.38824462890625,
|
|
"logps/ref_chosen": -305.82012939453125,
|
|
"logps/ref_rejected": -273.3159484863281,
|
|
"logps/rejected": -348.3529357910156,
|
|
"loss": 4.3427,
|
|
"margin_dpo/margin_mean": 55.4688720703125,
|
|
"margin_dpo/margin_std": 43.20487594604492,
|
|
"step": 328
|
|
},
|
|
{
|
|
"epoch": 0.6890052356020943,
|
|
"grad_norm": 51.00785827636719,
|
|
"learning_rate": 1.3463051491159093e-07,
|
|
"logits/chosen": 1.4473413228988647,
|
|
"logits/rejected": 1.7869523763656616,
|
|
"logps/chosen": -283.1918029785156,
|
|
"logps/ref_chosen": -258.7630615234375,
|
|
"logps/ref_rejected": -284.41131591796875,
|
|
"logps/rejected": -350.9872131347656,
|
|
"loss": 4.7045,
|
|
"margin_dpo/margin_mean": 42.1472053527832,
|
|
"margin_dpo/margin_std": 59.915836334228516,
|
|
"step": 329
|
|
},
|
|
{
|
|
"epoch": 0.6910994764397905,
|
|
"grad_norm": 47.54462432861328,
|
|
"learning_rate": 1.3300945667758012e-07,
|
|
"logits/chosen": 1.554024338722229,
|
|
"logits/rejected": 1.5038138628005981,
|
|
"logps/chosen": -360.6363830566406,
|
|
"logps/ref_chosen": -330.3982238769531,
|
|
"logps/ref_rejected": -274.9824523925781,
|
|
"logps/rejected": -335.7353515625,
|
|
"loss": 4.7699,
|
|
"margin_dpo/margin_mean": 30.514692306518555,
|
|
"margin_dpo/margin_std": 57.54186248779297,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.6931937172774869,
|
|
"grad_norm": 43.25348663330078,
|
|
"learning_rate": 1.3139467229135998e-07,
|
|
"logits/chosen": 1.2080715894699097,
|
|
"logits/rejected": 1.1073570251464844,
|
|
"logps/chosen": -306.7835693359375,
|
|
"logps/ref_chosen": -279.2760009765625,
|
|
"logps/ref_rejected": -220.27761840820312,
|
|
"logps/rejected": -285.9614562988281,
|
|
"loss": 4.4308,
|
|
"margin_dpo/margin_mean": 38.1762580871582,
|
|
"margin_dpo/margin_std": 54.596893310546875,
|
|
"step": 331
|
|
},
|
|
{
|
|
"epoch": 0.6952879581151833,
|
|
"grad_norm": 40.57438278198242,
|
|
"learning_rate": 1.2978624834891626e-07,
|
|
"logits/chosen": 1.1713310480117798,
|
|
"logits/rejected": 1.2179394960403442,
|
|
"logps/chosen": -245.34036254882812,
|
|
"logps/ref_chosen": -226.70223999023438,
|
|
"logps/ref_rejected": -205.92601013183594,
|
|
"logps/rejected": -280.2979736328125,
|
|
"loss": 4.2023,
|
|
"margin_dpo/margin_mean": 55.733863830566406,
|
|
"margin_dpo/margin_std": 48.06736755371094,
|
|
"step": 332
|
|
},
|
|
{
|
|
"epoch": 0.6973821989528796,
|
|
"grad_norm": 51.89149475097656,
|
|
"learning_rate": 1.281842711051438e-07,
|
|
"logits/chosen": 1.1539713144302368,
|
|
"logits/rejected": 1.0611777305603027,
|
|
"logps/chosen": -303.71087646484375,
|
|
"logps/ref_chosen": -280.1510009765625,
|
|
"logps/ref_rejected": -231.2144012451172,
|
|
"logps/rejected": -309.7812194824219,
|
|
"loss": 4.5953,
|
|
"margin_dpo/margin_mean": 55.00693130493164,
|
|
"margin_dpo/margin_std": 62.069034576416016,
|
|
"step": 333
|
|
},
|
|
{
|
|
"epoch": 0.6994764397905759,
|
|
"grad_norm": 38.94709014892578,
|
|
"learning_rate": 1.2658882646922033e-07,
|
|
"logits/chosen": 1.1479655504226685,
|
|
"logits/rejected": 1.1974003314971924,
|
|
"logps/chosen": -290.0271301269531,
|
|
"logps/ref_chosen": -269.64227294921875,
|
|
"logps/ref_rejected": -260.0500793457031,
|
|
"logps/rejected": -314.5991516113281,
|
|
"loss": 4.4026,
|
|
"margin_dpo/margin_mean": 34.16423416137695,
|
|
"margin_dpo/margin_std": 44.75538635253906,
|
|
"step": 334
|
|
},
|
|
{
|
|
"epoch": 0.7015706806282722,
|
|
"grad_norm": 49.46840286254883,
|
|
"learning_rate": 1.2500000000000005e-07,
|
|
"logits/chosen": 1.2762084007263184,
|
|
"logits/rejected": 1.3634967803955078,
|
|
"logps/chosen": -351.87103271484375,
|
|
"logps/ref_chosen": -304.7079162597656,
|
|
"logps/ref_rejected": -269.1751403808594,
|
|
"logps/rejected": -330.24505615234375,
|
|
"loss": 4.5817,
|
|
"margin_dpo/margin_mean": 13.906787872314453,
|
|
"margin_dpo/margin_std": 66.35944366455078,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.7036649214659686,
|
|
"grad_norm": 34.12943649291992,
|
|
"learning_rate": 1.2341787690142435e-07,
|
|
"logits/chosen": 1.5096263885498047,
|
|
"logits/rejected": 1.792067289352417,
|
|
"logps/chosen": -218.86050415039062,
|
|
"logps/ref_chosen": -210.38368225097656,
|
|
"logps/ref_rejected": -229.12037658691406,
|
|
"logps/rejected": -289.0048522949219,
|
|
"loss": 4.5624,
|
|
"margin_dpo/margin_mean": 51.40761947631836,
|
|
"margin_dpo/margin_std": 47.56304931640625,
|
|
"step": 336
|
|
},
|
|
{
|
|
"epoch": 0.7057591623036649,
|
|
"grad_norm": 52.81936264038086,
|
|
"learning_rate": 1.2184254201795363e-07,
|
|
"logits/chosen": 0.93461012840271,
|
|
"logits/rejected": 0.8486602306365967,
|
|
"logps/chosen": -299.5815734863281,
|
|
"logps/ref_chosen": -257.2767639160156,
|
|
"logps/ref_rejected": -297.5929260253906,
|
|
"logps/rejected": -374.0364074707031,
|
|
"loss": 4.4007,
|
|
"margin_dpo/margin_mean": 34.13860321044922,
|
|
"margin_dpo/margin_std": 52.06587219238281,
|
|
"step": 337
|
|
},
|
|
{
|
|
"epoch": 0.7078534031413612,
|
|
"grad_norm": 37.59018325805664,
|
|
"learning_rate": 1.202740798300168e-07,
|
|
"logits/chosen": 1.5364826917648315,
|
|
"logits/rejected": 1.5837008953094482,
|
|
"logps/chosen": -274.78564453125,
|
|
"logps/ref_chosen": -257.8255310058594,
|
|
"logps/ref_rejected": -216.51162719726562,
|
|
"logps/rejected": -298.27276611328125,
|
|
"loss": 4.1313,
|
|
"margin_dpo/margin_mean": 64.8010482788086,
|
|
"margin_dpo/margin_std": 52.778377532958984,
|
|
"step": 338
|
|
},
|
|
{
|
|
"epoch": 0.7099476439790576,
|
|
"grad_norm": 43.429386138916016,
|
|
"learning_rate": 1.1871257444948096e-07,
|
|
"logits/chosen": 1.5933047533035278,
|
|
"logits/rejected": 1.5398043394088745,
|
|
"logps/chosen": -267.3699645996094,
|
|
"logps/ref_chosen": -240.76815795898438,
|
|
"logps/ref_rejected": -244.97377014160156,
|
|
"logps/rejected": -315.84185791015625,
|
|
"loss": 4.1606,
|
|
"margin_dpo/margin_mean": 44.2662467956543,
|
|
"margin_dpo/margin_std": 55.80255126953125,
|
|
"step": 339
|
|
},
|
|
{
|
|
"epoch": 0.7120418848167539,
|
|
"grad_norm": 35.77497482299805,
|
|
"learning_rate": 1.1715810961514072e-07,
|
|
"logits/chosen": 0.9345113039016724,
|
|
"logits/rejected": 1.0999596118927002,
|
|
"logps/chosen": -204.77218627929688,
|
|
"logps/ref_chosen": -187.35751342773438,
|
|
"logps/ref_rejected": -232.0410614013672,
|
|
"logps/rejected": -292.81396484375,
|
|
"loss": 4.428,
|
|
"margin_dpo/margin_mean": 43.358245849609375,
|
|
"margin_dpo/margin_std": 77.42390441894531,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7141361256544503,
|
|
"grad_norm": 60.169677734375,
|
|
"learning_rate": 1.1561076868822755e-07,
|
|
"logits/chosen": 1.5765248537063599,
|
|
"logits/rejected": 1.8391637802124023,
|
|
"logps/chosen": -314.5309753417969,
|
|
"logps/ref_chosen": -283.4117736816406,
|
|
"logps/ref_rejected": -302.2451171875,
|
|
"logps/rejected": -380.4033203125,
|
|
"loss": 4.801,
|
|
"margin_dpo/margin_mean": 47.03903579711914,
|
|
"margin_dpo/margin_std": 53.40591049194336,
|
|
"step": 341
|
|
},
|
|
{
|
|
"epoch": 0.7162303664921466,
|
|
"grad_norm": 45.74888610839844,
|
|
"learning_rate": 1.1407063464793965e-07,
|
|
"logits/chosen": 1.175806999206543,
|
|
"logits/rejected": 1.3320945501327515,
|
|
"logps/chosen": -249.10618591308594,
|
|
"logps/ref_chosen": -221.50335693359375,
|
|
"logps/ref_rejected": -244.48382568359375,
|
|
"logps/rejected": -306.7912292480469,
|
|
"loss": 4.3427,
|
|
"margin_dpo/margin_mean": 34.704566955566406,
|
|
"margin_dpo/margin_std": 38.48919677734375,
|
|
"step": 342
|
|
},
|
|
{
|
|
"epoch": 0.7183246073298429,
|
|
"grad_norm": 33.789772033691406,
|
|
"learning_rate": 1.125377900869913e-07,
|
|
"logits/chosen": 1.5595121383666992,
|
|
"logits/rejected": 1.44582200050354,
|
|
"logps/chosen": -346.7395935058594,
|
|
"logps/ref_chosen": -340.46466064453125,
|
|
"logps/ref_rejected": -267.65313720703125,
|
|
"logps/rejected": -321.6587829589844,
|
|
"loss": 4.5708,
|
|
"margin_dpo/margin_mean": 47.7307243347168,
|
|
"margin_dpo/margin_std": 60.58958435058594,
|
|
"step": 343
|
|
},
|
|
{
|
|
"epoch": 0.7204188481675393,
|
|
"grad_norm": 31.717029571533203,
|
|
"learning_rate": 1.110123172071844e-07,
|
|
"logits/chosen": 1.2925912141799927,
|
|
"logits/rejected": 1.426027774810791,
|
|
"logps/chosen": -333.0621643066406,
|
|
"logps/ref_chosen": -310.25018310546875,
|
|
"logps/ref_rejected": -281.16302490234375,
|
|
"logps/rejected": -352.4907531738281,
|
|
"loss": 4.3502,
|
|
"margin_dpo/margin_mean": 48.51573181152344,
|
|
"margin_dpo/margin_std": 64.95976257324219,
|
|
"step": 344
|
|
},
|
|
{
|
|
"epoch": 0.7225130890052356,
|
|
"grad_norm": 51.54704284667969,
|
|
"learning_rate": 1.09494297815e-07,
|
|
"logits/chosen": 1.4484617710113525,
|
|
"logits/rejected": 1.585009217262268,
|
|
"logps/chosen": -307.612548828125,
|
|
"logps/ref_chosen": -284.6531066894531,
|
|
"logps/ref_rejected": -304.72369384765625,
|
|
"logps/rejected": -358.7782897949219,
|
|
"loss": 4.51,
|
|
"margin_dpo/margin_mean": 31.09515953063965,
|
|
"margin_dpo/margin_std": 56.86581039428711,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.724607329842932,
|
|
"grad_norm": 65.52984619140625,
|
|
"learning_rate": 1.0798381331721107e-07,
|
|
"logits/chosen": 0.9799513220787048,
|
|
"logits/rejected": 1.088678002357483,
|
|
"logps/chosen": -310.8472595214844,
|
|
"logps/ref_chosen": -255.6278076171875,
|
|
"logps/ref_rejected": -237.61305236816406,
|
|
"logps/rejected": -325.5954284667969,
|
|
"loss": 4.375,
|
|
"margin_dpo/margin_mean": 32.76295471191406,
|
|
"margin_dpo/margin_std": 52.9566535949707,
|
|
"step": 346
|
|
},
|
|
{
|
|
"epoch": 0.7267015706806282,
|
|
"grad_norm": 57.053504943847656,
|
|
"learning_rate": 1.0648094471651722e-07,
|
|
"logits/chosen": 1.3673675060272217,
|
|
"logits/rejected": 1.3742492198944092,
|
|
"logps/chosen": -315.336181640625,
|
|
"logps/ref_chosen": -287.71807861328125,
|
|
"logps/ref_rejected": -276.2634582519531,
|
|
"logps/rejected": -353.022705078125,
|
|
"loss": 4.5065,
|
|
"margin_dpo/margin_mean": 49.14113998413086,
|
|
"margin_dpo/margin_std": 64.81655883789062,
|
|
"step": 347
|
|
},
|
|
{
|
|
"epoch": 0.7287958115183246,
|
|
"grad_norm": 41.55851745605469,
|
|
"learning_rate": 1.0498577260720048e-07,
|
|
"logits/chosen": 1.4351952075958252,
|
|
"logits/rejected": 1.5722769498825073,
|
|
"logps/chosen": -296.4559020996094,
|
|
"logps/ref_chosen": -285.63232421875,
|
|
"logps/ref_rejected": -264.0018615722656,
|
|
"logps/rejected": -319.1177978515625,
|
|
"loss": 4.6334,
|
|
"margin_dpo/margin_mean": 44.29237365722656,
|
|
"margin_dpo/margin_std": 51.82339096069336,
|
|
"step": 348
|
|
},
|
|
{
|
|
"epoch": 0.7308900523560209,
|
|
"grad_norm": 34.51556396484375,
|
|
"learning_rate": 1.0349837717080347e-07,
|
|
"logits/chosen": 1.2128405570983887,
|
|
"logits/rejected": 1.2827637195587158,
|
|
"logps/chosen": -378.03369140625,
|
|
"logps/ref_chosen": -347.98370361328125,
|
|
"logps/ref_rejected": -328.8855895996094,
|
|
"logps/rejected": -408.9764709472656,
|
|
"loss": 4.2865,
|
|
"margin_dpo/margin_mean": 50.040870666503906,
|
|
"margin_dpo/margin_std": 62.83095169067383,
|
|
"step": 349
|
|
},
|
|
{
|
|
"epoch": 0.7329842931937173,
|
|
"grad_norm": 51.1704216003418,
|
|
"learning_rate": 1.0201883817182949e-07,
|
|
"logits/chosen": 1.6027448177337646,
|
|
"logits/rejected": 1.452850103378296,
|
|
"logps/chosen": -300.945556640625,
|
|
"logps/ref_chosen": -292.4501647949219,
|
|
"logps/ref_rejected": -188.39346313476562,
|
|
"logps/rejected": -268.33087158203125,
|
|
"loss": 4.4481,
|
|
"margin_dpo/margin_mean": 71.44198608398438,
|
|
"margin_dpo/margin_std": 57.31879806518555,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.7350785340314137,
|
|
"grad_norm": 54.72409439086914,
|
|
"learning_rate": 1.0054723495346482e-07,
|
|
"logits/chosen": 1.2379735708236694,
|
|
"logits/rejected": 1.2731664180755615,
|
|
"logps/chosen": -284.1971740722656,
|
|
"logps/ref_chosen": -267.4852294921875,
|
|
"logps/ref_rejected": -223.13552856445312,
|
|
"logps/rejected": -285.99163818359375,
|
|
"loss": 4.9619,
|
|
"margin_dpo/margin_mean": 46.14418029785156,
|
|
"margin_dpo/margin_std": 70.45701599121094,
|
|
"step": 351
|
|
},
|
|
{
|
|
"epoch": 0.7371727748691099,
|
|
"grad_norm": 65.03557586669922,
|
|
"learning_rate": 9.908364643332398e-08,
|
|
"logits/chosen": 1.2324098348617554,
|
|
"logits/rejected": 1.4873018264770508,
|
|
"logps/chosen": -282.592529296875,
|
|
"logps/ref_chosen": -257.07952880859375,
|
|
"logps/ref_rejected": -294.4090881347656,
|
|
"logps/rejected": -372.9225158691406,
|
|
"loss": 4.2427,
|
|
"margin_dpo/margin_mean": 53.00044631958008,
|
|
"margin_dpo/margin_std": 67.86283111572266,
|
|
"step": 352
|
|
},
|
|
{
|
|
"epoch": 0.7392670157068063,
|
|
"grad_norm": 39.74465560913086,
|
|
"learning_rate": 9.76281510992176e-08,
|
|
"logits/chosen": 1.1475257873535156,
|
|
"logits/rejected": 1.180965781211853,
|
|
"logps/chosen": -321.6578674316406,
|
|
"logps/ref_chosen": -290.9927062988281,
|
|
"logps/ref_rejected": -263.1128845214844,
|
|
"logps/rejected": -340.19183349609375,
|
|
"loss": 4.2739,
|
|
"margin_dpo/margin_mean": 46.413780212402344,
|
|
"margin_dpo/margin_std": 61.069339752197266,
|
|
"step": 353
|
|
},
|
|
{
|
|
"epoch": 0.7413612565445026,
|
|
"grad_norm": 48.98942184448242,
|
|
"learning_rate": 9.618082700494318e-08,
|
|
"logits/chosen": 1.084821343421936,
|
|
"logits/rejected": 1.1856131553649902,
|
|
"logps/chosen": -224.8162384033203,
|
|
"logps/ref_chosen": -196.65435791015625,
|
|
"logps/ref_rejected": -193.15533447265625,
|
|
"logps/rejected": -250.77032470703125,
|
|
"loss": 4.9672,
|
|
"margin_dpo/margin_mean": 29.453073501586914,
|
|
"margin_dpo/margin_std": 54.51377487182617,
|
|
"step": 354
|
|
},
|
|
{
|
|
"epoch": 0.743455497382199,
|
|
"grad_norm": 52.5598030090332,
|
|
"learning_rate": 9.474175176609956e-08,
|
|
"logits/chosen": 1.538980484008789,
|
|
"logits/rejected": 1.710750937461853,
|
|
"logps/chosen": -305.9091491699219,
|
|
"logps/ref_chosen": -277.7572937011719,
|
|
"logps/ref_rejected": -296.24908447265625,
|
|
"logps/rejected": -365.863525390625,
|
|
"loss": 4.304,
|
|
"margin_dpo/margin_mean": 41.46260452270508,
|
|
"margin_dpo/margin_std": 60.0238151550293,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.7455497382198953,
|
|
"grad_norm": 38.38217544555664,
|
|
"learning_rate": 9.331100255592436e-08,
|
|
"logits/chosen": 1.1549817323684692,
|
|
"logits/rejected": 1.270320177078247,
|
|
"logps/chosen": -250.93751525878906,
|
|
"logps/ref_chosen": -228.735595703125,
|
|
"logps/ref_rejected": -288.4073486328125,
|
|
"logps/rejected": -340.65838623046875,
|
|
"loss": 4.4604,
|
|
"margin_dpo/margin_mean": 30.04913902282715,
|
|
"margin_dpo/margin_std": 50.18395233154297,
|
|
"step": 356
|
|
},
|
|
{
|
|
"epoch": 0.7476439790575916,
|
|
"grad_norm": 48.77327346801758,
|
|
"learning_rate": 9.18886561011557e-08,
|
|
"logits/chosen": 1.2281720638275146,
|
|
"logits/rejected": 1.2340593338012695,
|
|
"logps/chosen": -345.0635986328125,
|
|
"logps/ref_chosen": -327.5565185546875,
|
|
"logps/ref_rejected": -286.9888610839844,
|
|
"logps/rejected": -364.4182434082031,
|
|
"loss": 4.4785,
|
|
"margin_dpo/margin_mean": 59.92234802246094,
|
|
"margin_dpo/margin_std": 68.7721939086914,
|
|
"step": 357
|
|
},
|
|
{
|
|
"epoch": 0.749738219895288,
|
|
"grad_norm": 33.63121795654297,
|
|
"learning_rate": 9.047478867791731e-08,
|
|
"logits/chosen": 1.2939711809158325,
|
|
"logits/rejected": 1.3026853799819946,
|
|
"logps/chosen": -300.43011474609375,
|
|
"logps/ref_chosen": -275.9919738769531,
|
|
"logps/ref_rejected": -226.95779418945312,
|
|
"logps/rejected": -304.18017578125,
|
|
"loss": 4.326,
|
|
"margin_dpo/margin_mean": 52.78423309326172,
|
|
"margin_dpo/margin_std": 65.29212951660156,
|
|
"step": 358
|
|
},
|
|
{
|
|
"epoch": 0.7518324607329843,
|
|
"grad_norm": 41.51667022705078,
|
|
"learning_rate": 8.906947610762825e-08,
|
|
"logits/chosen": 1.193036675453186,
|
|
"logits/rejected": 1.3061870336532593,
|
|
"logps/chosen": -288.68792724609375,
|
|
"logps/ref_chosen": -265.4796447753906,
|
|
"logps/ref_rejected": -269.9594421386719,
|
|
"logps/rejected": -336.7837219238281,
|
|
"loss": 4.2962,
|
|
"margin_dpo/margin_mean": 43.615962982177734,
|
|
"margin_dpo/margin_std": 57.33641052246094,
|
|
"step": 359
|
|
},
|
|
{
|
|
"epoch": 0.7539267015706806,
|
|
"grad_norm": 41.91855239868164,
|
|
"learning_rate": 8.76727937529367e-08,
|
|
"logits/chosen": 1.4477754831314087,
|
|
"logits/rejected": 1.4021023511886597,
|
|
"logps/chosen": -364.541015625,
|
|
"logps/ref_chosen": -336.95709228515625,
|
|
"logps/ref_rejected": -275.51239013671875,
|
|
"logps/rejected": -352.3710632324219,
|
|
"loss": 4.4654,
|
|
"margin_dpo/margin_mean": 49.27482604980469,
|
|
"margin_dpo/margin_std": 65.66383361816406,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.7560209424083769,
|
|
"grad_norm": 37.49740219116211,
|
|
"learning_rate": 8.628481651367875e-08,
|
|
"logits/chosen": 1.1130855083465576,
|
|
"logits/rejected": 1.3169794082641602,
|
|
"logps/chosen": -233.5477294921875,
|
|
"logps/ref_chosen": -223.0279541015625,
|
|
"logps/ref_rejected": -233.6653289794922,
|
|
"logps/rejected": -294.90289306640625,
|
|
"loss": 4.1403,
|
|
"margin_dpo/margin_mean": 50.717796325683594,
|
|
"margin_dpo/margin_std": 58.103668212890625,
|
|
"step": 361
|
|
},
|
|
{
|
|
"epoch": 0.7581151832460733,
|
|
"grad_norm": 75.54791259765625,
|
|
"learning_rate": 8.490561882286135e-08,
|
|
"logits/chosen": 1.167074203491211,
|
|
"logits/rejected": 1.180873155593872,
|
|
"logps/chosen": -337.9937744140625,
|
|
"logps/ref_chosen": -298.1035461425781,
|
|
"logps/ref_rejected": -230.74783325195312,
|
|
"logps/rejected": -312.3794250488281,
|
|
"loss": 4.3668,
|
|
"margin_dpo/margin_mean": 41.741363525390625,
|
|
"margin_dpo/margin_std": 74.15339660644531,
|
|
"step": 362
|
|
},
|
|
{
|
|
"epoch": 0.7602094240837697,
|
|
"grad_norm": 69.05213928222656,
|
|
"learning_rate": 8.353527464267104e-08,
|
|
"logits/chosen": 1.4149866104125977,
|
|
"logits/rejected": 1.3386046886444092,
|
|
"logps/chosen": -324.6624450683594,
|
|
"logps/ref_chosen": -315.25506591796875,
|
|
"logps/ref_rejected": -276.456298828125,
|
|
"logps/rejected": -352.84759521484375,
|
|
"loss": 4.1568,
|
|
"margin_dpo/margin_mean": 66.9839096069336,
|
|
"margin_dpo/margin_std": 64.92231750488281,
|
|
"step": 363
|
|
},
|
|
{
|
|
"epoch": 0.762303664921466,
|
|
"grad_norm": 31.97148895263672,
|
|
"learning_rate": 8.217385746050742e-08,
|
|
"logits/chosen": 1.5564781427383423,
|
|
"logits/rejected": 1.3156113624572754,
|
|
"logps/chosen": -380.2980041503906,
|
|
"logps/ref_chosen": -336.43798828125,
|
|
"logps/ref_rejected": -259.9676818847656,
|
|
"logps/rejected": -339.92596435546875,
|
|
"loss": 4.5886,
|
|
"margin_dpo/margin_mean": 36.0982666015625,
|
|
"margin_dpo/margin_std": 53.2963752746582,
|
|
"step": 364
|
|
},
|
|
{
|
|
"epoch": 0.7643979057591623,
|
|
"grad_norm": 53.09539794921875,
|
|
"learning_rate": 8.082144028504231e-08,
|
|
"logits/chosen": 1.0552072525024414,
|
|
"logits/rejected": 1.2801932096481323,
|
|
"logps/chosen": -227.60394287109375,
|
|
"logps/ref_chosen": -209.7356719970703,
|
|
"logps/ref_rejected": -294.5636291503906,
|
|
"logps/rejected": -367.4307861328125,
|
|
"loss": 4.2219,
|
|
"margin_dpo/margin_mean": 54.99891662597656,
|
|
"margin_dpo/margin_std": 50.34431457519531,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.7664921465968586,
|
|
"grad_norm": 56.880043029785156,
|
|
"learning_rate": 7.947809564230445e-08,
|
|
"logits/chosen": 1.2870928049087524,
|
|
"logits/rejected": 1.2019919157028198,
|
|
"logps/chosen": -343.0006408691406,
|
|
"logps/ref_chosen": -312.77142333984375,
|
|
"logps/ref_rejected": -273.8427734375,
|
|
"logps/rejected": -375.21856689453125,
|
|
"loss": 4.2553,
|
|
"margin_dpo/margin_mean": 71.1466293334961,
|
|
"margin_dpo/margin_std": 62.22167205810547,
|
|
"step": 366
|
|
},
|
|
{
|
|
"epoch": 0.768586387434555,
|
|
"grad_norm": 47.9551887512207,
|
|
"learning_rate": 7.814389557179016e-08,
|
|
"logits/chosen": 1.7175925970077515,
|
|
"logits/rejected": 1.4891177415847778,
|
|
"logps/chosen": -329.87542724609375,
|
|
"logps/ref_chosen": -284.1925964355469,
|
|
"logps/ref_rejected": -208.8526611328125,
|
|
"logps/rejected": -294.8234558105469,
|
|
"loss": 4.1368,
|
|
"margin_dpo/margin_mean": 40.28790283203125,
|
|
"margin_dpo/margin_std": 68.52880859375,
|
|
"step": 367
|
|
},
|
|
{
|
|
"epoch": 0.7706806282722513,
|
|
"grad_norm": 32.45009231567383,
|
|
"learning_rate": 7.681891162260015e-08,
|
|
"logits/chosen": 1.7018953561782837,
|
|
"logits/rejected": 1.5844436883926392,
|
|
"logps/chosen": -376.4222412109375,
|
|
"logps/ref_chosen": -360.64459228515625,
|
|
"logps/ref_rejected": -297.281005859375,
|
|
"logps/rejected": -367.4183654785156,
|
|
"loss": 3.9894,
|
|
"margin_dpo/margin_mean": 54.35973358154297,
|
|
"margin_dpo/margin_std": 49.44243621826172,
|
|
"step": 368
|
|
},
|
|
{
|
|
"epoch": 0.7727748691099476,
|
|
"grad_norm": 85.05940246582031,
|
|
"learning_rate": 7.550321484960251e-08,
|
|
"logits/chosen": 1.4974400997161865,
|
|
"logits/rejected": 1.5112247467041016,
|
|
"logps/chosen": -364.4231872558594,
|
|
"logps/ref_chosen": -340.94610595703125,
|
|
"logps/ref_rejected": -285.1484069824219,
|
|
"logps/rejected": -367.0999450683594,
|
|
"loss": 4.5603,
|
|
"margin_dpo/margin_mean": 58.47446060180664,
|
|
"margin_dpo/margin_std": 59.77842712402344,
|
|
"step": 369
|
|
},
|
|
{
|
|
"epoch": 0.774869109947644,
|
|
"grad_norm": 36.800376892089844,
|
|
"learning_rate": 7.419687580962222e-08,
|
|
"logits/chosen": 1.2962076663970947,
|
|
"logits/rejected": 1.5088801383972168,
|
|
"logps/chosen": -313.3735046386719,
|
|
"logps/ref_chosen": -276.9629211425781,
|
|
"logps/ref_rejected": -274.93865966796875,
|
|
"logps/rejected": -353.8816223144531,
|
|
"loss": 4.1904,
|
|
"margin_dpo/margin_mean": 42.532344818115234,
|
|
"margin_dpo/margin_std": 52.544273376464844,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.7769633507853403,
|
|
"grad_norm": 79.35717010498047,
|
|
"learning_rate": 7.289996455765748e-08,
|
|
"logits/chosen": 0.7718454599380493,
|
|
"logits/rejected": 1.0238559246063232,
|
|
"logps/chosen": -365.87713623046875,
|
|
"logps/ref_chosen": -323.23980712890625,
|
|
"logps/ref_rejected": -317.0935363769531,
|
|
"logps/rejected": -390.978759765625,
|
|
"loss": 4.4027,
|
|
"margin_dpo/margin_mean": 31.247907638549805,
|
|
"margin_dpo/margin_std": 54.091983795166016,
|
|
"step": 371
|
|
},
|
|
{
|
|
"epoch": 0.7790575916230367,
|
|
"grad_norm": 33.439125061035156,
|
|
"learning_rate": 7.161255064312283e-08,
|
|
"logits/chosen": 1.3146398067474365,
|
|
"logits/rejected": 1.290389060974121,
|
|
"logps/chosen": -338.93792724609375,
|
|
"logps/ref_chosen": -303.75262451171875,
|
|
"logps/ref_rejected": -205.62069702148438,
|
|
"logps/rejected": -309.5386962890625,
|
|
"loss": 4.0137,
|
|
"margin_dpo/margin_mean": 68.732666015625,
|
|
"margin_dpo/margin_std": 57.33296585083008,
|
|
"step": 372
|
|
},
|
|
{
|
|
"epoch": 0.7811518324607329,
|
|
"grad_norm": 45.49492263793945,
|
|
"learning_rate": 7.033470310611945e-08,
|
|
"logits/chosen": 1.4074095487594604,
|
|
"logits/rejected": 1.1533772945404053,
|
|
"logps/chosen": -377.86865234375,
|
|
"logps/ref_chosen": -346.5982666015625,
|
|
"logps/ref_rejected": -253.89280700683594,
|
|
"logps/rejected": -333.4144287109375,
|
|
"loss": 4.1968,
|
|
"margin_dpo/margin_mean": 48.25126266479492,
|
|
"margin_dpo/margin_std": 56.07286834716797,
|
|
"step": 373
|
|
},
|
|
{
|
|
"epoch": 0.7832460732984293,
|
|
"grad_norm": 57.437164306640625,
|
|
"learning_rate": 6.906649047373245e-08,
|
|
"logits/chosen": 1.4347639083862305,
|
|
"logits/rejected": 1.5623588562011719,
|
|
"logps/chosen": -292.1949768066406,
|
|
"logps/ref_chosen": -252.59971618652344,
|
|
"logps/ref_rejected": -249.61476135253906,
|
|
"logps/rejected": -319.74609375,
|
|
"loss": 4.7414,
|
|
"margin_dpo/margin_mean": 30.536096572875977,
|
|
"margin_dpo/margin_std": 58.64886474609375,
|
|
"step": 374
|
|
},
|
|
{
|
|
"epoch": 0.7853403141361257,
|
|
"grad_norm": 67.63172149658203,
|
|
"learning_rate": 6.780798075635675e-08,
|
|
"logits/chosen": 1.1573803424835205,
|
|
"logits/rejected": 1.0170570611953735,
|
|
"logps/chosen": -274.2476806640625,
|
|
"logps/ref_chosen": -247.3214569091797,
|
|
"logps/ref_rejected": -188.48236083984375,
|
|
"logps/rejected": -260.9528503417969,
|
|
"loss": 4.4518,
|
|
"margin_dpo/margin_mean": 45.54425811767578,
|
|
"margin_dpo/margin_std": 53.55020523071289,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.787434554973822,
|
|
"grad_norm": 43.886695861816406,
|
|
"learning_rate": 6.655924144404906e-08,
|
|
"logits/chosen": 1.1326963901519775,
|
|
"logits/rejected": 1.3887975215911865,
|
|
"logps/chosen": -327.3310852050781,
|
|
"logps/ref_chosen": -272.513916015625,
|
|
"logps/ref_rejected": -305.7491760253906,
|
|
"logps/rejected": -412.55767822265625,
|
|
"loss": 4.5376,
|
|
"margin_dpo/margin_mean": 51.99131393432617,
|
|
"margin_dpo/margin_std": 69.22034454345703,
|
|
"step": 376
|
|
},
|
|
{
|
|
"epoch": 0.7895287958115184,
|
|
"grad_norm": 47.22002029418945,
|
|
"learning_rate": 6.532033950290885e-08,
|
|
"logits/chosen": 1.356651782989502,
|
|
"logits/rejected": 1.438635230064392,
|
|
"logps/chosen": -323.85699462890625,
|
|
"logps/ref_chosen": -298.3796081542969,
|
|
"logps/ref_rejected": -273.41839599609375,
|
|
"logps/rejected": -348.69940185546875,
|
|
"loss": 4.5118,
|
|
"margin_dpo/margin_mean": 49.80352020263672,
|
|
"margin_dpo/margin_std": 56.26287841796875,
|
|
"step": 377
|
|
},
|
|
{
|
|
"epoch": 0.7916230366492146,
|
|
"grad_norm": 43.87843704223633,
|
|
"learning_rate": 6.409134137148736e-08,
|
|
"logits/chosen": 1.4002658128738403,
|
|
"logits/rejected": 1.4882569313049316,
|
|
"logps/chosen": -305.824462890625,
|
|
"logps/ref_chosen": -286.3173522949219,
|
|
"logps/ref_rejected": -271.7178955078125,
|
|
"logps/rejected": -340.6441345214844,
|
|
"loss": 4.5644,
|
|
"margin_dpo/margin_mean": 49.41912078857422,
|
|
"margin_dpo/margin_std": 56.88557434082031,
|
|
"step": 378
|
|
},
|
|
{
|
|
"epoch": 0.793717277486911,
|
|
"grad_norm": 57.02334213256836,
|
|
"learning_rate": 6.28723129572247e-08,
|
|
"logits/chosen": 1.439416527748108,
|
|
"logits/rejected": 1.3806058168411255,
|
|
"logps/chosen": -271.30230712890625,
|
|
"logps/ref_chosen": -233.71743774414062,
|
|
"logps/ref_rejected": -206.68927001953125,
|
|
"logps/rejected": -284.8748474121094,
|
|
"loss": 4.5262,
|
|
"margin_dpo/margin_mean": 40.600711822509766,
|
|
"margin_dpo/margin_std": 50.349300384521484,
|
|
"step": 379
|
|
},
|
|
{
|
|
"epoch": 0.7958115183246073,
|
|
"grad_norm": 72.12355041503906,
|
|
"learning_rate": 6.166331963291519e-08,
|
|
"logits/chosen": 1.7105507850646973,
|
|
"logits/rejected": 1.5248544216156006,
|
|
"logps/chosen": -387.1551513671875,
|
|
"logps/ref_chosen": -356.8863525390625,
|
|
"logps/ref_rejected": -354.776123046875,
|
|
"logps/rejected": -424.4413757324219,
|
|
"loss": 4.4649,
|
|
"margin_dpo/margin_mean": 39.396446228027344,
|
|
"margin_dpo/margin_std": 53.38307189941406,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.7979057591623037,
|
|
"grad_norm": 97.61827850341797,
|
|
"learning_rate": 6.046442623320145e-08,
|
|
"logits/chosen": 0.9507350325584412,
|
|
"logits/rejected": 1.0006842613220215,
|
|
"logps/chosen": -260.0545654296875,
|
|
"logps/ref_chosen": -235.81100463867188,
|
|
"logps/ref_rejected": -237.37062072753906,
|
|
"logps/rejected": -337.2359313964844,
|
|
"loss": 4.1799,
|
|
"margin_dpo/margin_mean": 75.62174987792969,
|
|
"margin_dpo/margin_std": 58.47461700439453,
|
|
"step": 381
|
|
},
|
|
{
|
|
"epoch": 0.8,
|
|
"grad_norm": 42.927345275878906,
|
|
"learning_rate": 5.9275697051098275e-08,
|
|
"logits/chosen": 1.312659740447998,
|
|
"logits/rejected": 1.322435975074768,
|
|
"logps/chosen": -294.5480651855469,
|
|
"logps/ref_chosen": -259.17388916015625,
|
|
"logps/ref_rejected": -230.83482360839844,
|
|
"logps/rejected": -323.2818298339844,
|
|
"loss": 3.9698,
|
|
"margin_dpo/margin_mean": 57.07281494140625,
|
|
"margin_dpo/margin_std": 65.82691955566406,
|
|
"step": 382
|
|
},
|
|
{
|
|
"epoch": 0.8020942408376963,
|
|
"grad_norm": 60.843833923339844,
|
|
"learning_rate": 5.809719583454414e-08,
|
|
"logits/chosen": 1.0891731977462769,
|
|
"logits/rejected": 1.3433376550674438,
|
|
"logps/chosen": -316.5683898925781,
|
|
"logps/ref_chosen": -269.8660583496094,
|
|
"logps/ref_rejected": -320.0415954589844,
|
|
"logps/rejected": -390.5964660644531,
|
|
"loss": 4.4135,
|
|
"margin_dpo/margin_mean": 23.852542877197266,
|
|
"margin_dpo/margin_std": 56.356651306152344,
|
|
"step": 383
|
|
},
|
|
{
|
|
"epoch": 0.8041884816753927,
|
|
"grad_norm": 70.86746978759766,
|
|
"learning_rate": 5.6928985782982524e-08,
|
|
"logits/chosen": 1.2213385105133057,
|
|
"logits/rejected": 1.5940483808517456,
|
|
"logps/chosen": -313.1070251464844,
|
|
"logps/ref_chosen": -280.7498779296875,
|
|
"logps/ref_rejected": -324.9134216308594,
|
|
"logps/rejected": -393.7688293457031,
|
|
"loss": 4.9522,
|
|
"margin_dpo/margin_mean": 36.498287200927734,
|
|
"margin_dpo/margin_std": 67.0685043334961,
|
|
"step": 384
|
|
},
|
|
{
|
|
"epoch": 0.806282722513089,
|
|
"grad_norm": 41.98042678833008,
|
|
"learning_rate": 5.57711295439732e-08,
|
|
"logits/chosen": 1.4527329206466675,
|
|
"logits/rejected": 1.5307879447937012,
|
|
"logps/chosen": -346.65673828125,
|
|
"logps/ref_chosen": -313.2212829589844,
|
|
"logps/ref_rejected": -256.9848937988281,
|
|
"logps/rejected": -341.84088134765625,
|
|
"loss": 4.4431,
|
|
"margin_dpo/margin_mean": 51.4205322265625,
|
|
"margin_dpo/margin_std": 66.022705078125,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.8083769633507853,
|
|
"grad_norm": 48.37839126586914,
|
|
"learning_rate": 5.4623689209832484e-08,
|
|
"logits/chosen": 1.6142921447753906,
|
|
"logits/rejected": 1.77769935131073,
|
|
"logps/chosen": -376.63311767578125,
|
|
"logps/ref_chosen": -342.4034423828125,
|
|
"logps/ref_rejected": -319.1665954589844,
|
|
"logps/rejected": -405.9571838378906,
|
|
"loss": 4.1689,
|
|
"margin_dpo/margin_mean": 52.56089401245117,
|
|
"margin_dpo/margin_std": 56.50434875488281,
|
|
"step": 386
|
|
},
|
|
{
|
|
"epoch": 0.8104712041884817,
|
|
"grad_norm": 37.86534118652344,
|
|
"learning_rate": 5.3486726314303175e-08,
|
|
"logits/chosen": 1.3724387884140015,
|
|
"logits/rejected": 1.450311303138733,
|
|
"logps/chosen": -249.56488037109375,
|
|
"logps/ref_chosen": -209.16738891601562,
|
|
"logps/ref_rejected": -225.61949157714844,
|
|
"logps/rejected": -305.92340087890625,
|
|
"loss": 4.3007,
|
|
"margin_dpo/margin_mean": 39.906436920166016,
|
|
"margin_dpo/margin_std": 77.46627807617188,
|
|
"step": 387
|
|
},
|
|
{
|
|
"epoch": 0.812565445026178,
|
|
"grad_norm": 86.58211517333984,
|
|
"learning_rate": 5.2360301829254745e-08,
|
|
"logits/chosen": 1.78019118309021,
|
|
"logits/rejected": 1.7659128904342651,
|
|
"logps/chosen": -381.0487365722656,
|
|
"logps/ref_chosen": -342.5128173828125,
|
|
"logps/ref_rejected": -296.8653564453125,
|
|
"logps/rejected": -390.75970458984375,
|
|
"loss": 4.5827,
|
|
"margin_dpo/margin_mean": 55.35844421386719,
|
|
"margin_dpo/margin_std": 75.9105224609375,
|
|
"step": 388
|
|
},
|
|
{
|
|
"epoch": 0.8146596858638744,
|
|
"grad_norm": 73.34844207763672,
|
|
"learning_rate": 5.1244476161413806e-08,
|
|
"logits/chosen": 1.6132086515426636,
|
|
"logits/rejected": 1.4335089921951294,
|
|
"logps/chosen": -354.9220886230469,
|
|
"logps/ref_chosen": -336.53912353515625,
|
|
"logps/ref_rejected": -237.36383056640625,
|
|
"logps/rejected": -318.2511901855469,
|
|
"loss": 4.2202,
|
|
"margin_dpo/margin_mean": 62.504356384277344,
|
|
"margin_dpo/margin_std": 76.5676498413086,
|
|
"step": 389
|
|
},
|
|
{
|
|
"epoch": 0.8167539267015707,
|
|
"grad_norm": 72.32861328125,
|
|
"learning_rate": 5.013930914912476e-08,
|
|
"logits/chosen": 1.4496684074401855,
|
|
"logits/rejected": 1.6118597984313965,
|
|
"logps/chosen": -313.1777648925781,
|
|
"logps/ref_chosen": -275.41680908203125,
|
|
"logps/ref_rejected": -300.94329833984375,
|
|
"logps/rejected": -397.3767395019531,
|
|
"loss": 4.4604,
|
|
"margin_dpo/margin_mean": 58.67253112792969,
|
|
"margin_dpo/margin_std": 57.53920364379883,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.818848167539267,
|
|
"grad_norm": 38.00588607788086,
|
|
"learning_rate": 4.904486005914027e-08,
|
|
"logits/chosen": 1.421841025352478,
|
|
"logits/rejected": 1.3273773193359375,
|
|
"logps/chosen": -301.99920654296875,
|
|
"logps/ref_chosen": -249.42276000976562,
|
|
"logps/ref_rejected": -187.71572875976562,
|
|
"logps/rejected": -270.2522888183594,
|
|
"loss": 4.2838,
|
|
"margin_dpo/margin_mean": 29.960115432739258,
|
|
"margin_dpo/margin_std": 43.918495178222656,
|
|
"step": 391
|
|
},
|
|
{
|
|
"epoch": 0.8209424083769633,
|
|
"grad_norm": 45.547096252441406,
|
|
"learning_rate": 4.796118758344353e-08,
|
|
"logits/chosen": 1.0780835151672363,
|
|
"logits/rejected": 1.0823677778244019,
|
|
"logps/chosen": -326.24456787109375,
|
|
"logps/ref_chosen": -290.30438232421875,
|
|
"logps/ref_rejected": -262.2787780761719,
|
|
"logps/rejected": -357.97216796875,
|
|
"loss": 3.8296,
|
|
"margin_dpo/margin_mean": 59.75324249267578,
|
|
"margin_dpo/margin_std": 60.47087860107422,
|
|
"step": 392
|
|
},
|
|
{
|
|
"epoch": 0.8230366492146597,
|
|
"grad_norm": 53.403141021728516,
|
|
"learning_rate": 4.688834983610082e-08,
|
|
"logits/chosen": 1.3256309032440186,
|
|
"logits/rejected": 1.1313904523849487,
|
|
"logps/chosen": -356.47021484375,
|
|
"logps/ref_chosen": -317.2633972167969,
|
|
"logps/ref_rejected": -237.91380310058594,
|
|
"logps/rejected": -326.9722900390625,
|
|
"loss": 4.5399,
|
|
"margin_dpo/margin_mean": 49.85166931152344,
|
|
"margin_dpo/margin_std": 60.655738830566406,
|
|
"step": 393
|
|
},
|
|
{
|
|
"epoch": 0.8251308900523561,
|
|
"grad_norm": 44.783817291259766,
|
|
"learning_rate": 4.582640435014459e-08,
|
|
"logits/chosen": 1.5566421747207642,
|
|
"logits/rejected": 1.6724714040756226,
|
|
"logps/chosen": -406.4788818359375,
|
|
"logps/ref_chosen": -377.4843444824219,
|
|
"logps/ref_rejected": -298.2265319824219,
|
|
"logps/rejected": -381.8688659667969,
|
|
"loss": 4.4044,
|
|
"margin_dpo/margin_mean": 54.6478271484375,
|
|
"margin_dpo/margin_std": 60.59336853027344,
|
|
"step": 394
|
|
},
|
|
{
|
|
"epoch": 0.8272251308900523,
|
|
"grad_norm": 43.559539794921875,
|
|
"learning_rate": 4.477540807448832e-08,
|
|
"logits/chosen": 1.251692771911621,
|
|
"logits/rejected": 1.2946186065673828,
|
|
"logps/chosen": -310.1583557128906,
|
|
"logps/ref_chosen": -281.3030090332031,
|
|
"logps/ref_rejected": -272.98968505859375,
|
|
"logps/rejected": -343.1010437011719,
|
|
"loss": 4.3814,
|
|
"margin_dpo/margin_mean": 41.256004333496094,
|
|
"margin_dpo/margin_std": 55.48289108276367,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.8293193717277487,
|
|
"grad_norm": 173.74839782714844,
|
|
"learning_rate": 4.373541737087263e-08,
|
|
"logits/chosen": 1.4800870418548584,
|
|
"logits/rejected": 1.4773489236831665,
|
|
"logps/chosen": -338.620849609375,
|
|
"logps/ref_chosen": -295.05364990234375,
|
|
"logps/ref_rejected": -255.04061889648438,
|
|
"logps/rejected": -347.4020080566406,
|
|
"loss": 4.402,
|
|
"margin_dpo/margin_mean": 48.79420471191406,
|
|
"margin_dpo/margin_std": 69.13119506835938,
|
|
"step": 396
|
|
},
|
|
{
|
|
"epoch": 0.831413612565445,
|
|
"grad_norm": 48.23088836669922,
|
|
"learning_rate": 4.270648801084295e-08,
|
|
"logits/chosen": 1.4665961265563965,
|
|
"logits/rejected": 1.5797699689865112,
|
|
"logps/chosen": -311.6304016113281,
|
|
"logps/ref_chosen": -288.0824890136719,
|
|
"logps/ref_rejected": -270.2839050292969,
|
|
"logps/rejected": -332.85223388671875,
|
|
"loss": 4.47,
|
|
"margin_dpo/margin_mean": 39.020355224609375,
|
|
"margin_dpo/margin_std": 48.824378967285156,
|
|
"step": 397
|
|
},
|
|
{
|
|
"epoch": 0.8335078534031414,
|
|
"grad_norm": 92.45420837402344,
|
|
"learning_rate": 4.168867517275806e-08,
|
|
"logits/chosen": 1.2100859880447388,
|
|
"logits/rejected": 1.4831223487854004,
|
|
"logps/chosen": -297.94940185546875,
|
|
"logps/ref_chosen": -252.48330688476562,
|
|
"logps/ref_rejected": -273.87139892578125,
|
|
"logps/rejected": -354.4050598144531,
|
|
"loss": 4.9205,
|
|
"margin_dpo/margin_mean": 35.06761932373047,
|
|
"margin_dpo/margin_std": 80.69606018066406,
|
|
"step": 398
|
|
},
|
|
{
|
|
"epoch": 0.8356020942408376,
|
|
"grad_norm": 82.78327941894531,
|
|
"learning_rate": 4.0682033438831584e-08,
|
|
"logits/chosen": 1.3988953828811646,
|
|
"logits/rejected": 1.5033973455429077,
|
|
"logps/chosen": -330.5037536621094,
|
|
"logps/ref_chosen": -277.4305419921875,
|
|
"logps/ref_rejected": -271.1197204589844,
|
|
"logps/rejected": -356.5560607910156,
|
|
"loss": 4.4852,
|
|
"margin_dpo/margin_mean": 32.36311340332031,
|
|
"margin_dpo/margin_std": 66.02288818359375,
|
|
"step": 399
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"grad_norm": 37.495018005371094,
|
|
"learning_rate": 3.968661679220467e-08,
|
|
"logits/chosen": 1.2931967973709106,
|
|
"logits/rejected": 1.240352988243103,
|
|
"logps/chosen": -299.0211181640625,
|
|
"logps/ref_chosen": -266.20025634765625,
|
|
"logps/ref_rejected": -217.865966796875,
|
|
"logps/rejected": -301.7166748046875,
|
|
"loss": 4.2739,
|
|
"margin_dpo/margin_mean": 51.02983093261719,
|
|
"margin_dpo/margin_std": 63.865108489990234,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"eval_logits/chosen": 1.193334937095642,
|
|
"eval_logits/rejected": 1.2366639375686646,
|
|
"eval_logps/chosen": -316.0413513183594,
|
|
"eval_logps/ref_chosen": -281.4588928222656,
|
|
"eval_logps/ref_rejected": -261.84954833984375,
|
|
"eval_logps/rejected": -345.1450500488281,
|
|
"eval_loss": 0.5601758360862732,
|
|
"eval_margin_dpo/margin_mean": 48.71305465698242,
|
|
"eval_margin_dpo/margin_std": 68.15460205078125,
|
|
"eval_runtime": 92.6746,
|
|
"eval_samples_per_second": 21.581,
|
|
"eval_steps_per_second": 1.349,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.8397905759162304,
|
|
"grad_norm": 42.00017547607422,
|
|
"learning_rate": 3.8702478614051345e-08,
|
|
"logits/chosen": 1.1790591478347778,
|
|
"logits/rejected": 1.3527344465255737,
|
|
"logps/chosen": -331.7237854003906,
|
|
"logps/ref_chosen": -293.3692932128906,
|
|
"logps/ref_rejected": -299.9609069824219,
|
|
"logps/rejected": -400.61529541015625,
|
|
"loss": 3.9949,
|
|
"margin_dpo/margin_mean": 62.299903869628906,
|
|
"margin_dpo/margin_std": 44.7177619934082,
|
|
"step": 401
|
|
},
|
|
{
|
|
"epoch": 0.8418848167539267,
|
|
"grad_norm": 58.57191848754883,
|
|
"learning_rate": 3.772967168071517e-08,
|
|
"logits/chosen": 1.4228373765945435,
|
|
"logits/rejected": 1.3410090208053589,
|
|
"logps/chosen": -332.11737060546875,
|
|
"logps/ref_chosen": -279.55889892578125,
|
|
"logps/ref_rejected": -259.6942138671875,
|
|
"logps/rejected": -331.76983642578125,
|
|
"loss": 4.3682,
|
|
"margin_dpo/margin_mean": 19.51715660095215,
|
|
"margin_dpo/margin_std": 81.67544555664062,
|
|
"step": 402
|
|
},
|
|
{
|
|
"epoch": 0.8439790575916231,
|
|
"grad_norm": 46.684837341308594,
|
|
"learning_rate": 3.676824816087978e-08,
|
|
"logits/chosen": 1.5173556804656982,
|
|
"logits/rejected": 1.6118801832199097,
|
|
"logps/chosen": -406.491455078125,
|
|
"logps/ref_chosen": -372.2436828613281,
|
|
"logps/ref_rejected": -285.5693359375,
|
|
"logps/rejected": -390.88140869140625,
|
|
"loss": 3.7148,
|
|
"margin_dpo/margin_mean": 71.0643081665039,
|
|
"margin_dpo/margin_std": 51.123374938964844,
|
|
"step": 403
|
|
},
|
|
{
|
|
"epoch": 0.8460732984293193,
|
|
"grad_norm": 38.361358642578125,
|
|
"learning_rate": 3.581825961277074e-08,
|
|
"logits/chosen": 1.4560322761535645,
|
|
"logits/rejected": 1.349548578262329,
|
|
"logps/chosen": -372.926513671875,
|
|
"logps/ref_chosen": -328.00860595703125,
|
|
"logps/ref_rejected": -278.35009765625,
|
|
"logps/rejected": -352.3278503417969,
|
|
"loss": 4.4734,
|
|
"margin_dpo/margin_mean": 29.05979347229004,
|
|
"margin_dpo/margin_std": 75.3277816772461,
|
|
"step": 404
|
|
},
|
|
{
|
|
"epoch": 0.8481675392670157,
|
|
"grad_norm": 50.2591438293457,
|
|
"learning_rate": 3.487975698139084e-08,
|
|
"logits/chosen": 1.3894712924957275,
|
|
"logits/rejected": 1.5616024732589722,
|
|
"logps/chosen": -270.36468505859375,
|
|
"logps/ref_chosen": -228.44178771972656,
|
|
"logps/ref_rejected": -244.33395385742188,
|
|
"logps/rejected": -330.79925537109375,
|
|
"loss": 4.2189,
|
|
"margin_dpo/margin_mean": 44.542388916015625,
|
|
"margin_dpo/margin_std": 59.48860168457031,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.8502617801047121,
|
|
"grad_norm": 63.63072204589844,
|
|
"learning_rate": 3.3952790595787986e-08,
|
|
"logits/chosen": 1.3200132846832275,
|
|
"logits/rejected": 1.2781833410263062,
|
|
"logps/chosen": -353.1100158691406,
|
|
"logps/ref_chosen": -321.44598388671875,
|
|
"logps/ref_rejected": -268.0445861816406,
|
|
"logps/rejected": -349.9507751464844,
|
|
"loss": 4.6261,
|
|
"margin_dpo/margin_mean": 50.24217987060547,
|
|
"margin_dpo/margin_std": 67.54312896728516,
|
|
"step": 406
|
|
},
|
|
{
|
|
"epoch": 0.8523560209424084,
|
|
"grad_norm": 53.97494888305664,
|
|
"learning_rate": 3.303741016635614e-08,
|
|
"logits/chosen": 1.3076931238174438,
|
|
"logits/rejected": 1.1266769170761108,
|
|
"logps/chosen": -309.05596923828125,
|
|
"logps/ref_chosen": -247.0522918701172,
|
|
"logps/ref_rejected": -186.8645782470703,
|
|
"logps/rejected": -270.8433837890625,
|
|
"loss": 4.3225,
|
|
"margin_dpo/margin_mean": 21.975143432617188,
|
|
"margin_dpo/margin_std": 54.5270881652832,
|
|
"step": 407
|
|
},
|
|
{
|
|
"epoch": 0.8544502617801047,
|
|
"grad_norm": 53.08890151977539,
|
|
"learning_rate": 3.2133664782169944e-08,
|
|
"logits/chosen": 1.1703412532806396,
|
|
"logits/rejected": 1.2470866441726685,
|
|
"logps/chosen": -253.00314331054688,
|
|
"logps/ref_chosen": -213.5513458251953,
|
|
"logps/ref_rejected": -267.9826354980469,
|
|
"logps/rejected": -361.67535400390625,
|
|
"loss": 4.2575,
|
|
"margin_dpo/margin_mean": 54.240936279296875,
|
|
"margin_dpo/margin_std": 45.82322692871094,
|
|
"step": 408
|
|
},
|
|
{
|
|
"epoch": 0.856544502617801,
|
|
"grad_norm": 48.80266571044922,
|
|
"learning_rate": 3.12416029083514e-08,
|
|
"logits/chosen": 1.4997256994247437,
|
|
"logits/rejected": 1.6531615257263184,
|
|
"logps/chosen": -320.41851806640625,
|
|
"logps/ref_chosen": -280.0785217285156,
|
|
"logps/ref_rejected": -304.26910400390625,
|
|
"logps/rejected": -392.4356994628906,
|
|
"loss": 4.3055,
|
|
"margin_dpo/margin_mean": 47.82660675048828,
|
|
"margin_dpo/margin_std": 75.04901885986328,
|
|
"step": 409
|
|
},
|
|
{
|
|
"epoch": 0.8586387434554974,
|
|
"grad_norm": 63.39336013793945,
|
|
"learning_rate": 3.036127238347164e-08,
|
|
"logits/chosen": 1.5989128351211548,
|
|
"logits/rejected": 1.5641684532165527,
|
|
"logps/chosen": -292.51336669921875,
|
|
"logps/ref_chosen": -260.9378662109375,
|
|
"logps/ref_rejected": -296.7695007324219,
|
|
"logps/rejected": -368.9515380859375,
|
|
"loss": 4.4542,
|
|
"margin_dpo/margin_mean": 40.606529235839844,
|
|
"margin_dpo/margin_std": 67.39288330078125,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.8607329842931937,
|
|
"grad_norm": 40.665950775146484,
|
|
"learning_rate": 2.9492720416985e-08,
|
|
"logits/chosen": 1.3025436401367188,
|
|
"logits/rejected": 1.4121453762054443,
|
|
"logps/chosen": -354.9593200683594,
|
|
"logps/ref_chosen": -330.0611877441406,
|
|
"logps/ref_rejected": -310.21368408203125,
|
|
"logps/rejected": -398.6880187988281,
|
|
"loss": 4.1892,
|
|
"margin_dpo/margin_mean": 63.576229095458984,
|
|
"margin_dpo/margin_std": 51.235904693603516,
|
|
"step": 411
|
|
},
|
|
{
|
|
"epoch": 0.86282722513089,
|
|
"grad_norm": 55.58880615234375,
|
|
"learning_rate": 2.863599358669755e-08,
|
|
"logits/chosen": 1.1845945119857788,
|
|
"logits/rejected": 1.3782296180725098,
|
|
"logps/chosen": -288.62408447265625,
|
|
"logps/ref_chosen": -254.76255798339844,
|
|
"logps/ref_rejected": -315.16156005859375,
|
|
"logps/rejected": -383.30535888671875,
|
|
"loss": 4.4912,
|
|
"margin_dpo/margin_mean": 34.28225326538086,
|
|
"margin_dpo/margin_std": 53.833274841308594,
|
|
"step": 412
|
|
},
|
|
{
|
|
"epoch": 0.8649214659685864,
|
|
"grad_norm": 81.09705352783203,
|
|
"learning_rate": 2.7791137836269158e-08,
|
|
"logits/chosen": 1.3580859899520874,
|
|
"logits/rejected": 1.3456140756607056,
|
|
"logps/chosen": -294.88671875,
|
|
"logps/ref_chosen": -260.4962463378906,
|
|
"logps/ref_rejected": -250.442626953125,
|
|
"logps/rejected": -319.46563720703125,
|
|
"loss": 4.5037,
|
|
"margin_dpo/margin_mean": 34.632568359375,
|
|
"margin_dpo/margin_std": 63.56281280517578,
|
|
"step": 413
|
|
},
|
|
{
|
|
"epoch": 0.8670157068062827,
|
|
"grad_norm": 51.592769622802734,
|
|
"learning_rate": 2.6958198472749717e-08,
|
|
"logits/chosen": 1.6022746562957764,
|
|
"logits/rejected": 1.5306450128555298,
|
|
"logps/chosen": -428.2533264160156,
|
|
"logps/ref_chosen": -391.94610595703125,
|
|
"logps/ref_rejected": -287.8221130371094,
|
|
"logps/rejected": -345.431396484375,
|
|
"loss": 4.3943,
|
|
"margin_dpo/margin_mean": 21.30208396911621,
|
|
"margin_dpo/margin_std": 81.18660736083984,
|
|
"step": 414
|
|
},
|
|
{
|
|
"epoch": 0.8691099476439791,
|
|
"grad_norm": 49.326297760009766,
|
|
"learning_rate": 2.613722016414943e-08,
|
|
"logits/chosen": 0.8005275726318359,
|
|
"logits/rejected": 0.8713321685791016,
|
|
"logps/chosen": -245.8915557861328,
|
|
"logps/ref_chosen": -223.82870483398438,
|
|
"logps/ref_rejected": -206.082763671875,
|
|
"logps/rejected": -291.24188232421875,
|
|
"loss": 4.0595,
|
|
"margin_dpo/margin_mean": 63.09626007080078,
|
|
"margin_dpo/margin_std": 63.13758087158203,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.8712041884816754,
|
|
"grad_norm": 48.53297424316406,
|
|
"learning_rate": 2.5328246937043525e-08,
|
|
"logits/chosen": 1.264346718788147,
|
|
"logits/rejected": 1.331539273262024,
|
|
"logps/chosen": -316.0765686035156,
|
|
"logps/ref_chosen": -303.91656494140625,
|
|
"logps/ref_rejected": -267.0210266113281,
|
|
"logps/rejected": -343.7052917480469,
|
|
"loss": 4.154,
|
|
"margin_dpo/margin_mean": 64.5242919921875,
|
|
"margin_dpo/margin_std": 58.65679168701172,
|
|
"step": 416
|
|
},
|
|
{
|
|
"epoch": 0.8732984293193717,
|
|
"grad_norm": 43.959312438964844,
|
|
"learning_rate": 2.4531322174210973e-08,
|
|
"logits/chosen": 1.207326054573059,
|
|
"logits/rejected": 1.3471075296401978,
|
|
"logps/chosen": -298.7107849121094,
|
|
"logps/ref_chosen": -249.8788604736328,
|
|
"logps/ref_rejected": -210.48683166503906,
|
|
"logps/rejected": -290.503662109375,
|
|
"loss": 4.4556,
|
|
"margin_dpo/margin_mean": 31.184890747070312,
|
|
"margin_dpo/margin_std": 68.83541107177734,
|
|
"step": 417
|
|
},
|
|
{
|
|
"epoch": 0.875392670157068,
|
|
"grad_norm": 57.87160110473633,
|
|
"learning_rate": 2.3746488612308295e-08,
|
|
"logits/chosen": 1.1209101676940918,
|
|
"logits/rejected": 1.000208854675293,
|
|
"logps/chosen": -411.3262023925781,
|
|
"logps/ref_chosen": -355.4482421875,
|
|
"logps/ref_rejected": -344.9490661621094,
|
|
"logps/rejected": -433.7691650390625,
|
|
"loss": 4.5932,
|
|
"margin_dpo/margin_mean": 32.94209671020508,
|
|
"margin_dpo/margin_std": 72.00234985351562,
|
|
"step": 418
|
|
},
|
|
{
|
|
"epoch": 0.8774869109947644,
|
|
"grad_norm": 55.62238311767578,
|
|
"learning_rate": 2.297378833957761e-08,
|
|
"logits/chosen": 1.6920902729034424,
|
|
"logits/rejected": 1.618727445602417,
|
|
"logps/chosen": -433.79205322265625,
|
|
"logps/ref_chosen": -381.6947021484375,
|
|
"logps/ref_rejected": -322.436767578125,
|
|
"logps/rejected": -426.5209045410156,
|
|
"loss": 4.2079,
|
|
"margin_dpo/margin_mean": 51.986717224121094,
|
|
"margin_dpo/margin_std": 64.11946868896484,
|
|
"step": 419
|
|
},
|
|
{
|
|
"epoch": 0.8795811518324608,
|
|
"grad_norm": 42.748905181884766,
|
|
"learning_rate": 2.2213262793589482e-08,
|
|
"logits/chosen": 1.0813815593719482,
|
|
"logits/rejected": 1.1282932758331299,
|
|
"logps/chosen": -290.3770446777344,
|
|
"logps/ref_chosen": -255.09539794921875,
|
|
"logps/ref_rejected": -254.58306884765625,
|
|
"logps/rejected": -328.9988708496094,
|
|
"loss": 4.2003,
|
|
"margin_dpo/margin_mean": 39.13414001464844,
|
|
"margin_dpo/margin_std": 68.49993896484375,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.881675392670157,
|
|
"grad_norm": 62.450592041015625,
|
|
"learning_rate": 2.1464952759020856e-08,
|
|
"logits/chosen": 1.2170071601867676,
|
|
"logits/rejected": 1.0278328657150269,
|
|
"logps/chosen": -303.1274719238281,
|
|
"logps/ref_chosen": -280.7524719238281,
|
|
"logps/ref_rejected": -187.39218139648438,
|
|
"logps/rejected": -256.837646484375,
|
|
"loss": 4.0246,
|
|
"margin_dpo/margin_mean": 47.07042694091797,
|
|
"margin_dpo/margin_std": 56.84251403808594,
|
|
"step": 421
|
|
},
|
|
{
|
|
"epoch": 0.8837696335078534,
|
|
"grad_norm": 56.18370056152344,
|
|
"learning_rate": 2.07288983654679e-08,
|
|
"logits/chosen": 1.3475362062454224,
|
|
"logits/rejected": 1.4008029699325562,
|
|
"logps/chosen": -307.1512145996094,
|
|
"logps/ref_chosen": -278.18890380859375,
|
|
"logps/ref_rejected": -250.71591186523438,
|
|
"logps/rejected": -318.3471984863281,
|
|
"loss": 4.6777,
|
|
"margin_dpo/margin_mean": 38.6689453125,
|
|
"margin_dpo/margin_std": 72.85610961914062,
|
|
"step": 422
|
|
},
|
|
{
|
|
"epoch": 0.8858638743455497,
|
|
"grad_norm": 64.5829849243164,
|
|
"learning_rate": 2.0005139085293942e-08,
|
|
"logits/chosen": 1.3403944969177246,
|
|
"logits/rejected": 1.4666098356246948,
|
|
"logps/chosen": -301.6260986328125,
|
|
"logps/ref_chosen": -281.21820068359375,
|
|
"logps/ref_rejected": -296.73907470703125,
|
|
"logps/rejected": -367.79888916015625,
|
|
"loss": 4.1139,
|
|
"margin_dpo/margin_mean": 50.65192413330078,
|
|
"margin_dpo/margin_std": 72.86007690429688,
|
|
"step": 423
|
|
},
|
|
{
|
|
"epoch": 0.8879581151832461,
|
|
"grad_norm": 43.40376281738281,
|
|
"learning_rate": 1.9293713731512673e-08,
|
|
"logits/chosen": 1.1475857496261597,
|
|
"logits/rejected": 1.005727767944336,
|
|
"logps/chosen": -361.1445617675781,
|
|
"logps/ref_chosen": -339.32550048828125,
|
|
"logps/ref_rejected": -274.3222351074219,
|
|
"logps/rejected": -354.5352478027344,
|
|
"loss": 4.1116,
|
|
"margin_dpo/margin_mean": 58.39393615722656,
|
|
"margin_dpo/margin_std": 50.71818542480469,
|
|
"step": 424
|
|
},
|
|
{
|
|
"epoch": 0.8900523560209425,
|
|
"grad_norm": 46.304969787597656,
|
|
"learning_rate": 1.8594660455706763e-08,
|
|
"logits/chosen": 1.2751210927963257,
|
|
"logits/rejected": 1.5024828910827637,
|
|
"logps/chosen": -283.94232177734375,
|
|
"logps/ref_chosen": -254.7490997314453,
|
|
"logps/ref_rejected": -266.1432800292969,
|
|
"logps/rejected": -349.154541015625,
|
|
"loss": 4.3925,
|
|
"margin_dpo/margin_mean": 53.81803894042969,
|
|
"margin_dpo/margin_std": 49.791236877441406,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.8921465968586387,
|
|
"grad_norm": 41.32174301147461,
|
|
"learning_rate": 1.7908016745981856e-08,
|
|
"logits/chosen": 0.896172285079956,
|
|
"logits/rejected": 1.0513912439346313,
|
|
"logps/chosen": -284.55767822265625,
|
|
"logps/ref_chosen": -264.97216796875,
|
|
"logps/ref_rejected": -244.05935668945312,
|
|
"logps/rejected": -316.9269714355469,
|
|
"loss": 4.2069,
|
|
"margin_dpo/margin_mean": 53.282081604003906,
|
|
"margin_dpo/margin_std": 63.170196533203125,
|
|
"step": 426
|
|
},
|
|
{
|
|
"epoch": 0.8942408376963351,
|
|
"grad_norm": 42.068763732910156,
|
|
"learning_rate": 1.7233819424956247e-08,
|
|
"logits/chosen": 1.2293376922607422,
|
|
"logits/rejected": 1.2114195823669434,
|
|
"logps/chosen": -329.5334167480469,
|
|
"logps/ref_chosen": -301.6879577636719,
|
|
"logps/ref_rejected": -265.6783752441406,
|
|
"logps/rejected": -380.6156311035156,
|
|
"loss": 3.7144,
|
|
"margin_dpo/margin_mean": 87.09181213378906,
|
|
"margin_dpo/margin_std": 52.67485046386719,
|
|
"step": 427
|
|
},
|
|
{
|
|
"epoch": 0.8963350785340314,
|
|
"grad_norm": 55.713130950927734,
|
|
"learning_rate": 1.6572104647786245e-08,
|
|
"logits/chosen": 1.5100287199020386,
|
|
"logits/rejected": 1.70786714553833,
|
|
"logps/chosen": -415.50616455078125,
|
|
"logps/ref_chosen": -376.43316650390625,
|
|
"logps/ref_rejected": -334.24676513671875,
|
|
"logps/rejected": -440.6484375,
|
|
"loss": 3.7971,
|
|
"margin_dpo/margin_mean": 67.32867431640625,
|
|
"margin_dpo/margin_std": 66.57231140136719,
|
|
"step": 428
|
|
},
|
|
{
|
|
"epoch": 0.8984293193717278,
|
|
"grad_norm": 70.71985626220703,
|
|
"learning_rate": 1.5922907900227017e-08,
|
|
"logits/chosen": 1.4469355344772339,
|
|
"logits/rejected": 1.4792401790618896,
|
|
"logps/chosen": -247.02073669433594,
|
|
"logps/ref_chosen": -218.89503479003906,
|
|
"logps/ref_rejected": -236.546630859375,
|
|
"logps/rejected": -322.55419921875,
|
|
"loss": 4.4589,
|
|
"margin_dpo/margin_mean": 57.881866455078125,
|
|
"margin_dpo/margin_std": 76.20121002197266,
|
|
"step": 429
|
|
},
|
|
{
|
|
"epoch": 0.900523560209424,
|
|
"grad_norm": 92.4472427368164,
|
|
"learning_rate": 1.5286263996730026e-08,
|
|
"logits/chosen": 1.3988643884658813,
|
|
"logits/rejected": 1.5516958236694336,
|
|
"logps/chosen": -318.7999267578125,
|
|
"logps/ref_chosen": -281.9652099609375,
|
|
"logps/ref_rejected": -266.40411376953125,
|
|
"logps/rejected": -329.5230712890625,
|
|
"loss": 4.4918,
|
|
"margin_dpo/margin_mean": 26.284305572509766,
|
|
"margin_dpo/margin_std": 64.46965789794922,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.9026178010471204,
|
|
"grad_norm": 42.31532287597656,
|
|
"learning_rate": 1.4662207078575684e-08,
|
|
"logits/chosen": 1.7694166898727417,
|
|
"logits/rejected": 1.8366968631744385,
|
|
"logps/chosen": -332.827880859375,
|
|
"logps/ref_chosen": -286.10888671875,
|
|
"logps/ref_rejected": -274.0017395019531,
|
|
"logps/rejected": -361.9835510253906,
|
|
"loss": 4.5107,
|
|
"margin_dpo/margin_mean": 41.2628173828125,
|
|
"margin_dpo/margin_std": 59.571441650390625,
|
|
"step": 431
|
|
},
|
|
{
|
|
"epoch": 0.9047120418848168,
|
|
"grad_norm": 37.992881774902344,
|
|
"learning_rate": 1.40507706120426e-08,
|
|
"logits/chosen": 1.4250589609146118,
|
|
"logits/rejected": 1.6473501920700073,
|
|
"logps/chosen": -348.860595703125,
|
|
"logps/ref_chosen": -316.9443359375,
|
|
"logps/ref_rejected": -358.3585510253906,
|
|
"logps/rejected": -438.4804382324219,
|
|
"loss": 4.1495,
|
|
"margin_dpo/margin_mean": 48.20561981201172,
|
|
"margin_dpo/margin_std": 72.83356475830078,
|
|
"step": 432
|
|
},
|
|
{
|
|
"epoch": 0.9068062827225131,
|
|
"grad_norm": 56.42523193359375,
|
|
"learning_rate": 1.345198738661285e-08,
|
|
"logits/chosen": 1.288971185684204,
|
|
"logits/rejected": 1.2402310371398926,
|
|
"logps/chosen": -312.3082275390625,
|
|
"logps/ref_chosen": -282.2297668457031,
|
|
"logps/ref_rejected": -258.5012512207031,
|
|
"logps/rejected": -328.95513916015625,
|
|
"loss": 4.5137,
|
|
"margin_dpo/margin_mean": 40.375389099121094,
|
|
"margin_dpo/margin_std": 55.306190490722656,
|
|
"step": 433
|
|
},
|
|
{
|
|
"epoch": 0.9089005235602095,
|
|
"grad_norm": 34.96892547607422,
|
|
"learning_rate": 1.2865889513213628e-08,
|
|
"logits/chosen": 1.7188708782196045,
|
|
"logits/rejected": 1.7233338356018066,
|
|
"logps/chosen": -339.9571533203125,
|
|
"logps/ref_chosen": -313.5975646972656,
|
|
"logps/ref_rejected": -293.44097900390625,
|
|
"logps/rejected": -374.973876953125,
|
|
"loss": 4.2523,
|
|
"margin_dpo/margin_mean": 55.173301696777344,
|
|
"margin_dpo/margin_std": 61.733543395996094,
|
|
"step": 434
|
|
},
|
|
{
|
|
"epoch": 0.9109947643979057,
|
|
"grad_norm": 84.76171875,
|
|
"learning_rate": 1.2292508422495157e-08,
|
|
"logits/chosen": 1.584928035736084,
|
|
"logits/rejected": 1.718322515487671,
|
|
"logps/chosen": -211.07582092285156,
|
|
"logps/ref_chosen": -191.58889770507812,
|
|
"logps/ref_rejected": -206.38133239746094,
|
|
"logps/rejected": -269.05389404296875,
|
|
"loss": 4.3653,
|
|
"margin_dpo/margin_mean": 43.18561553955078,
|
|
"margin_dpo/margin_std": 55.161441802978516,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.9130890052356021,
|
|
"grad_norm": 57.35254669189453,
|
|
"learning_rate": 1.1731874863145142e-08,
|
|
"logits/chosen": 1.098213791847229,
|
|
"logits/rejected": 1.1476788520812988,
|
|
"logps/chosen": -349.2528076171875,
|
|
"logps/ref_chosen": -329.4399719238281,
|
|
"logps/ref_rejected": -310.59783935546875,
|
|
"logps/rejected": -374.02459716796875,
|
|
"loss": 4.8351,
|
|
"margin_dpo/margin_mean": 43.61388397216797,
|
|
"margin_dpo/margin_std": 60.17463302612305,
|
|
"step": 436
|
|
},
|
|
{
|
|
"epoch": 0.9151832460732985,
|
|
"grad_norm": 33.499061584472656,
|
|
"learning_rate": 1.118401890024001e-08,
|
|
"logits/chosen": 1.4528710842132568,
|
|
"logits/rejected": 1.5862656831741333,
|
|
"logps/chosen": -284.3026123046875,
|
|
"logps/ref_chosen": -245.99761962890625,
|
|
"logps/ref_rejected": -340.40283203125,
|
|
"logps/rejected": -442.78497314453125,
|
|
"loss": 4.1697,
|
|
"margin_dpo/margin_mean": 64.07716369628906,
|
|
"margin_dpo/margin_std": 62.12276077270508,
|
|
"step": 437
|
|
},
|
|
{
|
|
"epoch": 0.9172774869109948,
|
|
"grad_norm": 139.29515075683594,
|
|
"learning_rate": 1.06489699136324e-08,
|
|
"logits/chosen": 1.1451233625411987,
|
|
"logits/rejected": 1.288071870803833,
|
|
"logps/chosen": -289.2056884765625,
|
|
"logps/ref_chosen": -264.5708923339844,
|
|
"logps/ref_rejected": -263.8647155761719,
|
|
"logps/rejected": -315.4420471191406,
|
|
"loss": 5.1452,
|
|
"margin_dpo/margin_mean": 26.942535400390625,
|
|
"margin_dpo/margin_std": 64.0194320678711,
|
|
"step": 438
|
|
},
|
|
{
|
|
"epoch": 0.9193717277486911,
|
|
"grad_norm": 45.723548889160156,
|
|
"learning_rate": 1.0126756596375685e-08,
|
|
"logits/chosen": 1.449920892715454,
|
|
"logits/rejected": 1.4370176792144775,
|
|
"logps/chosen": -275.9388427734375,
|
|
"logps/ref_chosen": -236.2272491455078,
|
|
"logps/ref_rejected": -260.26531982421875,
|
|
"logps/rejected": -337.4900207519531,
|
|
"loss": 4.4005,
|
|
"margin_dpo/margin_mean": 37.513099670410156,
|
|
"margin_dpo/margin_std": 69.3291244506836,
|
|
"step": 439
|
|
},
|
|
{
|
|
"epoch": 0.9214659685863874,
|
|
"grad_norm": 51.19523620605469,
|
|
"learning_rate": 9.617406953185136e-09,
|
|
"logits/chosen": 1.4269630908966064,
|
|
"logits/rejected": 1.2528316974639893,
|
|
"logps/chosen": -450.7151794433594,
|
|
"logps/ref_chosen": -402.7833557128906,
|
|
"logps/ref_rejected": -285.3439636230469,
|
|
"logps/rejected": -360.1721496582031,
|
|
"loss": 4.77,
|
|
"margin_dpo/margin_mean": 26.89635467529297,
|
|
"margin_dpo/margin_std": 57.31180191040039,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.9235602094240838,
|
|
"grad_norm": 45.438751220703125,
|
|
"learning_rate": 9.12094829893642e-09,
|
|
"logits/chosen": 1.4254412651062012,
|
|
"logits/rejected": 1.6387895345687866,
|
|
"logps/chosen": -382.6518859863281,
|
|
"logps/ref_chosen": -348.18212890625,
|
|
"logps/ref_rejected": -375.537353515625,
|
|
"logps/rejected": -471.38824462890625,
|
|
"loss": 4.0819,
|
|
"margin_dpo/margin_mean": 61.381141662597656,
|
|
"margin_dpo/margin_std": 54.50360870361328,
|
|
"step": 441
|
|
},
|
|
{
|
|
"epoch": 0.9256544502617801,
|
|
"grad_norm": 53.282371520996094,
|
|
"learning_rate": 8.637407257200496e-09,
|
|
"logits/chosen": 1.2402985095977783,
|
|
"logits/rejected": 1.3638098239898682,
|
|
"logps/chosen": -275.3695373535156,
|
|
"logps/ref_chosen": -232.696044921875,
|
|
"logps/ref_rejected": -204.60752868652344,
|
|
"logps/rejected": -279.0164489746094,
|
|
"loss": 4.6929,
|
|
"margin_dpo/margin_mean": 31.735416412353516,
|
|
"margin_dpo/margin_std": 70.92927551269531,
|
|
"step": 442
|
|
},
|
|
{
|
|
"epoch": 0.9277486910994764,
|
|
"grad_norm": 41.01181411743164,
|
|
"learning_rate": 8.166809758815895e-09,
|
|
"logits/chosen": 1.3078192472457886,
|
|
"logits/rejected": 1.3253602981567383,
|
|
"logps/chosen": -330.3334655761719,
|
|
"logps/ref_chosen": -275.13873291015625,
|
|
"logps/ref_rejected": -264.1988830566406,
|
|
"logps/rejected": -356.9568176269531,
|
|
"loss": 4.5058,
|
|
"margin_dpo/margin_mean": 37.56320571899414,
|
|
"margin_dpo/margin_std": 54.74559783935547,
|
|
"step": 443
|
|
},
|
|
{
|
|
"epoch": 0.9298429319371728,
|
|
"grad_norm": 51.980506896972656,
|
|
"learning_rate": 7.709181040498253e-09,
|
|
"logits/chosen": 0.7922985553741455,
|
|
"logits/rejected": 0.9772998690605164,
|
|
"logps/chosen": -340.13336181640625,
|
|
"logps/ref_chosen": -305.7708740234375,
|
|
"logps/ref_rejected": -292.68731689453125,
|
|
"logps/rejected": -366.3361511230469,
|
|
"loss": 4.3732,
|
|
"margin_dpo/margin_mean": 39.28638458251953,
|
|
"margin_dpo/margin_std": 50.5058479309082,
|
|
"step": 444
|
|
},
|
|
{
|
|
"epoch": 0.9319371727748691,
|
|
"grad_norm": 50.610992431640625,
|
|
"learning_rate": 7.2645456434869965e-09,
|
|
"logits/chosen": 1.336693286895752,
|
|
"logits/rejected": 1.4031049013137817,
|
|
"logps/chosen": -257.5855712890625,
|
|
"logps/ref_chosen": -240.496337890625,
|
|
"logps/ref_rejected": -226.0730743408203,
|
|
"logps/rejected": -288.43756103515625,
|
|
"loss": 4.6998,
|
|
"margin_dpo/margin_mean": 45.27527618408203,
|
|
"margin_dpo/margin_std": 74.52619934082031,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.9340314136125655,
|
|
"grad_norm": 62.949615478515625,
|
|
"learning_rate": 6.832927412229017e-09,
|
|
"logits/chosen": 1.3089869022369385,
|
|
"logits/rejected": 1.283385992050171,
|
|
"logps/chosen": -267.8168029785156,
|
|
"logps/ref_chosen": -244.18284606933594,
|
|
"logps/ref_rejected": -211.3776397705078,
|
|
"logps/rejected": -278.58465576171875,
|
|
"loss": 4.3051,
|
|
"margin_dpo/margin_mean": 43.57305145263672,
|
|
"margin_dpo/margin_std": 57.471893310546875,
|
|
"step": 446
|
|
},
|
|
{
|
|
"epoch": 0.9361256544502617,
|
|
"grad_norm": 52.591312408447266,
|
|
"learning_rate": 6.414349493100129e-09,
|
|
"logits/chosen": 1.3533639907836914,
|
|
"logits/rejected": 1.4261730909347534,
|
|
"logps/chosen": -258.5954895019531,
|
|
"logps/ref_chosen": -237.29592895507812,
|
|
"logps/ref_rejected": -246.57034301757812,
|
|
"logps/rejected": -316.0662841796875,
|
|
"loss": 4.0452,
|
|
"margin_dpo/margin_mean": 48.1964225769043,
|
|
"margin_dpo/margin_std": 65.75294494628906,
|
|
"step": 447
|
|
},
|
|
{
|
|
"epoch": 0.9382198952879581,
|
|
"grad_norm": 46.954933166503906,
|
|
"learning_rate": 6.0088343331638756e-09,
|
|
"logits/chosen": 1.6584538221359253,
|
|
"logits/rejected": 1.7075550556182861,
|
|
"logps/chosen": -336.6059265136719,
|
|
"logps/ref_chosen": -308.6024475097656,
|
|
"logps/ref_rejected": -277.87921142578125,
|
|
"logps/rejected": -346.0470275878906,
|
|
"loss": 4.3113,
|
|
"margin_dpo/margin_mean": 40.164310455322266,
|
|
"margin_dpo/margin_std": 57.308650970458984,
|
|
"step": 448
|
|
},
|
|
{
|
|
"epoch": 0.9403141361256544,
|
|
"grad_norm": 54.72856140136719,
|
|
"learning_rate": 5.616403678967624e-09,
|
|
"logits/chosen": 1.8703218698501587,
|
|
"logits/rejected": 1.5918076038360596,
|
|
"logps/chosen": -386.6697998046875,
|
|
"logps/ref_chosen": -376.94281005859375,
|
|
"logps/ref_rejected": -267.11236572265625,
|
|
"logps/rejected": -342.6089782714844,
|
|
"loss": 4.1562,
|
|
"margin_dpo/margin_mean": 65.76960754394531,
|
|
"margin_dpo/margin_std": 61.83976745605469,
|
|
"step": 449
|
|
},
|
|
{
|
|
"epoch": 0.9424083769633508,
|
|
"grad_norm": 65.85668182373047,
|
|
"learning_rate": 5.2370785753763356e-09,
|
|
"logits/chosen": 1.6628509759902954,
|
|
"logits/rejected": 1.4311879873275757,
|
|
"logps/chosen": -327.4361572265625,
|
|
"logps/ref_chosen": -312.619384765625,
|
|
"logps/ref_rejected": -215.62857055664062,
|
|
"logps/rejected": -281.2243957519531,
|
|
"loss": 4.5566,
|
|
"margin_dpo/margin_mean": 50.77903366088867,
|
|
"margin_dpo/margin_std": 56.17271041870117,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.9445026178010472,
|
|
"grad_norm": 55.37830352783203,
|
|
"learning_rate": 4.8708793644441086e-09,
|
|
"logits/chosen": 1.4510207176208496,
|
|
"logits/rejected": 1.5802251100540161,
|
|
"logps/chosen": -330.12811279296875,
|
|
"logps/ref_chosen": -296.6983947753906,
|
|
"logps/ref_rejected": -312.4747619628906,
|
|
"logps/rejected": -392.5194396972656,
|
|
"loss": 4.3406,
|
|
"margin_dpo/margin_mean": 46.6149787902832,
|
|
"margin_dpo/margin_std": 63.69156265258789,
|
|
"step": 451
|
|
},
|
|
{
|
|
"epoch": 0.9465968586387434,
|
|
"grad_norm": 50.63416290283203,
|
|
"learning_rate": 4.517825684323323e-09,
|
|
"logits/chosen": 1.3996615409851074,
|
|
"logits/rejected": 1.581308364868164,
|
|
"logps/chosen": -329.9542541503906,
|
|
"logps/ref_chosen": -294.50958251953125,
|
|
"logps/ref_rejected": -295.56097412109375,
|
|
"logps/rejected": -362.0646667480469,
|
|
"loss": 4.3711,
|
|
"margin_dpo/margin_mean": 31.059064865112305,
|
|
"margin_dpo/margin_std": 63.657875061035156,
|
|
"step": 452
|
|
},
|
|
{
|
|
"epoch": 0.9486910994764398,
|
|
"grad_norm": 44.21456527709961,
|
|
"learning_rate": 4.1779364682113794e-09,
|
|
"logits/chosen": 1.4765185117721558,
|
|
"logits/rejected": 1.6153349876403809,
|
|
"logps/chosen": -341.7298278808594,
|
|
"logps/ref_chosen": -308.21917724609375,
|
|
"logps/ref_rejected": -328.357421875,
|
|
"logps/rejected": -414.666748046875,
|
|
"loss": 4.1702,
|
|
"margin_dpo/margin_mean": 52.798675537109375,
|
|
"margin_dpo/margin_std": 59.44575881958008,
|
|
"step": 453
|
|
},
|
|
{
|
|
"epoch": 0.9507853403141361,
|
|
"grad_norm": 36.86773681640625,
|
|
"learning_rate": 3.851229943335393e-09,
|
|
"logits/chosen": 1.8053613901138306,
|
|
"logits/rejected": 1.7590644359588623,
|
|
"logps/chosen": -365.978759765625,
|
|
"logps/ref_chosen": -332.5453796386719,
|
|
"logps/ref_rejected": -267.130615234375,
|
|
"logps/rejected": -339.3133544921875,
|
|
"loss": 4.3145,
|
|
"margin_dpo/margin_mean": 38.749393463134766,
|
|
"margin_dpo/margin_std": 65.05109405517578,
|
|
"step": 454
|
|
},
|
|
{
|
|
"epoch": 0.9528795811518325,
|
|
"grad_norm": 49.20968246459961,
|
|
"learning_rate": 3.5377236299748147e-09,
|
|
"logits/chosen": 1.3919376134872437,
|
|
"logits/rejected": 1.5252500772476196,
|
|
"logps/chosen": -261.4302978515625,
|
|
"logps/ref_chosen": -240.13719177246094,
|
|
"logps/ref_rejected": -233.28451538085938,
|
|
"logps/rejected": -313.81097412109375,
|
|
"loss": 4.5531,
|
|
"margin_dpo/margin_mean": 59.233402252197266,
|
|
"margin_dpo/margin_std": 58.3806037902832,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.9549738219895288,
|
|
"grad_norm": 42.70205307006836,
|
|
"learning_rate": 3.2374343405217884e-09,
|
|
"logits/chosen": 1.635735034942627,
|
|
"logits/rejected": 1.8064732551574707,
|
|
"logps/chosen": -370.0901184082031,
|
|
"logps/ref_chosen": -334.82666015625,
|
|
"logps/ref_rejected": -313.20404052734375,
|
|
"logps/rejected": -386.83837890625,
|
|
"loss": 4.1309,
|
|
"margin_dpo/margin_mean": 38.370887756347656,
|
|
"margin_dpo/margin_std": 76.44358825683594,
|
|
"step": 456
|
|
},
|
|
{
|
|
"epoch": 0.9570680628272251,
|
|
"grad_norm": 44.188438415527344,
|
|
"learning_rate": 2.9503781785795713e-09,
|
|
"logits/chosen": 1.4283052682876587,
|
|
"logits/rejected": 1.370866298675537,
|
|
"logps/chosen": -319.96136474609375,
|
|
"logps/ref_chosen": -299.60650634765625,
|
|
"logps/ref_rejected": -263.5287780761719,
|
|
"logps/rejected": -360.9966125488281,
|
|
"loss": 4.2728,
|
|
"margin_dpo/margin_mean": 77.11296844482422,
|
|
"margin_dpo/margin_std": 65.07032775878906,
|
|
"step": 457
|
|
},
|
|
{
|
|
"epoch": 0.9591623036649215,
|
|
"grad_norm": 53.56594467163086,
|
|
"learning_rate": 2.6765705380989432e-09,
|
|
"logits/chosen": 1.6065071821212769,
|
|
"logits/rejected": 1.4925872087478638,
|
|
"logps/chosen": -309.9053955078125,
|
|
"logps/ref_chosen": -272.7044372558594,
|
|
"logps/ref_rejected": -235.6636962890625,
|
|
"logps/rejected": -300.68353271484375,
|
|
"loss": 4.3749,
|
|
"margin_dpo/margin_mean": 27.818876266479492,
|
|
"margin_dpo/margin_std": 55.92146301269531,
|
|
"step": 458
|
|
},
|
|
{
|
|
"epoch": 0.9612565445026178,
|
|
"grad_norm": 38.86277389526367,
|
|
"learning_rate": 2.416026102552732e-09,
|
|
"logits/chosen": 1.428498387336731,
|
|
"logits/rejected": 1.3140032291412354,
|
|
"logps/chosen": -328.8189697265625,
|
|
"logps/ref_chosen": -280.32196044921875,
|
|
"logps/ref_rejected": -217.7216339111328,
|
|
"logps/rejected": -297.50244140625,
|
|
"loss": 4.8445,
|
|
"margin_dpo/margin_mean": 31.283798217773438,
|
|
"margin_dpo/margin_std": 54.594871520996094,
|
|
"step": 459
|
|
},
|
|
{
|
|
"epoch": 0.9633507853403142,
|
|
"grad_norm": 91.57032012939453,
|
|
"learning_rate": 2.168758844148272e-09,
|
|
"logits/chosen": 1.443167805671692,
|
|
"logits/rejected": 1.4732171297073364,
|
|
"logps/chosen": -426.33984375,
|
|
"logps/ref_chosen": -387.5949401855469,
|
|
"logps/ref_rejected": -289.61505126953125,
|
|
"logps/rejected": -362.8568115234375,
|
|
"loss": 4.8551,
|
|
"margin_dpo/margin_mean": 34.49686050415039,
|
|
"margin_dpo/margin_std": 72.81037902832031,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.9654450261780104,
|
|
"grad_norm": 57.414451599121094,
|
|
"learning_rate": 1.9347820230782295e-09,
|
|
"logits/chosen": 1.5660542249679565,
|
|
"logits/rejected": 1.520784616470337,
|
|
"logps/chosen": -268.8662414550781,
|
|
"logps/ref_chosen": -247.67520141601562,
|
|
"logps/ref_rejected": -227.18458557128906,
|
|
"logps/rejected": -311.7144775390625,
|
|
"loss": 4.5036,
|
|
"margin_dpo/margin_mean": 63.33882141113281,
|
|
"margin_dpo/margin_std": 73.68379974365234,
|
|
"step": 461
|
|
},
|
|
{
|
|
"epoch": 0.9675392670157068,
|
|
"grad_norm": 49.87493896484375,
|
|
"learning_rate": 1.7141081868094209e-09,
|
|
"logits/chosen": 1.4014174938201904,
|
|
"logits/rejected": 1.3472931385040283,
|
|
"logps/chosen": -378.84088134765625,
|
|
"logps/ref_chosen": -350.8253173828125,
|
|
"logps/ref_rejected": -262.5743713378906,
|
|
"logps/rejected": -355.7335205078125,
|
|
"loss": 4.116,
|
|
"margin_dpo/margin_mean": 65.14358520507812,
|
|
"margin_dpo/margin_std": 64.9459457397461,
|
|
"step": 462
|
|
},
|
|
{
|
|
"epoch": 0.9696335078534032,
|
|
"grad_norm": 84.11515808105469,
|
|
"learning_rate": 1.5067491694100153e-09,
|
|
"logits/chosen": 1.2984429597854614,
|
|
"logits/rejected": 1.3639535903930664,
|
|
"logps/chosen": -271.6904602050781,
|
|
"logps/ref_chosen": -229.31683349609375,
|
|
"logps/ref_rejected": -227.2904815673828,
|
|
"logps/rejected": -293.2787170410156,
|
|
"loss": 4.4943,
|
|
"margin_dpo/margin_mean": 23.614595413208008,
|
|
"margin_dpo/margin_std": 70.91625213623047,
|
|
"step": 463
|
|
},
|
|
{
|
|
"epoch": 0.9717277486910995,
|
|
"grad_norm": 51.687442779541016,
|
|
"learning_rate": 1.3127160909147672e-09,
|
|
"logits/chosen": 1.7068381309509277,
|
|
"logits/rejected": 1.6888086795806885,
|
|
"logps/chosen": -248.65789794921875,
|
|
"logps/ref_chosen": -226.55776977539062,
|
|
"logps/ref_rejected": -208.3471221923828,
|
|
"logps/rejected": -285.44244384765625,
|
|
"loss": 4.4572,
|
|
"margin_dpo/margin_mean": 54.995182037353516,
|
|
"margin_dpo/margin_std": 61.694210052490234,
|
|
"step": 464
|
|
},
|
|
{
|
|
"epoch": 0.9738219895287958,
|
|
"grad_norm": 37.04083251953125,
|
|
"learning_rate": 1.1320193567288527e-09,
|
|
"logits/chosen": 1.1953487396240234,
|
|
"logits/rejected": 1.2031378746032715,
|
|
"logps/chosen": -311.39080810546875,
|
|
"logps/ref_chosen": -287.9401550292969,
|
|
"logps/ref_rejected": -305.5926818847656,
|
|
"logps/rejected": -382.921630859375,
|
|
"loss": 4.0925,
|
|
"margin_dpo/margin_mean": 53.878334045410156,
|
|
"margin_dpo/margin_std": 47.030704498291016,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.9759162303664921,
|
|
"grad_norm": 40.89207458496094,
|
|
"learning_rate": 9.64668657069706e-10,
|
|
"logits/chosen": 1.327850580215454,
|
|
"logits/rejected": 1.417038917541504,
|
|
"logps/chosen": -233.49664306640625,
|
|
"logps/ref_chosen": -224.32131958007812,
|
|
"logps/ref_rejected": -205.79212951660156,
|
|
"logps/rejected": -285.48529052734375,
|
|
"loss": 3.8998,
|
|
"margin_dpo/margin_mean": 70.51785278320312,
|
|
"margin_dpo/margin_std": 54.0175895690918,
|
|
"step": 466
|
|
},
|
|
{
|
|
"epoch": 0.9780104712041885,
|
|
"grad_norm": 66.9071273803711,
|
|
"learning_rate": 8.106729664475176e-10,
|
|
"logits/chosen": 0.6929614543914795,
|
|
"logits/rejected": 0.9394963979721069,
|
|
"logps/chosen": -260.9166259765625,
|
|
"logps/ref_chosen": -227.0828094482422,
|
|
"logps/ref_rejected": -285.5081481933594,
|
|
"logps/rejected": -364.97393798828125,
|
|
"loss": 4.4527,
|
|
"margin_dpo/margin_mean": 45.63193893432617,
|
|
"margin_dpo/margin_std": 61.40578079223633,
|
|
"step": 467
|
|
},
|
|
{
|
|
"epoch": 0.9801047120418848,
|
|
"grad_norm": 43.40538787841797,
|
|
"learning_rate": 6.700405431837585e-10,
|
|
"logits/chosen": 1.376441478729248,
|
|
"logits/rejected": 1.1604515314102173,
|
|
"logps/chosen": -349.08599853515625,
|
|
"logps/ref_chosen": -314.6758117675781,
|
|
"logps/ref_rejected": -290.43023681640625,
|
|
"logps/rejected": -357.7455749511719,
|
|
"loss": 4.5586,
|
|
"margin_dpo/margin_mean": 32.90521240234375,
|
|
"margin_dpo/margin_std": 61.592552185058594,
|
|
"step": 468
|
|
},
|
|
{
|
|
"epoch": 0.9821989528795811,
|
|
"grad_norm": 48.64304733276367,
|
|
"learning_rate": 5.427789289685347e-10,
|
|
"logits/chosen": 1.2290468215942383,
|
|
"logits/rejected": 1.205275058746338,
|
|
"logps/chosen": -290.6042175292969,
|
|
"logps/ref_chosen": -269.7442321777344,
|
|
"logps/ref_rejected": -237.1964874267578,
|
|
"logps/rejected": -307.8065185546875,
|
|
"loss": 4.2722,
|
|
"margin_dpo/margin_mean": 49.750064849853516,
|
|
"margin_dpo/margin_std": 63.16607666015625,
|
|
"step": 469
|
|
},
|
|
{
|
|
"epoch": 0.9842931937172775,
|
|
"grad_norm": 48.205284118652344,
|
|
"learning_rate": 4.288949484559934e-10,
|
|
"logits/chosen": 0.8344168066978455,
|
|
"logits/rejected": 0.8336673378944397,
|
|
"logps/chosen": -342.15057373046875,
|
|
"logps/ref_chosen": -326.9454650878906,
|
|
"logps/ref_rejected": -283.81768798828125,
|
|
"logps/rejected": -360.6470031738281,
|
|
"loss": 4.0516,
|
|
"margin_dpo/margin_mean": 61.624237060546875,
|
|
"margin_dpo/margin_std": 61.775753021240234,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.9863874345549738,
|
|
"grad_norm": 58.954227447509766,
|
|
"learning_rate": 3.2839470889836627e-10,
|
|
"logits/chosen": 1.4265129566192627,
|
|
"logits/rejected": 1.394487977027893,
|
|
"logps/chosen": -337.0513000488281,
|
|
"logps/ref_chosen": -309.4604797363281,
|
|
"logps/ref_rejected": -292.0646057128906,
|
|
"logps/rejected": -356.0199890136719,
|
|
"loss": 4.5083,
|
|
"margin_dpo/margin_mean": 36.36451721191406,
|
|
"margin_dpo/margin_std": 59.396270751953125,
|
|
"step": 471
|
|
},
|
|
{
|
|
"epoch": 0.9884816753926702,
|
|
"grad_norm": 61.63766098022461,
|
|
"learning_rate": 2.412835998185092e-10,
|
|
"logits/chosen": 1.1254520416259766,
|
|
"logits/rejected": 1.199691891670227,
|
|
"logps/chosen": -210.82077026367188,
|
|
"logps/ref_chosen": -185.00701904296875,
|
|
"logps/ref_rejected": -208.1576385498047,
|
|
"logps/rejected": -272.89483642578125,
|
|
"loss": 4.1475,
|
|
"margin_dpo/margin_mean": 38.92341613769531,
|
|
"margin_dpo/margin_std": 71.17210388183594,
|
|
"step": 472
|
|
},
|
|
{
|
|
"epoch": 0.9905759162303664,
|
|
"grad_norm": 56.59166717529297,
|
|
"learning_rate": 1.6756629272085544e-10,
|
|
"logits/chosen": 1.3166826963424683,
|
|
"logits/rejected": 1.1217594146728516,
|
|
"logps/chosen": -350.744140625,
|
|
"logps/ref_chosen": -330.0291442871094,
|
|
"logps/ref_rejected": -224.92051696777344,
|
|
"logps/rejected": -302.14898681640625,
|
|
"loss": 4.1337,
|
|
"margin_dpo/margin_mean": 56.51344299316406,
|
|
"margin_dpo/margin_std": 51.90734100341797,
|
|
"step": 473
|
|
},
|
|
{
|
|
"epoch": 0.9926701570680628,
|
|
"grad_norm": 59.311004638671875,
|
|
"learning_rate": 1.072467408408384e-10,
|
|
"logits/chosen": 1.2557741403579712,
|
|
"logits/rejected": 1.3725204467773438,
|
|
"logps/chosen": -355.0524597167969,
|
|
"logps/ref_chosen": -315.9046936035156,
|
|
"logps/ref_rejected": -340.7234191894531,
|
|
"logps/rejected": -411.844482421875,
|
|
"loss": 4.5535,
|
|
"margin_dpo/margin_mean": 31.97334861755371,
|
|
"margin_dpo/margin_std": 64.91607666015625,
|
|
"step": 474
|
|
},
|
|
{
|
|
"epoch": 0.9947643979057592,
|
|
"grad_norm": 54.42702102661133,
|
|
"learning_rate": 6.032817893297793e-11,
|
|
"logits/chosen": 0.872796356678009,
|
|
"logits/rejected": 0.9358187913894653,
|
|
"logps/chosen": -227.73483276367188,
|
|
"logps/ref_chosen": -202.84310913085938,
|
|
"logps/ref_rejected": -175.70704650878906,
|
|
"logps/rejected": -250.8938751220703,
|
|
"loss": 4.4903,
|
|
"margin_dpo/margin_mean": 50.29510498046875,
|
|
"margin_dpo/margin_std": 74.1104965209961,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.9968586387434555,
|
|
"grad_norm": 50.788307189941406,
|
|
"learning_rate": 2.6813123097352287e-11,
|
|
"logits/chosen": 1.07370924949646,
|
|
"logits/rejected": 1.213090181350708,
|
|
"logps/chosen": -292.42132568359375,
|
|
"logps/ref_chosen": -276.843505859375,
|
|
"logps/ref_rejected": -309.07757568359375,
|
|
"logps/rejected": -368.9504089355469,
|
|
"loss": 4.2622,
|
|
"margin_dpo/margin_mean": 44.29502868652344,
|
|
"margin_dpo/margin_std": 54.164615631103516,
|
|
"step": 476
|
|
},
|
|
{
|
|
"epoch": 0.9989528795811519,
|
|
"grad_norm": 40.021400451660156,
|
|
"learning_rate": 6.7033706447061635e-12,
|
|
"logits/chosen": 0.8247851729393005,
|
|
"logits/rejected": 0.9125658869743347,
|
|
"logps/chosen": -297.199951171875,
|
|
"logps/ref_chosen": -262.76971435546875,
|
|
"logps/ref_rejected": -272.2499694824219,
|
|
"logps/rejected": -356.56658935546875,
|
|
"loss": 4.216,
|
|
"margin_dpo/margin_mean": 49.88636779785156,
|
|
"margin_dpo/margin_std": 77.48868560791016,
|
|
"step": 477
|
|
},
|
|
{
|
|
"epoch": 0.9989528795811519,
|
|
"step": 477,
|
|
"total_flos": 0.0,
|
|
"train_loss": 4.779813265150698,
|
|
"train_runtime": 7822.2821,
|
|
"train_samples_per_second": 7.815,
|
|
"train_steps_per_second": 0.061
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 477,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|