2924 lines
100 KiB
JSON
2924 lines
100 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 2112.0,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -1.9987274408340454,
|
|
"logits/rejected": -2.014296054840088,
|
|
"logps/chosen": -0.27934736013412476,
|
|
"logps/rejected": -0.2821010947227478,
|
|
"loss": 4425.13330078125,
|
|
"loss/core": 4425.13330078125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4205944538116455,
|
|
"rewards/margins": 1.4615087509155273,
|
|
"rewards/rejected": 0.9590854644775391,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 11584.0,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -2.3447604179382324,
|
|
"logits/rejected": -2.391680955886841,
|
|
"logps/chosen": -0.2895873188972473,
|
|
"logps/rejected": -0.28846535086631775,
|
|
"loss": 4443.54931640625,
|
|
"loss/core": 4443.54931640625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6877996921539307,
|
|
"rewards/margins": 0.0745888203382492,
|
|
"rewards/rejected": 1.613210916519165,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 2416.0,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -2.229412317276001,
|
|
"logits/rejected": -2.1927170753479004,
|
|
"logps/chosen": -0.27719348669052124,
|
|
"logps/rejected": -0.2841333746910095,
|
|
"loss": 4431.30712890625,
|
|
"loss/core": 4431.30712890625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.136369228363037,
|
|
"rewards/margins": 0.9970209002494812,
|
|
"rewards/rejected": 1.1393483877182007,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 1632.0,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -2.1783900260925293,
|
|
"logits/rejected": -2.219207286834717,
|
|
"logps/chosen": -0.30116215348243713,
|
|
"logps/rejected": -0.2862641215324402,
|
|
"loss": 4430.82421875,
|
|
"loss/core": 4430.82421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.917729377746582,
|
|
"rewards/margins": 1.0351120233535767,
|
|
"rewards/rejected": 0.8826172947883606,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 1272.0,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -2.075169086456299,
|
|
"logits/rejected": -2.1347451210021973,
|
|
"logps/chosen": -0.27128368616104126,
|
|
"logps/rejected": -0.25881752371788025,
|
|
"loss": 4416.3876953125,
|
|
"loss/core": 4416.3876953125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.8423290252685547,
|
|
"rewards/margins": 2.2059836387634277,
|
|
"rewards/rejected": 1.6363451480865479,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 3216.0,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -2.348489284515381,
|
|
"logits/rejected": -2.3322410583496094,
|
|
"logps/chosen": -0.30126723647117615,
|
|
"logps/rejected": -0.3248973786830902,
|
|
"loss": 4428.22412109375,
|
|
"loss/core": 4428.22412109375,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2666220664978027,
|
|
"rewards/margins": 1.2462260723114014,
|
|
"rewards/rejected": 1.0203959941864014,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 696.0,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -2.2013611793518066,
|
|
"logits/rejected": -2.183184862136841,
|
|
"logps/chosen": -0.26950961351394653,
|
|
"logps/rejected": -0.2648638188838959,
|
|
"loss": 4425.138671875,
|
|
"loss/core": 4425.138671875,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6406617164611816,
|
|
"rewards/margins": 1.477004885673523,
|
|
"rewards/rejected": 1.1636569499969482,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 2256.0,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -2.0045735836029053,
|
|
"logits/rejected": -2.0654070377349854,
|
|
"logps/chosen": -0.2931990921497345,
|
|
"logps/rejected": -0.280799925327301,
|
|
"loss": 4407.8447265625,
|
|
"loss/core": 4407.8447265625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.472355842590332,
|
|
"rewards/margins": 2.8589112758636475,
|
|
"rewards/rejected": 1.6134445667266846,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 1032.0,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -2.20025372505188,
|
|
"logits/rejected": -2.3626952171325684,
|
|
"logps/chosen": -0.2890096604824066,
|
|
"logps/rejected": -0.28794199228286743,
|
|
"loss": 4427.9345703125,
|
|
"loss/core": 4427.9345703125,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.0934605598449707,
|
|
"rewards/margins": 1.246740460395813,
|
|
"rewards/rejected": 0.8467203378677368,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 5600.0,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -2.1670279502868652,
|
|
"logits/rejected": -2.163269519805908,
|
|
"logps/chosen": -0.2812585234642029,
|
|
"logps/rejected": -0.28595098853111267,
|
|
"loss": 4425.6552734375,
|
|
"loss/core": 4425.6552734375,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.883915662765503,
|
|
"rewards/margins": 1.4437261819839478,
|
|
"rewards/rejected": 1.4401893615722656,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 440.0,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -2.197618007659912,
|
|
"logits/rejected": -2.1856415271759033,
|
|
"logps/chosen": -0.28887230157852173,
|
|
"logps/rejected": -0.28767216205596924,
|
|
"loss": 4442.45166015625,
|
|
"loss/core": 4442.45166015625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2596747875213623,
|
|
"rewards/margins": 0.2099606990814209,
|
|
"rewards/rejected": 2.0497140884399414,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 18688.0,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -2.091808319091797,
|
|
"logits/rejected": -2.120185375213623,
|
|
"logps/chosen": -0.27423757314682007,
|
|
"logps/rejected": -0.2837786376476288,
|
|
"loss": 4436.55517578125,
|
|
"loss/core": 4436.55517578125,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.158466100692749,
|
|
"rewards/margins": 0.602463960647583,
|
|
"rewards/rejected": 1.556002140045166,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 888.0,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -1.9588630199432373,
|
|
"logits/rejected": -2.0604052543640137,
|
|
"logps/chosen": -0.3092983365058899,
|
|
"logps/rejected": -0.30547577142715454,
|
|
"loss": 4420.9130859375,
|
|
"loss/core": 4420.9130859375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.9688713550567627,
|
|
"rewards/margins": 1.7887201309204102,
|
|
"rewards/rejected": 1.180151104927063,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 2208.0,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -1.9690837860107422,
|
|
"logits/rejected": -1.95412278175354,
|
|
"logps/chosen": -0.3884851932525635,
|
|
"logps/rejected": -0.2990027964115143,
|
|
"loss": 4427.15771484375,
|
|
"loss/core": 4427.15771484375,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.5522398948669434,
|
|
"rewards/margins": 1.389047384262085,
|
|
"rewards/rejected": 2.1631925106048584,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 676.0,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -2.0886988639831543,
|
|
"logits/rejected": -2.0533549785614014,
|
|
"logps/chosen": -0.3141383230686188,
|
|
"logps/rejected": -0.28872591257095337,
|
|
"loss": 4431.21728515625,
|
|
"loss/core": 4431.21728515625,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.615492582321167,
|
|
"rewards/margins": 1.0181725025177002,
|
|
"rewards/rejected": 1.597320318222046,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 760.0,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -2.0682148933410645,
|
|
"logits/rejected": -2.1878905296325684,
|
|
"logps/chosen": -0.2604108154773712,
|
|
"logps/rejected": -0.2718687355518341,
|
|
"loss": 4418.80078125,
|
|
"loss/core": 4418.80078125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2428879737854004,
|
|
"rewards/margins": 1.9694182872772217,
|
|
"rewards/rejected": 1.2734694480895996,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 1072.0,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -2.1902594566345215,
|
|
"logits/rejected": -2.2378265857696533,
|
|
"logps/chosen": -0.3089032471179962,
|
|
"logps/rejected": -0.2776981294155121,
|
|
"loss": 4416.4970703125,
|
|
"loss/core": 4416.4970703125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.4842758178710938,
|
|
"rewards/margins": 2.1533937454223633,
|
|
"rewards/rejected": 1.3308818340301514,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 436.0,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -2.421081066131592,
|
|
"logits/rejected": -2.4086060523986816,
|
|
"logps/chosen": -0.2822219729423523,
|
|
"logps/rejected": -0.29552292823791504,
|
|
"loss": 4432.59619140625,
|
|
"loss/core": 4432.59619140625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.7451679706573486,
|
|
"rewards/margins": 0.8951616287231445,
|
|
"rewards/rejected": 0.8500064015388489,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 2064.0,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -2.0548970699310303,
|
|
"logits/rejected": -2.1117641925811768,
|
|
"logps/chosen": -0.26747360825538635,
|
|
"logps/rejected": -0.27913323044776917,
|
|
"loss": 4430.07275390625,
|
|
"loss/core": 4430.07275390625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2089908123016357,
|
|
"rewards/margins": 1.1066198348999023,
|
|
"rewards/rejected": 2.1023709774017334,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 1104.0,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -2.234778881072998,
|
|
"logits/rejected": -2.280416250228882,
|
|
"logps/chosen": -0.25745129585266113,
|
|
"logps/rejected": -0.2470175325870514,
|
|
"loss": 4428.8994140625,
|
|
"loss/core": 4428.8994140625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.4600722789764404,
|
|
"rewards/margins": 1.177268147468567,
|
|
"rewards/rejected": 1.282804250717163,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 4160.0,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -2.339501142501831,
|
|
"logits/rejected": -2.4241232872009277,
|
|
"logps/chosen": -0.2842095196247101,
|
|
"logps/rejected": -0.27984419465065,
|
|
"loss": 4428.7177734375,
|
|
"loss/core": 4428.7177734375,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.997147798538208,
|
|
"rewards/margins": 1.1996146440505981,
|
|
"rewards/rejected": 0.7975330948829651,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 944.0,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -1.907444953918457,
|
|
"logits/rejected": -1.858888030052185,
|
|
"logps/chosen": -0.3371734619140625,
|
|
"logps/rejected": -0.2982056736946106,
|
|
"loss": 4410.75390625,
|
|
"loss/core": 4410.75390625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.483543395996094,
|
|
"rewards/margins": 2.5996921062469482,
|
|
"rewards/rejected": 1.8838508129119873,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 5120.0,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -2.087294816970825,
|
|
"logits/rejected": -2.0307278633117676,
|
|
"logps/chosen": -0.2998853623867035,
|
|
"logps/rejected": -0.2985258996486664,
|
|
"loss": 4440.2822265625,
|
|
"loss/core": 4440.2822265625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7805169820785522,
|
|
"rewards/margins": 0.33072328567504883,
|
|
"rewards/rejected": 1.4497936964035034,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 8576.0,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -2.0372519493103027,
|
|
"logits/rejected": -2.075047492980957,
|
|
"logps/chosen": -0.30229058861732483,
|
|
"logps/rejected": -0.2958064079284668,
|
|
"loss": 4435.390625,
|
|
"loss/core": 4435.390625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.808635950088501,
|
|
"rewards/margins": 0.6832652688026428,
|
|
"rewards/rejected": 1.1253705024719238,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 712.0,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -2.1084086894989014,
|
|
"logits/rejected": -2.138524055480957,
|
|
"logps/chosen": -0.29876020550727844,
|
|
"logps/rejected": -0.306560218334198,
|
|
"loss": 4430.5625,
|
|
"loss/core": 4430.5625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.067880153656006,
|
|
"rewards/margins": 1.0501195192337036,
|
|
"rewards/rejected": 1.0177606344223022,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 1504.0,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -2.001467227935791,
|
|
"logits/rejected": -2.094036817550659,
|
|
"logps/chosen": -0.2802198827266693,
|
|
"logps/rejected": -0.27933305501937866,
|
|
"loss": 4432.16796875,
|
|
"loss/core": 4432.16796875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.805000901222229,
|
|
"rewards/margins": 0.9235280752182007,
|
|
"rewards/rejected": 0.8814727067947388,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 2192.0,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -2.0947656631469727,
|
|
"logits/rejected": -2.167937994003296,
|
|
"logps/chosen": -0.3018343448638916,
|
|
"logps/rejected": -0.29743117094039917,
|
|
"loss": 4417.396484375,
|
|
"loss/core": 4417.396484375,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.3767879009246826,
|
|
"rewards/margins": 2.0548617839813232,
|
|
"rewards/rejected": 1.3219258785247803,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 860.0,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -2.454082489013672,
|
|
"logits/rejected": -2.513239622116089,
|
|
"logps/chosen": -0.282596230506897,
|
|
"logps/rejected": -0.2890459895133972,
|
|
"loss": 4432.37744140625,
|
|
"loss/core": 4432.37744140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6272993087768555,
|
|
"rewards/margins": 0.908282458782196,
|
|
"rewards/rejected": 0.7190167307853699,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 884.0,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.781502366065979,
|
|
"logits/rejected": -1.8124113082885742,
|
|
"logps/chosen": -0.29897254705429077,
|
|
"logps/rejected": -0.3044045567512512,
|
|
"loss": 4420.0517578125,
|
|
"loss/core": 4420.0517578125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.627842664718628,
|
|
"rewards/margins": 1.8691842555999756,
|
|
"rewards/rejected": 1.7586586475372314,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 2224.0,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -2.2341880798339844,
|
|
"logits/rejected": -2.2840676307678223,
|
|
"logps/chosen": -0.3164155185222626,
|
|
"logps/rejected": -0.3073708713054657,
|
|
"loss": 4415.888671875,
|
|
"loss/core": 4415.888671875,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.64851713180542,
|
|
"rewards/margins": 2.182638168334961,
|
|
"rewards/rejected": 1.4658793210983276,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 1064.0,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -2.100919008255005,
|
|
"logits/rejected": -2.178107738494873,
|
|
"logps/chosen": -0.28356048464775085,
|
|
"logps/rejected": -0.2917582392692566,
|
|
"loss": 4417.8515625,
|
|
"loss/core": 4417.8515625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8598971366882324,
|
|
"rewards/margins": 2.027747392654419,
|
|
"rewards/rejected": 0.8321496248245239,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 5024.0,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -2.0568435192108154,
|
|
"logits/rejected": -2.0316061973571777,
|
|
"logps/chosen": -0.31080225110054016,
|
|
"logps/rejected": -0.3151562511920929,
|
|
"loss": 4421.96240234375,
|
|
"loss/core": 4421.96240234375,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.147416591644287,
|
|
"rewards/margins": 1.7445529699325562,
|
|
"rewards/rejected": 1.4028639793395996,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 1232.0,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -2.222330331802368,
|
|
"logits/rejected": -2.154171943664551,
|
|
"logps/chosen": -0.3106082081794739,
|
|
"logps/rejected": -0.3049444556236267,
|
|
"loss": 4433.68994140625,
|
|
"loss/core": 4433.68994140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6072232723236084,
|
|
"rewards/margins": 0.810656726360321,
|
|
"rewards/rejected": 0.7965666055679321,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 5184.0,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -2.189422130584717,
|
|
"logits/rejected": -2.1332600116729736,
|
|
"logps/chosen": -0.2748359739780426,
|
|
"logps/rejected": -0.2597963213920593,
|
|
"loss": 4447.82421875,
|
|
"loss/core": 4447.82421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5958924293518066,
|
|
"rewards/margins": -0.17370417714118958,
|
|
"rewards/rejected": 2.769596576690674,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 7424.0,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -2.1792263984680176,
|
|
"logits/rejected": -2.2180469036102295,
|
|
"logps/chosen": -0.28667014837265015,
|
|
"logps/rejected": -0.30290576815605164,
|
|
"loss": 4426.7734375,
|
|
"loss/core": 4426.7734375,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.5856590270996094,
|
|
"rewards/margins": 1.3379803895950317,
|
|
"rewards/rejected": 1.2476783990859985,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 8320.0,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -2.2014615535736084,
|
|
"logits/rejected": -2.1121859550476074,
|
|
"logps/chosen": -0.27293267846107483,
|
|
"logps/rejected": -0.2612500488758087,
|
|
"loss": 4422.0908203125,
|
|
"loss/core": 4422.0908203125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2250092029571533,
|
|
"rewards/margins": 1.7280479669570923,
|
|
"rewards/rejected": 1.4969611167907715,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 844.0,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -2.20589017868042,
|
|
"logits/rejected": -2.162938117980957,
|
|
"logps/chosen": -0.288524866104126,
|
|
"logps/rejected": -0.2919921875,
|
|
"loss": 4438.9091796875,
|
|
"loss/core": 4438.9091796875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.0444185733795166,
|
|
"rewards/margins": 0.4165409207344055,
|
|
"rewards/rejected": 0.6278777122497559,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 1832.0,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -2.3000447750091553,
|
|
"logits/rejected": -2.2772605419158936,
|
|
"logps/chosen": -0.2829676866531372,
|
|
"logps/rejected": -0.28464275598526,
|
|
"loss": 4423.5048828125,
|
|
"loss/core": 4423.5048828125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.844500780105591,
|
|
"rewards/margins": 1.6092599630355835,
|
|
"rewards/rejected": 1.2352410554885864,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 1216.0,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -1.9864647388458252,
|
|
"logits/rejected": -2.007460594177246,
|
|
"logps/chosen": -0.2905668616294861,
|
|
"logps/rejected": -0.2901652455329895,
|
|
"loss": 4422.435546875,
|
|
"loss/core": 4422.435546875,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.672790288925171,
|
|
"rewards/margins": 1.7109134197235107,
|
|
"rewards/rejected": 0.9618767499923706,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 1080.0,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -1.8949693441390991,
|
|
"logits/rejected": -1.9352257251739502,
|
|
"logps/chosen": -0.3231773376464844,
|
|
"logps/rejected": -0.31668004393577576,
|
|
"loss": 4425.9697265625,
|
|
"loss/core": 4425.9697265625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.618408441543579,
|
|
"rewards/margins": 1.3984508514404297,
|
|
"rewards/rejected": 1.2199573516845703,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 752.0,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -2.1187617778778076,
|
|
"logits/rejected": -2.1708757877349854,
|
|
"logps/chosen": -0.2496471405029297,
|
|
"logps/rejected": -0.26736050844192505,
|
|
"loss": 4429.56640625,
|
|
"loss/core": 4429.56640625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.2940521240234375,
|
|
"rewards/margins": 1.1699793338775635,
|
|
"rewards/rejected": 2.124073028564453,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 1952.0,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -2.1127567291259766,
|
|
"logits/rejected": -2.137141227722168,
|
|
"logps/chosen": -0.31239503622055054,
|
|
"logps/rejected": -0.322940468788147,
|
|
"loss": 4426.55859375,
|
|
"loss/core": 4426.55859375,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8321475982666016,
|
|
"rewards/margins": 1.3591885566711426,
|
|
"rewards/rejected": 1.4729589223861694,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 1848.0,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -2.0887973308563232,
|
|
"logits/rejected": -2.079000949859619,
|
|
"logps/chosen": -0.27418142557144165,
|
|
"logps/rejected": -0.2853352427482605,
|
|
"loss": 4425.31494140625,
|
|
"loss/core": 4425.31494140625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.2043747901916504,
|
|
"rewards/margins": 1.5126924514770508,
|
|
"rewards/rejected": 1.6916821002960205,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 1888.0,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -2.104156017303467,
|
|
"logits/rejected": -2.01837420463562,
|
|
"logps/chosen": -0.3019450902938843,
|
|
"logps/rejected": -0.3014443516731262,
|
|
"loss": 4433.5771484375,
|
|
"loss/core": 4433.5771484375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8912023305892944,
|
|
"rewards/margins": 0.8183046579360962,
|
|
"rewards/rejected": 1.0728976726531982,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 1040.0,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -2.190114974975586,
|
|
"logits/rejected": -2.1780142784118652,
|
|
"logps/chosen": -0.2899002134799957,
|
|
"logps/rejected": -0.2752569317817688,
|
|
"loss": 4415.31396484375,
|
|
"loss/core": 4415.31396484375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6290431022644043,
|
|
"rewards/margins": 2.217606782913208,
|
|
"rewards/rejected": 1.4114364385604858,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 10112.0,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -2.2766060829162598,
|
|
"logits/rejected": -2.2850639820098877,
|
|
"logps/chosen": -0.28285735845565796,
|
|
"logps/rejected": -0.3064045310020447,
|
|
"loss": 4430.21240234375,
|
|
"loss/core": 4430.21240234375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8430675268173218,
|
|
"rewards/margins": 1.0752555131912231,
|
|
"rewards/rejected": 0.7678121328353882,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 1328.0,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -2.0158145427703857,
|
|
"logits/rejected": -2.0498404502868652,
|
|
"logps/chosen": -0.2925497591495514,
|
|
"logps/rejected": -0.28553271293640137,
|
|
"loss": 4437.0009765625,
|
|
"loss/core": 4437.0009765625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9056549072265625,
|
|
"rewards/margins": 0.5666888952255249,
|
|
"rewards/rejected": 1.3389660120010376,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 1200.0,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -2.028263568878174,
|
|
"logits/rejected": -2.0478172302246094,
|
|
"logps/chosen": -0.29387903213500977,
|
|
"logps/rejected": -0.2936249375343323,
|
|
"loss": 4434.69873046875,
|
|
"loss/core": 4434.69873046875,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7645080089569092,
|
|
"rewards/margins": 0.732918381690979,
|
|
"rewards/rejected": 1.0315898656845093,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 10048.0,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -2.0705394744873047,
|
|
"logits/rejected": -2.0584795475006104,
|
|
"logps/chosen": -0.2776476740837097,
|
|
"logps/rejected": -0.2890920341014862,
|
|
"loss": 4413.03466796875,
|
|
"loss/core": 4413.03466796875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6031315326690674,
|
|
"rewards/margins": 2.431091547012329,
|
|
"rewards/rejected": 1.1720398664474487,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 1512.0,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -1.9100065231323242,
|
|
"logits/rejected": -1.989465355873108,
|
|
"logps/chosen": -0.29988300800323486,
|
|
"logps/rejected": -0.2941280007362366,
|
|
"loss": 4427.71923828125,
|
|
"loss/core": 4427.71923828125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3385276794433594,
|
|
"rewards/margins": 1.2655916213989258,
|
|
"rewards/rejected": 1.0729362964630127,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 916.0,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -2.2194790840148926,
|
|
"logits/rejected": -2.2149035930633545,
|
|
"logps/chosen": -0.2853752374649048,
|
|
"logps/rejected": -0.2876095473766327,
|
|
"loss": 4433.2822265625,
|
|
"loss/core": 4433.2822265625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4470086097717285,
|
|
"rewards/margins": 0.8454030156135559,
|
|
"rewards/rejected": 1.6016056537628174,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 458.0,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -2.1570799350738525,
|
|
"logits/rejected": -2.2298617362976074,
|
|
"logps/chosen": -0.28772681951522827,
|
|
"logps/rejected": -0.26979178190231323,
|
|
"loss": 4433.95703125,
|
|
"loss/core": 4433.95703125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3997950553894043,
|
|
"rewards/margins": 0.8103548288345337,
|
|
"rewards/rejected": 1.5894405841827393,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 448.0,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.0369317531585693,
|
|
"logits/rejected": -2.0460586547851562,
|
|
"logps/chosen": -0.29983657598495483,
|
|
"logps/rejected": -0.3035128712654114,
|
|
"loss": 4443.8427734375,
|
|
"loss/core": 4443.8427734375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.2268575429916382,
|
|
"rewards/margins": 0.0723171979188919,
|
|
"rewards/rejected": 1.1545404195785522,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 868.0,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -1.9414504766464233,
|
|
"logits/rejected": -2.0309865474700928,
|
|
"logps/chosen": -0.28761979937553406,
|
|
"logps/rejected": -0.28891196846961975,
|
|
"loss": 4429.82275390625,
|
|
"loss/core": 4429.82275390625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.290050506591797,
|
|
"rewards/margins": 1.1456321477890015,
|
|
"rewards/rejected": 2.144418716430664,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 8704.0,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -2.0923826694488525,
|
|
"logits/rejected": -2.0464677810668945,
|
|
"logps/chosen": -0.29613757133483887,
|
|
"logps/rejected": -0.30172640085220337,
|
|
"loss": 4428.80517578125,
|
|
"loss/core": 4428.80517578125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.328432083129883,
|
|
"rewards/margins": 1.1818605661392212,
|
|
"rewards/rejected": 1.146571397781372,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 760.0,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -2.060886859893799,
|
|
"logits/rejected": -1.9781144857406616,
|
|
"logps/chosen": -0.2877367436885834,
|
|
"logps/rejected": -0.30556520819664,
|
|
"loss": 4427.63134765625,
|
|
"loss/core": 4427.63134765625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.049919605255127,
|
|
"rewards/margins": 1.2704269886016846,
|
|
"rewards/rejected": 0.7794923782348633,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 4608.0,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -2.0870399475097656,
|
|
"logits/rejected": -2.082958221435547,
|
|
"logps/chosen": -0.29648491740226746,
|
|
"logps/rejected": -0.2911376357078552,
|
|
"loss": 4431.78466796875,
|
|
"loss/core": 4431.78466796875,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.612039089202881,
|
|
"rewards/margins": 0.9668930768966675,
|
|
"rewards/rejected": 1.6451460123062134,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 10176.0,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -2.1645455360412598,
|
|
"logits/rejected": -2.1757516860961914,
|
|
"logps/chosen": -0.2864384949207306,
|
|
"logps/rejected": -0.28760045766830444,
|
|
"loss": 4444.3701171875,
|
|
"loss/core": 4444.3701171875,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.4827371835708618,
|
|
"rewards/margins": 0.028162401169538498,
|
|
"rewards/rejected": 1.4545748233795166,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 924.0,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -2.5665838718414307,
|
|
"logits/rejected": -2.5299837589263916,
|
|
"logps/chosen": -0.24450433254241943,
|
|
"logps/rejected": -0.2568889260292053,
|
|
"loss": 4435.9375,
|
|
"loss/core": 4435.9375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.310236930847168,
|
|
"rewards/margins": 0.6390492916107178,
|
|
"rewards/rejected": 0.6711876392364502,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 12928.0,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.9790928363800049,
|
|
"logits/rejected": -1.9473092555999756,
|
|
"logps/chosen": -0.3141786456108093,
|
|
"logps/rejected": -0.3128449320793152,
|
|
"loss": 4410.31787109375,
|
|
"loss/core": 4410.31787109375,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.7060604095458984,
|
|
"rewards/margins": 2.605220317840576,
|
|
"rewards/rejected": 1.1008400917053223,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 1080.0,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -2.1226131916046143,
|
|
"logits/rejected": -2.1846141815185547,
|
|
"logps/chosen": -0.2690519690513611,
|
|
"logps/rejected": -0.26041924953460693,
|
|
"loss": 4427.720703125,
|
|
"loss/core": 4427.720703125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1630797386169434,
|
|
"rewards/margins": 1.2623918056488037,
|
|
"rewards/rejected": 0.9006881713867188,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 13568.0,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -2.1288938522338867,
|
|
"logits/rejected": -2.0386104583740234,
|
|
"logps/chosen": -0.27233630418777466,
|
|
"logps/rejected": -0.279468834400177,
|
|
"loss": 4437.134765625,
|
|
"loss/core": 4437.134765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6192855834960938,
|
|
"rewards/margins": 0.5920935869216919,
|
|
"rewards/rejected": 2.027191638946533,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 840.0,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -1.9616706371307373,
|
|
"logits/rejected": -2.0680670738220215,
|
|
"logps/chosen": -0.2692381739616394,
|
|
"logps/rejected": -0.27612730860710144,
|
|
"loss": 4423.5947265625,
|
|
"loss/core": 4423.5947265625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.912264347076416,
|
|
"rewards/margins": 1.5709517002105713,
|
|
"rewards/rejected": 1.3413128852844238,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 708.0,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -2.000558614730835,
|
|
"logits/rejected": -2.0157294273376465,
|
|
"logps/chosen": -0.2892259359359741,
|
|
"logps/rejected": -0.29263320565223694,
|
|
"loss": 4439.705078125,
|
|
"loss/core": 4439.705078125,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5899919271469116,
|
|
"rewards/margins": 0.36150193214416504,
|
|
"rewards/rejected": 1.2284899950027466,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 7104.0,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -2.257622718811035,
|
|
"logits/rejected": -2.2041261196136475,
|
|
"logps/chosen": -0.2870021462440491,
|
|
"logps/rejected": -0.2847207188606262,
|
|
"loss": 4416.87060546875,
|
|
"loss/core": 4416.87060546875,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.6918609142303467,
|
|
"rewards/margins": 2.167919635772705,
|
|
"rewards/rejected": 1.5239416360855103,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 568.0,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -2.2275662422180176,
|
|
"logits/rejected": -2.2161643505096436,
|
|
"logps/chosen": -0.2746240496635437,
|
|
"logps/rejected": -0.271222323179245,
|
|
"loss": 4431.955078125,
|
|
"loss/core": 4431.955078125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.07051682472229,
|
|
"rewards/margins": 0.9463024139404297,
|
|
"rewards/rejected": 1.12421452999115,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 17920.0,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -2.0493054389953613,
|
|
"logits/rejected": -2.0343852043151855,
|
|
"logps/chosen": -0.3021050691604614,
|
|
"logps/rejected": -0.32243406772613525,
|
|
"loss": 4425.26953125,
|
|
"loss/core": 4425.26953125,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1218528747558594,
|
|
"rewards/margins": 1.4721834659576416,
|
|
"rewards/rejected": 0.6496694684028625,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 5632.0,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -2.0967724323272705,
|
|
"logits/rejected": -2.2191970348358154,
|
|
"logps/chosen": -0.2803352475166321,
|
|
"logps/rejected": -0.2822166979312897,
|
|
"loss": 4428.95849609375,
|
|
"loss/core": 4428.95849609375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.037506580352783,
|
|
"rewards/margins": 1.1726973056793213,
|
|
"rewards/rejected": 0.8648093342781067,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 1392.0,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -2.3210458755493164,
|
|
"logits/rejected": -2.3820576667785645,
|
|
"logps/chosen": -0.2815735936164856,
|
|
"logps/rejected": -0.2927202582359314,
|
|
"loss": 4433.78076171875,
|
|
"loss/core": 4433.78076171875,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6497310400009155,
|
|
"rewards/margins": 0.8072711825370789,
|
|
"rewards/rejected": 0.8424596786499023,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 1080.0,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -2.062190532684326,
|
|
"logits/rejected": -2.041247844696045,
|
|
"logps/chosen": -0.2606438100337982,
|
|
"logps/rejected": -0.2558349072933197,
|
|
"loss": 4430.6279296875,
|
|
"loss/core": 4430.6279296875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.995518445968628,
|
|
"rewards/margins": 1.0421174764633179,
|
|
"rewards/rejected": 0.9534010887145996,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 1096.0,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -2.1064131259918213,
|
|
"logits/rejected": -2.233365535736084,
|
|
"logps/chosen": -0.2812296450138092,
|
|
"logps/rejected": -0.29490309953689575,
|
|
"loss": 4437.8212890625,
|
|
"loss/core": 4437.8212890625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8455944061279297,
|
|
"rewards/margins": 0.5175111889839172,
|
|
"rewards/rejected": 1.3280833959579468,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 7392.0,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -2.132842540740967,
|
|
"logits/rejected": -2.196658134460449,
|
|
"logps/chosen": -0.2782808542251587,
|
|
"logps/rejected": -0.28051522374153137,
|
|
"loss": 4426.58642578125,
|
|
"loss/core": 4426.58642578125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.652100086212158,
|
|
"rewards/margins": 1.363356351852417,
|
|
"rewards/rejected": 1.2887436151504517,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 6272.0,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -2.0724411010742188,
|
|
"logits/rejected": -2.1809439659118652,
|
|
"logps/chosen": -0.2859554886817932,
|
|
"logps/rejected": -0.28384047746658325,
|
|
"loss": 4417.919921875,
|
|
"loss/core": 4417.919921875,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.864577054977417,
|
|
"rewards/margins": 2.0546889305114746,
|
|
"rewards/rejected": 0.8098880648612976,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 2048.0,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.2656424045562744,
|
|
"logits/rejected": -2.311187744140625,
|
|
"logps/chosen": -0.28399044275283813,
|
|
"logps/rejected": -0.28443023562431335,
|
|
"loss": 4436.71484375,
|
|
"loss/core": 4436.71484375,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6527982950210571,
|
|
"rewards/margins": 0.5823618173599243,
|
|
"rewards/rejected": 1.0704364776611328,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 1048.0,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -2.04667067527771,
|
|
"logits/rejected": -2.024685859680176,
|
|
"logps/chosen": -0.2581968307495117,
|
|
"logps/rejected": -0.2569906711578369,
|
|
"loss": 4434.41162109375,
|
|
"loss/core": 4434.41162109375,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0193209648132324,
|
|
"rewards/margins": 0.787818968296051,
|
|
"rewards/rejected": 2.231502056121826,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 1440.0,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -2.1206488609313965,
|
|
"logits/rejected": -2.1304726600646973,
|
|
"logps/chosen": -0.2783791422843933,
|
|
"logps/rejected": -0.2880426049232483,
|
|
"loss": 4438.98291015625,
|
|
"loss/core": 4438.98291015625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.067422866821289,
|
|
"rewards/margins": 0.4526546597480774,
|
|
"rewards/rejected": 1.6147682666778564,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 1912.0,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -2.113156318664551,
|
|
"logits/rejected": -2.1326403617858887,
|
|
"logps/chosen": -0.2700130343437195,
|
|
"logps/rejected": -0.27478334307670593,
|
|
"loss": 4435.85009765625,
|
|
"loss/core": 4435.85009765625,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.1801278591156006,
|
|
"rewards/margins": 0.6569812893867493,
|
|
"rewards/rejected": 1.523146390914917,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 584.0,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -1.8711488246917725,
|
|
"logits/rejected": -1.948341727256775,
|
|
"logps/chosen": -0.3067759573459625,
|
|
"logps/rejected": -0.3026246428489685,
|
|
"loss": 4432.92138671875,
|
|
"loss/core": 4432.92138671875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7659752368927002,
|
|
"rewards/margins": 0.8714345693588257,
|
|
"rewards/rejected": 0.8945406079292297,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 1744.0,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -2.2217893600463867,
|
|
"logits/rejected": -2.32525897026062,
|
|
"logps/chosen": -0.29468852281570435,
|
|
"logps/rejected": -0.2896806001663208,
|
|
"loss": 4432.77392578125,
|
|
"loss/core": 4432.77392578125,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 1.685217261314392,
|
|
"rewards/margins": 0.8789496421813965,
|
|
"rewards/rejected": 0.8062676191329956,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 1272.0,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -2.1023941040039062,
|
|
"logits/rejected": -2.1372156143188477,
|
|
"logps/chosen": -0.29075393080711365,
|
|
"logps/rejected": -0.29359036684036255,
|
|
"loss": 4417.08642578125,
|
|
"loss/core": 4417.08642578125,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9458072185516357,
|
|
"rewards/margins": 2.109001636505127,
|
|
"rewards/rejected": 0.836805522441864,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 940.0,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -2.132859706878662,
|
|
"logits/rejected": -2.1874256134033203,
|
|
"logps/chosen": -0.31743866205215454,
|
|
"logps/rejected": -0.3030361235141754,
|
|
"loss": 4430.7001953125,
|
|
"loss/core": 4430.7001953125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.2906603813171387,
|
|
"rewards/margins": 1.0478847026824951,
|
|
"rewards/rejected": 1.242775797843933,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 1176.0,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -2.2498955726623535,
|
|
"logits/rejected": -2.280546188354492,
|
|
"logps/chosen": -0.2766816020011902,
|
|
"logps/rejected": -0.28772997856140137,
|
|
"loss": 4432.49267578125,
|
|
"loss/core": 4432.49267578125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0912842750549316,
|
|
"rewards/margins": 0.8995159268379211,
|
|
"rewards/rejected": 1.1917682886123657,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 1160.0,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -2.11924409866333,
|
|
"logits/rejected": -2.1979384422302246,
|
|
"logps/chosen": -0.3298892378807068,
|
|
"logps/rejected": -0.32105594873428345,
|
|
"loss": 4436.1533203125,
|
|
"loss/core": 4436.1533203125,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 0.9991945028305054,
|
|
"rewards/margins": 0.6242727041244507,
|
|
"rewards/rejected": 0.37492185831069946,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 3312.0,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -2.350698947906494,
|
|
"logits/rejected": -2.317554473876953,
|
|
"logps/chosen": -0.283860981464386,
|
|
"logps/rejected": -0.2982264757156372,
|
|
"loss": 4427.1689453125,
|
|
"loss/core": 4427.1689453125,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.040308952331543,
|
|
"rewards/margins": 1.3134760856628418,
|
|
"rewards/rejected": 0.7268326282501221,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 4608.0,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -2.014967679977417,
|
|
"logits/rejected": -2.074613094329834,
|
|
"logps/chosen": -0.2912288010120392,
|
|
"logps/rejected": -0.26653727889060974,
|
|
"loss": 4436.998046875,
|
|
"loss/core": 4436.998046875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.238948345184326,
|
|
"rewards/margins": 0.6072303652763367,
|
|
"rewards/rejected": 1.6317180395126343,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 5952.0,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -2.0681891441345215,
|
|
"logits/rejected": -2.092362403869629,
|
|
"logps/chosen": -0.2822319567203522,
|
|
"logps/rejected": -0.26612913608551025,
|
|
"loss": 4406.2177734375,
|
|
"loss/core": 4406.2177734375,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.376003265380859,
|
|
"rewards/margins": 2.9532055854797363,
|
|
"rewards/rejected": 1.422797679901123,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 1256.0,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.350803852081299,
|
|
"logits/rejected": -2.3343265056610107,
|
|
"logps/chosen": -0.27796900272369385,
|
|
"logps/rejected": -0.2925384044647217,
|
|
"loss": 4436.28271484375,
|
|
"loss/core": 4436.28271484375,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6284303665161133,
|
|
"rewards/margins": 0.6162401437759399,
|
|
"rewards/rejected": 1.0121902227401733,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 5120.0,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -2.056842565536499,
|
|
"logits/rejected": -2.1276373863220215,
|
|
"logps/chosen": -0.2973445951938629,
|
|
"logps/rejected": -0.300473153591156,
|
|
"loss": 4422.72900390625,
|
|
"loss/core": 4422.72900390625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0658655166625977,
|
|
"rewards/margins": 1.684059500694275,
|
|
"rewards/rejected": 1.3818061351776123,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 2016.0,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -1.9376929998397827,
|
|
"logits/rejected": -1.9126249551773071,
|
|
"logps/chosen": -0.2954959571361542,
|
|
"logps/rejected": -0.2955567538738251,
|
|
"loss": 4426.203125,
|
|
"loss/core": 4426.203125,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.961683988571167,
|
|
"rewards/margins": 1.3829456567764282,
|
|
"rewards/rejected": 1.5787384510040283,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 8832.0,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -1.97494375705719,
|
|
"logits/rejected": -1.992226004600525,
|
|
"logps/chosen": -0.2974433898925781,
|
|
"logps/rejected": -0.28895604610443115,
|
|
"loss": 4420.197265625,
|
|
"loss/core": 4420.197265625,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.466543197631836,
|
|
"rewards/margins": 1.8552677631378174,
|
|
"rewards/rejected": 1.6112754344940186,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 3072.0,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -1.9352457523345947,
|
|
"logits/rejected": -2.1265578269958496,
|
|
"logps/chosen": -0.32452231645584106,
|
|
"logps/rejected": -0.31393322348594666,
|
|
"loss": 4418.81005859375,
|
|
"loss/core": 4418.81005859375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4475817680358887,
|
|
"rewards/margins": 1.9687623977661133,
|
|
"rewards/rejected": 1.4788196086883545,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 2192.0,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -2.062018871307373,
|
|
"logits/rejected": -2.0496106147766113,
|
|
"logps/chosen": -0.29210391640663147,
|
|
"logps/rejected": -0.30656731128692627,
|
|
"loss": 4420.3466796875,
|
|
"loss/core": 4420.3466796875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.7037289142608643,
|
|
"rewards/margins": 1.8343826532363892,
|
|
"rewards/rejected": 1.8693459033966064,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 892.0,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -1.8918297290802002,
|
|
"logits/rejected": -1.9900699853897095,
|
|
"logps/chosen": -0.29650312662124634,
|
|
"logps/rejected": -0.2915659546852112,
|
|
"loss": 4426.185546875,
|
|
"loss/core": 4426.185546875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.307586908340454,
|
|
"rewards/margins": 1.3769824504852295,
|
|
"rewards/rejected": 0.9306044578552246,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 10496.0,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -2.1318187713623047,
|
|
"logits/rejected": -2.199493646621704,
|
|
"logps/chosen": -0.27676263451576233,
|
|
"logps/rejected": -0.26010042428970337,
|
|
"loss": 4420.11279296875,
|
|
"loss/core": 4420.11279296875,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.0329606533050537,
|
|
"rewards/margins": 1.8643920421600342,
|
|
"rewards/rejected": 1.1685688495635986,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 3424.0,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.0941619873046875,
|
|
"logits/rejected": -2.0339064598083496,
|
|
"logps/chosen": -0.294299840927124,
|
|
"logps/rejected": -0.3125583529472351,
|
|
"loss": 4440.9580078125,
|
|
"loss/core": 4440.9580078125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8425333499908447,
|
|
"rewards/margins": 0.29318827390670776,
|
|
"rewards/rejected": 1.5493448972702026,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 996.0,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -2.262141704559326,
|
|
"logits/rejected": -2.3195528984069824,
|
|
"logps/chosen": -0.275444895029068,
|
|
"logps/rejected": -0.2715856432914734,
|
|
"loss": 4427.11572265625,
|
|
"loss/core": 4427.11572265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.358044147491455,
|
|
"rewards/margins": 1.3245257139205933,
|
|
"rewards/rejected": 1.033518671989441,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 884.0,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -2.2081265449523926,
|
|
"logits/rejected": -2.2153499126434326,
|
|
"logps/chosen": -0.2904752790927887,
|
|
"logps/rejected": -0.29524365067481995,
|
|
"loss": 4432.99755859375,
|
|
"loss/core": 4432.99755859375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.753950834274292,
|
|
"rewards/margins": 0.8617108464241028,
|
|
"rewards/rejected": 0.892240047454834,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 1744.0,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -2.158295154571533,
|
|
"logits/rejected": -2.2005717754364014,
|
|
"logps/chosen": -0.3060145378112793,
|
|
"logps/rejected": -0.29342374205589294,
|
|
"loss": 4430.84033203125,
|
|
"loss/core": 4430.84033203125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.088890314102173,
|
|
"rewards/margins": 1.0260167121887207,
|
|
"rewards/rejected": 1.0628736019134521,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 1896.0,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -2.228339433670044,
|
|
"logits/rejected": -2.3221817016601562,
|
|
"logps/chosen": -0.2925832271575928,
|
|
"logps/rejected": -0.29101797938346863,
|
|
"loss": 4421.3046875,
|
|
"loss/core": 4421.3046875,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.663743495941162,
|
|
"rewards/margins": 1.77371084690094,
|
|
"rewards/rejected": 0.8900324702262878,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 2352.0,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -2.023076295852661,
|
|
"logits/rejected": -1.9727592468261719,
|
|
"logps/chosen": -0.2806738018989563,
|
|
"logps/rejected": -0.29887765645980835,
|
|
"loss": 4417.65283203125,
|
|
"loss/core": 4417.65283203125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.167788505554199,
|
|
"rewards/margins": 2.10418963432312,
|
|
"rewards/rejected": 2.0635993480682373,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 7104.0,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -2.1479763984680176,
|
|
"logits/rejected": -2.180921792984009,
|
|
"logps/chosen": -0.2815966010093689,
|
|
"logps/rejected": -0.27384239435195923,
|
|
"loss": 4422.50537109375,
|
|
"loss/core": 4422.50537109375,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.650578022003174,
|
|
"rewards/margins": 1.758052110671997,
|
|
"rewards/rejected": 1.8925259113311768,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 324.0,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -2.1493828296661377,
|
|
"logits/rejected": -2.2821457386016846,
|
|
"logps/chosen": -0.3144438862800598,
|
|
"logps/rejected": -0.29132357239723206,
|
|
"loss": 4428.4169921875,
|
|
"loss/core": 4428.4169921875,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.025472640991211,
|
|
"rewards/margins": 1.2251968383789062,
|
|
"rewards/rejected": 0.8002756834030151,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 1976.0,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -2.0333778858184814,
|
|
"logits/rejected": -2.035680055618286,
|
|
"logps/chosen": -0.28305697441101074,
|
|
"logps/rejected": -0.30175352096557617,
|
|
"loss": 4433.373046875,
|
|
"loss/core": 4433.373046875,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.540903091430664,
|
|
"rewards/margins": 0.8402018547058105,
|
|
"rewards/rejected": 1.7007014751434326,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 2752.0,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -2.0594825744628906,
|
|
"logits/rejected": -2.229219675064087,
|
|
"logps/chosen": -0.3165486454963684,
|
|
"logps/rejected": -0.33373939990997314,
|
|
"loss": 4416.2041015625,
|
|
"loss/core": 4416.2041015625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1289918422698975,
|
|
"rewards/margins": 2.1620774269104004,
|
|
"rewards/rejected": 0.9669145345687866,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 7648.0,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -2.1951212882995605,
|
|
"logits/rejected": -2.163513660430908,
|
|
"logps/chosen": -0.29069000482559204,
|
|
"logps/rejected": -0.29456523060798645,
|
|
"loss": 4407.59716796875,
|
|
"loss/core": 4407.59716796875,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.40447998046875,
|
|
"rewards/margins": 2.884629726409912,
|
|
"rewards/rejected": 1.519850254058838,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 368.0,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -2.039048433303833,
|
|
"logits/rejected": -2.001903772354126,
|
|
"logps/chosen": -0.2846403121948242,
|
|
"logps/rejected": -0.28684523701667786,
|
|
"loss": 4424.06103515625,
|
|
"loss/core": 4424.06103515625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.0643351078033447,
|
|
"rewards/margins": 1.5574977397918701,
|
|
"rewards/rejected": 1.5068371295928955,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 3616.0,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -2.187757968902588,
|
|
"logits/rejected": -2.110003709793091,
|
|
"logps/chosen": -0.28276944160461426,
|
|
"logps/rejected": -0.27659210562705994,
|
|
"loss": 4418.234375,
|
|
"loss/core": 4418.234375,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.121807098388672,
|
|
"rewards/margins": 2.0180087089538574,
|
|
"rewards/rejected": 1.1037986278533936,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 560.0,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -2.1087703704833984,
|
|
"logits/rejected": -2.086918592453003,
|
|
"logps/chosen": -0.2788047194480896,
|
|
"logps/rejected": -0.28258147835731506,
|
|
"loss": 4444.259765625,
|
|
"loss/core": 4444.259765625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.587753176689148,
|
|
"rewards/margins": 0.029676269739866257,
|
|
"rewards/rejected": 1.5580768585205078,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 10176.0,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.3330397605895996,
|
|
"logits/rejected": -2.172541379928589,
|
|
"logps/chosen": -0.28407856822013855,
|
|
"logps/rejected": -0.2892422080039978,
|
|
"loss": 4422.20458984375,
|
|
"loss/core": 4422.20458984375,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6327457427978516,
|
|
"rewards/margins": 1.716529130935669,
|
|
"rewards/rejected": 0.9162166714668274,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 1416.0,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -1.896632194519043,
|
|
"logits/rejected": -1.9163414239883423,
|
|
"logps/chosen": -0.2775726616382599,
|
|
"logps/rejected": -0.2957395911216736,
|
|
"loss": 4429.7470703125,
|
|
"loss/core": 4429.7470703125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.7539432048797607,
|
|
"rewards/margins": 1.1101391315460205,
|
|
"rewards/rejected": 1.6438043117523193,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 756.0,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -2.0946218967437744,
|
|
"logits/rejected": -2.152243137359619,
|
|
"logps/chosen": -0.3078138530254364,
|
|
"logps/rejected": -0.2756267786026001,
|
|
"loss": 4413.80322265625,
|
|
"loss/core": 4413.80322265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3336167335510254,
|
|
"rewards/margins": 2.344905376434326,
|
|
"rewards/rejected": 0.9887111783027649,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 2208.0,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -2.1799540519714355,
|
|
"logits/rejected": -2.1663684844970703,
|
|
"logps/chosen": -0.2761627435684204,
|
|
"logps/rejected": -0.28063830733299255,
|
|
"loss": 4427.53564453125,
|
|
"loss/core": 4427.53564453125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.810899496078491,
|
|
"rewards/margins": 1.2789033651351929,
|
|
"rewards/rejected": 1.5319960117340088,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 2080.0,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -2.0762481689453125,
|
|
"logits/rejected": -2.101166009902954,
|
|
"logps/chosen": -0.28619739413261414,
|
|
"logps/rejected": -0.30411672592163086,
|
|
"loss": 4429.6884765625,
|
|
"loss/core": 4429.6884765625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.794001579284668,
|
|
"rewards/margins": 1.1226036548614502,
|
|
"rewards/rejected": 1.6713975667953491,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 5824.0,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -2.077721118927002,
|
|
"logits/rejected": -2.128166913986206,
|
|
"logps/chosen": -0.28693076968193054,
|
|
"logps/rejected": -0.2858390212059021,
|
|
"loss": 4422.75732421875,
|
|
"loss/core": 4422.75732421875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.361377000808716,
|
|
"rewards/margins": 1.646521806716919,
|
|
"rewards/rejected": 1.7148549556732178,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 588.0,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.156691074371338,
|
|
"logits/rejected": -2.1882190704345703,
|
|
"logps/chosen": -0.3002692461013794,
|
|
"logps/rejected": -0.3224543035030365,
|
|
"loss": 4427.705078125,
|
|
"loss/core": 4427.705078125,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8466622829437256,
|
|
"rewards/margins": 1.278574824333191,
|
|
"rewards/rejected": 0.5680874586105347,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 402.0,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -2.4301846027374268,
|
|
"logits/rejected": -2.4865431785583496,
|
|
"logps/chosen": -0.26671233773231506,
|
|
"logps/rejected": -0.27991726994514465,
|
|
"loss": 4438.52734375,
|
|
"loss/core": 4438.52734375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4181255102157593,
|
|
"rewards/margins": 0.4488455653190613,
|
|
"rewards/rejected": 0.9692800641059875,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 1120.0,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -2.1363842487335205,
|
|
"logits/rejected": -2.100966453552246,
|
|
"logps/chosen": -0.2700468599796295,
|
|
"logps/rejected": -0.2819743752479553,
|
|
"loss": 4433.23828125,
|
|
"loss/core": 4433.23828125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.22441029548645,
|
|
"rewards/margins": 0.8479706048965454,
|
|
"rewards/rejected": 1.3764398097991943,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 688.0,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -2.0266470909118652,
|
|
"logits/rejected": -2.1219282150268555,
|
|
"logps/chosen": -0.2864200174808502,
|
|
"logps/rejected": -0.2867039442062378,
|
|
"loss": 4410.529296875,
|
|
"loss/core": 4410.529296875,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.6321520805358887,
|
|
"rewards/margins": 2.5971732139587402,
|
|
"rewards/rejected": 1.0349791049957275,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 940.0,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -2.047276020050049,
|
|
"logits/rejected": -2.041769027709961,
|
|
"logps/chosen": -0.27496907114982605,
|
|
"logps/rejected": -0.27177929878234863,
|
|
"loss": 4433.5009765625,
|
|
"loss/core": 4433.5009765625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.713202714920044,
|
|
"rewards/margins": 0.8228942155838013,
|
|
"rewards/rejected": 0.8903085589408875,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 1688.0,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -2.2743325233459473,
|
|
"logits/rejected": -2.264472484588623,
|
|
"logps/chosen": -0.2655165493488312,
|
|
"logps/rejected": -0.28108400106430054,
|
|
"loss": 4434.0869140625,
|
|
"loss/core": 4434.0869140625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8811838626861572,
|
|
"rewards/margins": 0.7859585881233215,
|
|
"rewards/rejected": 1.0952253341674805,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 3680.0,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -2.117419481277466,
|
|
"logits/rejected": -2.192072868347168,
|
|
"logps/chosen": -0.27613067626953125,
|
|
"logps/rejected": -0.2757434844970703,
|
|
"loss": 4405.54541015625,
|
|
"loss/core": 4405.54541015625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.223845481872559,
|
|
"rewards/margins": 3.018414258956909,
|
|
"rewards/rejected": 1.205431342124939,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 2688.0,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -2.1161704063415527,
|
|
"logits/rejected": -2.132699489593506,
|
|
"logps/chosen": -0.2725360095500946,
|
|
"logps/rejected": -0.27246853709220886,
|
|
"loss": 4413.8681640625,
|
|
"loss/core": 4413.8681640625,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 4.309195518493652,
|
|
"rewards/margins": 2.3448550701141357,
|
|
"rewards/rejected": 1.9643408060073853,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 1112.0,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -1.9690643548965454,
|
|
"logits/rejected": -2.0342695713043213,
|
|
"logps/chosen": -0.3134908080101013,
|
|
"logps/rejected": -0.3014810383319855,
|
|
"loss": 4428.2001953125,
|
|
"loss/core": 4428.2001953125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.745121717453003,
|
|
"rewards/margins": 1.2340598106384277,
|
|
"rewards/rejected": 1.5110619068145752,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 828.0,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -2.141674518585205,
|
|
"logits/rejected": -2.1674609184265137,
|
|
"logps/chosen": -0.28517812490463257,
|
|
"logps/rejected": -0.29004770517349243,
|
|
"loss": 4441.734375,
|
|
"loss/core": 4441.734375,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6989895105361938,
|
|
"rewards/margins": 0.22279176115989685,
|
|
"rewards/rejected": 1.4761978387832642,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 19968.0,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.933323621749878,
|
|
"logits/rejected": -1.9620736837387085,
|
|
"logps/chosen": -0.3021884560585022,
|
|
"logps/rejected": -0.3325551450252533,
|
|
"loss": 4414.35498046875,
|
|
"loss/core": 4414.35498046875,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.912782669067383,
|
|
"rewards/margins": 2.36540150642395,
|
|
"rewards/rejected": 1.5473812818527222,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 1264.0,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -1.9333274364471436,
|
|
"logits/rejected": -2.048060894012451,
|
|
"logps/chosen": -0.27091148495674133,
|
|
"logps/rejected": -0.2834457755088806,
|
|
"loss": 4421.9296875,
|
|
"loss/core": 4421.9296875,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9585728645324707,
|
|
"rewards/margins": 1.7152507305145264,
|
|
"rewards/rejected": 1.2433220148086548,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 5376.0,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -1.983690857887268,
|
|
"logits/rejected": -2.0693118572235107,
|
|
"logps/chosen": -0.3024982810020447,
|
|
"logps/rejected": -0.29896295070648193,
|
|
"loss": 4427.31494140625,
|
|
"loss/core": 4427.31494140625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.329944372177124,
|
|
"rewards/margins": 1.3042433261871338,
|
|
"rewards/rejected": 1.0257014036178589,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 668.0,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -2.2102410793304443,
|
|
"logits/rejected": -2.2873520851135254,
|
|
"logps/chosen": -0.29883983731269836,
|
|
"logps/rejected": -0.2977871298789978,
|
|
"loss": 4428.97705078125,
|
|
"loss/core": 4428.97705078125,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.9840648174285889,
|
|
"rewards/margins": 1.1769111156463623,
|
|
"rewards/rejected": 0.807153582572937,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 1560.0,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -2.1556880474090576,
|
|
"logits/rejected": -2.131955623626709,
|
|
"logps/chosen": -0.2893293499946594,
|
|
"logps/rejected": -0.2930451035499573,
|
|
"loss": 4438.78759765625,
|
|
"loss/core": 4438.78759765625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6869512796401978,
|
|
"rewards/margins": 0.43390756845474243,
|
|
"rewards/rejected": 1.2530437707901,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 932.0,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -2.222175359725952,
|
|
"logits/rejected": -2.172823429107666,
|
|
"logps/chosen": -0.2886146605014801,
|
|
"logps/rejected": -0.29807931184768677,
|
|
"loss": 4437.9853515625,
|
|
"loss/core": 4437.9853515625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.3422349691390991,
|
|
"rewards/margins": 0.49204936623573303,
|
|
"rewards/rejected": 0.8501856923103333,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 4224.0,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -2.060539960861206,
|
|
"logits/rejected": -2.123028039932251,
|
|
"logps/chosen": -0.28187817335128784,
|
|
"logps/rejected": -0.2881320118904114,
|
|
"loss": 4431.12353515625,
|
|
"loss/core": 4431.12353515625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0083515644073486,
|
|
"rewards/margins": 1.0040746927261353,
|
|
"rewards/rejected": 1.004276990890503,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 616.0,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -2.032500982284546,
|
|
"logits/rejected": -2.027933359146118,
|
|
"logps/chosen": -0.29038962721824646,
|
|
"logps/rejected": -0.30487364530563354,
|
|
"loss": 4425.306640625,
|
|
"loss/core": 4425.306640625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8122401237487793,
|
|
"rewards/margins": 1.49310302734375,
|
|
"rewards/rejected": 1.3191370964050293,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 2960.0,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -2.1880297660827637,
|
|
"logits/rejected": -2.1642916202545166,
|
|
"logps/chosen": -0.28237125277519226,
|
|
"logps/rejected": -0.27983298897743225,
|
|
"loss": 4433.11083984375,
|
|
"loss/core": 4433.11083984375,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.367220163345337,
|
|
"rewards/margins": 0.8713071942329407,
|
|
"rewards/rejected": 1.495913028717041,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 588.0,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -1.982226014137268,
|
|
"logits/rejected": -2.0288047790527344,
|
|
"logps/chosen": -0.2729211151599884,
|
|
"logps/rejected": -0.31506603956222534,
|
|
"loss": 4413.7041015625,
|
|
"loss/core": 4413.7041015625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.6589457988739014,
|
|
"rewards/margins": 2.350693702697754,
|
|
"rewards/rejected": 1.308251976966858,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 1032.0,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.0349762439727783,
|
|
"logits/rejected": -2.077690601348877,
|
|
"logps/chosen": -0.28860360383987427,
|
|
"logps/rejected": -0.2868594527244568,
|
|
"loss": 4434.82080078125,
|
|
"loss/core": 4434.82080078125,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6204445362091064,
|
|
"rewards/margins": 0.7260267734527588,
|
|
"rewards/rejected": 0.8944176435470581,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 596.0,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -2.1599338054656982,
|
|
"logits/rejected": -2.0785868167877197,
|
|
"logps/chosen": -0.28200381994247437,
|
|
"logps/rejected": -0.29134702682495117,
|
|
"loss": 4434.482421875,
|
|
"loss/core": 4434.482421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1464552879333496,
|
|
"rewards/margins": 0.7586153149604797,
|
|
"rewards/rejected": 1.3878400325775146,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 780.0,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -2.0853731632232666,
|
|
"logits/rejected": -2.0709526538848877,
|
|
"logps/chosen": -0.25362181663513184,
|
|
"logps/rejected": -0.2743147313594818,
|
|
"loss": 4427.447265625,
|
|
"loss/core": 4427.447265625,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.885993719100952,
|
|
"rewards/margins": 1.3240457773208618,
|
|
"rewards/rejected": 1.5619480609893799,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 708.0,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -1.8858239650726318,
|
|
"logits/rejected": -1.9080051183700562,
|
|
"logps/chosen": -0.30707043409347534,
|
|
"logps/rejected": -0.31979310512542725,
|
|
"loss": 4436.5107421875,
|
|
"loss/core": 4436.5107421875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7793325185775757,
|
|
"rewards/margins": 0.6050835251808167,
|
|
"rewards/rejected": 1.1742490530014038,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 916.0,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -2.273056745529175,
|
|
"logits/rejected": -2.28229022026062,
|
|
"logps/chosen": -0.30402398109436035,
|
|
"logps/rejected": -0.29174989461898804,
|
|
"loss": 4427.4306640625,
|
|
"loss/core": 4427.4306640625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9931259155273438,
|
|
"rewards/margins": 1.2991578578948975,
|
|
"rewards/rejected": 0.6939681768417358,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 14528.0,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -2.105236768722534,
|
|
"logits/rejected": -2.20990252494812,
|
|
"logps/chosen": -0.2958459258079529,
|
|
"logps/rejected": -0.31115972995758057,
|
|
"loss": 4437.0419921875,
|
|
"loss/core": 4437.0419921875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.240123748779297,
|
|
"rewards/margins": 0.5702623128890991,
|
|
"rewards/rejected": 1.6698611974716187,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 2880.0,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -2.197855234146118,
|
|
"logits/rejected": -2.154949903488159,
|
|
"logps/chosen": -0.28139424324035645,
|
|
"logps/rejected": -0.2661704421043396,
|
|
"loss": 4420.2060546875,
|
|
"loss/core": 4420.2060546875,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.394221067428589,
|
|
"rewards/margins": 1.8499267101287842,
|
|
"rewards/rejected": 1.5442945957183838,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 9536.0,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -1.889145851135254,
|
|
"logits/rejected": -1.876465082168579,
|
|
"logps/chosen": -0.27592915296554565,
|
|
"logps/rejected": -0.2931953966617584,
|
|
"loss": 4422.54833984375,
|
|
"loss/core": 4422.54833984375,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.609574794769287,
|
|
"rewards/margins": 1.6662747859954834,
|
|
"rewards/rejected": 1.9433000087738037,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 792.0,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -2.253756284713745,
|
|
"logits/rejected": -2.234802484512329,
|
|
"logps/chosen": -0.2832790017127991,
|
|
"logps/rejected": -0.3265189826488495,
|
|
"loss": 4430.35302734375,
|
|
"loss/core": 4430.35302734375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8739410638809204,
|
|
"rewards/margins": 1.0730499029159546,
|
|
"rewards/rejected": 0.8008909225463867,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 2096.0,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -1.9960432052612305,
|
|
"logits/rejected": -2.0051181316375732,
|
|
"logps/chosen": -0.2970682680606842,
|
|
"logps/rejected": -0.29701724648475647,
|
|
"loss": 4437.2138671875,
|
|
"loss/core": 4437.2138671875,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0947940349578857,
|
|
"rewards/margins": 0.5625554919242859,
|
|
"rewards/rejected": 1.5322383642196655,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 932.0,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -2.124847888946533,
|
|
"logits/rejected": -2.2395145893096924,
|
|
"logps/chosen": -0.28501376509666443,
|
|
"logps/rejected": -0.27728787064552307,
|
|
"loss": 4433.05029296875,
|
|
"loss/core": 4433.05029296875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5915935039520264,
|
|
"rewards/margins": 0.8602860569953918,
|
|
"rewards/rejected": 0.7313073873519897,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 6816.0,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -2.2217555046081543,
|
|
"logits/rejected": -2.272338628768921,
|
|
"logps/chosen": -0.28637298941612244,
|
|
"logps/rejected": -0.2902549207210541,
|
|
"loss": 4443.1611328125,
|
|
"loss/core": 4443.1611328125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.0200834274291992,
|
|
"rewards/margins": 0.1075645312666893,
|
|
"rewards/rejected": 0.9125189781188965,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 2208.0,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -2.0483860969543457,
|
|
"logits/rejected": -2.0965161323547363,
|
|
"logps/chosen": -0.3195610046386719,
|
|
"logps/rejected": -0.3177678883075714,
|
|
"loss": 4434.37109375,
|
|
"loss/core": 4434.37109375,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.4221794605255127,
|
|
"rewards/margins": 0.7587219476699829,
|
|
"rewards/rejected": 0.6634576320648193,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 972.0,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -2.103890895843506,
|
|
"logits/rejected": -2.122037172317505,
|
|
"logps/chosen": -0.3093516230583191,
|
|
"logps/rejected": -0.31388068199157715,
|
|
"loss": 4435.8095703125,
|
|
"loss/core": 4435.8095703125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.601134777069092,
|
|
"rewards/margins": 0.7491642236709595,
|
|
"rewards/rejected": 1.8519706726074219,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 5824.0,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -2.053230047225952,
|
|
"logits/rejected": -2.097982406616211,
|
|
"logps/chosen": -0.28634682297706604,
|
|
"logps/rejected": -0.28023213148117065,
|
|
"loss": 4423.0634765625,
|
|
"loss/core": 4423.0634765625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1622960567474365,
|
|
"rewards/margins": 1.6293554306030273,
|
|
"rewards/rejected": 1.5329406261444092,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 2544.0,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -2.1885745525360107,
|
|
"logits/rejected": -2.1799445152282715,
|
|
"logps/chosen": -0.3118075728416443,
|
|
"logps/rejected": -0.3031180500984192,
|
|
"loss": 4445.6220703125,
|
|
"loss/core": 4445.6220703125,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.2049453258514404,
|
|
"rewards/margins": -0.05652730539441109,
|
|
"rewards/rejected": 1.261472463607788,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 2000.0,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -1.9294588565826416,
|
|
"logits/rejected": -1.839930534362793,
|
|
"logps/chosen": -0.3078926205635071,
|
|
"logps/rejected": -0.29328030347824097,
|
|
"loss": 4435.65625,
|
|
"loss/core": 4435.65625,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.3930587768554688,
|
|
"rewards/margins": 0.6730942726135254,
|
|
"rewards/rejected": 1.719964623451233,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 1056.0,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -2.1844775676727295,
|
|
"logits/rejected": -2.2019870281219482,
|
|
"logps/chosen": -0.30291616916656494,
|
|
"logps/rejected": -0.292309045791626,
|
|
"loss": 4429.7392578125,
|
|
"loss/core": 4429.7392578125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.988659143447876,
|
|
"rewards/margins": 1.112038254737854,
|
|
"rewards/rejected": 0.8766206502914429,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 1656.0,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -2.0841526985168457,
|
|
"logits/rejected": -2.1565730571746826,
|
|
"logps/chosen": -0.2759454846382141,
|
|
"logps/rejected": -0.2714347541332245,
|
|
"loss": 4434.8095703125,
|
|
"loss/core": 4434.8095703125,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7954429388046265,
|
|
"rewards/margins": 0.7274172902107239,
|
|
"rewards/rejected": 1.0680259466171265,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 2832.0,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -1.9811954498291016,
|
|
"logits/rejected": -1.9468730688095093,
|
|
"logps/chosen": -0.3071630895137787,
|
|
"logps/rejected": -0.2786198556423187,
|
|
"loss": 4427.37744140625,
|
|
"loss/core": 4427.37744140625,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.7594974040985107,
|
|
"rewards/margins": 1.3463454246520996,
|
|
"rewards/rejected": 2.4131522178649902,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 412.0,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -2.095184803009033,
|
|
"logits/rejected": -2.1245932579040527,
|
|
"logps/chosen": -0.30588921904563904,
|
|
"logps/rejected": -0.29908287525177,
|
|
"loss": 4436.82177734375,
|
|
"loss/core": 4436.82177734375,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5300137996673584,
|
|
"rewards/margins": 0.5759037733078003,
|
|
"rewards/rejected": 0.9541099667549133,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 1480.0,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -2.1838436126708984,
|
|
"logits/rejected": -2.197448492050171,
|
|
"logps/chosen": -0.30141058564186096,
|
|
"logps/rejected": -0.29812097549438477,
|
|
"loss": 4436.86181640625,
|
|
"loss/core": 4436.86181640625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8982967138290405,
|
|
"rewards/margins": 0.5740464925765991,
|
|
"rewards/rejected": 1.3242502212524414,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 564.0,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.9970121383666992,
|
|
"logits/rejected": -1.9551795721054077,
|
|
"logps/chosen": -0.31093713641166687,
|
|
"logps/rejected": -0.2889935374259949,
|
|
"loss": 4418.28271484375,
|
|
"loss/core": 4418.28271484375,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.431575298309326,
|
|
"rewards/margins": 2.0386598110198975,
|
|
"rewards/rejected": 1.3929154872894287,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 19840.0,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -2.1677725315093994,
|
|
"logits/rejected": -1.9948818683624268,
|
|
"logps/chosen": -0.2858007550239563,
|
|
"logps/rejected": -0.31171754002571106,
|
|
"loss": 4436.4150390625,
|
|
"loss/core": 4436.4150390625,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.6222074031829834,
|
|
"rewards/margins": 0.6268441081047058,
|
|
"rewards/rejected": 1.9953632354736328,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 2176.0,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -1.9716495275497437,
|
|
"logits/rejected": -1.9900119304656982,
|
|
"logps/chosen": -0.3067142367362976,
|
|
"logps/rejected": -0.29664263129234314,
|
|
"loss": 4428.5654296875,
|
|
"loss/core": 4428.5654296875,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3875348567962646,
|
|
"rewards/margins": 1.2067521810531616,
|
|
"rewards/rejected": 1.1807825565338135,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 824.0,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.1559529304504395,
|
|
"logits/rejected": -2.1331565380096436,
|
|
"logps/chosen": -0.289201557636261,
|
|
"logps/rejected": -0.28383710980415344,
|
|
"loss": 4430.1826171875,
|
|
"loss/core": 4430.1826171875,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2973222732543945,
|
|
"rewards/margins": 1.0815565586090088,
|
|
"rewards/rejected": 1.2157660722732544,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 1240.0,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -2.0271267890930176,
|
|
"logits/rejected": -2.0145370960235596,
|
|
"logps/chosen": -0.27510371804237366,
|
|
"logps/rejected": -0.2665782868862152,
|
|
"loss": 4441.60107421875,
|
|
"loss/core": 4441.60107421875,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.2546658515930176,
|
|
"rewards/margins": 0.22854623198509216,
|
|
"rewards/rejected": 2.0261194705963135,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 840.0,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -2.2316105365753174,
|
|
"logits/rejected": -2.135343074798584,
|
|
"logps/chosen": -0.30771583318710327,
|
|
"logps/rejected": -0.31672921776771545,
|
|
"loss": 4443.7978515625,
|
|
"loss/core": 4443.7978515625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.536110520362854,
|
|
"rewards/margins": 0.05860384181141853,
|
|
"rewards/rejected": 1.4775065183639526,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 1800.0,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -2.0983340740203857,
|
|
"logits/rejected": -2.187591552734375,
|
|
"logps/chosen": -0.2833598256111145,
|
|
"logps/rejected": -0.28382277488708496,
|
|
"loss": 4430.76611328125,
|
|
"loss/core": 4430.76611328125,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1276965141296387,
|
|
"rewards/margins": 1.0290817022323608,
|
|
"rewards/rejected": 1.0986146926879883,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 636.0,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -2.1024179458618164,
|
|
"logits/rejected": -2.1378636360168457,
|
|
"logps/chosen": -0.28758540749549866,
|
|
"logps/rejected": -0.31663960218429565,
|
|
"loss": 4427.5947265625,
|
|
"loss/core": 4427.5947265625,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.9671586751937866,
|
|
"rewards/margins": 1.2754833698272705,
|
|
"rewards/rejected": 0.6916751265525818,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 2944.0,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -2.1443867683410645,
|
|
"logits/rejected": -2.248300075531006,
|
|
"logps/chosen": -0.2994143068790436,
|
|
"logps/rejected": -0.28394490480422974,
|
|
"loss": 4427.640625,
|
|
"loss/core": 4427.640625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.490103006362915,
|
|
"rewards/margins": 1.267347812652588,
|
|
"rewards/rejected": 1.2227551937103271,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 1272.0,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -2.18145489692688,
|
|
"logits/rejected": -2.130202293395996,
|
|
"logps/chosen": -0.2789270877838135,
|
|
"logps/rejected": -0.3010575771331787,
|
|
"loss": 4444.62744140625,
|
|
"loss/core": 4444.62744140625,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7224094867706299,
|
|
"rewards/margins": 0.0003525257052388042,
|
|
"rewards/rejected": 1.7220569849014282,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 7616.0,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -2.0989551544189453,
|
|
"logits/rejected": -2.0986316204071045,
|
|
"logps/chosen": -0.25472623109817505,
|
|
"logps/rejected": -0.3020690977573395,
|
|
"loss": 4423.72119140625,
|
|
"loss/core": 4423.72119140625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4041237831115723,
|
|
"rewards/margins": 1.5831953287124634,
|
|
"rewards/rejected": 0.8209284543991089,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 21504.0,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -2.036388874053955,
|
|
"logits/rejected": -2.085162401199341,
|
|
"logps/chosen": -0.32136353850364685,
|
|
"logps/rejected": -0.3007509410381317,
|
|
"loss": 4421.49072265625,
|
|
"loss/core": 4421.49072265625,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8142402172088623,
|
|
"rewards/margins": 1.7705316543579102,
|
|
"rewards/rejected": 1.0437085628509521,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 764.0,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -2.175672769546509,
|
|
"logits/rejected": -2.2510218620300293,
|
|
"logps/chosen": -0.2935303747653961,
|
|
"logps/rejected": -0.3111540675163269,
|
|
"loss": 4431.751953125,
|
|
"loss/core": 4431.751953125,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.4188427925109863,
|
|
"rewards/margins": 0.9629634618759155,
|
|
"rewards/rejected": 0.4558793008327484,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 3712.0,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -2.2930002212524414,
|
|
"logits/rejected": -2.3410279750823975,
|
|
"logps/chosen": -0.251552551984787,
|
|
"logps/rejected": -0.24615788459777832,
|
|
"loss": 4435.8291015625,
|
|
"loss/core": 4435.8291015625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.5625412464141846,
|
|
"rewards/margins": 0.6484128832817078,
|
|
"rewards/rejected": 0.9141284227371216,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 2384.0,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -2.006430149078369,
|
|
"logits/rejected": -2.0440707206726074,
|
|
"logps/chosen": -0.301272451877594,
|
|
"logps/rejected": -0.29304254055023193,
|
|
"loss": 4425.640625,
|
|
"loss/core": 4425.640625,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6843559741973877,
|
|
"rewards/margins": 1.433323621749878,
|
|
"rewards/rejected": 1.2510322332382202,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 508.0,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -1.952792763710022,
|
|
"logits/rejected": -2.0415608882904053,
|
|
"logps/chosen": -0.2997278571128845,
|
|
"logps/rejected": -0.29428717494010925,
|
|
"loss": 4416.9169921875,
|
|
"loss/core": 4416.9169921875,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.084506034851074,
|
|
"rewards/margins": 2.108910083770752,
|
|
"rewards/rejected": 1.975595474243164,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 1544.0,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -1.961377739906311,
|
|
"logits/rejected": -2.034229040145874,
|
|
"logps/chosen": -0.2911316156387329,
|
|
"logps/rejected": -0.2944122850894928,
|
|
"loss": 4433.12158203125,
|
|
"loss/core": 4433.12158203125,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.4497056007385254,
|
|
"rewards/margins": 0.8692841529846191,
|
|
"rewards/rejected": 1.5804215669631958,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 1304.0,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -2.1136727333068848,
|
|
"logits/rejected": -2.121825695037842,
|
|
"logps/chosen": -0.3133166432380676,
|
|
"logps/rejected": -0.29426655173301697,
|
|
"loss": 4415.9951171875,
|
|
"loss/core": 4415.9951171875,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.1630001068115234,
|
|
"rewards/margins": 2.198111057281494,
|
|
"rewards/rejected": 0.9648886919021606,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 968.0,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -2.1847896575927734,
|
|
"logits/rejected": -2.2328908443450928,
|
|
"logps/chosen": -0.3081599771976471,
|
|
"logps/rejected": -0.3277169167995453,
|
|
"loss": 4434.0029296875,
|
|
"loss/core": 4434.0029296875,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8598581552505493,
|
|
"rewards/margins": 0.8051584959030151,
|
|
"rewards/rejected": 1.0546996593475342,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 8000.0,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -2.1225297451019287,
|
|
"logits/rejected": -2.1040260791778564,
|
|
"logps/chosen": -0.290223628282547,
|
|
"logps/rejected": -0.32425063848495483,
|
|
"loss": 4436.90234375,
|
|
"loss/core": 4436.90234375,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.4622364044189453,
|
|
"rewards/margins": 0.6243095993995667,
|
|
"rewards/rejected": 1.8379265069961548,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 1528.0,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -2.043444871902466,
|
|
"logits/rejected": -2.1212260723114014,
|
|
"logps/chosen": -0.2810841202735901,
|
|
"logps/rejected": -0.277138888835907,
|
|
"loss": 4423.04150390625,
|
|
"loss/core": 4423.04150390625,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.739732265472412,
|
|
"rewards/margins": 1.617659568786621,
|
|
"rewards/rejected": 1.122072458267212,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 5856.0,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -1.9764293432235718,
|
|
"logits/rejected": -2.0967178344726562,
|
|
"logps/chosen": -0.29462671279907227,
|
|
"logps/rejected": -0.27400484681129456,
|
|
"loss": 4430.4931640625,
|
|
"loss/core": 4430.4931640625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5097718238830566,
|
|
"rewards/margins": 1.0680214166641235,
|
|
"rewards/rejected": 1.4417507648468018,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 2288.0,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -2.046212673187256,
|
|
"logits/rejected": -2.080132007598877,
|
|
"logps/chosen": -0.2724604308605194,
|
|
"logps/rejected": -0.2843073904514313,
|
|
"loss": 4434.88623046875,
|
|
"loss/core": 4434.88623046875,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.3006958961486816,
|
|
"rewards/margins": 0.7250384092330933,
|
|
"rewards/rejected": 1.5756577253341675,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 2272.0,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -2.057281970977783,
|
|
"logits/rejected": -2.177447557449341,
|
|
"logps/chosen": -0.2676500976085663,
|
|
"logps/rejected": -0.26869815587997437,
|
|
"loss": 4414.8955078125,
|
|
"loss/core": 4414.8955078125,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.1301474571228027,
|
|
"rewards/margins": 2.2849082946777344,
|
|
"rewards/rejected": 0.8452390432357788,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 4428.528517795139,
|
|
"train_runtime": 7133.5294,
|
|
"train_samples_per_second": 2.019,
|
|
"train_steps_per_second": 0.126
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|