1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 10.78728199005127,
|
|
"kl": 6.045640468597412,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37436245.333333336,
|
|
"logits/rejected": -39276200.65882353,
|
|
"logps/chosen": -485.2478125,
|
|
"logps/rejected": -358.6040670955882,
|
|
"loss": 0.4642,
|
|
"loss/base_kto": 3.7135796546936035,
|
|
"metrics/advantage_var": 399.55609130859375,
|
|
"rewards/chosen": -0.16165313720703126,
|
|
"rewards/margins": 0.2856965906479779,
|
|
"rewards/rejected": -0.4473497278550092,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 10.290058135986328,
|
|
"kl": 7.082744121551514,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36603852.558869705,
|
|
"logits/rejected": -37564747.247278385,
|
|
"logps/chosen": -464.4825843799058,
|
|
"logps/rejected": -382.692675933126,
|
|
"loss": 0.4521,
|
|
"loss/base_kto": 3.61698842048645,
|
|
"metrics/advantage_var": 725.0475463867188,
|
|
"rewards/chosen": 0.0748193522263172,
|
|
"rewards/margins": 0.4525030006516671,
|
|
"rewards/rejected": -0.3776836484253499,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 11.527665138244629,
|
|
"kl": 5.735081672668457,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36186875.10828026,
|
|
"logits/rejected": -38534866.45398773,
|
|
"logps/chosen": -479.4423765923567,
|
|
"logps/rejected": -381.6414877300613,
|
|
"loss": 0.4266,
|
|
"loss/base_kto": 3.412820816040039,
|
|
"metrics/advantage_var": 935.1497192382812,
|
|
"rewards/chosen": 0.272069019876468,
|
|
"rewards/margins": 0.7197896469883593,
|
|
"rewards/rejected": -0.4477206271118913,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 8.851114273071289,
|
|
"kl": 5.344668865203857,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -38228702.0032,
|
|
"logits/rejected": -39082499.908396944,
|
|
"logps/chosen": -478.27695,
|
|
"logps/rejected": -376.62304389312976,
|
|
"loss": 0.4305,
|
|
"loss/base_kto": 3.4443156719207764,
|
|
"metrics/advantage_var": 1630.0986328125,
|
|
"rewards/chosen": 0.14659896240234374,
|
|
"rewards/margins": 0.7242343765841185,
|
|
"rewards/rejected": -0.5776354141817748,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 10.306970596313477,
|
|
"kl": 16.163732528686523,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -37611377.270664506,
|
|
"logits/rejected": -38828652.88687783,
|
|
"logps/chosen": -441.5427471636953,
|
|
"logps/rejected": -363.75973322021116,
|
|
"loss": 0.4155,
|
|
"loss/base_kto": 3.3236663341522217,
|
|
"metrics/advantage_var": 2278.3662109375,
|
|
"rewards/chosen": 0.639388667126722,
|
|
"rewards/margins": 0.949373107304536,
|
|
"rewards/rejected": -0.3099844401778139,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 8.703822135925293,
|
|
"kl": 19.105472564697266,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -38187107.38823529,
|
|
"logits/rejected": -38690228.906666666,
|
|
"logps/chosen": -477.3513786764706,
|
|
"logps/rejected": -371.125703125,
|
|
"loss": 0.4164,
|
|
"loss/base_kto": 3.331439256668091,
|
|
"metrics/advantage_var": 3020.926513671875,
|
|
"rewards/chosen": 0.6670391307157628,
|
|
"rewards/margins": 0.9879135101916743,
|
|
"rewards/rejected": -0.32087437947591146,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 7.378037452697754,
|
|
"kl": 47.66223907470703,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -38584833.528358206,
|
|
"logits/rejected": -39298165.50819672,
|
|
"logps/chosen": -460.7433768656716,
|
|
"logps/rejected": -353.40066598360653,
|
|
"loss": 0.3796,
|
|
"loss/base_kto": 3.0371625423431396,
|
|
"metrics/advantage_var": 2964.495361328125,
|
|
"rewards/chosen": 1.6767709305037313,
|
|
"rewards/margins": 1.3247292044695356,
|
|
"rewards/rejected": 0.3520417260341957,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 7.053978443145752,
|
|
"kl": 10.050368309020996,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -41215980.679245286,
|
|
"logits/rejected": -40242430.40993789,
|
|
"logps/chosen": -470.062106918239,
|
|
"logps/rejected": -374.5430415372671,
|
|
"loss": 0.3795,
|
|
"loss/base_kto": 3.0357117652893066,
|
|
"metrics/advantage_var": 4414.76513671875,
|
|
"rewards/chosen": 0.6166294025924971,
|
|
"rewards/margins": 1.6501214020587236,
|
|
"rewards/rejected": -1.0334919994662266,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 10.467313766479492,
|
|
"kl": 13.880614280700684,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -40612920.26373626,
|
|
"logits/rejected": -40778782.25816485,
|
|
"logps/chosen": -464.33070054945057,
|
|
"logps/rejected": -358.9967680793157,
|
|
"loss": 0.3659,
|
|
"loss/base_kto": 2.9274158477783203,
|
|
"metrics/advantage_var": 5033.04248046875,
|
|
"rewards/chosen": 0.90423583984375,
|
|
"rewards/margins": 1.9941266713950718,
|
|
"rewards/rejected": -1.0898908315513218,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 8.401222229003906,
|
|
"kl": 34.443275451660156,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -42970434.54434251,
|
|
"logits/rejected": -41785458.50479233,
|
|
"logps/chosen": -495.66800458715596,
|
|
"logps/rejected": -364.3836861022364,
|
|
"loss": 0.3681,
|
|
"loss/base_kto": 2.944542407989502,
|
|
"metrics/advantage_var": 4823.06640625,
|
|
"rewards/chosen": 1.6523476696889334,
|
|
"rewards/margins": 1.8160656281819072,
|
|
"rewards/rejected": -0.16371795849297374,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 7.976482391357422,
|
|
"kl": 16.6918888092041,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -42382349.98482549,
|
|
"logits/rejected": -43137208.68276972,
|
|
"logps/chosen": -486.696936646434,
|
|
"logps/rejected": -382.6753723832528,
|
|
"loss": 0.371,
|
|
"loss/base_kto": 2.96830153465271,
|
|
"metrics/advantage_var": 5592.57373046875,
|
|
"rewards/chosen": 1.4093304693427542,
|
|
"rewards/margins": 1.9811324826969712,
|
|
"rewards/rejected": -0.571802013354217,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 7.520793914794922,
|
|
"kl": 30.6082820892334,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -41710281.45567652,
|
|
"logits/rejected": -41567834.82574568,
|
|
"logps/chosen": -460.29806570762054,
|
|
"logps/rejected": -371.8941326530612,
|
|
"loss": 0.3473,
|
|
"loss/base_kto": 2.7783279418945312,
|
|
"metrics/advantage_var": 6056.45361328125,
|
|
"rewards/chosen": 1.6413401459831842,
|
|
"rewards/margins": 2.216954176902042,
|
|
"rewards/rejected": -0.5756140309188579,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 8.243804931640625,
|
|
"kl": 14.915915489196777,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -42622353.537007876,
|
|
"logits/rejected": -42973828.56434108,
|
|
"logps/chosen": -479.83922244094487,
|
|
"logps/rejected": -391.5109738372093,
|
|
"loss": 0.3694,
|
|
"loss/base_kto": 2.9554238319396973,
|
|
"metrics/advantage_var": 6400.8681640625,
|
|
"rewards/chosen": 0.871806275375246,
|
|
"rewards/margins": 2.077575848261632,
|
|
"rewards/rejected": -1.2057695728863858,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 7.776176929473877,
|
|
"kl": 19.597034454345703,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -40272461.824,
|
|
"logits/rejected": -41789380.59236641,
|
|
"logps/chosen": -469.8292,
|
|
"logps/rejected": -394.7070610687023,
|
|
"loss": 0.3643,
|
|
"loss/base_kto": 2.9144322872161865,
|
|
"metrics/advantage_var": 6827.59619140625,
|
|
"rewards/chosen": 1.0211619140625,
|
|
"rewards/margins": 2.1597158188215646,
|
|
"rewards/rejected": -1.138553904759065,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 7.238597393035889,
|
|
"kl": 14.930597305297852,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -41993617.38271605,
|
|
"logits/rejected": -42283986.63291139,
|
|
"logps/chosen": -468.4034047067901,
|
|
"logps/rejected": -399.5572339794304,
|
|
"loss": 0.3445,
|
|
"loss/base_kto": 2.7562012672424316,
|
|
"metrics/advantage_var": 6334.48095703125,
|
|
"rewards/chosen": 1.5196476689091436,
|
|
"rewards/margins": 2.4504255928794265,
|
|
"rewards/rejected": -0.9307779239702828,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 7.258980751037598,
|
|
"kl": 36.62384033203125,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -44909393.996992484,
|
|
"logits/rejected": -43970276.94308943,
|
|
"logps/chosen": -451.784492481203,
|
|
"logps/rejected": -391.7923526422764,
|
|
"loss": 0.34,
|
|
"loss/base_kto": 2.720338821411133,
|
|
"metrics/advantage_var": 6735.33935546875,
|
|
"rewards/chosen": 2.063958235432331,
|
|
"rewards/margins": 2.3941122822120566,
|
|
"rewards/rejected": -0.3301540467797256,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 8.35556411743164,
|
|
"kl": 19.140226364135742,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -46471030.614664584,
|
|
"logits/rejected": -44903640.33802817,
|
|
"logps/chosen": -494.1961290951638,
|
|
"logps/rejected": -376.35766334115806,
|
|
"loss": 0.3404,
|
|
"loss/base_kto": 2.722824811935425,
|
|
"metrics/advantage_var": 8471.8046875,
|
|
"rewards/chosen": 1.7114452058551093,
|
|
"rewards/margins": 2.770817989868216,
|
|
"rewards/rejected": -1.0593727840131064,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 9.343971252441406,
|
|
"kl": 13.9060697555542,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -43613165.54954955,
|
|
"logits/rejected": -43185035.25732899,
|
|
"logps/chosen": -478.3201951951952,
|
|
"logps/rejected": -399.224730252443,
|
|
"loss": 0.3565,
|
|
"loss/base_kto": 2.8517332077026367,
|
|
"metrics/advantage_var": 8707.2890625,
|
|
"rewards/chosen": 1.4123950305285755,
|
|
"rewards/margins": 2.5237740512940476,
|
|
"rewards/rejected": -1.1113790207654723,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 8.0005521774292,
|
|
"kl": 31.26420021057129,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -42496073.14285714,
|
|
"logits/rejected": -44363827.52201258,
|
|
"logps/chosen": -478.9863159937888,
|
|
"logps/rejected": -366.2966047562893,
|
|
"loss": 0.3337,
|
|
"loss/base_kto": 2.6695454120635986,
|
|
"metrics/advantage_var": 8384.955078125,
|
|
"rewards/chosen": 1.8745265036636258,
|
|
"rewards/margins": 2.831779246023099,
|
|
"rewards/rejected": -0.9572527423594732,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 9.551956176757812,
|
|
"kl": 30.651647567749023,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -44210673.64485981,
|
|
"logits/rejected": -45153241.479623824,
|
|
"logps/chosen": -462.75515965732086,
|
|
"logps/rejected": -386.5633571708464,
|
|
"loss": 0.3509,
|
|
"loss/base_kto": 2.8070526123046875,
|
|
"metrics/advantage_var": 7753.9462890625,
|
|
"rewards/chosen": 1.7811591127579829,
|
|
"rewards/margins": 2.4744187555473394,
|
|
"rewards/rejected": -0.6932596427893564,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 8.593188285827637,
|
|
"kl": 33.22036361694336,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -44223064.493827164,
|
|
"logits/rejected": -43488385.62025317,
|
|
"logps/chosen": -467.7560763888889,
|
|
"logps/rejected": -389.8130933544304,
|
|
"loss": 0.3453,
|
|
"loss/base_kto": 2.7623486518859863,
|
|
"metrics/advantage_var": 9086.478515625,
|
|
"rewards/chosen": 1.87820585274402,
|
|
"rewards/margins": 2.616768424595533,
|
|
"rewards/rejected": -0.7385625718515131,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 13.203508377075195,
|
|
"kl": 18.998483657836914,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -45640985.03470032,
|
|
"logits/rejected": -46091413.00309598,
|
|
"logps/chosen": -476.49191640378547,
|
|
"logps/rejected": -430.8054373065016,
|
|
"loss": 0.3401,
|
|
"loss/base_kto": 2.721027374267578,
|
|
"metrics/advantage_var": 9048.0234375,
|
|
"rewards/chosen": 1.0728805012507394,
|
|
"rewards/margins": 2.8573349414355302,
|
|
"rewards/rejected": -1.784454440184791,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 8.664673805236816,
|
|
"kl": 28.86256217956543,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -42183146.09191759,
|
|
"logits/rejected": -43393292.22804315,
|
|
"logps/chosen": -485.44958399366084,
|
|
"logps/rejected": -378.23045069337445,
|
|
"loss": 0.3378,
|
|
"loss/base_kto": 2.702357053756714,
|
|
"metrics/advantage_var": 8466.8408203125,
|
|
"rewards/chosen": 1.8133991803684628,
|
|
"rewards/margins": 2.888673302515516,
|
|
"rewards/rejected": -1.0752741221470532,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 7.829504489898682,
|
|
"kl": 37.354576110839844,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -45578671.807228915,
|
|
"logits/rejected": -45010784.415584415,
|
|
"logps/chosen": -464.59892695783134,
|
|
"logps/rejected": -383.26407771915586,
|
|
"loss": 0.3198,
|
|
"loss/base_kto": 2.55859375,
|
|
"metrics/advantage_var": 8599.970703125,
|
|
"rewards/chosen": 2.297871968832361,
|
|
"rewards/margins": 2.9343021890028154,
|
|
"rewards/rejected": -0.6364302201704546,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 9.73172664642334,
|
|
"kl": 29.568395614624023,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -45679112.6779661,
|
|
"logits/rejected": -46415651.296354994,
|
|
"logps/chosen": -490.24913328197226,
|
|
"logps/rejected": -382.99640946909665,
|
|
"loss": 0.3263,
|
|
"loss/base_kto": 2.610485792160034,
|
|
"metrics/advantage_var": 9661.892578125,
|
|
"rewards/chosen": 2.01228038629141,
|
|
"rewards/margins": 3.09106845443102,
|
|
"rewards/rejected": -1.0787880681396098,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 8.299348831176758,
|
|
"kl": 25.486047744750977,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -45759279.8699187,
|
|
"logits/rejected": -47092001.49172933,
|
|
"logps/chosen": -488.6234756097561,
|
|
"logps/rejected": -358.422227443609,
|
|
"loss": 0.3261,
|
|
"loss/base_kto": 2.608835458755493,
|
|
"metrics/advantage_var": 9507.947265625,
|
|
"rewards/chosen": 1.7613670287093497,
|
|
"rewards/margins": 3.0828396555890487,
|
|
"rewards/rejected": -1.3214726268796992,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 7.881642818450928,
|
|
"kl": 47.24177932739258,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -45693681.538931295,
|
|
"logits/rejected": -45166647.7056,
|
|
"logps/chosen": -446.0384541984733,
|
|
"logps/rejected": -400.538975,
|
|
"loss": 0.3247,
|
|
"loss/base_kto": 2.5978293418884277,
|
|
"metrics/advantage_var": 10352.3291015625,
|
|
"rewards/chosen": 2.509184346672233,
|
|
"rewards/margins": 3.2096743857347327,
|
|
"rewards/rejected": -0.7004900390625,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 7.793478488922119,
|
|
"kl": 35.15746307373047,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -44939849.615508884,
|
|
"logits/rejected": -46229974.94704992,
|
|
"logps/chosen": -464.09995961227787,
|
|
"logps/rejected": -372.51092095310133,
|
|
"loss": 0.3203,
|
|
"loss/base_kto": 2.562568426132202,
|
|
"metrics/advantage_var": 9777.9736328125,
|
|
"rewards/chosen": 2.1015311442977587,
|
|
"rewards/margins": 3.1540492646291693,
|
|
"rewards/rejected": -1.0525181203314107,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 8.537753105163574,
|
|
"kl": 21.723495483398438,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -45649936.56957929,
|
|
"logits/rejected": -45545744.24169184,
|
|
"logps/chosen": -466.2949534789644,
|
|
"logps/rejected": -369.1769967900302,
|
|
"loss": 0.3378,
|
|
"loss/base_kto": 2.702486753463745,
|
|
"metrics/advantage_var": 10140.341796875,
|
|
"rewards/chosen": 1.6531990322866101,
|
|
"rewards/margins": 2.910111309830983,
|
|
"rewards/rejected": -1.256912277544373,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 7.486123561859131,
|
|
"kl": 29.603656768798828,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -46145442.05504587,
|
|
"logits/rejected": -46587112.28115016,
|
|
"logps/chosen": -468.3471903669725,
|
|
"logps/rejected": -391.3940195686901,
|
|
"loss": 0.3374,
|
|
"loss/base_kto": 2.699201822280884,
|
|
"metrics/advantage_var": 10859.9677734375,
|
|
"rewards/chosen": 2.073822674765864,
|
|
"rewards/margins": 3.0097128481447077,
|
|
"rewards/rejected": -0.9358901733788438,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 9.535039901733398,
|
|
"kl": 24.991872787475586,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -46978959.326860845,
|
|
"logits/rejected": -47023280.33836858,
|
|
"logps/chosen": -478.23391990291265,
|
|
"logps/rejected": -400.0674093655589,
|
|
"loss": 0.3257,
|
|
"loss/base_kto": 2.605404853820801,
|
|
"metrics/advantage_var": 12756.2958984375,
|
|
"rewards/chosen": 1.9604095162697208,
|
|
"rewards/margins": 3.527833264735544,
|
|
"rewards/rejected": -1.5674237484658233,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 7.184971809387207,
|
|
"kl": 24.45808219909668,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -46707373.783866055,
|
|
"logits/rejected": -47414063.255216695,
|
|
"logps/chosen": -445.0344368340944,
|
|
"logps/rejected": -383.34402588282506,
|
|
"loss": 0.332,
|
|
"loss/base_kto": 2.656392812728882,
|
|
"metrics/advantage_var": 10849.8701171875,
|
|
"rewards/chosen": 2.03060100212852,
|
|
"rewards/margins": 3.1241544748035803,
|
|
"rewards/rejected": -1.0935534726750602,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 7.227497100830078,
|
|
"kl": 37.22367477416992,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -47398545.05359878,
|
|
"logits/rejected": -46995420.07017544,
|
|
"logps/chosen": -472.5529287901991,
|
|
"logps/rejected": -374.84190590111643,
|
|
"loss": 0.3065,
|
|
"loss/base_kto": 2.451979160308838,
|
|
"metrics/advantage_var": 11067.1982421875,
|
|
"rewards/chosen": 2.5833558904814318,
|
|
"rewards/margins": 3.576648642736715,
|
|
"rewards/rejected": -0.9932927522552831,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 7.81859016418457,
|
|
"kl": 22.77915382385254,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -47092352.80503145,
|
|
"logits/rejected": -46101885.61490683,
|
|
"logps/chosen": -465.61576257861634,
|
|
"logps/rejected": -376.53144409937886,
|
|
"loss": 0.3287,
|
|
"loss/base_kto": 2.629446506500244,
|
|
"metrics/advantage_var": 11029.83203125,
|
|
"rewards/chosen": 2.13449231033805,
|
|
"rewards/margins": 3.29012447154365,
|
|
"rewards/rejected": -1.1556321612055998,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 9.141914367675781,
|
|
"kl": 27.3914852142334,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -48338286.78119935,
|
|
"logits/rejected": -48161626.73906486,
|
|
"logps/chosen": -471.70006077795784,
|
|
"logps/rejected": -404.33300339366514,
|
|
"loss": 0.3152,
|
|
"loss/base_kto": 2.5217628479003906,
|
|
"metrics/advantage_var": 13034.7158203125,
|
|
"rewards/chosen": 1.950958796026641,
|
|
"rewards/margins": 3.6061690810120295,
|
|
"rewards/rejected": -1.6552102849853885,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 7.617023468017578,
|
|
"kl": 18.97572898864746,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -45358883.44615385,
|
|
"logits/rejected": -47993508.16507936,
|
|
"logps/chosen": -444.6782692307692,
|
|
"logps/rejected": -399.07599206349204,
|
|
"loss": 0.3121,
|
|
"loss/base_kto": 2.4965858459472656,
|
|
"metrics/advantage_var": 12820.9453125,
|
|
"rewards/chosen": 1.9479655573918269,
|
|
"rewards/margins": 3.761633557143811,
|
|
"rewards/rejected": -1.8136679997519842,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 8.201563835144043,
|
|
"kl": 22.254520416259766,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -47446659.926380366,
|
|
"logits/rejected": -47940621.04458599,
|
|
"logps/chosen": -477.0963861196319,
|
|
"logps/rejected": -373.2238505175159,
|
|
"loss": 0.3392,
|
|
"loss/base_kto": 2.7132928371429443,
|
|
"metrics/advantage_var": 12771.9248046875,
|
|
"rewards/chosen": 2.1605134741660277,
|
|
"rewards/margins": 3.263844633040928,
|
|
"rewards/rejected": -1.1033311588749004,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 9.163492202758789,
|
|
"kl": 28.68179702758789,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -46050212.71961102,
|
|
"logits/rejected": -46882056.108597286,
|
|
"logps/chosen": -482.05085089141005,
|
|
"logps/rejected": -386.1900452488688,
|
|
"loss": 0.3257,
|
|
"loss/base_kto": 2.6057794094085693,
|
|
"metrics/advantage_var": 12102.6533203125,
|
|
"rewards/chosen": 1.9439724963913088,
|
|
"rewards/margins": 3.1843624512013577,
|
|
"rewards/rejected": -1.240389954810049,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 6.806163787841797,
|
|
"kl": 25.385343551635742,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -47408088.36129032,
|
|
"logits/rejected": -47495398.32218845,
|
|
"logps/chosen": -497.61275201612904,
|
|
"logps/rejected": -389.4937072568389,
|
|
"loss": 0.3046,
|
|
"loss/base_kto": 2.4370665550231934,
|
|
"metrics/advantage_var": 12872.2626953125,
|
|
"rewards/chosen": 2.1698866321194554,
|
|
"rewards/margins": 3.769127569975648,
|
|
"rewards/rejected": -1.599240937856193,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 6.389303684234619,
|
|
"kl": 17.691709518432617,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -48771504.17637795,
|
|
"logits/rejected": -48449129.5751938,
|
|
"logps/chosen": -465.32273622047245,
|
|
"logps/rejected": -400.1376937984496,
|
|
"loss": 0.2682,
|
|
"loss/base_kto": 2.1456105709075928,
|
|
"metrics/advantage_var": 13492.072265625,
|
|
"rewards/chosen": 1.9728844349778543,
|
|
"rewards/margins": 4.646865100554405,
|
|
"rewards/rejected": -2.6739806655765506,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 6.119379043579102,
|
|
"kl": 19.591279983520508,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -48208287.806060605,
|
|
"logits/rejected": -48525847.12258065,
|
|
"logps/chosen": -458.76458333333335,
|
|
"logps/rejected": -387.18455141129033,
|
|
"loss": 0.2534,
|
|
"loss/base_kto": 2.027486562728882,
|
|
"metrics/advantage_var": 13143.611328125,
|
|
"rewards/chosen": 2.6688169537168562,
|
|
"rewards/margins": 4.806769007648308,
|
|
"rewards/rejected": -2.1379520539314516,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 7.692666053771973,
|
|
"kl": 34.7623405456543,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -49127913.95215311,
|
|
"logits/rejected": -49812977.10260337,
|
|
"logps/chosen": -456.0070773524721,
|
|
"logps/rejected": -436.2947454058193,
|
|
"loss": 0.26,
|
|
"loss/base_kto": 2.0802910327911377,
|
|
"metrics/advantage_var": 14263.2255859375,
|
|
"rewards/chosen": 2.7569722201455344,
|
|
"rewards/margins": 4.696360290828919,
|
|
"rewards/rejected": -1.9393880706833844,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 7.434744834899902,
|
|
"kl": 12.84818172454834,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -50147663.30658106,
|
|
"logits/rejected": -50447433.25418569,
|
|
"logps/chosen": -444.97396669341896,
|
|
"logps/rejected": -407.2007229832572,
|
|
"loss": 0.2614,
|
|
"loss/base_kto": 2.0915536880493164,
|
|
"metrics/advantage_var": 15056.923828125,
|
|
"rewards/chosen": 1.9565668733697834,
|
|
"rewards/margins": 4.866010210448836,
|
|
"rewards/rejected": -2.9094433370790527,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 6.86375093460083,
|
|
"kl": 20.340803146362305,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -50494052.54380665,
|
|
"logits/rejected": -50472234.25242718,
|
|
"logps/chosen": -453.93249622356495,
|
|
"logps/rejected": -396.7928802588997,
|
|
"loss": 0.2556,
|
|
"loss/base_kto": 2.045093536376953,
|
|
"metrics/advantage_var": 13487.6455078125,
|
|
"rewards/chosen": 2.620534844989143,
|
|
"rewards/margins": 4.822372576014123,
|
|
"rewards/rejected": -2.20183773102498,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 7.61278772354126,
|
|
"kl": 11.203639030456543,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -49845101.26348228,
|
|
"logits/rejected": -50176350.52931854,
|
|
"logps/chosen": -462.35275423728814,
|
|
"logps/rejected": -387.9735538827258,
|
|
"loss": 0.2587,
|
|
"loss/base_kto": 2.069570541381836,
|
|
"metrics/advantage_var": 14044.2470703125,
|
|
"rewards/chosen": 2.3151443551979005,
|
|
"rewards/margins": 4.664339735794176,
|
|
"rewards/rejected": -2.3491953805962758,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 6.555257797241211,
|
|
"kl": 21.843231201171875,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -51787978.54945055,
|
|
"logits/rejected": -52390773.44945568,
|
|
"logps/chosen": -467.1393249607535,
|
|
"logps/rejected": -391.87551030326597,
|
|
"loss": 0.2603,
|
|
"loss/base_kto": 2.082411289215088,
|
|
"metrics/advantage_var": 14247.1123046875,
|
|
"rewards/chosen": 2.302504223594486,
|
|
"rewards/margins": 4.600613975246897,
|
|
"rewards/rejected": -2.2981097516524107,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 9.335948944091797,
|
|
"kl": 23.824705123901367,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -50488478.283911675,
|
|
"logits/rejected": -49709978.551083595,
|
|
"logps/chosen": -470.3002760252366,
|
|
"logps/rejected": -389.8466282894737,
|
|
"loss": 0.2699,
|
|
"loss/base_kto": 2.1592204570770264,
|
|
"metrics/advantage_var": 13056.5576171875,
|
|
"rewards/chosen": 2.5590252319721016,
|
|
"rewards/margins": 4.345054681529474,
|
|
"rewards/rejected": -1.7860294495573723,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 5.899699687957764,
|
|
"kl": 18.416074752807617,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -51300281.48982786,
|
|
"logits/rejected": -51180459.332293294,
|
|
"logps/chosen": -427.04631259780905,
|
|
"logps/rejected": -393.7709877145086,
|
|
"loss": 0.2491,
|
|
"loss/base_kto": 1.9927533864974976,
|
|
"metrics/advantage_var": 15910.4501953125,
|
|
"rewards/chosen": 2.523304349753032,
|
|
"rewards/margins": 5.119925382563094,
|
|
"rewards/rejected": -2.5966210328100625,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 6.821751594543457,
|
|
"kl": 28.01690673828125,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -50323961.43589743,
|
|
"logits/rejected": -50340445.65853658,
|
|
"logps/chosen": -475.4713541666667,
|
|
"logps/rejected": -385.641982660061,
|
|
"loss": 0.2514,
|
|
"loss/base_kto": 2.011336326599121,
|
|
"metrics/advantage_var": 13328.1220703125,
|
|
"rewards/chosen": 2.7215857872596154,
|
|
"rewards/margins": 4.759149948606796,
|
|
"rewards/rejected": -2.0375641613471798,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 7.234650135040283,
|
|
"kl": 25.949859619140625,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -51581178.65074627,
|
|
"logits/rejected": -51804243.93442623,
|
|
"logps/chosen": -473.0375,
|
|
"logps/rejected": -406.4485911885246,
|
|
"loss": 0.2553,
|
|
"loss/base_kto": 2.0423760414123535,
|
|
"metrics/advantage_var": 13629.1279296875,
|
|
"rewards/chosen": 2.8382909063083024,
|
|
"rewards/margins": 4.770606720217113,
|
|
"rewards/rejected": -1.9323158139088115,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 5.109222888946533,
|
|
"kl": 19.274444580078125,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -50138758.7368421,
|
|
"logits/rejected": -50881342.1829653,
|
|
"logps/chosen": -448.0441660216718,
|
|
"logps/rejected": -384.5434739747634,
|
|
"loss": 0.2523,
|
|
"loss/base_kto": 2.018216371536255,
|
|
"metrics/advantage_var": 12616.2705078125,
|
|
"rewards/chosen": 2.5717410627902475,
|
|
"rewards/margins": 4.748237983691766,
|
|
"rewards/rejected": -2.176496920901518,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 8.433963775634766,
|
|
"kl": 19.19364356994629,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -51527698.61818182,
|
|
"logits/rejected": -49797925.98709677,
|
|
"logps/chosen": -481.74583333333334,
|
|
"logps/rejected": -386.2210181451613,
|
|
"loss": 0.2463,
|
|
"loss/base_kto": 1.970375418663025,
|
|
"metrics/advantage_var": 13977.0859375,
|
|
"rewards/chosen": 2.6085739598129734,
|
|
"rewards/margins": 4.939082481108336,
|
|
"rewards/rejected": -2.330508521295363,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 7.160104274749756,
|
|
"kl": 19.392568588256836,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -51063385.21821036,
|
|
"logits/rejected": -52179596.93934681,
|
|
"logps/chosen": -487.41012558869704,
|
|
"logps/rejected": -395.7419809486781,
|
|
"loss": 0.2474,
|
|
"loss/base_kto": 1.9794098138809204,
|
|
"metrics/advantage_var": 13968.1201171875,
|
|
"rewards/chosen": 2.3944735684237637,
|
|
"rewards/margins": 4.971222572117387,
|
|
"rewards/rejected": -2.5767490036936236,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 6.981439113616943,
|
|
"kl": 20.005552291870117,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -50218137.52086553,
|
|
"logits/rejected": -50817425.18799368,
|
|
"logps/chosen": -451.67610123647603,
|
|
"logps/rejected": -392.22013230647707,
|
|
"loss": 0.2563,
|
|
"loss/base_kto": 2.0505902767181396,
|
|
"metrics/advantage_var": 12431.8095703125,
|
|
"rewards/chosen": 2.4141296669725656,
|
|
"rewards/margins": 4.498154539232637,
|
|
"rewards/rejected": -2.084024872260071,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 6.792326927185059,
|
|
"kl": 11.254304885864258,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -50549148.17610063,
|
|
"logits/rejected": -50661592.2484472,
|
|
"logps/chosen": -448.44752358490564,
|
|
"logps/rejected": -397.70450795807454,
|
|
"loss": 0.2511,
|
|
"loss/base_kto": 2.0086920261383057,
|
|
"metrics/advantage_var": 14288.7939453125,
|
|
"rewards/chosen": 2.2998998869889937,
|
|
"rewards/margins": 4.963305216533346,
|
|
"rewards/rejected": -2.663405329544352,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 9.0466890335083,
|
|
"kl": 16.8918514251709,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -49829091.376377955,
|
|
"logits/rejected": -51076523.063565895,
|
|
"logps/chosen": -457.84104330708664,
|
|
"logps/rejected": -399.90581395348835,
|
|
"loss": 0.2621,
|
|
"loss/base_kto": 2.0966598987579346,
|
|
"metrics/advantage_var": 13364.3486328125,
|
|
"rewards/chosen": 2.2398460568405514,
|
|
"rewards/margins": 4.591905165367683,
|
|
"rewards/rejected": -2.352059108527132,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 8.03991413116455,
|
|
"kl": 15.425745010375977,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -50888763.7471637,
|
|
"logits/rejected": -53589345.689291105,
|
|
"logps/chosen": -461.6916531604538,
|
|
"logps/rejected": -386.0625471342383,
|
|
"loss": 0.2486,
|
|
"loss/base_kto": 1.9890469312667847,
|
|
"metrics/advantage_var": 13639.9716796875,
|
|
"rewards/chosen": 2.305420515409745,
|
|
"rewards/margins": 4.8323844297314835,
|
|
"rewards/rejected": -2.526963914321738,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 6.370032787322998,
|
|
"kl": 26.204471588134766,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -51099644.680713125,
|
|
"logits/rejected": -50851358.11764706,
|
|
"logps/chosen": -466.8704923014587,
|
|
"logps/rejected": -391.82850207390646,
|
|
"loss": 0.2509,
|
|
"loss/base_kto": 2.0070436000823975,
|
|
"metrics/advantage_var": 14002.6279296875,
|
|
"rewards/chosen": 2.7109810258812805,
|
|
"rewards/margins": 4.972625760397683,
|
|
"rewards/rejected": -2.261644734516403,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 8.21532917022705,
|
|
"kl": 25.85410499572754,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -50496609.37423313,
|
|
"logits/rejected": -52054051.87261146,
|
|
"logps/chosen": -466.07419478527606,
|
|
"logps/rejected": -400.6569715366242,
|
|
"loss": 0.2562,
|
|
"loss/base_kto": 2.049304723739624,
|
|
"metrics/advantage_var": 14128.8779296875,
|
|
"rewards/chosen": 2.778415796946894,
|
|
"rewards/margins": 4.772426187696297,
|
|
"rewards/rejected": -1.994010390749403,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 8.312483787536621,
|
|
"kl": 14.62979793548584,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -50865794.60406886,
|
|
"logits/rejected": -50939977.48517941,
|
|
"logps/chosen": -452.6448552425665,
|
|
"logps/rejected": -407.5865347113885,
|
|
"loss": 0.2436,
|
|
"loss/base_kto": 1.9488365650177002,
|
|
"metrics/advantage_var": 13803.8154296875,
|
|
"rewards/chosen": 2.395782900528169,
|
|
"rewards/margins": 5.012799006953676,
|
|
"rewards/rejected": -2.617016106425507,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 9.717442512512207,
|
|
"kl": 26.102514266967773,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -52090053.047318615,
|
|
"logits/rejected": -51147164.136222914,
|
|
"logps/chosen": -447.686415615142,
|
|
"logps/rejected": -426.5359907120743,
|
|
"loss": 0.2489,
|
|
"loss/base_kto": 1.991180419921875,
|
|
"metrics/advantage_var": 13899.3818359375,
|
|
"rewards/chosen": 2.9700094036499407,
|
|
"rewards/margins": 4.8545486505781525,
|
|
"rewards/rejected": -1.884539246928212,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 7.653444766998291,
|
|
"kl": 24.483278274536133,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -52653258.08291874,
|
|
"logits/rejected": -52675346.52880354,
|
|
"logps/chosen": -477.04477611940297,
|
|
"logps/rejected": -373.59608105613,
|
|
"loss": 0.2556,
|
|
"loss/base_kto": 2.044577121734619,
|
|
"metrics/advantage_var": 14217.7216796875,
|
|
"rewards/chosen": 2.2566705453850537,
|
|
"rewards/margins": 4.839719108092772,
|
|
"rewards/rejected": -2.583048562707718,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 11.300180435180664,
|
|
"kl": 18.176010131835938,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -51462939.58118361,
|
|
"logits/rejected": -49455661.346215785,
|
|
"logps/chosen": -484.98852427921094,
|
|
"logps/rejected": -430.2469806763285,
|
|
"loss": 0.2575,
|
|
"loss/base_kto": 2.0600175857543945,
|
|
"metrics/advantage_var": 13463.3359375,
|
|
"rewards/chosen": 2.6043593735181148,
|
|
"rewards/margins": 4.869201039870269,
|
|
"rewards/rejected": -2.264841666352154,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 7.970808029174805,
|
|
"kl": 24.66413688659668,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -52837982.63507109,
|
|
"logits/rejected": -52683818.73261206,
|
|
"logps/chosen": -474.78041074249603,
|
|
"logps/rejected": -406.2067233384853,
|
|
"loss": 0.2559,
|
|
"loss/base_kto": 2.0468387603759766,
|
|
"metrics/advantage_var": 14646.4560546875,
|
|
"rewards/chosen": 2.4331729641834516,
|
|
"rewards/margins": 4.927368869865928,
|
|
"rewards/rejected": -2.494195905682477,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 7.020127296447754,
|
|
"kl": 19.669519424438477,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -51798627.2969697,
|
|
"logits/rejected": -53283086.86451613,
|
|
"logps/chosen": -476.07651515151514,
|
|
"logps/rejected": -379.72452116935483,
|
|
"loss": 0.2366,
|
|
"loss/base_kto": 1.8930256366729736,
|
|
"metrics/advantage_var": 13605.1845703125,
|
|
"rewards/chosen": 2.772806433475379,
|
|
"rewards/margins": 5.087551739674975,
|
|
"rewards/rejected": -2.3147453061995966,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 11.152055740356445,
|
|
"kl": 17.177064895629883,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -48855737.461300306,
|
|
"logits/rejected": -51137997.93059937,
|
|
"logps/chosen": -483.4895510835913,
|
|
"logps/rejected": -382.77481762618294,
|
|
"loss": 0.2547,
|
|
"loss/base_kto": 2.037907361984253,
|
|
"metrics/advantage_var": 13901.1376953125,
|
|
"rewards/chosen": 2.4697624655330883,
|
|
"rewards/margins": 5.0859972779837985,
|
|
"rewards/rejected": -2.6162348124507098,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 7.5428948402404785,
|
|
"kl": 18.95989227294922,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -50488190.81846154,
|
|
"logits/rejected": -52109048.68571428,
|
|
"logps/chosen": -467.8317307692308,
|
|
"logps/rejected": -389.0001488095238,
|
|
"loss": 0.2513,
|
|
"loss/base_kto": 2.010078191757202,
|
|
"metrics/advantage_var": 14102.5361328125,
|
|
"rewards/chosen": 2.512865459735577,
|
|
"rewards/margins": 5.080259510454823,
|
|
"rewards/rejected": -2.567394050719246,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 8.855107307434082,
|
|
"kl": 21.101959228515625,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -52115810.77165354,
|
|
"logits/rejected": -52631317.03565892,
|
|
"logps/chosen": -454.15187007874016,
|
|
"logps/rejected": -393.9096414728682,
|
|
"loss": 0.2497,
|
|
"loss/base_kto": 1.9974640607833862,
|
|
"metrics/advantage_var": 13606.6240234375,
|
|
"rewards/chosen": 2.5867137518454726,
|
|
"rewards/margins": 4.852088555018922,
|
|
"rewards/rejected": -2.2653748031734495,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 8.883349418640137,
|
|
"kl": 16.477609634399414,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -52024444.411214955,
|
|
"logits/rejected": -51389911.87460815,
|
|
"logps/chosen": -467.2625097352025,
|
|
"logps/rejected": -381.31132445141066,
|
|
"loss": 0.2475,
|
|
"loss/base_kto": 1.9803965091705322,
|
|
"metrics/advantage_var": 12575.1845703125,
|
|
"rewards/chosen": 2.3062188929857865,
|
|
"rewards/margins": 4.80757085039664,
|
|
"rewards/rejected": -2.5013519574108543,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 8.243531227111816,
|
|
"kl": 17.660175323486328,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -52642564.14239482,
|
|
"logits/rejected": -52224761.03927492,
|
|
"logps/chosen": -465.48553802589,
|
|
"logps/rejected": -408.9011518126888,
|
|
"loss": 0.2633,
|
|
"loss/base_kto": 2.1064612865448,
|
|
"metrics/advantage_var": 14548.8095703125,
|
|
"rewards/chosen": 2.3217147283955297,
|
|
"rewards/margins": 4.738218819041772,
|
|
"rewards/rejected": -2.4165040906462423,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 9.128072738647461,
|
|
"kl": 29.375097274780273,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -50154355.14064915,
|
|
"logits/rejected": -49508727.304897316,
|
|
"logps/chosen": -436.9555158423493,
|
|
"logps/rejected": -403.6753060821485,
|
|
"loss": 0.2548,
|
|
"loss/base_kto": 2.0382015705108643,
|
|
"metrics/advantage_var": 13969.5693359375,
|
|
"rewards/chosen": 2.3645096886471215,
|
|
"rewards/margins": 4.911086165987169,
|
|
"rewards/rejected": -2.5465764773400474,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 9.225976943969727,
|
|
"kl": 22.192514419555664,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -53056889.77080063,
|
|
"logits/rejected": -53704936.510108866,
|
|
"logps/chosen": -470.7689364207221,
|
|
"logps/rejected": -397.0832280326594,
|
|
"loss": 0.2515,
|
|
"loss/base_kto": 2.0117104053497314,
|
|
"metrics/advantage_var": 13975.228515625,
|
|
"rewards/chosen": 2.679721227433281,
|
|
"rewards/margins": 4.832321647825971,
|
|
"rewards/rejected": -2.1526004203926905,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 6.5635223388671875,
|
|
"kl": 17.10091781616211,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -52599308.8,
|
|
"logits/rejected": -52682214.4,
|
|
"logps/chosen": -462.09482421875,
|
|
"logps/rejected": -398.41201171875,
|
|
"loss": 0.2551,
|
|
"loss/base_kto": 2.040544033050537,
|
|
"metrics/advantage_var": 15413.7578125,
|
|
"rewards/chosen": 2.1388565063476563,
|
|
"rewards/margins": 4.8688764572143555,
|
|
"rewards/rejected": -2.730019950866699,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 8.384824752807617,
|
|
"kl": 21.283100128173828,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -53873727.209876545,
|
|
"logits/rejected": -52510023.29113924,
|
|
"logps/chosen": -441.6039737654321,
|
|
"logps/rejected": -390.2602600870253,
|
|
"loss": 0.2497,
|
|
"loss/base_kto": 1.9978981018066406,
|
|
"metrics/advantage_var": 14391.3466796875,
|
|
"rewards/chosen": 2.6387082323615934,
|
|
"rewards/margins": 4.906072541165489,
|
|
"rewards/rejected": -2.2673643088038964,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 9.146220207214355,
|
|
"kl": 20.053346633911133,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -50999080.585365854,
|
|
"logits/rejected": -51287384.615384616,
|
|
"logps/chosen": -451.2948742378049,
|
|
"logps/rejected": -393.21314102564105,
|
|
"loss": 0.2599,
|
|
"loss/base_kto": 2.078908681869507,
|
|
"metrics/advantage_var": 14017.6591796875,
|
|
"rewards/chosen": 2.7764029153963414,
|
|
"rewards/margins": 4.802241914044178,
|
|
"rewards/rejected": -2.0258389986478367,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 8.22525691986084,
|
|
"kl": 20.598541259765625,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -52111584.25764895,
|
|
"logits/rejected": -53519566.6646434,
|
|
"logps/chosen": -478.40841384863126,
|
|
"logps/rejected": -376.4315724582701,
|
|
"loss": 0.2603,
|
|
"loss/base_kto": 2.082355260848999,
|
|
"metrics/advantage_var": 13044.7890625,
|
|
"rewards/chosen": 2.4278499584842996,
|
|
"rewards/margins": 4.723116927554386,
|
|
"rewards/rejected": -2.295266969070087,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 8.048925399780273,
|
|
"kl": 16.850187301635742,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -50448719.01234568,
|
|
"logits/rejected": -51392998.07594936,
|
|
"logps/chosen": -421.76957947530866,
|
|
"logps/rejected": -394.25674940664555,
|
|
"loss": 0.2535,
|
|
"loss/base_kto": 2.0280842781066895,
|
|
"metrics/advantage_var": 13453.8984375,
|
|
"rewards/chosen": 2.5178375244140625,
|
|
"rewards/margins": 4.781443921825554,
|
|
"rewards/rejected": -2.2636063974114915,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 8.18259048461914,
|
|
"kl": 22.198036193847656,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -52280503.45276873,
|
|
"logits/rejected": -51407273.638554215,
|
|
"logps/chosen": -429.43297027687294,
|
|
"logps/rejected": -378.1626270707831,
|
|
"loss": 0.2277,
|
|
"loss/base_kto": 1.821925163269043,
|
|
"metrics/advantage_var": 14607.0263671875,
|
|
"rewards/chosen": 2.6215007173121947,
|
|
"rewards/margins": 5.2629680760047775,
|
|
"rewards/rejected": -2.6414673586925828,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 7.484233379364014,
|
|
"kl": 12.324950218200684,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -51822439.590697676,
|
|
"logits/rejected": -51960907.792125985,
|
|
"logps/chosen": -461.7236918604651,
|
|
"logps/rejected": -412.37598425196853,
|
|
"loss": 0.2154,
|
|
"loss/base_kto": 1.722840666770935,
|
|
"metrics/advantage_var": 15416.4501953125,
|
|
"rewards/chosen": 2.524923351199128,
|
|
"rewards/margins": 5.516593734442238,
|
|
"rewards/rejected": -2.99167038324311,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 7.056107521057129,
|
|
"kl": 13.95053768157959,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -52132353.625396825,
|
|
"logits/rejected": -53285191.68,
|
|
"logps/chosen": -464.37132936507936,
|
|
"logps/rejected": -383.63980769230767,
|
|
"loss": 0.1986,
|
|
"loss/base_kto": 1.5886918306350708,
|
|
"metrics/advantage_var": 15551.6064453125,
|
|
"rewards/chosen": 2.8239622085813494,
|
|
"rewards/margins": 5.818112223605388,
|
|
"rewards/rejected": -2.9941500150240383,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 7.444598197937012,
|
|
"kl": 12.505131721496582,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -52138753.187017,
|
|
"logits/rejected": -52800827.45023697,
|
|
"logps/chosen": -445.2485510046368,
|
|
"logps/rejected": -409.02399289099526,
|
|
"loss": 0.2243,
|
|
"loss/base_kto": 1.794498085975647,
|
|
"metrics/advantage_var": 14556.9033203125,
|
|
"rewards/chosen": 2.469342427791731,
|
|
"rewards/margins": 5.232089615353441,
|
|
"rewards/rejected": -2.7627471875617102,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 5.655614376068115,
|
|
"kl": 13.41953182220459,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -52270431.52238806,
|
|
"logits/rejected": -52245782.66229508,
|
|
"logps/chosen": -466.24118470149256,
|
|
"logps/rejected": -414.2294057377049,
|
|
"loss": 0.2034,
|
|
"loss/base_kto": 1.627501368522644,
|
|
"metrics/advantage_var": 14324.3251953125,
|
|
"rewards/chosen": 2.5005901279734144,
|
|
"rewards/margins": 5.578346115422185,
|
|
"rewards/rejected": -3.0777559874487705,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 7.273128509521484,
|
|
"kl": 26.179365158081055,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -50385255.872204475,
|
|
"logits/rejected": -52798060.036697246,
|
|
"logps/chosen": -443.0647963258786,
|
|
"logps/rejected": -388.5483323776758,
|
|
"loss": 0.2091,
|
|
"loss/base_kto": 1.6730972528457642,
|
|
"metrics/advantage_var": 14618.921875,
|
|
"rewards/chosen": 2.985624758199381,
|
|
"rewards/margins": 5.507712951946992,
|
|
"rewards/rejected": -2.522088193747611,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 6.6122283935546875,
|
|
"kl": 22.36004066467285,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -52280876.75259259,
|
|
"logits/rejected": -51246538.68429752,
|
|
"logps/chosen": -463.63194444444446,
|
|
"logps/rejected": -403.38119834710744,
|
|
"loss": 0.1987,
|
|
"loss/base_kto": 1.5892845392227173,
|
|
"metrics/advantage_var": 14510.6015625,
|
|
"rewards/chosen": 2.9543858506944445,
|
|
"rewards/margins": 5.739317733442379,
|
|
"rewards/rejected": -2.7849318827479337,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 7.424135684967041,
|
|
"kl": 21.711545944213867,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -52776490.12658228,
|
|
"logits/rejected": -52171839.209876545,
|
|
"logps/chosen": -462.67948971518985,
|
|
"logps/rejected": -395.7844569830247,
|
|
"loss": 0.2083,
|
|
"loss/base_kto": 1.666551947593689,
|
|
"metrics/advantage_var": 15177.5224609375,
|
|
"rewards/chosen": 2.850885125655162,
|
|
"rewards/margins": 5.720151776950011,
|
|
"rewards/rejected": -2.8692666512948497,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 9.241097450256348,
|
|
"kl": 17.936187744140625,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -52626990.39745628,
|
|
"logits/rejected": -54468249.3640553,
|
|
"logps/chosen": -476.0680147058824,
|
|
"logps/rejected": -378.2085733486943,
|
|
"loss": 0.2002,
|
|
"loss/base_kto": 1.6016310453414917,
|
|
"metrics/advantage_var": 13864.609375,
|
|
"rewards/chosen": 2.5726293130216615,
|
|
"rewards/margins": 5.62121029720465,
|
|
"rewards/rejected": -3.048580984182988,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 8.677379608154297,
|
|
"kl": 17.456422805786133,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -52103591.39969372,
|
|
"logits/rejected": -51538292.36363637,
|
|
"logps/chosen": -459.0336906584992,
|
|
"logps/rejected": -414.3886313795853,
|
|
"loss": 0.2033,
|
|
"loss/base_kto": 1.6267732381820679,
|
|
"metrics/advantage_var": 14518.798828125,
|
|
"rewards/chosen": 2.7230377898281968,
|
|
"rewards/margins": 5.754052215540916,
|
|
"rewards/rejected": -3.031014425712719,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 6.065794467926025,
|
|
"kl": 23.13743782043457,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -51528764.8,
|
|
"logits/rejected": -52346649.6,
|
|
"logps/chosen": -478.83916015625,
|
|
"logps/rejected": -425.9013671875,
|
|
"loss": 0.1956,
|
|
"loss/base_kto": 1.5644174814224243,
|
|
"metrics/advantage_var": 15502.2568359375,
|
|
"rewards/chosen": 3.003139877319336,
|
|
"rewards/margins": 5.866967582702637,
|
|
"rewards/rejected": -2.863827705383301,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 8.646085739135742,
|
|
"kl": 19.484638214111328,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -53728719.08280255,
|
|
"logits/rejected": -53097283.53374233,
|
|
"logps/chosen": -474.18222531847135,
|
|
"logps/rejected": -397.84077837423314,
|
|
"loss": 0.2045,
|
|
"loss/base_kto": 1.6361578702926636,
|
|
"metrics/advantage_var": 16025.0693359375,
|
|
"rewards/chosen": 2.7687875541152467,
|
|
"rewards/margins": 5.923817599887869,
|
|
"rewards/rejected": -3.1550300457726226,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 7.306393146514893,
|
|
"kl": 18.7791690826416,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -52419255.54716981,
|
|
"logits/rejected": -53539369.34161491,
|
|
"logps/chosen": -458.7684257075472,
|
|
"logps/rejected": -397.62533967391306,
|
|
"loss": 0.2064,
|
|
"loss/base_kto": 1.6515058279037476,
|
|
"metrics/advantage_var": 14051.9248046875,
|
|
"rewards/chosen": 2.7090047200520835,
|
|
"rewards/margins": 5.527774281630112,
|
|
"rewards/rejected": -2.818769561578028,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 7.299766540527344,
|
|
"kl": 19.348020553588867,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -51917969.331158236,
|
|
"logits/rejected": -53623284.48575712,
|
|
"logps/chosen": -441.7658034257749,
|
|
"logps/rejected": -408.68988943028484,
|
|
"loss": 0.1881,
|
|
"loss/base_kto": 1.5046846866607666,
|
|
"metrics/advantage_var": 15916.6005859375,
|
|
"rewards/chosen": 2.6475585140956364,
|
|
"rewards/margins": 5.903848703961641,
|
|
"rewards/rejected": -3.2562901898660046,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 9.079314231872559,
|
|
"kl": 17.321130752563477,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -52036113.32923077,
|
|
"logits/rejected": -52050241.82857143,
|
|
"logps/chosen": -455.6421634615385,
|
|
"logps/rejected": -395.63385416666665,
|
|
"loss": 0.2054,
|
|
"loss/base_kto": 1.6432907581329346,
|
|
"metrics/advantage_var": 14563.306640625,
|
|
"rewards/chosen": 2.6505673452524037,
|
|
"rewards/margins": 5.668501411782662,
|
|
"rewards/rejected": -3.017934066530258,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 7.00515079498291,
|
|
"kl": 17.82425308227539,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -51968006.11343284,
|
|
"logits/rejected": -54259913.44262295,
|
|
"logps/chosen": -458.44272388059704,
|
|
"logps/rejected": -416.29928278688527,
|
|
"loss": 0.2076,
|
|
"loss/base_kto": 1.6604769229888916,
|
|
"metrics/advantage_var": 14892.939453125,
|
|
"rewards/chosen": 2.832628119169776,
|
|
"rewards/margins": 5.514739215161066,
|
|
"rewards/rejected": -2.682111095991291,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 8.637384414672852,
|
|
"kl": 26.615041732788086,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -51986283.17219589,
|
|
"logits/rejected": -51022384.27202473,
|
|
"logps/chosen": -472.0436413902054,
|
|
"logps/rejected": -394.59010336166926,
|
|
"loss": 0.2016,
|
|
"loss/base_kto": 1.6125751733779907,
|
|
"metrics/advantage_var": 14819.5888671875,
|
|
"rewards/chosen": 3.209165648449842,
|
|
"rewards/margins": 5.748421347831604,
|
|
"rewards/rejected": -2.539255699381762,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 6.848017692565918,
|
|
"kl": 9.814329147338867,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -53291416.371814094,
|
|
"logits/rejected": -52309788.815660685,
|
|
"logps/chosen": -464.25159295352324,
|
|
"logps/rejected": -418.0154465742251,
|
|
"loss": 0.2048,
|
|
"loss/base_kto": 1.6382410526275635,
|
|
"metrics/advantage_var": 16801.287109375,
|
|
"rewards/chosen": 2.736009680706522,
|
|
"rewards/margins": 5.989718387884336,
|
|
"rewards/rejected": -3.253708707177814,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 7.6930036544799805,
|
|
"kl": 16.38731575012207,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -51957545.636070855,
|
|
"logits/rejected": -52754142.98027314,
|
|
"logps/chosen": -444.3054549114332,
|
|
"logps/rejected": -400.0576631259484,
|
|
"loss": 0.21,
|
|
"loss/base_kto": 1.680148959159851,
|
|
"metrics/advantage_var": 15314.6279296875,
|
|
"rewards/chosen": 2.6964651897141705,
|
|
"rewards/margins": 5.63595980750628,
|
|
"rewards/rejected": -2.9394946177921093,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 6.98551082611084,
|
|
"kl": 24.062509536743164,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -53703912.44171779,
|
|
"logits/rejected": -53829821.14649682,
|
|
"logps/chosen": -446.0058953220859,
|
|
"logps/rejected": -421.1812798566879,
|
|
"loss": 0.2096,
|
|
"loss/base_kto": 1.6764150857925415,
|
|
"metrics/advantage_var": 16047.712890625,
|
|
"rewards/chosen": 2.5807792803992524,
|
|
"rewards/margins": 5.571978166371585,
|
|
"rewards/rejected": -2.9911988859723326,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 7.3181562423706055,
|
|
"kl": 15.073596000671387,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -54002159.69426752,
|
|
"logits/rejected": -53693295.50920245,
|
|
"logps/chosen": -464.53712181528664,
|
|
"logps/rejected": -402.6618577453988,
|
|
"loss": 0.2111,
|
|
"loss/base_kto": 1.6886459589004517,
|
|
"metrics/advantage_var": 15776.330078125,
|
|
"rewards/chosen": 2.7690118680334397,
|
|
"rewards/margins": 5.763737756525579,
|
|
"rewards/rejected": -2.9947258884921397,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 6.888214588165283,
|
|
"kl": 15.283075332641602,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -53212281.38601824,
|
|
"logits/rejected": -53768587.11254019,
|
|
"logps/chosen": -465.1954787234043,
|
|
"logps/rejected": -408.79119774919616,
|
|
"loss": 0.1973,
|
|
"loss/base_kto": 1.5781694650650024,
|
|
"metrics/advantage_var": 14977.0615234375,
|
|
"rewards/chosen": 2.866462313295498,
|
|
"rewards/margins": 5.860386273303536,
|
|
"rewards/rejected": -2.9939239600080385,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 7.8260297775268555,
|
|
"kl": 9.290362358093262,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -52013418.85885886,
|
|
"logits/rejected": -52967787.570032574,
|
|
"logps/chosen": -463.9862049549549,
|
|
"logps/rejected": -396.20549165309444,
|
|
"loss": 0.2102,
|
|
"loss/base_kto": 1.6817753314971924,
|
|
"metrics/advantage_var": 15169.3505859375,
|
|
"rewards/chosen": 2.7301703559027777,
|
|
"rewards/margins": 5.604926016466194,
|
|
"rewards/rejected": -2.874755660563416,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 6.119743824005127,
|
|
"kl": 13.453554153442383,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -53113685.0719755,
|
|
"logits/rejected": -52189785.00797448,
|
|
"logps/chosen": -470.26804173047475,
|
|
"logps/rejected": -397.8858153907496,
|
|
"loss": 0.2208,
|
|
"loss/base_kto": 1.7666877508163452,
|
|
"metrics/advantage_var": 16165.1123046875,
|
|
"rewards/chosen": 2.440948252656011,
|
|
"rewards/margins": 5.580639521844607,
|
|
"rewards/rejected": -3.1396912691885963,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 7.360220432281494,
|
|
"kl": 23.2429256439209,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -52083296.25276461,
|
|
"logits/rejected": -53096606.26893354,
|
|
"logps/chosen": -433.90496642969987,
|
|
"logps/rejected": -402.75613408037094,
|
|
"loss": 0.215,
|
|
"loss/base_kto": 1.7202949523925781,
|
|
"metrics/advantage_var": 15276.4013671875,
|
|
"rewards/chosen": 2.571273249284163,
|
|
"rewards/margins": 5.496104916775952,
|
|
"rewards/rejected": -2.924831667491789,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 6.64047384262085,
|
|
"kl": 15.690417289733887,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -53221443.516483516,
|
|
"logits/rejected": -54514887.06687403,
|
|
"logps/chosen": -462.11803375196234,
|
|
"logps/rejected": -408.3264968895801,
|
|
"loss": 0.2134,
|
|
"loss/base_kto": 1.706957221031189,
|
|
"metrics/advantage_var": 14106.6533203125,
|
|
"rewards/chosen": 2.615590812647174,
|
|
"rewards/margins": 5.379577830289087,
|
|
"rewards/rejected": -2.763987017641913,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 6.458062171936035,
|
|
"kl": 12.019271850585938,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -51360204.881141044,
|
|
"logits/rejected": -53352842.45300462,
|
|
"logps/chosen": -472.2121632329636,
|
|
"logps/rejected": -391.82140793528504,
|
|
"loss": 0.212,
|
|
"loss/base_kto": 1.695715308189392,
|
|
"metrics/advantage_var": 14661.03125,
|
|
"rewards/chosen": 2.5609312707384113,
|
|
"rewards/margins": 5.544705887937949,
|
|
"rewards/rejected": -2.9837746171995376,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 8.161032676696777,
|
|
"kl": 15.232950210571289,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -53328244.07453416,
|
|
"logits/rejected": -53104240.704402514,
|
|
"logps/chosen": -461.04396350931677,
|
|
"logps/rejected": -410.7378390330189,
|
|
"loss": 0.2054,
|
|
"loss/base_kto": 1.6435362100601196,
|
|
"metrics/advantage_var": 14669.265625,
|
|
"rewards/chosen": 2.6601926436335406,
|
|
"rewards/margins": 5.704665775880986,
|
|
"rewards/rejected": -3.044473132247445,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 9.841913223266602,
|
|
"kl": 13.96495246887207,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -52528856.79279279,
|
|
"logits/rejected": -53861933.02931596,
|
|
"logps/chosen": -466.80259947447445,
|
|
"logps/rejected": -394.4695897801303,
|
|
"loss": 0.1968,
|
|
"loss/base_kto": 1.5746790170669556,
|
|
"metrics/advantage_var": 15459.4501953125,
|
|
"rewards/chosen": 2.742815080705706,
|
|
"rewards/margins": 5.83864663798889,
|
|
"rewards/rejected": -3.095831557283184,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 7.300918102264404,
|
|
"kl": 20.875574111938477,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -51876761.6,
|
|
"logits/rejected": -53363110.012121215,
|
|
"logps/chosen": -465.5453629032258,
|
|
"logps/rejected": -411.0222537878788,
|
|
"loss": 0.207,
|
|
"loss/base_kto": 1.655659556388855,
|
|
"metrics/advantage_var": 15438.736328125,
|
|
"rewards/chosen": 2.7058333858366934,
|
|
"rewards/margins": 5.918691088843323,
|
|
"rewards/rejected": -3.212857703006629,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 9.059916496276855,
|
|
"kl": 17.868297576904297,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -52797909.52639517,
|
|
"logits/rejected": -53588980.38249595,
|
|
"logps/chosen": -470.601338612368,
|
|
"logps/rejected": -413.7048723662885,
|
|
"loss": 0.2025,
|
|
"loss/base_kto": 1.619645118713379,
|
|
"metrics/advantage_var": 13850.1376953125,
|
|
"rewards/chosen": 2.6566224709535255,
|
|
"rewards/margins": 5.526776061917869,
|
|
"rewards/rejected": -2.8701535909643434,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 8.5919189453125,
|
|
"kl": 16.793075561523438,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -54307853.04458599,
|
|
"logits/rejected": -54554624.0,
|
|
"logps/chosen": -469.29140127388536,
|
|
"logps/rejected": -393.99012653374234,
|
|
"loss": 0.2167,
|
|
"loss/base_kto": 1.7337658405303955,
|
|
"metrics/advantage_var": 14650.9267578125,
|
|
"rewards/chosen": 2.7456175202776674,
|
|
"rewards/margins": 5.435390050113749,
|
|
"rewards/rejected": -2.6897725298360813,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 6.7300214767456055,
|
|
"kl": 17.726110458374023,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -54416618.3322884,
|
|
"logits/rejected": -53772032.797507785,
|
|
"logps/chosen": -463.94734521943576,
|
|
"logps/rejected": -392.9740800233645,
|
|
"loss": 0.2173,
|
|
"loss/base_kto": 1.7380943298339844,
|
|
"metrics/advantage_var": 15046.2314453125,
|
|
"rewards/chosen": 2.5639112705721003,
|
|
"rewards/margins": 5.547979573665462,
|
|
"rewards/rejected": -2.984068303093361,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 7.305049896240234,
|
|
"kl": 21.30866813659668,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -52208295.368760064,
|
|
"logits/rejected": -52240744.497723825,
|
|
"logps/chosen": -469.00946054750403,
|
|
"logps/rejected": -408.43996585735965,
|
|
"loss": 0.2164,
|
|
"loss/base_kto": 1.731019377708435,
|
|
"metrics/advantage_var": 15013.5537109375,
|
|
"rewards/chosen": 2.6099717881944446,
|
|
"rewards/margins": 5.480226235332628,
|
|
"rewards/rejected": -2.870254447138183,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 8.381829261779785,
|
|
"kl": 14.172942161560059,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -52908790.88673139,
|
|
"logits/rejected": -52410337.06344411,
|
|
"logps/chosen": -431.7904530744337,
|
|
"logps/rejected": -416.3897753021148,
|
|
"loss": 0.2108,
|
|
"loss/base_kto": 1.6866282224655151,
|
|
"metrics/advantage_var": 16544.642578125,
|
|
"rewards/chosen": 2.800618094533778,
|
|
"rewards/margins": 5.837725625571354,
|
|
"rewards/rejected": -3.0371075310375755,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 6.838298797607422,
|
|
"kl": 19.05963897705078,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -52345202.84502447,
|
|
"logits/rejected": -53091056.26386806,
|
|
"logps/chosen": -448.6211256117455,
|
|
"logps/rejected": -375.99433095952025,
|
|
"loss": 0.212,
|
|
"loss/base_kto": 1.695879340171814,
|
|
"metrics/advantage_var": 14115.3251953125,
|
|
"rewards/chosen": 2.662252951990722,
|
|
"rewards/margins": 5.491003493712048,
|
|
"rewards/rejected": -2.828750541721327,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.845784664154053,
|
|
"kl": 15.954147338867188,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -53338313.030674845,
|
|
"logits/rejected": -50993624.866242036,
|
|
"logps/chosen": -424.1664110429448,
|
|
"logps/rejected": -384.4034136146497,
|
|
"loss": 0.2162,
|
|
"loss/base_kto": 1.729690432548523,
|
|
"metrics/advantage_var": 14194.8515625,
|
|
"rewards/chosen": 2.4134079634777605,
|
|
"rewards/margins": 5.440207450937951,
|
|
"rewards/rejected": -3.026799487460191,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 5.568167686462402,
|
|
"kl": 14.366287231445312,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -53023664.98267716,
|
|
"logits/rejected": -53818925.24651163,
|
|
"logps/chosen": -446.8387303149606,
|
|
"logps/rejected": -400.86308139534884,
|
|
"loss": 0.2198,
|
|
"loss/base_kto": 1.7587683200836182,
|
|
"metrics/advantage_var": 14852.1162109375,
|
|
"rewards/chosen": 2.6796111820250985,
|
|
"rewards/margins": 5.5208675410367265,
|
|
"rewards/rejected": -2.841256359011628,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.97585917447635e+17,
|
|
"train_loss": 0.27257865101150486,
|
|
"train_runtime": 11039.5355,
|
|
"train_samples_per_second": 13.426,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.97585917447635e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|