1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 12.468659400939941,
|
|
"kl": 29.8879337310791,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37374458.35590551,
|
|
"logits/rejected": -38401060.51472868,
|
|
"logps/chosen": -477.19443897637797,
|
|
"logps/rejected": -352.93997093023256,
|
|
"loss": 0.4634,
|
|
"loss/base_kto": 3.707191228866577,
|
|
"metrics/advantage_var": 531.9630126953125,
|
|
"rewards/chosen": 0.5312210683747539,
|
|
"rewards/margins": 0.2895921110255533,
|
|
"rewards/rejected": 0.2416289573492006,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 10.705573081970215,
|
|
"kl": 7.3258957862854,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -37314006.4,
|
|
"logits/rejected": -37823193.6,
|
|
"logps/chosen": -475.64013671875,
|
|
"logps/rejected": -414.605908203125,
|
|
"loss": 0.4578,
|
|
"loss/base_kto": 3.662597417831421,
|
|
"metrics/advantage_var": 619.570068359375,
|
|
"rewards/chosen": 0.08362504839897156,
|
|
"rewards/margins": 0.38068429827690126,
|
|
"rewards/rejected": -0.2970592498779297,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 9.74264907836914,
|
|
"kl": 7.171512126922607,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -37287739.57232705,
|
|
"logits/rejected": -39389584.69565217,
|
|
"logps/chosen": -486.7311320754717,
|
|
"logps/rejected": -376.1077251552795,
|
|
"loss": 0.449,
|
|
"loss/base_kto": 3.5922298431396484,
|
|
"metrics/advantage_var": 765.0824584960938,
|
|
"rewards/chosen": 0.156025748582756,
|
|
"rewards/margins": 0.48436920515908555,
|
|
"rewards/rejected": -0.3283434565763296,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 11.212510108947754,
|
|
"kl": 30.199316024780273,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -39358154.86792453,
|
|
"logits/rejected": -38897091.57763975,
|
|
"logps/chosen": -496.57139347484275,
|
|
"logps/rejected": -381.1135238742236,
|
|
"loss": 0.4283,
|
|
"loss/base_kto": 3.426403760910034,
|
|
"metrics/advantage_var": 1222.6826171875,
|
|
"rewards/chosen": 0.9127416070902122,
|
|
"rewards/margins": 0.6644606850120145,
|
|
"rewards/rejected": 0.2482809220781978,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 9.89513874053955,
|
|
"kl": 27.885400772094727,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -38983038.80373832,
|
|
"logits/rejected": -39081836.338557996,
|
|
"logps/chosen": -500.18433605919006,
|
|
"logps/rejected": -378.2253379702194,
|
|
"loss": 0.4129,
|
|
"loss/base_kto": 3.303199529647827,
|
|
"metrics/advantage_var": 1563.1497802734375,
|
|
"rewards/chosen": 0.9240179804626655,
|
|
"rewards/margins": 0.8425314548822384,
|
|
"rewards/rejected": 0.08148652558042711,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 7.232778072357178,
|
|
"kl": 26.04160499572754,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -39328741.0942813,
|
|
"logits/rejected": -40328264.79620853,
|
|
"logps/chosen": -462.09457109737247,
|
|
"logps/rejected": -370.9913605845182,
|
|
"loss": 0.4159,
|
|
"loss/base_kto": 3.327594041824341,
|
|
"metrics/advantage_var": 2382.521484375,
|
|
"rewards/chosen": 0.8227335297527048,
|
|
"rewards/margins": 0.9499181881949564,
|
|
"rewards/rejected": -0.12718465844225169,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 8.212059020996094,
|
|
"kl": 10.217510223388672,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -39942811.00917431,
|
|
"logits/rejected": -40622629.62300319,
|
|
"logps/chosen": -475.33371559633025,
|
|
"logps/rejected": -390.1128943690096,
|
|
"loss": 0.4073,
|
|
"loss/base_kto": 3.258467197418213,
|
|
"metrics/advantage_var": 2937.027099609375,
|
|
"rewards/chosen": 0.5755939075341647,
|
|
"rewards/margins": 1.1537541051242706,
|
|
"rewards/rejected": -0.5781601975901058,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 7.369364261627197,
|
|
"kl": 15.60644817352295,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -39963147.02769231,
|
|
"logits/rejected": -40619862.95873016,
|
|
"logps/chosen": -486.4833173076923,
|
|
"logps/rejected": -367.47792658730157,
|
|
"loss": 0.376,
|
|
"loss/base_kto": 3.008207321166992,
|
|
"metrics/advantage_var": 3660.644287109375,
|
|
"rewards/chosen": 0.8883199368990384,
|
|
"rewards/margins": 1.604301556597699,
|
|
"rewards/rejected": -0.7159816196986607,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 8.021268844604492,
|
|
"kl": 29.99273681640625,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -39923759.92511701,
|
|
"logits/rejected": -40120464.225352116,
|
|
"logps/chosen": -513.3507215288612,
|
|
"logps/rejected": -368.61678403755866,
|
|
"loss": 0.3689,
|
|
"loss/base_kto": 2.9508259296417236,
|
|
"metrics/advantage_var": 4051.784423828125,
|
|
"rewards/chosen": 1.4986730328588143,
|
|
"rewards/margins": 1.6529032969165767,
|
|
"rewards/rejected": -0.15423026405776238,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 8.382246971130371,
|
|
"kl": 33.228641510009766,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -40510072.13858268,
|
|
"logits/rejected": -41386263.417054266,
|
|
"logps/chosen": -448.5313976377953,
|
|
"logps/rejected": -377.7218265503876,
|
|
"loss": 0.3599,
|
|
"loss/base_kto": 2.8793251514434814,
|
|
"metrics/advantage_var": 5004.10888671875,
|
|
"rewards/chosen": 1.4610107421875,
|
|
"rewards/margins": 1.873297355710998,
|
|
"rewards/rejected": -0.41228661352349805,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 9.600776672363281,
|
|
"kl": 22.71444320678711,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -41444090.347003154,
|
|
"logits/rejected": -42096608.29721362,
|
|
"logps/chosen": -463.39880717665613,
|
|
"logps/rejected": -382.3298664860681,
|
|
"loss": 0.3704,
|
|
"loss/base_kto": 2.9630680084228516,
|
|
"metrics/advantage_var": 5203.06201171875,
|
|
"rewards/chosen": 1.451242729692429,
|
|
"rewards/margins": 1.8382827409773193,
|
|
"rewards/rejected": -0.3870400112848902,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 6.537970066070557,
|
|
"kl": 15.437616348266602,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -39280118.930756845,
|
|
"logits/rejected": -40898034.79210926,
|
|
"logps/chosen": -473.702999194847,
|
|
"logps/rejected": -360.13294290591807,
|
|
"loss": 0.3601,
|
|
"loss/base_kto": 2.880817174911499,
|
|
"metrics/advantage_var": 5864.35546875,
|
|
"rewards/chosen": 0.9577954180190721,
|
|
"rewards/margins": 2.0404741593352234,
|
|
"rewards/rejected": -1.0826787413161514,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 9.806045532226562,
|
|
"kl": 19.706167221069336,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -42583149.829652995,
|
|
"logits/rejected": -42101725.12693498,
|
|
"logps/chosen": -473.08142744479494,
|
|
"logps/rejected": -396.03990905572755,
|
|
"loss": 0.3702,
|
|
"loss/base_kto": 2.961411952972412,
|
|
"metrics/advantage_var": 5851.69775390625,
|
|
"rewards/chosen": 1.1792202056018335,
|
|
"rewards/margins": 1.917102884693721,
|
|
"rewards/rejected": -0.7378826790918875,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 10.039080619812012,
|
|
"kl": 15.152414321899414,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -42944540.18348624,
|
|
"logits/rejected": -43340040.99680511,
|
|
"logps/chosen": -482.88054281345563,
|
|
"logps/rejected": -386.05176717252397,
|
|
"loss": 0.3623,
|
|
"loss/base_kto": 2.8985350131988525,
|
|
"metrics/advantage_var": 5968.19091796875,
|
|
"rewards/chosen": 1.2328592002938648,
|
|
"rewards/margins": 2.250269527364302,
|
|
"rewards/rejected": -1.0174103270704373,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 9.22908878326416,
|
|
"kl": 34.14554977416992,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -41032895.6928,
|
|
"logits/rejected": -41480729.79541985,
|
|
"logps/chosen": -479.8308,
|
|
"logps/rejected": -400.47323473282444,
|
|
"loss": 0.3558,
|
|
"loss/base_kto": 2.846527576446533,
|
|
"metrics/advantage_var": 5473.09716796875,
|
|
"rewards/chosen": 1.7591283203125,
|
|
"rewards/margins": 2.1426396728515624,
|
|
"rewards/rejected": -0.3835113525390625,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 9.44801139831543,
|
|
"kl": 5.6502299308776855,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -41668222.79623824,
|
|
"logits/rejected": -41938136.92211838,
|
|
"logps/chosen": -464.014302507837,
|
|
"logps/rejected": -380.6324230919003,
|
|
"loss": 0.3456,
|
|
"loss/base_kto": 2.7647082805633545,
|
|
"metrics/advantage_var": 6785.89453125,
|
|
"rewards/chosen": 0.8642528378477664,
|
|
"rewards/margins": 2.4993839132225717,
|
|
"rewards/rejected": -1.6351310753748054,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 8.25853443145752,
|
|
"kl": 27.141498565673828,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -42697218.29596413,
|
|
"logits/rejected": -42946503.01800327,
|
|
"logps/chosen": -440.3116591928251,
|
|
"logps/rejected": -367.64192921440264,
|
|
"loss": 0.3266,
|
|
"loss/base_kto": 2.6130597591400146,
|
|
"metrics/advantage_var": 7005.6376953125,
|
|
"rewards/chosen": 1.804233157581278,
|
|
"rewards/margins": 2.6989783802567078,
|
|
"rewards/rejected": -0.8947452226754297,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 10.800997734069824,
|
|
"kl": 18.460763931274414,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -42541564.71794872,
|
|
"logits/rejected": -42184426.146341465,
|
|
"logps/chosen": -474.4635416666667,
|
|
"logps/rejected": -387.2720322027439,
|
|
"loss": 0.3465,
|
|
"loss/base_kto": 2.771848440170288,
|
|
"metrics/advantage_var": 7674.00732421875,
|
|
"rewards/chosen": 1.1859318659855769,
|
|
"rewards/margins": 2.5386900409748585,
|
|
"rewards/rejected": -1.3527581749892816,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 7.289452075958252,
|
|
"kl": 19.30964469909668,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -44670307.96937213,
|
|
"logits/rejected": -43826612.05741627,
|
|
"logps/chosen": -469.2593797856049,
|
|
"logps/rejected": -383.1033692185008,
|
|
"loss": 0.3388,
|
|
"loss/base_kto": 2.7102856636047363,
|
|
"metrics/advantage_var": 7789.09375,
|
|
"rewards/chosen": 1.5137171139213246,
|
|
"rewards/margins": 2.627849546387433,
|
|
"rewards/rejected": -1.1141324324661084,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 11.180095672607422,
|
|
"kl": 8.698614120483398,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -43742993.234811164,
|
|
"logits/rejected": -44725041.97913562,
|
|
"logps/chosen": -488.38161945812806,
|
|
"logps/rejected": -398.0107116244411,
|
|
"loss": 0.3474,
|
|
"loss/base_kto": 2.779360294342041,
|
|
"metrics/advantage_var": 9631.0576171875,
|
|
"rewards/chosen": 0.4435019908084462,
|
|
"rewards/margins": 2.8141602172195865,
|
|
"rewards/rejected": -2.37065822641114,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 8.808432579040527,
|
|
"kl": 19.872203826904297,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -43714582.02150538,
|
|
"logits/rejected": -43255111.22416534,
|
|
"logps/chosen": -474.78638632872503,
|
|
"logps/rejected": -376.85358704292526,
|
|
"loss": 0.3379,
|
|
"loss/base_kto": 2.7029435634613037,
|
|
"metrics/advantage_var": 8687.8916015625,
|
|
"rewards/chosen": 1.690362558203725,
|
|
"rewards/margins": 2.747389965608832,
|
|
"rewards/rejected": -1.0570274074051074,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 8.22370433807373,
|
|
"kl": 12.967694282531738,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -44093455.375375375,
|
|
"logits/rejected": -45716296.54723127,
|
|
"logps/chosen": -494.614771021021,
|
|
"logps/rejected": -378.99407064332246,
|
|
"loss": 0.3262,
|
|
"loss/base_kto": 2.609593629837036,
|
|
"metrics/advantage_var": 9493.0263671875,
|
|
"rewards/chosen": 1.4646418574336055,
|
|
"rewards/margins": 3.2769947368615373,
|
|
"rewards/rejected": -1.8123528794279316,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 9.493188858032227,
|
|
"kl": 28.234481811523438,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -43742895.61352657,
|
|
"logits/rejected": -44186697.03186646,
|
|
"logps/chosen": -454.02123590982285,
|
|
"logps/rejected": -362.98392450682854,
|
|
"loss": 0.3324,
|
|
"loss/base_kto": 2.659339189529419,
|
|
"metrics/advantage_var": 10535.546875,
|
|
"rewards/chosen": 2.081637126044183,
|
|
"rewards/margins": 3.0719833833943344,
|
|
"rewards/rejected": -0.9903462573501517,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 8.444293022155762,
|
|
"kl": 12.679133415222168,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -45683208.23151126,
|
|
"logits/rejected": -45501421.32522796,
|
|
"logps/chosen": -472.53878617363347,
|
|
"logps/rejected": -407.61084726443767,
|
|
"loss": 0.3209,
|
|
"loss/base_kto": 2.5670223236083984,
|
|
"metrics/advantage_var": 11255.380859375,
|
|
"rewards/chosen": 1.2575028103647508,
|
|
"rewards/margins": 3.4119206425740973,
|
|
"rewards/rejected": -2.1544178322093464,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 6.553300380706787,
|
|
"kl": 21.521692276000977,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -44957182.41486068,
|
|
"logits/rejected": -45334599.066246055,
|
|
"logps/chosen": -486.2410023219814,
|
|
"logps/rejected": -369.3547417192429,
|
|
"loss": 0.3386,
|
|
"loss/base_kto": 2.7085347175598145,
|
|
"metrics/advantage_var": 9516.822265625,
|
|
"rewards/chosen": 1.316057801615712,
|
|
"rewards/margins": 2.804156138686887,
|
|
"rewards/rejected": -1.4880983370711751,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 8.086736679077148,
|
|
"kl": 25.90850257873535,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -44885738.30508474,
|
|
"logits/rejected": -44230659.245641835,
|
|
"logps/chosen": -468.65167565485365,
|
|
"logps/rejected": -380.25049524564184,
|
|
"loss": 0.3197,
|
|
"loss/base_kto": 2.5576512813568115,
|
|
"metrics/advantage_var": 10727.76171875,
|
|
"rewards/chosen": 1.882302400327427,
|
|
"rewards/margins": 3.272062655007498,
|
|
"rewards/rejected": -1.3897602546800714,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 7.290363311767578,
|
|
"kl": 46.841773986816406,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -46048638.45783132,
|
|
"logits/rejected": -45202997.1948052,
|
|
"logps/chosen": -459.4590549698795,
|
|
"logps/rejected": -363.3772575081169,
|
|
"loss": 0.3306,
|
|
"loss/base_kto": 2.644582986831665,
|
|
"metrics/advantage_var": 8934.9560546875,
|
|
"rewards/chosen": 2.396558830536992,
|
|
"rewards/margins": 2.956978681072199,
|
|
"rewards/rejected": -0.5604198505352069,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 9.326342582702637,
|
|
"kl": 34.701419830322266,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -46096527.6097561,
|
|
"logits/rejected": -46722238.35897436,
|
|
"logps/chosen": -448.9080125762195,
|
|
"logps/rejected": -356.9432592147436,
|
|
"loss": 0.3373,
|
|
"loss/base_kto": 2.698012351989746,
|
|
"metrics/advantage_var": 8884.0751953125,
|
|
"rewards/chosen": 2.0205548914467415,
|
|
"rewards/margins": 2.9187025722673043,
|
|
"rewards/rejected": -0.898147680820563,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 7.9763569831848145,
|
|
"kl": 30.290571212768555,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -47435029.08235294,
|
|
"logits/rejected": -48232424.10666667,
|
|
"logps/chosen": -453.71875,
|
|
"logps/rejected": -392.1332552083333,
|
|
"loss": 0.3347,
|
|
"loss/base_kto": 2.6772916316986084,
|
|
"metrics/advantage_var": 9017.8701171875,
|
|
"rewards/chosen": 2.336901496438419,
|
|
"rewards/margins": 2.92593978582644,
|
|
"rewards/rejected": -0.5890382893880208,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 8.198352813720703,
|
|
"kl": 42.193519592285156,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -45948783.064615384,
|
|
"logits/rejected": -45057124.77460317,
|
|
"logps/chosen": -428.4231730769231,
|
|
"logps/rejected": -398.4294642857143,
|
|
"loss": 0.3403,
|
|
"loss/base_kto": 2.72217059135437,
|
|
"metrics/advantage_var": 9118.69140625,
|
|
"rewards/chosen": 2.434313401442308,
|
|
"rewards/margins": 2.8220942600422583,
|
|
"rewards/rejected": -0.3877808585999504,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 7.566533088684082,
|
|
"kl": 24.0041561126709,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -44345575.8490566,
|
|
"logits/rejected": -45049067.32919255,
|
|
"logps/chosen": -472.1415585691824,
|
|
"logps/rejected": -404.17330648291926,
|
|
"loss": 0.3262,
|
|
"loss/base_kto": 2.609588384628296,
|
|
"metrics/advantage_var": 10468.0185546875,
|
|
"rewards/chosen": 1.6340403046997838,
|
|
"rewards/margins": 3.2025497086630503,
|
|
"rewards/rejected": -1.5685094039632668,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 6.906414031982422,
|
|
"kl": 29.112890243530273,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -44022749.26009693,
|
|
"logits/rejected": -44602871.4795764,
|
|
"logps/chosen": -474.4382067851373,
|
|
"logps/rejected": -381.9731467473525,
|
|
"loss": 0.3129,
|
|
"loss/base_kto": 2.5033178329467773,
|
|
"metrics/advantage_var": 11300.2333984375,
|
|
"rewards/chosen": 2.0221373229881867,
|
|
"rewards/margins": 3.574880931748588,
|
|
"rewards/rejected": -1.552743608760401,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 7.620474338531494,
|
|
"kl": 24.406091690063477,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -45466683.49847095,
|
|
"logits/rejected": -46020585.09904154,
|
|
"logps/chosen": -477.24870986238534,
|
|
"logps/rejected": -381.3620207667732,
|
|
"loss": 0.3229,
|
|
"loss/base_kto": 2.583230495452881,
|
|
"metrics/advantage_var": 9361.6982421875,
|
|
"rewards/chosen": 2.0213107887758026,
|
|
"rewards/margins": 3.1422668684233663,
|
|
"rewards/rejected": -1.120956079647564,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 11.329891204833984,
|
|
"kl": 26.5103816986084,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -43904365.3632,
|
|
"logits/rejected": -45610836.03053435,
|
|
"logps/chosen": -458.59245,
|
|
"logps/rejected": -390.3072996183206,
|
|
"loss": 0.3241,
|
|
"loss/base_kto": 2.5929224491119385,
|
|
"metrics/advantage_var": 11055.96875,
|
|
"rewards/chosen": 1.6681978515625,
|
|
"rewards/margins": 3.202909989265267,
|
|
"rewards/rejected": -1.5347121377027673,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 8.696304321289062,
|
|
"kl": 15.144403457641602,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -44265897.30351438,
|
|
"logits/rejected": -46273955.62079511,
|
|
"logps/chosen": -468.4798821884984,
|
|
"logps/rejected": -381.43926796636083,
|
|
"loss": 0.3069,
|
|
"loss/base_kto": 2.4550883769989014,
|
|
"metrics/advantage_var": 12411.6318359375,
|
|
"rewards/chosen": 1.7593678239816295,
|
|
"rewards/margins": 3.7511917261700853,
|
|
"rewards/rejected": -1.9918239021884556,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 9.53442096710205,
|
|
"kl": 13.843756675720215,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -46577823.03645008,
|
|
"logits/rejected": -47045128.6779661,
|
|
"logps/chosen": -467.5065372424723,
|
|
"logps/rejected": -386.2795165639445,
|
|
"loss": 0.3309,
|
|
"loss/base_kto": 2.6470870971679688,
|
|
"metrics/advantage_var": 12080.7734375,
|
|
"rewards/chosen": 1.70285745128021,
|
|
"rewards/margins": 3.3992557939829258,
|
|
"rewards/rejected": -1.6963983427027156,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 6.700616359710693,
|
|
"kl": 23.6582088470459,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -48391386.96855346,
|
|
"logits/rejected": -48613763.97515528,
|
|
"logps/chosen": -496.71255896226415,
|
|
"logps/rejected": -369.32375776397515,
|
|
"loss": 0.3146,
|
|
"loss/base_kto": 2.5171942710876465,
|
|
"metrics/advantage_var": 13365.2724609375,
|
|
"rewards/chosen": 2.063871755539996,
|
|
"rewards/margins": 3.7268028305472747,
|
|
"rewards/rejected": -1.6629310750072788,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 6.985821723937988,
|
|
"kl": 23.956912994384766,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -48765427.751196176,
|
|
"logits/rejected": -47536871.301684536,
|
|
"logps/chosen": -473.1124401913876,
|
|
"logps/rejected": -416.80647013782544,
|
|
"loss": 0.3062,
|
|
"loss/base_kto": 2.449873685836792,
|
|
"metrics/advantage_var": 13375.8642578125,
|
|
"rewards/chosen": 1.7407265500398723,
|
|
"rewards/margins": 3.816490153121802,
|
|
"rewards/rejected": -2.0757636030819295,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 8.316628456115723,
|
|
"kl": 27.635818481445312,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -47696105.15692308,
|
|
"logits/rejected": -47566792.560509555,
|
|
"logps/chosen": -459.5833653846154,
|
|
"logps/rejected": -418.2669685509554,
|
|
"loss": 0.2826,
|
|
"loss/base_kto": 2.2607383728027344,
|
|
"metrics/advantage_var": 14893.017578125,
|
|
"rewards/chosen": 2.2947506009615384,
|
|
"rewards/margins": 4.421246934187056,
|
|
"rewards/rejected": -2.1264963332255173,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 8.086918830871582,
|
|
"kl": 18.65052604675293,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -47266483.120743036,
|
|
"logits/rejected": -46471193.84227129,
|
|
"logps/chosen": -447.69045085139317,
|
|
"logps/rejected": -406.4086652208202,
|
|
"loss": 0.2638,
|
|
"loss/base_kto": 2.110551118850708,
|
|
"metrics/advantage_var": 14927.890625,
|
|
"rewards/chosen": 2.344045727614648,
|
|
"rewards/margins": 4.815705776042289,
|
|
"rewards/rejected": -2.471660048427642,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 9.235642433166504,
|
|
"kl": 14.318849563598633,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -47330193.20655738,
|
|
"logits/rejected": -48266255.28358209,
|
|
"logps/chosen": -478.18575819672134,
|
|
"logps/rejected": -398.50046641791045,
|
|
"loss": 0.2707,
|
|
"loss/base_kto": 2.165867805480957,
|
|
"metrics/advantage_var": 16995.412109375,
|
|
"rewards/chosen": 1.6457587570440573,
|
|
"rewards/margins": 4.922583252729692,
|
|
"rewards/rejected": -3.2768244956856343,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 6.9475836753845215,
|
|
"kl": 18.910648345947266,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -48754031.46411483,
|
|
"logits/rejected": -48983170.156202145,
|
|
"logps/chosen": -449.504485645933,
|
|
"logps/rejected": -377.55764261102604,
|
|
"loss": 0.2572,
|
|
"loss/base_kto": 2.057651996612549,
|
|
"metrics/advantage_var": 16152.619140625,
|
|
"rewards/chosen": 2.3972954514304226,
|
|
"rewards/margins": 5.001059703560017,
|
|
"rewards/rejected": -2.603764252129594,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 7.026848793029785,
|
|
"kl": 18.607139587402344,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -46669212.90322581,
|
|
"logits/rejected": -47008243.587878786,
|
|
"logps/chosen": -477.9242943548387,
|
|
"logps/rejected": -392.5854640151515,
|
|
"loss": 0.2558,
|
|
"loss/base_kto": 2.0461461544036865,
|
|
"metrics/advantage_var": 13921.1953125,
|
|
"rewards/chosen": 2.056893034904234,
|
|
"rewards/margins": 4.811471226488041,
|
|
"rewards/rejected": -2.754578191583807,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 7.500204563140869,
|
|
"kl": 20.632123947143555,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -48184230.28398792,
|
|
"logits/rejected": -48479195.54692557,
|
|
"logps/chosen": -472.469458081571,
|
|
"logps/rejected": -404.5207827669903,
|
|
"loss": 0.2658,
|
|
"loss/base_kto": 2.126520872116089,
|
|
"metrics/advantage_var": 14187.9189453125,
|
|
"rewards/chosen": 2.431369009334875,
|
|
"rewards/margins": 4.628053911489001,
|
|
"rewards/rejected": -2.1966849021541264,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 11.07607650756836,
|
|
"kl": 19.66994285583496,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -49061644.641975306,
|
|
"logits/rejected": -49284076.55696203,
|
|
"logps/chosen": -453.7958140432099,
|
|
"logps/rejected": -417.20149327531647,
|
|
"loss": 0.2483,
|
|
"loss/base_kto": 1.9867771863937378,
|
|
"metrics/advantage_var": 14991.2001953125,
|
|
"rewards/chosen": 2.400873631606867,
|
|
"rewards/margins": 4.973577705355878,
|
|
"rewards/rejected": -2.5727040737490112,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 7.448911666870117,
|
|
"kl": 15.462172508239746,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -48214747.88714734,
|
|
"logits/rejected": -49171829.23364486,
|
|
"logps/chosen": -455.8012343260188,
|
|
"logps/rejected": -429.9959598909657,
|
|
"loss": 0.2591,
|
|
"loss/base_kto": 2.072408676147461,
|
|
"metrics/advantage_var": 14586.7568359375,
|
|
"rewards/chosen": 2.2276615154780566,
|
|
"rewards/margins": 4.818913447307301,
|
|
"rewards/rejected": -2.5912519318292446,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 6.72298002243042,
|
|
"kl": 23.561777114868164,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -47956512.0,
|
|
"logits/rejected": -49148969.6,
|
|
"logps/chosen": -450.888916015625,
|
|
"logps/rejected": -404.3291015625,
|
|
"loss": 0.2624,
|
|
"loss/base_kto": 2.099287509918213,
|
|
"metrics/advantage_var": 13910.462890625,
|
|
"rewards/chosen": 2.486882781982422,
|
|
"rewards/margins": 4.68602180480957,
|
|
"rewards/rejected": -2.1991390228271483,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 6.384796619415283,
|
|
"kl": 18.171024322509766,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -49109873.3374613,
|
|
"logits/rejected": -48812248.42902208,
|
|
"logps/chosen": -462.2097523219814,
|
|
"logps/rejected": -390.073738170347,
|
|
"loss": 0.2643,
|
|
"loss/base_kto": 2.114152431488037,
|
|
"metrics/advantage_var": 14593.4248046875,
|
|
"rewards/chosen": 2.215952208905766,
|
|
"rewards/margins": 4.692166981801074,
|
|
"rewards/rejected": -2.4762147728953074,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 6.847940921783447,
|
|
"kl": 17.695087432861328,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -50544042.93416928,
|
|
"logits/rejected": -48933128.77258567,
|
|
"logps/chosen": -456.51489028213166,
|
|
"logps/rejected": -395.2147098909657,
|
|
"loss": 0.2613,
|
|
"loss/base_kto": 2.0907790660858154,
|
|
"metrics/advantage_var": 12825.6767578125,
|
|
"rewards/chosen": 2.1193005792025863,
|
|
"rewards/margins": 4.628709690439443,
|
|
"rewards/rejected": -2.5094091112368573,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 6.785857677459717,
|
|
"kl": 23.224470138549805,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -48401680.440366976,
|
|
"logits/rejected": -47403619.782747604,
|
|
"logps/chosen": -463.9189124617737,
|
|
"logps/rejected": -383.25039936102235,
|
|
"loss": 0.2638,
|
|
"loss/base_kto": 2.110729694366455,
|
|
"metrics/advantage_var": 13884.853515625,
|
|
"rewards/chosen": 2.27902361668578,
|
|
"rewards/margins": 4.50523706579221,
|
|
"rewards/rejected": -2.22621344910643,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 8.263350486755371,
|
|
"kl": 29.09259033203125,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -50162247.24919094,
|
|
"logits/rejected": -51190753.06344411,
|
|
"logps/chosen": -466.3296925566343,
|
|
"logps/rejected": -381.7652473564955,
|
|
"loss": 0.2527,
|
|
"loss/base_kto": 2.0212230682373047,
|
|
"metrics/advantage_var": 13454.7890625,
|
|
"rewards/chosen": 2.5315537931078076,
|
|
"rewards/margins": 4.647886247731581,
|
|
"rewards/rejected": -2.1163324546237727,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 7.771735191345215,
|
|
"kl": 27.689910888671875,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -49508589.54231975,
|
|
"logits/rejected": -50506238.40498442,
|
|
"logps/chosen": -458.10452586206895,
|
|
"logps/rejected": -415.4616433021807,
|
|
"loss": 0.2494,
|
|
"loss/base_kto": 1.9951435327529907,
|
|
"metrics/advantage_var": 15280.3505859375,
|
|
"rewards/chosen": 2.685712951851489,
|
|
"rewards/margins": 4.9627353641129766,
|
|
"rewards/rejected": -2.2770224122614877,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 6.784241676330566,
|
|
"kl": 18.85918426513672,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -50499053.278219394,
|
|
"logits/rejected": -51514501.70199693,
|
|
"logps/chosen": -453.55176868044515,
|
|
"logps/rejected": -394.48804723502303,
|
|
"loss": 0.2559,
|
|
"loss/base_kto": 2.047353744506836,
|
|
"metrics/advantage_var": 14907.603515625,
|
|
"rewards/chosen": 2.608219116405008,
|
|
"rewards/margins": 4.9608688109854615,
|
|
"rewards/rejected": -2.352649694580453,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 9.232665061950684,
|
|
"kl": 20.594545364379883,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -49949756.04938272,
|
|
"logits/rejected": -50106912.405063294,
|
|
"logps/chosen": -474.619212962963,
|
|
"logps/rejected": -386.55075652689874,
|
|
"loss": 0.2712,
|
|
"loss/base_kto": 2.1697463989257812,
|
|
"metrics/advantage_var": 13390.572265625,
|
|
"rewards/chosen": 2.2712521023220487,
|
|
"rewards/margins": 4.5504941229411155,
|
|
"rewards/rejected": -2.2792420206190664,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 7.8734869956970215,
|
|
"kl": 20.734603881835938,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -49311651.469879515,
|
|
"logits/rejected": -49996404.36363637,
|
|
"logps/chosen": -454.7604951054217,
|
|
"logps/rejected": -383.43514103084414,
|
|
"loss": 0.241,
|
|
"loss/base_kto": 1.9277294874191284,
|
|
"metrics/advantage_var": 13632.861328125,
|
|
"rewards/chosen": 2.6894562502941453,
|
|
"rewards/margins": 5.044658665066873,
|
|
"rewards/rejected": -2.355202414772727,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 8.654077529907227,
|
|
"kl": 20.20318603515625,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -48451214.73939394,
|
|
"logits/rejected": -49571747.50967742,
|
|
"logps/chosen": -453.94166666666666,
|
|
"logps/rejected": -407.38770161290324,
|
|
"loss": 0.2628,
|
|
"loss/base_kto": 2.102159023284912,
|
|
"metrics/advantage_var": 15591.119140625,
|
|
"rewards/chosen": 2.485277210582386,
|
|
"rewards/margins": 4.870609478097507,
|
|
"rewards/rejected": -2.385332267515121,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 8.510467529296875,
|
|
"kl": 17.95111656188965,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -48232398.61093248,
|
|
"logits/rejected": -48532613.835866265,
|
|
"logps/chosen": -474.1833802250804,
|
|
"logps/rejected": -380.45203267477206,
|
|
"loss": 0.2508,
|
|
"loss/base_kto": 2.0065157413482666,
|
|
"metrics/advantage_var": 14855.84375,
|
|
"rewards/chosen": 2.3578408784038385,
|
|
"rewards/margins": 4.872037767057904,
|
|
"rewards/rejected": -2.5141968886540655,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 8.991572380065918,
|
|
"kl": 15.149621963500977,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -49015907.62360447,
|
|
"logits/rejected": -50663179.36906585,
|
|
"logps/chosen": -475.938995215311,
|
|
"logps/rejected": -412.8200612557427,
|
|
"loss": 0.2499,
|
|
"loss/base_kto": 1.9995607137680054,
|
|
"metrics/advantage_var": 15994.5146484375,
|
|
"rewards/chosen": 2.0744765188895533,
|
|
"rewards/margins": 5.248389139726077,
|
|
"rewards/rejected": -3.1739126208365236,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 7.84478235244751,
|
|
"kl": 21.839262008666992,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -48684938.084848486,
|
|
"logits/rejected": -50561935.69032258,
|
|
"logps/chosen": -474.6498106060606,
|
|
"logps/rejected": -399.6490927419355,
|
|
"loss": 0.2454,
|
|
"loss/base_kto": 1.9630470275878906,
|
|
"metrics/advantage_var": 14052.4287109375,
|
|
"rewards/chosen": 2.6722438003077653,
|
|
"rewards/margins": 5.023397955423693,
|
|
"rewards/rejected": -2.3511541551159274,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 9.335622787475586,
|
|
"kl": 15.088180541992188,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -47908844.24437299,
|
|
"logits/rejected": -50457746.28571428,
|
|
"logps/chosen": -465.9122286977492,
|
|
"logps/rejected": -383.3266052431611,
|
|
"loss": 0.2499,
|
|
"loss/base_kto": 1.9995310306549072,
|
|
"metrics/advantage_var": 13934.4267578125,
|
|
"rewards/chosen": 2.2671612803959005,
|
|
"rewards/margins": 4.926837782936744,
|
|
"rewards/rejected": -2.6596765025408433,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 6.023534297943115,
|
|
"kl": 22.312055587768555,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -48825570.66248038,
|
|
"logits/rejected": -49317209.58009331,
|
|
"logps/chosen": -439.3760302197802,
|
|
"logps/rejected": -393.66628596423016,
|
|
"loss": 0.2703,
|
|
"loss/base_kto": 2.162353277206421,
|
|
"metrics/advantage_var": 14146.5986328125,
|
|
"rewards/chosen": 2.341197446526688,
|
|
"rewards/margins": 4.739072625740334,
|
|
"rewards/rejected": -2.397875179213647,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 8.418098449707031,
|
|
"kl": 28.46211814880371,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -49135524.76804916,
|
|
"logits/rejected": -48735933.659777425,
|
|
"logps/chosen": -460.48915130568355,
|
|
"logps/rejected": -390.8551768680445,
|
|
"loss": 0.2433,
|
|
"loss/base_kto": 1.9464820623397827,
|
|
"metrics/advantage_var": 12723.3603515625,
|
|
"rewards/chosen": 2.8656859038978495,
|
|
"rewards/margins": 4.710928111581176,
|
|
"rewards/rejected": -1.8452422076833268,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 7.899602890014648,
|
|
"kl": 18.49418067932129,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -51717907.0681458,
|
|
"logits/rejected": -51133725.583975345,
|
|
"logps/chosen": -465.9952456418383,
|
|
"logps/rejected": -404.7318952234206,
|
|
"loss": 0.2494,
|
|
"loss/base_kto": 1.9950428009033203,
|
|
"metrics/advantage_var": 14188.0537109375,
|
|
"rewards/chosen": 2.3086831263619256,
|
|
"rewards/margins": 5.01768047956156,
|
|
"rewards/rejected": -2.708997353199634,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 10.677668571472168,
|
|
"kl": 34.446258544921875,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -50640544.0,
|
|
"logits/rejected": -50915161.6,
|
|
"logps/chosen": -467.531201171875,
|
|
"logps/rejected": -394.303173828125,
|
|
"loss": 0.2485,
|
|
"loss/base_kto": 1.9881908893585205,
|
|
"metrics/advantage_var": 11588.8076171875,
|
|
"rewards/chosen": 2.739125442504883,
|
|
"rewards/margins": 4.502296447753906,
|
|
"rewards/rejected": -1.7631710052490235,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 11.427660942077637,
|
|
"kl": 19.136369705200195,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -51027703.54354354,
|
|
"logits/rejected": -50779572.95114007,
|
|
"logps/chosen": -447.70481418918916,
|
|
"logps/rejected": -390.48531657166126,
|
|
"loss": 0.262,
|
|
"loss/base_kto": 2.096342086791992,
|
|
"metrics/advantage_var": 12463.3466796875,
|
|
"rewards/chosen": 2.589214336406719,
|
|
"rewards/margins": 4.624389672318771,
|
|
"rewards/rejected": -2.035175335912052,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 7.118358612060547,
|
|
"kl": 29.556894302368164,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -50797718.87147336,
|
|
"logits/rejected": -52695827.937694706,
|
|
"logps/chosen": -459.33473746081506,
|
|
"logps/rejected": -397.45083722741435,
|
|
"loss": 0.2511,
|
|
"loss/base_kto": 2.0086159706115723,
|
|
"metrics/advantage_var": 13190.4873046875,
|
|
"rewards/chosen": 2.7182460294621866,
|
|
"rewards/margins": 4.6446517014345385,
|
|
"rewards/rejected": -1.9264056719723521,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 7.0251383781433105,
|
|
"kl": 21.34518814086914,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -49025282.761171035,
|
|
"logits/rejected": -50635216.12678289,
|
|
"logps/chosen": -459.6283224191063,
|
|
"logps/rejected": -376.3868116085578,
|
|
"loss": 0.2538,
|
|
"loss/base_kto": 2.0301125049591064,
|
|
"metrics/advantage_var": 12765.095703125,
|
|
"rewards/chosen": 2.7072889330941834,
|
|
"rewards/margins": 4.747839298028316,
|
|
"rewards/rejected": -2.0405503649341323,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 7.253071308135986,
|
|
"kl": 15.496813774108887,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -49935774.09404389,
|
|
"logits/rejected": -49520655.950155765,
|
|
"logps/chosen": -451.2373628526646,
|
|
"logps/rejected": -389.37940517912773,
|
|
"loss": 0.2642,
|
|
"loss/base_kto": 2.1133599281311035,
|
|
"metrics/advantage_var": 14677.015625,
|
|
"rewards/chosen": 2.3958952613758817,
|
|
"rewards/margins": 4.980968381873351,
|
|
"rewards/rejected": -2.585073120497469,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 10.220635414123535,
|
|
"kl": 18.263525009155273,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -50836060.110210694,
|
|
"logits/rejected": -50945405.49019608,
|
|
"logps/chosen": -478.1290012155592,
|
|
"logps/rejected": -402.84605957767724,
|
|
"loss": 0.2563,
|
|
"loss/base_kto": 2.0507876873016357,
|
|
"metrics/advantage_var": 13821.056640625,
|
|
"rewards/chosen": 2.3436550344408427,
|
|
"rewards/margins": 4.7163410379287765,
|
|
"rewards/rejected": -2.372686003487934,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 10.060528755187988,
|
|
"kl": 17.743131637573242,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -52369721.56937799,
|
|
"logits/rejected": -50915011.23430321,
|
|
"logps/chosen": -481.9745813397129,
|
|
"logps/rejected": -410.11581163859114,
|
|
"loss": 0.2446,
|
|
"loss/base_kto": 1.9567173719406128,
|
|
"metrics/advantage_var": 14335.0966796875,
|
|
"rewards/chosen": 2.371380332934609,
|
|
"rewards/margins": 5.096604732939395,
|
|
"rewards/rejected": -2.7252244000047856,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 8.237143516540527,
|
|
"kl": 16.771970748901367,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -49415204.51472868,
|
|
"logits/rejected": -49890866.79685039,
|
|
"logps/chosen": -462.5661821705426,
|
|
"logps/rejected": -418.36003937007877,
|
|
"loss": 0.2713,
|
|
"loss/base_kto": 2.170379400253296,
|
|
"metrics/advantage_var": 13692.6533203125,
|
|
"rewards/chosen": 1.8580134901889536,
|
|
"rewards/margins": 4.5056849268892485,
|
|
"rewards/rejected": -2.6476714367002954,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 7.028975963592529,
|
|
"kl": 17.3207950592041,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -50510410.51643193,
|
|
"logits/rejected": -49549935.026521064,
|
|
"logps/chosen": -456.42703442879497,
|
|
"logps/rejected": -386.7444910296412,
|
|
"loss": 0.2477,
|
|
"loss/base_kto": 1.9812153577804565,
|
|
"metrics/advantage_var": 14027.7607421875,
|
|
"rewards/chosen": 2.5905593609399453,
|
|
"rewards/margins": 4.8752099998951905,
|
|
"rewards/rejected": -2.2846506389552457,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 11.530803680419922,
|
|
"kl": 31.405414581298828,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -51228454.4,
|
|
"logits/rejected": -50197628.8,
|
|
"logps/chosen": -466.95244140625,
|
|
"logps/rejected": -384.535595703125,
|
|
"loss": 0.2648,
|
|
"loss/base_kto": 2.11802339553833,
|
|
"metrics/advantage_var": 13980.767578125,
|
|
"rewards/chosen": 2.5955698013305666,
|
|
"rewards/margins": 4.664482307434082,
|
|
"rewards/rejected": -2.0689125061035156,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 7.988217353820801,
|
|
"kl": 19.561134338378906,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -51357387.52795031,
|
|
"logits/rejected": -50171961.96226415,
|
|
"logps/chosen": -456.13480201863354,
|
|
"logps/rejected": -402.56245086477986,
|
|
"loss": 0.2583,
|
|
"loss/base_kto": 2.0667009353637695,
|
|
"metrics/advantage_var": 12997.6875,
|
|
"rewards/chosen": 2.3391037461180124,
|
|
"rewards/margins": 4.620794811492423,
|
|
"rewards/rejected": -2.2816910653744102,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 9.072982788085938,
|
|
"kl": 10.457106590270996,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -49079869.44,
|
|
"logits/rejected": -49365005.0031746,
|
|
"logps/chosen": -479.4330769230769,
|
|
"logps/rejected": -389.29305555555555,
|
|
"loss": 0.2524,
|
|
"loss/base_kto": 2.019576072692871,
|
|
"metrics/advantage_var": 13046.0380859375,
|
|
"rewards/chosen": 2.1333349609375,
|
|
"rewards/margins": 4.809057500930059,
|
|
"rewards/rejected": -2.6757225399925595,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 6.771179676055908,
|
|
"kl": 18.836105346679688,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -51580368.56230032,
|
|
"logits/rejected": -51782458.71559633,
|
|
"logps/chosen": -458.86077276357827,
|
|
"logps/rejected": -387.5057339449541,
|
|
"loss": 0.2481,
|
|
"loss/base_kto": 1.9848469495773315,
|
|
"metrics/advantage_var": 13515.59375,
|
|
"rewards/chosen": 2.6073875549121404,
|
|
"rewards/margins": 4.905324230657554,
|
|
"rewards/rejected": -2.297936675745413,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 9.2738037109375,
|
|
"kl": 20.875213623046875,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -49941718.99222395,
|
|
"logits/rejected": -49673548.75981162,
|
|
"logps/chosen": -477.6462869362364,
|
|
"logps/rejected": -393.92337127158555,
|
|
"loss": 0.2624,
|
|
"loss/base_kto": 2.0992538928985596,
|
|
"metrics/advantage_var": 13856.025390625,
|
|
"rewards/chosen": 2.3467542965712482,
|
|
"rewards/margins": 4.769467883207826,
|
|
"rewards/rejected": -2.422713586636578,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 9.066880226135254,
|
|
"kl": 15.743685722351074,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -49764477.83717358,
|
|
"logits/rejected": -50455547.10047847,
|
|
"logps/chosen": -475.1092069892473,
|
|
"logps/rejected": -391.1201405502392,
|
|
"loss": 0.2159,
|
|
"loss/base_kto": 1.7273807525634766,
|
|
"metrics/advantage_var": 14440.111328125,
|
|
"rewards/chosen": 2.4454221945204493,
|
|
"rewards/margins": 5.453206464543874,
|
|
"rewards/rejected": -3.007784270023425,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 9.389199256896973,
|
|
"kl": 14.396817207336426,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -49846694.853620954,
|
|
"logits/rejected": -49367530.09191759,
|
|
"logps/chosen": -454.39897919876734,
|
|
"logps/rejected": -431.447999207607,
|
|
"loss": 0.1951,
|
|
"loss/base_kto": 1.5605216026306152,
|
|
"metrics/advantage_var": 14475.1787109375,
|
|
"rewards/chosen": 2.7826467552243837,
|
|
"rewards/margins": 5.7794783438485915,
|
|
"rewards/rejected": -2.9968315886242074,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 9.054242134094238,
|
|
"kl": 17.919559478759766,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -48817131.61408882,
|
|
"logits/rejected": -49831112.880382776,
|
|
"logps/chosen": -472.0823602603369,
|
|
"logps/rejected": -417.2660735645933,
|
|
"loss": 0.223,
|
|
"loss/base_kto": 1.7838348150253296,
|
|
"metrics/advantage_var": 16574.927734375,
|
|
"rewards/chosen": 2.5876096576497893,
|
|
"rewards/margins": 5.747501519317453,
|
|
"rewards/rejected": -3.1598918616676634,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 6.2732720375061035,
|
|
"kl": 25.5059814453125,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -51527312.93802345,
|
|
"logits/rejected": -51728000.18740849,
|
|
"logps/chosen": -459.5612960636516,
|
|
"logps/rejected": -430.1048682284041,
|
|
"loss": 0.207,
|
|
"loss/base_kto": 1.6557525396347046,
|
|
"metrics/advantage_var": 15799.3505859375,
|
|
"rewards/chosen": 2.7100296403894473,
|
|
"rewards/margins": 5.6499131391998425,
|
|
"rewards/rejected": -2.9398834988103952,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 4.447214126586914,
|
|
"kl": 14.574604988098145,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -49083068.97160883,
|
|
"logits/rejected": -50236628.408668734,
|
|
"logps/chosen": -458.88761829653,
|
|
"logps/rejected": -392.84940982972137,
|
|
"loss": 0.2058,
|
|
"loss/base_kto": 1.6461609601974487,
|
|
"metrics/advantage_var": 15221.9443359375,
|
|
"rewards/chosen": 2.56582015846806,
|
|
"rewards/margins": 5.746134260877115,
|
|
"rewards/rejected": -3.1803141024090555,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 7.516998291015625,
|
|
"kl": 12.84099292755127,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -51319764.45984252,
|
|
"logits/rejected": -49929865.327131785,
|
|
"logps/chosen": -460.7167322834646,
|
|
"logps/rejected": -405.8479893410853,
|
|
"loss": 0.2265,
|
|
"loss/base_kto": 1.811827540397644,
|
|
"metrics/advantage_var": 15491.5888671875,
|
|
"rewards/chosen": 2.40770523191437,
|
|
"rewards/margins": 5.415428781454098,
|
|
"rewards/rejected": -3.0077235495397288,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 7.593645095825195,
|
|
"kl": 11.170424461364746,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -51468312.56371814,
|
|
"logits/rejected": -52534046.48613377,
|
|
"logps/chosen": -462.5133526986507,
|
|
"logps/rejected": -403.46454424959217,
|
|
"loss": 0.2075,
|
|
"loss/base_kto": 1.6599674224853516,
|
|
"metrics/advantage_var": 16793.154296875,
|
|
"rewards/chosen": 2.6445632774433094,
|
|
"rewards/margins": 6.016063264698611,
|
|
"rewards/rejected": -3.371499987255302,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 8.261808395385742,
|
|
"kl": 21.838741302490234,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -51434695.60697306,
|
|
"logits/rejected": -50746618.87211094,
|
|
"logps/chosen": -457.355883518225,
|
|
"logps/rejected": -381.01047284283516,
|
|
"loss": 0.2137,
|
|
"loss/base_kto": 1.709884524345398,
|
|
"metrics/advantage_var": 14293.1083984375,
|
|
"rewards/chosen": 2.6543518163133912,
|
|
"rewards/margins": 5.4538469079206715,
|
|
"rewards/rejected": -2.7994950916072803,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 6.802206516265869,
|
|
"kl": 22.52808380126953,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -49329979.88069074,
|
|
"logits/rejected": -49380143.377916016,
|
|
"logps/chosen": -430.70692700156985,
|
|
"logps/rejected": -378.99448386469675,
|
|
"loss": 0.2146,
|
|
"loss/base_kto": 1.7167484760284424,
|
|
"metrics/advantage_var": 14634.306640625,
|
|
"rewards/chosen": 2.905923457123234,
|
|
"rewards/margins": 5.455109060192344,
|
|
"rewards/rejected": -2.54918560306911,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 8.124829292297363,
|
|
"kl": 18.092741012573242,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -49389036.679245286,
|
|
"logits/rejected": -52162470.95652174,
|
|
"logps/chosen": -451.3704304245283,
|
|
"logps/rejected": -398.70661878881987,
|
|
"loss": 0.2241,
|
|
"loss/base_kto": 1.7931416034698486,
|
|
"metrics/advantage_var": 14365.892578125,
|
|
"rewards/chosen": 2.5793134581367925,
|
|
"rewards/margins": 5.32185048865892,
|
|
"rewards/rejected": -2.7425370305221275,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 6.912888526916504,
|
|
"kl": 18.40091896057129,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -50879499.39586645,
|
|
"logits/rejected": -51614092.38709678,
|
|
"logps/chosen": -469.5434220985692,
|
|
"logps/rejected": -401.07716493855605,
|
|
"loss": 0.1912,
|
|
"loss/base_kto": 1.5298614501953125,
|
|
"metrics/advantage_var": 15943.1455078125,
|
|
"rewards/chosen": 2.826442409206081,
|
|
"rewards/margins": 5.9957837470517035,
|
|
"rewards/rejected": -3.1693413378456223,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 7.5973076820373535,
|
|
"kl": 16.810462951660156,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -52416382.018575855,
|
|
"logits/rejected": -50373166.83911672,
|
|
"logps/chosen": -448.53125,
|
|
"logps/rejected": -416.87638012618294,
|
|
"loss": 0.2206,
|
|
"loss/base_kto": 1.7645885944366455,
|
|
"metrics/advantage_var": 15707.8125,
|
|
"rewards/chosen": 2.6600234087775734,
|
|
"rewards/margins": 5.533431488677021,
|
|
"rewards/rejected": -2.873408079899448,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 8.159750938415527,
|
|
"kl": 15.953948974609375,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -50439779.90243903,
|
|
"logits/rejected": -52401309.538461536,
|
|
"logps/chosen": -433.3176448170732,
|
|
"logps/rejected": -398.44851762820514,
|
|
"loss": 0.2066,
|
|
"loss/base_kto": 1.653062105178833,
|
|
"metrics/advantage_var": 14817.1083984375,
|
|
"rewards/chosen": 2.8851549102038874,
|
|
"rewards/margins": 5.672613281097316,
|
|
"rewards/rejected": -2.7874583708934293,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 8.426179885864258,
|
|
"kl": 21.919878005981445,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -53184847.127272725,
|
|
"logits/rejected": -53741422.658064514,
|
|
"logps/chosen": -479.77836174242424,
|
|
"logps/rejected": -384.43377016129034,
|
|
"loss": 0.2202,
|
|
"loss/base_kto": 1.7617501020431519,
|
|
"metrics/advantage_var": 13528.0146484375,
|
|
"rewards/chosen": 2.8051628343986743,
|
|
"rewards/margins": 5.240481516669339,
|
|
"rewards/rejected": -2.4353186822706654,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 8.882457733154297,
|
|
"kl": 14.964949607849121,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -51817944.85626911,
|
|
"logits/rejected": -53017380.80511182,
|
|
"logps/chosen": -478.57282110091745,
|
|
"logps/rejected": -419.58541333865816,
|
|
"loss": 0.2156,
|
|
"loss/base_kto": 1.7244430780410767,
|
|
"metrics/advantage_var": 15768.9912109375,
|
|
"rewards/chosen": 2.7158899336415327,
|
|
"rewards/margins": 5.506087336546884,
|
|
"rewards/rejected": -2.7901974029053513,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 9.163458824157715,
|
|
"kl": 15.542861938476562,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -52963602.73170732,
|
|
"logits/rejected": -52902058.666666664,
|
|
"logps/chosen": -471.7673399390244,
|
|
"logps/rejected": -374.76407251602564,
|
|
"loss": 0.2272,
|
|
"loss/base_kto": 1.8178703784942627,
|
|
"metrics/advantage_var": 13268.478515625,
|
|
"rewards/chosen": 2.2558530481850227,
|
|
"rewards/margins": 5.160528262903572,
|
|
"rewards/rejected": -2.9046752147185497,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 8.934484481811523,
|
|
"kl": 20.741098403930664,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -51061180.37735849,
|
|
"logits/rejected": -51750101.06832298,
|
|
"logps/chosen": -447.9060534591195,
|
|
"logps/rejected": -389.9256599378882,
|
|
"loss": 0.1976,
|
|
"loss/base_kto": 1.580649733543396,
|
|
"metrics/advantage_var": 14590.390625,
|
|
"rewards/chosen": 2.9136214346255898,
|
|
"rewards/margins": 5.722851883662372,
|
|
"rewards/rejected": -2.809230449036782,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 8.323333740234375,
|
|
"kl": 18.002378463745117,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -51800554.764976956,
|
|
"logits/rejected": -51589393.50079491,
|
|
"logps/chosen": -459.7103974654378,
|
|
"logps/rejected": -413.1946045310016,
|
|
"loss": 0.2154,
|
|
"loss/base_kto": 1.723293662071228,
|
|
"metrics/advantage_var": 14929.03125,
|
|
"rewards/chosen": 2.601546561479935,
|
|
"rewards/margins": 5.695510427114871,
|
|
"rewards/rejected": -3.093963865634936,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 7.7876386642456055,
|
|
"kl": 8.304442405700684,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -49807206.24624625,
|
|
"logits/rejected": -52539025.094462544,
|
|
"logps/chosen": -478.9786974474475,
|
|
"logps/rejected": -405.7814026872964,
|
|
"loss": 0.2193,
|
|
"loss/base_kto": 1.7541526556015015,
|
|
"metrics/advantage_var": 15291.994140625,
|
|
"rewards/chosen": 2.579624118747654,
|
|
"rewards/margins": 5.542451521663981,
|
|
"rewards/rejected": -2.9628274029163273,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 8.908504486083984,
|
|
"kl": 14.945642471313477,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -50482227.52201258,
|
|
"logits/rejected": -52607968.19875777,
|
|
"logps/chosen": -443.0347877358491,
|
|
"logps/rejected": -399.8195603649068,
|
|
"loss": 0.2169,
|
|
"loss/base_kto": 1.735451579093933,
|
|
"metrics/advantage_var": 14073.5625,
|
|
"rewards/chosen": 2.4197395372690647,
|
|
"rewards/margins": 5.358329261223354,
|
|
"rewards/rejected": -2.9385897239542897,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 6.341398239135742,
|
|
"kl": 17.427776336669922,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -49973613.24840765,
|
|
"logits/rejected": -49779624.04907975,
|
|
"logps/chosen": -439.78841560509557,
|
|
"logps/rejected": -381.7407496165644,
|
|
"loss": 0.2208,
|
|
"loss/base_kto": 1.7662585973739624,
|
|
"metrics/advantage_var": 14243.8115234375,
|
|
"rewards/chosen": 2.6234202779782048,
|
|
"rewards/margins": 5.432613520525176,
|
|
"rewards/rejected": -2.809193242546971,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 7.262148857116699,
|
|
"kl": 14.746039390563965,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -53409282.403755866,
|
|
"logits/rejected": -53459969.5975039,
|
|
"logps/chosen": -476.1471048513302,
|
|
"logps/rejected": -378.63133775351014,
|
|
"loss": 0.2166,
|
|
"loss/base_kto": 1.7325859069824219,
|
|
"metrics/advantage_var": 13408.611328125,
|
|
"rewards/chosen": 2.5891344431607,
|
|
"rewards/margins": 5.4351666438237265,
|
|
"rewards/rejected": -2.8460322006630263,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 7.136597633361816,
|
|
"kl": 17.677648544311523,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -52757400.216216214,
|
|
"logits/rejected": -53190644.093023255,
|
|
"logps/chosen": -452.89711782094594,
|
|
"logps/rejected": -439.1735555959302,
|
|
"loss": 0.2207,
|
|
"loss/base_kto": 1.765488862991333,
|
|
"metrics/advantage_var": 15374.1005859375,
|
|
"rewards/chosen": 2.442764694626267,
|
|
"rewards/margins": 5.516843631535787,
|
|
"rewards/rejected": -3.0740789369095203,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 9.145492553710938,
|
|
"kl": 17.41663932800293,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -52410463.403726704,
|
|
"logits/rejected": -52548765.786163524,
|
|
"logps/chosen": -464.3349184782609,
|
|
"logps/rejected": -403.373034591195,
|
|
"loss": 0.2086,
|
|
"loss/base_kto": 1.6685141324996948,
|
|
"metrics/advantage_var": 15616.8876953125,
|
|
"rewards/chosen": 2.643056549640916,
|
|
"rewards/margins": 5.672259376758644,
|
|
"rewards/rejected": -3.029202827117728,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 7.5655131340026855,
|
|
"kl": 15.2339448928833,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -51871648.66773676,
|
|
"logits/rejected": -52101949.175038055,
|
|
"logps/chosen": -466.7584269662921,
|
|
"logps/rejected": -413.64906773211567,
|
|
"loss": 0.2043,
|
|
"loss/base_kto": 1.634189248085022,
|
|
"metrics/advantage_var": 14743.259765625,
|
|
"rewards/chosen": 2.6696169931280096,
|
|
"rewards/margins": 5.812133804272416,
|
|
"rewards/rejected": -3.1425168111444064,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 6.279407978057861,
|
|
"kl": 14.870948791503906,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -53818698.7820711,
|
|
"logits/rejected": -53071774.93838862,
|
|
"logps/chosen": -491.1116692426584,
|
|
"logps/rejected": -396.6323805292259,
|
|
"loss": 0.2063,
|
|
"loss/base_kto": 1.650800108909607,
|
|
"metrics/advantage_var": 17707.349609375,
|
|
"rewards/chosen": 2.7372101631931027,
|
|
"rewards/margins": 5.961800446627641,
|
|
"rewards/rejected": -3.224590283434538,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 8.242992401123047,
|
|
"kl": 23.719873428344727,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -54485708.8,
|
|
"logits/rejected": -52339644.8,
|
|
"logps/chosen": -465.237744140625,
|
|
"logps/rejected": -403.1852294921875,
|
|
"loss": 0.2125,
|
|
"loss/base_kto": 1.6999969482421875,
|
|
"metrics/advantage_var": 15859.7861328125,
|
|
"rewards/chosen": 2.6175846099853515,
|
|
"rewards/margins": 5.560161018371582,
|
|
"rewards/rejected": -2.9425764083862305,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 7.003545761108398,
|
|
"kl": 20.206554412841797,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -51759656.01275917,
|
|
"logits/rejected": -52200769.47013783,
|
|
"logps/chosen": -468.85930023923447,
|
|
"logps/rejected": -387.87416251914243,
|
|
"loss": 0.2183,
|
|
"loss/base_kto": 1.7463781833648682,
|
|
"metrics/advantage_var": 14242.609375,
|
|
"rewards/chosen": 2.620401239471192,
|
|
"rewards/margins": 5.27650524113419,
|
|
"rewards/rejected": -2.6561040016629978,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 8.930207252502441,
|
|
"kl": 14.447230339050293,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -49582002.47318612,
|
|
"logits/rejected": -51979628.58204334,
|
|
"logps/chosen": -437.5366226340694,
|
|
"logps/rejected": -408.49970975232196,
|
|
"loss": 0.2252,
|
|
"loss/base_kto": 1.801300048828125,
|
|
"metrics/advantage_var": 14933.4375,
|
|
"rewards/chosen": 2.586171628548896,
|
|
"rewards/margins": 5.44742983711604,
|
|
"rewards/rejected": -2.861258208567144,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 8.271646499633789,
|
|
"kl": 15.560394287109375,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -52691174.4,
|
|
"logits/rejected": -52273539.2,
|
|
"logps/chosen": -449.1541015625,
|
|
"logps/rejected": -425.691455078125,
|
|
"loss": 0.2191,
|
|
"loss/base_kto": 1.7531509399414062,
|
|
"metrics/advantage_var": 14178.3193359375,
|
|
"rewards/chosen": 2.5278388977050783,
|
|
"rewards/margins": 5.39668254852295,
|
|
"rewards/rejected": -2.868843650817871,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 8.030889511108398,
|
|
"kl": 16.55352210998535,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -51556539.07692308,
|
|
"logits/rejected": -52023102.43902439,
|
|
"logps/chosen": -468.71634615384613,
|
|
"logps/rejected": -404.0106707317073,
|
|
"loss": 0.2067,
|
|
"loss/base_kto": 1.6536568403244019,
|
|
"metrics/advantage_var": 15274.6171875,
|
|
"rewards/chosen": 2.796620099972456,
|
|
"rewards/margins": 5.835180734678534,
|
|
"rewards/rejected": -3.0385606347060783,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 8.587590217590332,
|
|
"kl": 16.787067413330078,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -50933763.22012579,
|
|
"logits/rejected": -51906292.86956522,
|
|
"logps/chosen": -450.91283411949684,
|
|
"logps/rejected": -395.3308909161491,
|
|
"loss": 0.2252,
|
|
"loss/base_kto": 1.801880121231079,
|
|
"metrics/advantage_var": 13920.283203125,
|
|
"rewards/chosen": 2.509590964647209,
|
|
"rewards/margins": 5.2165346424665024,
|
|
"rewards/rejected": -2.7069436778192935,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 6.320584774017334,
|
|
"kl": 20.709028244018555,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -51578837.93996841,
|
|
"logits/rejected": -50881885.774343126,
|
|
"logps/chosen": -459.2451125592417,
|
|
"logps/rejected": -404.6876207496136,
|
|
"loss": 0.2161,
|
|
"loss/base_kto": 1.7288354635238647,
|
|
"metrics/advantage_var": 13952.947265625,
|
|
"rewards/chosen": 2.7227630856289493,
|
|
"rewards/margins": 5.459844009001245,
|
|
"rewards/rejected": -2.7370809233722952,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 8.064732551574707,
|
|
"kl": 10.20324993133545,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -52426484.32826748,
|
|
"logits/rejected": -50288877.06752411,
|
|
"logps/chosen": -456.69006458966567,
|
|
"logps/rejected": -402.67941117363347,
|
|
"loss": 0.218,
|
|
"loss/base_kto": 1.7438405752182007,
|
|
"metrics/advantage_var": 14851.8125,
|
|
"rewards/chosen": 2.6717870648508737,
|
|
"rewards/margins": 5.595717242202159,
|
|
"rewards/rejected": -2.923930177351286,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 7.492672920227051,
|
|
"kl": 13.49139404296875,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -53041129.494505495,
|
|
"logits/rejected": -53300661.947122864,
|
|
"logps/chosen": -470.7699175824176,
|
|
"logps/rejected": -392.91937208398133,
|
|
"loss": 0.1995,
|
|
"loss/base_kto": 1.5957868099212646,
|
|
"metrics/advantage_var": 14841.4296875,
|
|
"rewards/chosen": 2.8295043753679354,
|
|
"rewards/margins": 5.935396099949973,
|
|
"rewards/rejected": -3.105891724582037,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 9.252459526062012,
|
|
"kl": 13.560362815856934,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -52936039.783783786,
|
|
"logits/rejected": -52383545.80645161,
|
|
"logps/chosen": -459.2036963434022,
|
|
"logps/rejected": -415.0323540706605,
|
|
"loss": 0.2088,
|
|
"loss/base_kto": 1.6702415943145752,
|
|
"metrics/advantage_var": 14908.0498046875,
|
|
"rewards/chosen": 2.6849551542006163,
|
|
"rewards/margins": 5.633000079075808,
|
|
"rewards/rejected": -2.948044924875192,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.345427513122559,
|
|
"kl": 17.02194595336914,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -52286718.32679739,
|
|
"logits/rejected": -52460586.922155686,
|
|
"logps/chosen": -440.9954044117647,
|
|
"logps/rejected": -395.601188248503,
|
|
"loss": 0.2281,
|
|
"loss/base_kto": 1.8249582052230835,
|
|
"metrics/advantage_var": 14355.4287109375,
|
|
"rewards/chosen": 2.5418543597452,
|
|
"rewards/margins": 5.323924862294788,
|
|
"rewards/rejected": -2.782070502549588,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 6.615070343017578,
|
|
"kl": 12.28957462310791,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -52443794.28571428,
|
|
"logits/rejected": -52651681.684210524,
|
|
"logps/chosen": -479.6156994047619,
|
|
"logps/rejected": -385.69053248355266,
|
|
"loss": 0.2065,
|
|
"loss/base_kto": 1.6521881818771362,
|
|
"metrics/advantage_var": 15773.1162109375,
|
|
"rewards/chosen": 2.806511470249721,
|
|
"rewards/margins": 5.816383505226078,
|
|
"rewards/rejected": -3.009872034976357,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_loss": 0.2751038508917589,
|
|
"train_runtime": 11048.6004,
|
|
"train_samples_per_second": 13.415,
|
|
"train_steps_per_second": 0.21
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.969850461966828e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|