Gueule-d-ange/llama32-3b-redo-dpo_simple
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
51c9b49e18686750421d6c49b11b2ebdd7984b3a
llama32-3b-redo-dpo_simple/trainer_state.json

2037 lines
78 KiB
JSON
Raw Normal View History

初始化项目，由ModelHub XC社区提供模型 Model: Gueule-d-ange/llama32-3b-redo-dpo_simple Source: Original Platform
2026-08-29 12:08:17 +08:00
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 300,
"global_step": 927,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.010794764539198488,
"grad_norm": 24.147397994995117,
"learning_rate": 4.8387096774193546e-08,
"logits/chosen": -1.4572813510894775,
"logits/rejected": -1.5238107442855835,
"logps/chosen": -328.8013610839844,
"logps/rejected": -253.98362731933594,
"loss": 0.6928,
"metrics/advantage_var": 0.2102208137512207,
"rewards/accuracies": 0.40937501192092896,
"rewards/chosen": 0.0015822252025827765,
"rewards/margins": 0.0017194593092426658,
"rewards/rejected": -0.0001372337428620085,
"step": 10
},
{
"epoch": 0.021589529078396976,
"grad_norm": 25.48040199279785,
"learning_rate": 1.0215053763440861e-07,
"logits/chosen": -1.4601737260818481,
"logits/rejected": -1.495823860168457,
"logps/chosen": -329.26373291015625,
"logps/rejected": -269.0686950683594,
"loss": 0.6933,
"metrics/advantage_var": 0.26926887035369873,
"rewards/accuracies": 0.5140625238418579,
"rewards/chosen": 0.0026884928811341524,
"rewards/margins": 0.0011197883868589997,
"rewards/rejected": 0.0015687048435211182,
"stability/repetition_rate_mean": 0.8257812261581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.434472680091858,
"step": 20
},
{
"epoch": 0.03238429361759547,
"grad_norm": 19.910463333129883,
"learning_rate": 1.5591397849462365e-07,
"logits/chosen": -1.4658056497573853,
"logits/rejected": -1.5197511911392212,
"logps/chosen": -316.17425537109375,
"logps/rejected": -256.8540954589844,
"loss": 0.6925,
"metrics/advantage_var": 0.24051065742969513,
"rewards/accuracies": 0.5140625238418579,
"rewards/chosen": 0.004414338618516922,
"rewards/margins": 0.0025372428353875875,
"rewards/rejected": 0.0018770955502986908,
"stability/repetition_rate_mean": 0.8824218511581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.364843726158142,
"step": 30
},
{
"epoch": 0.04317905815679395,
"grad_norm": 23.751585006713867,
"learning_rate": 2.0967741935483871e-07,
"logits/chosen": -1.4785693883895874,
"logits/rejected": -1.5407034158706665,
"logps/chosen": -329.3841247558594,
"logps/rejected": -258.57672119140625,
"loss": 0.6913,
"metrics/advantage_var": 0.2763865292072296,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.005997015163302422,
"rewards/margins": 0.005208877846598625,
"rewards/rejected": 0.0007881380734033883,
"stability/repetition_rate_mean": 0.8031250238418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.0068359375,
"step": 40
},
{
"epoch": 0.05397382269599244,
"grad_norm": 20.962650299072266,
"learning_rate": 2.6344086021505376e-07,
"logits/chosen": -1.438301682472229,
"logits/rejected": -1.503927230834961,
"logps/chosen": -313.481689453125,
"logps/rejected": -258.3254089355469,
"loss": 0.6885,
"metrics/advantage_var": 0.281412810087204,
"rewards/accuracies": 0.546875,
"rewards/chosen": 0.01573883555829525,
"rewards/margins": 0.010696313343942165,
"rewards/rejected": 0.005042524076998234,
"stability/repetition_rate_mean": 0.8675781488418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.07568359375,
"step": 50
},
{
"epoch": 0.06476858723519094,
"grad_norm": 20.89899444580078,
"learning_rate": 3.172043010752688e-07,
"logits/chosen": -1.4596624374389648,
"logits/rejected": -1.501914620399475,
"logps/chosen": -318.685302734375,
"logps/rejected": -265.6797180175781,
"loss": 0.6846,
"metrics/advantage_var": 0.43760356307029724,
"rewards/accuracies": 0.604687511920929,
"rewards/chosen": 0.02920054830610752,
"rewards/margins": 0.019200993701815605,
"rewards/rejected": 0.009999553672969341,
"stability/repetition_rate_mean": 0.7318359613418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.2527344226837158,
"step": 60
},
{
"epoch": 0.07556335177438941,
"grad_norm": 22.085494995117188,
"learning_rate": 3.7096774193548384e-07,
"logits/chosen": -1.5026830434799194,
"logits/rejected": -1.5432065725326538,
"logps/chosen": -339.0155334472656,
"logps/rejected": -273.65655517578125,
"loss": 0.6787,
"metrics/advantage_var": 0.8573139309883118,
"rewards/accuracies": 0.609375,
"rewards/chosen": 0.06427477300167084,
"rewards/margins": 0.03314407169818878,
"rewards/rejected": 0.03113069012761116,
"stability/repetition_rate_mean": 0.8822265863418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.3330078125,
"step": 70
},
{
"epoch": 0.0863581163135879,
"grad_norm": 19.948606491088867,
"learning_rate": 4.247311827956989e-07,
"logits/chosen": -1.4859027862548828,
"logits/rejected": -1.5006340742111206,
"logps/chosen": -325.33526611328125,
"logps/rejected": -254.3518524169922,
"loss": 0.6616,
"metrics/advantage_var": 1.5368133783340454,
"rewards/accuracies": 0.643750011920929,
"rewards/chosen": 0.1002170592546463,
"rewards/margins": 0.07156896591186523,
"rewards/rejected": 0.028648091480135918,
"stability/repetition_rate_mean": 0.8583984375,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.825048804283142,
"step": 80
},
{
"epoch": 0.0971528808527864,
"grad_norm": 16.032991409301758,
"learning_rate": 4.78494623655914e-07,
"logits/chosen": -1.4689861536026,
"logits/rejected": -1.4905648231506348,
"logps/chosen": -295.8207702636719,
"logps/rejected": -248.85952758789062,
"loss": 0.6497,
"metrics/advantage_var": 3.12138295173645,
"rewards/accuracies": 0.6953125,
"rewards/chosen": 0.14759615063667297,
"rewards/margins": 0.10207539796829224,
"rewards/rejected": 0.04552074521780014,
"stability/repetition_rate_mean": 0.8740234375,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.663671851158142,
"step": 90
},
{
"epoch": 0.10794764539198488,
"grad_norm": 19.689159393310547,
"learning_rate": 4.999361498869529e-07,
"logits/chosen": -1.4919625520706177,
"logits/rejected": -1.5506508350372314,
"logps/chosen": -314.435791015625,
"logps/rejected": -249.51695251464844,
"loss": 0.6413,
"metrics/advantage_var": 5.264100551605225,
"rewards/accuracies": 0.6890625357627869,
"rewards/chosen": 0.19397814571857452,
"rewards/margins": 0.12933655083179474,
"rewards/rejected": 0.06464159488677979,
"stability/repetition_rate_mean": 0.9400390386581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.4281249046325684,
"step": 100
},
{
"epoch": 0.11874240993118337,
"grad_norm": 17.759952545166016,
"learning_rate": 4.995460728562402e-07,
"logits/chosen": -1.4327154159545898,
"logits/rejected": -1.5077978372573853,
"logps/chosen": -319.29156494140625,
"logps/rejected": -253.4399871826172,
"loss": 0.6214,
"metrics/advantage_var": 8.01412582397461,
"rewards/accuracies": 0.714062511920929,
"rewards/chosen": 0.24892178177833557,
"rewards/margins": 0.18393009901046753,
"rewards/rejected": 0.06499166041612625,
"stability/repetition_rate_mean": 0.809374988079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.246875047683716,
"step": 110
},
{
"epoch": 0.12953717447038188,
"grad_norm": 20.968534469604492,
"learning_rate": 4.988019438437758e-07,
"logits/chosen": -1.4564826488494873,
"logits/rejected": -1.5484386682510376,
"logps/chosen": -330.7845764160156,
"logps/rejected": -248.1776885986328,
"loss": 0.6032,
"metrics/advantage_var": 14.087158203125,
"rewards/accuracies": 0.739062488079071,
"rewards/chosen": 0.3554464876651764,
"rewards/margins": 0.2473072111606598,
"rewards/rejected": 0.1081392914056778,
"stability/repetition_rate_mean": 0.7470703125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.070214867591858,
"step": 120
},
{
"epoch": 0.14033193900958035,
"grad_norm": 19.773218154907227,
"learning_rate": 4.977048186079155e-07,
"logits/chosen": -1.4577782154083252,
"logits/rejected": -1.508943796157837,
"logps/chosen": -317.5703125,
"logps/rejected": -253.7754364013672,
"loss": 0.5951,
"metrics/advantage_var": 16.9871883392334,
"rewards/accuracies": 0.721875011920929,
"rewards/chosen": 0.40177080035209656,
"rewards/margins": 0.28298041224479675,
"rewards/rejected": 0.11879038065671921,
"stability/repetition_rate_mean": 0.773242175579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.4833984375,
"step": 130
},
{
"epoch": 0.15112670354877883,
"grad_norm": 13.235716819763184,
"learning_rate": 4.962562537324176e-07,
"logits/chosen": -1.4774430990219116,
"logits/rejected": -1.512833833694458,
"logps/chosen": -343.68548583984375,
"logps/rejected": -272.2403869628906,
"loss": 0.6036,
"metrics/advantage_var": 24.688919067382812,
"rewards/accuracies": 0.6890625357627869,
"rewards/chosen": 0.42533954977989197,
"rewards/margins": 0.2894915044307709,
"rewards/rejected": 0.1358480602502823,
"stability/repetition_rate_mean": 0.8072265386581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.019140601158142,
"step": 140
},
{
"epoch": 0.16192146808797733,
"grad_norm": 19.26375961303711,
"learning_rate": 4.944583044179871e-07,
"logits/chosen": -1.4802706241607666,
"logits/rejected": -1.5120983123779297,
"logps/chosen": -320.7796936035156,
"logps/rejected": -268.3492126464844,
"loss": 0.6074,
"metrics/advantage_var": 21.159940719604492,
"rewards/accuracies": 0.6968750357627869,
"rewards/chosen": 0.412137508392334,
"rewards/margins": 0.2721858024597168,
"rewards/rejected": 0.13995173573493958,
"stability/repetition_rate_mean": 0.807421863079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.5673828125,
"step": 150
},
{
"epoch": 0.1727162326271758,
"grad_norm": 24.66869354248047,
"learning_rate": 4.923135215663896e-07,
"logits/chosen": -1.479994773864746,
"logits/rejected": -1.5399045944213867,
"logps/chosen": -317.5917663574219,
"logps/rejected": -243.84231567382812,
"loss": 0.5807,
"metrics/advantage_var": 26.341684341430664,
"rewards/accuracies": 0.7484375238418579,
"rewards/chosen": 0.46426326036453247,
"rewards/margins": 0.3650646209716797,
"rewards/rejected": 0.09919857233762741,
"stability/repetition_rate_mean": 0.789257824420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.4421875476837158,
"step": 160
},
{
"epoch": 0.1835109971663743,
"grad_norm": 22.35947036743164,
"learning_rate": 4.89824948161273e-07,
"logits/chosen": -1.4407405853271484,
"logits/rejected": -1.5115913152694702,
"logps/chosen": -308.2352600097656,
"logps/rejected": -256.36358642578125,
"loss": 0.5752,
"metrics/advantage_var": 30.53228759765625,
"rewards/accuracies": 0.706250011920929,
"rewards/chosen": 0.47893524169921875,
"rewards/margins": 0.39258310198783875,
"rewards/rejected": 0.08635219186544418,
"stability/repetition_rate_mean": 0.7845703363418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.5763671398162842,
"step": 170
},
{
"epoch": 0.1943057617055728,
"grad_norm": 14.936376571655273,
"learning_rate": 4.8699611495083e-07,
"logits/chosen": -1.504378318786621,
"logits/rejected": -1.5372484922409058,
"logps/chosen": -317.1926574707031,
"logps/rejected": -261.8941650390625,
"loss": 0.5744,
"metrics/advantage_var": 36.89189910888672,
"rewards/accuracies": 0.7109375,
"rewards/chosen": 0.5145285725593567,
"rewards/margins": 0.40907564759254456,
"rewards/rejected": 0.10545291751623154,
"stability/repetition_rate_mean": 0.7490234375,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.5693359375,
"step": 180
},
{
"epoch": 0.2051005262447713,
"grad_norm": 20.977313995361328,
"learning_rate": 4.838310354384302e-07,
"logits/chosen": -1.4963879585266113,
"logits/rejected": -1.5239351987838745,
"logps/chosen": -307.32684326171875,
"logps/rejected": -263.76373291015625,
"loss": 0.5629,
"metrics/advantage_var": 36.270076751708984,
"rewards/accuracies": 0.707812488079071,
"rewards/chosen": 0.5523476600646973,
"rewards/margins": 0.4342571198940277,
"rewards/rejected": 0.11809053272008896,
"stability/repetition_rate_mean": 0.850781261920929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.1296875476837158,
"step": 190
},
{
"epoch": 0.21589529078396977,
"grad_norm": 13.247014045715332,
"learning_rate": 4.803342001883246e-07,
"logits/chosen": -1.5078216791152954,
"logits/rejected": -1.541197419166565,
"logps/chosen": -318.7608337402344,
"logps/rejected": -267.5777282714844,
"loss": 0.5524,
"metrics/advantage_var": 35.416255950927734,
"rewards/accuracies": 0.745312511920929,
"rewards/chosen": 0.556470513343811,
"rewards/margins": 0.4586350917816162,
"rewards/rejected": 0.09783537685871124,
"stability/repetition_rate_mean": 0.796679675579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.4445312023162842,
"step": 200
},
{
"epoch": 0.22669005532316827,
"grad_norm": 16.3123722076416,
"learning_rate": 4.7651057045450515e-07,
"logits/chosen": -1.4529699087142944,
"logits/rejected": -1.5274174213409424,
"logps/chosen": -326.5751647949219,
"logps/rejected": -245.41970825195312,
"loss": 0.5515,
"metrics/advantage_var": 36.527748107910156,
"rewards/accuracies": 0.7484375238418579,
"rewards/chosen": 0.5690556764602661,
"rewards/margins": 0.4599929749965668,
"rewards/rejected": 0.10906264930963516,
"stability/repetition_rate_mean": 0.81640625,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.3254883289337158,
"step": 210
},
{
"epoch": 0.23748481986236675,
"grad_norm": 21.77730369567871,
"learning_rate": 4.72365571141757e-07,
"logits/chosen": -1.4564087390899658,
"logits/rejected": -1.4975181818008423,
"logps/chosen": -331.7127380371094,
"logps/rejected": -264.9964294433594,
"loss": 0.5548,
"metrics/advantage_var": 43.936622619628906,
"rewards/accuracies": 0.746874988079071,
"rewards/chosen": 0.5794717669487,
"rewards/margins": 0.4975171983242035,
"rewards/rejected": 0.08195456117391586,
"stability/repetition_rate_mean": 0.7367187738418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.763671875,
"step": 220
},
{
"epoch": 0.24827958440156525,
"grad_norm": 18.723806381225586,
"learning_rate": 4.6790508310889007e-07,
"logits/chosen": -1.4929759502410889,
"logits/rejected": -1.5188223123550415,
"logps/chosen": -311.7873229980469,
"logps/rejected": -257.8661193847656,
"loss": 0.5575,
"metrics/advantage_var": 45.32672882080078,
"rewards/accuracies": 0.7359375357627869,
"rewards/chosen": 0.5666539072990417,
"rewards/margins": 0.48853203654289246,
"rewards/rejected": 0.07812191545963287,
"stability/repetition_rate_mean": 0.8841797113418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.4400391578674316,
"step": 230
},
{
"epoch": 0.25907434894076375,
"grad_norm": 12.753899574279785,
"learning_rate": 4.6313543482507056e-07,
"logits/chosen": -1.4668247699737549,
"logits/rejected": -1.50505530834198,
"logps/chosen": -319.4496154785156,
"logps/rejected": -261.7314453125,
"loss": 0.5496,
"metrics/advantage_var": 46.35799789428711,
"rewards/accuracies": 0.7281250357627869,
"rewards/chosen": 0.5762147307395935,
"rewards/margins": 0.5230244398117065,
"rewards/rejected": 0.05319035053253174,
"stability/repetition_rate_mean": 0.802539050579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.0635743141174316,
"step": 240
},
{
"epoch": 0.26986911347996223,
"grad_norm": 17.50574493408203,
"learning_rate": 4.580633933910901e-07,
"logits/chosen": -1.436219334602356,
"logits/rejected": -1.46418035030365,
"logps/chosen": -305.2557678222656,
"logps/rejected": -253.59902954101562,
"loss": 0.5612,
"metrics/advantage_var": 47.63227081298828,
"rewards/accuracies": 0.723437488079071,
"rewards/chosen": 0.5559987425804138,
"rewards/margins": 0.4999403655529022,
"rewards/rejected": 0.0560583770275116,
"stability/repetition_rate_mean": 0.811718761920929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.45166015625,
"step": 250
},
{
"epoch": 0.2806638780191607,
"grad_norm": 15.86179256439209,
"learning_rate": 4.526961549383108e-07,
"logits/chosen": -1.4230726957321167,
"logits/rejected": -1.4305095672607422,
"logps/chosen": -325.4045104980469,
"logps/rejected": -272.9486389160156,
"loss": 0.5574,
"metrics/advantage_var": 50.00086212158203,
"rewards/accuracies": 0.7203125357627869,
"rewards/chosen": 0.6053926348686218,
"rewards/margins": 0.5237736105918884,
"rewards/rejected": 0.08161897957324982,
"stability/repetition_rate_mean": 0.813281238079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.894628882408142,
"step": 260
},
{
"epoch": 0.2914586425583592,
"grad_norm": 15.887772560119629,
"learning_rate": 4.470413344189098e-07,
"logits/chosen": -1.4393119812011719,
"logits/rejected": -1.467215895652771,
"logps/chosen": -323.3600158691406,
"logps/rejected": -270.2887878417969,
"loss": 0.5356,
"metrics/advantage_var": 55.44666290283203,
"rewards/accuracies": 0.7437500357627869,
"rewards/chosen": 0.6630539894104004,
"rewards/margins": 0.5985837578773499,
"rewards/rejected": 0.06447024643421173,
"stability/repetition_rate_mean": 0.828906238079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.030859351158142,
"step": 270
},
{
"epoch": 0.30225340709755766,
"grad_norm": 15.110116958618164,
"learning_rate": 4.4110695480190597e-07,
"logits/chosen": -1.4905976057052612,
"logits/rejected": -1.5187627077102661,
"logps/chosen": -330.3580627441406,
"logps/rejected": -267.19561767578125,
"loss": 0.5221,
"metrics/advantage_var": 60.70363235473633,
"rewards/accuracies": 0.7671875357627869,
"rewards/chosen": 0.7377834916114807,
"rewards/margins": 0.6288355588912964,
"rewards/rejected": 0.10894794762134552,
"stability/repetition_rate_mean": 0.8851562738418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.202734351158142,
"step": 280
},
{
"epoch": 0.3130481716367562,
"grad_norm": 15.053842544555664,
"learning_rate": 4.3490143569030017e-07,
"logits/chosen": -1.4678796529769897,
"logits/rejected": -1.5179208517074585,
"logps/chosen": -320.11309814453125,
"logps/rejected": -252.4005584716797,
"loss": 0.5299,
"metrics/advantage_var": 55.2855110168457,
"rewards/accuracies": 0.746874988079071,
"rewards/chosen": 0.6983198523521423,
"rewards/margins": 0.5931033492088318,
"rewards/rejected": 0.10521648079156876,
"stability/repetition_rate_mean": 0.8099609613418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.6193358898162842,
"step": 290
},
{
"epoch": 0.32384293617595467,
"grad_norm": 21.189123153686523,
"learning_rate": 4.284335813754769e-07,
"logits/chosen": -1.4394444227218628,
"logits/rejected": -1.4822733402252197,
"logps/chosen": -321.5018005371094,
"logps/rejected": -276.88409423828125,
"loss": 0.5031,
"metrics/advantage_var": 60.6578483581543,
"rewards/accuracies": 0.770312488079071,
"rewards/chosen": 0.6781904101371765,
"rewards/margins": 0.6781806349754333,
"rewards/rejected": 9.752810001373291e-06,
"stability/repetition_rate_mean": 0.7289062738418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.1236329078674316,
"step": 300
},
{
"epoch": 0.32384293617595467,
"eval_logits/chosen": -1.4329925775527954,
"eval_logits/rejected": -1.4661825895309448,
"eval_logps/chosen": -319.7928161621094,
"eval_logps/rejected": -254.64100646972656,
"eval_loss": 0.513998806476593,
"eval_metrics/advantage_var": 65.03108215332031,
"eval_rewards/accuracies": 0.7623737454414368,
"eval_rewards/chosen": 0.7029823660850525,
"eval_rewards/margins": 0.680192232131958,
"eval_rewards/rejected": 0.022790152579545975,
"eval_runtime": 272.245,
"eval_samples_per_second": 14.524,
"eval_stability/repetition_rate_mean": 0.9048827886581421,
"eval_stability/response_length_mean": 512.0,
"eval_stability/response_length_std": 0.0,
"eval_stability/response_length_var": 0.0,
"eval_stability/token_entropy_mean": 1.9105956554412842,
"eval_steps_per_second": 1.818,
"step": 300
},
{
"epoch": 0.33463770071515314,
"grad_norm": 20.66907501220703,
"learning_rate": 4.217125683458161e-07,
"logits/chosen": -1.4233049154281616,
"logits/rejected": -1.452370285987854,
"logps/chosen": -315.331298828125,
"logps/rejected": -258.9646911621094,
"loss": 0.534,
"metrics/advantage_var": 62.50199508666992,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.6554938554763794,
"rewards/margins": 0.6286452412605286,
"rewards/rejected": 0.0268485676497221,
"stability/repetition_rate_mean": 0.799023449420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 0.908203125,
"step": 310
},
{
"epoch": 0.3454324652543516,
"grad_norm": 15.366827011108398,
"learning_rate": 4.1474793226723825e-07,
"logits/chosen": -1.448975682258606,
"logits/rejected": -1.4607841968536377,
"logps/chosen": -327.7336120605469,
"logps/rejected": -262.0153503417969,
"loss": 0.5117,
"metrics/advantage_var": 68.2557373046875,
"rewards/accuracies": 0.760937511920929,
"rewards/chosen": 0.7362013459205627,
"rewards/margins": 0.6878542304039001,
"rewards/rejected": 0.048347149044275284,
"stability/repetition_rate_mean": 0.9072265625,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.083886742591858,
"step": 320
},
{
"epoch": 0.35622722979355015,
"grad_norm": 19.474281311035156,
"learning_rate": 4.0754955445415396e-07,
"logits/chosen": -1.447295069694519,
"logits/rejected": -1.47311270236969,
"logps/chosen": -330.9326477050781,
"logps/rejected": -256.48638916015625,
"loss": 0.4903,
"metrics/advantage_var": 77.84224700927734,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": 0.7997223734855652,
"rewards/margins": 0.7868985533714294,
"rewards/rejected": 0.012823845259845257,
"stability/repetition_rate_mean": 0.955273449420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.7205078601837158,
"step": 330
},
{
"epoch": 0.3670219943327486,
"grad_norm": 13.608744621276855,
"learning_rate": 4.001276478500126e-07,
"logits/chosen": -1.4461034536361694,
"logits/rejected": -1.4699530601501465,
"logps/chosen": -316.4836120605469,
"logps/rejected": -258.7716064453125,
"loss": 0.4981,
"metrics/advantage_var": 72.29796600341797,
"rewards/accuracies": 0.770312488079071,
"rewards/chosen": 0.7650503516197205,
"rewards/margins": 0.738609790802002,
"rewards/rejected": 0.026440534740686417,
"stability/repetition_rate_mean": 0.7232421636581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.8572266101837158,
"step": 340
},
{
"epoch": 0.3778167588719471,
"grad_norm": 13.565873146057129,
"learning_rate": 3.9249274253734164e-07,
"logits/chosen": -1.4219696521759033,
"logits/rejected": -1.4259979724884033,
"logps/chosen": -314.6985778808594,
"logps/rejected": -269.089599609375,
"loss": 0.5247,
"metrics/advantage_var": 80.2893295288086,
"rewards/accuracies": 0.7359375357627869,
"rewards/chosen": 0.7249447107315063,
"rewards/margins": 0.712753176689148,
"rewards/rejected": 0.012191593647003174,
"stability/repetition_rate_mean": 0.76953125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.2517578601837158,
"step": 350
},
{
"epoch": 0.3886115234111456,
"grad_norm": 13.095726013183594,
"learning_rate": 3.846556707978337e-07,
"logits/chosen": -1.427175760269165,
"logits/rejected": -1.461297869682312,
"logps/chosen": -334.6982727050781,
"logps/rejected": -252.6654815673828,
"loss": 0.5044,
"metrics/advantage_var": 77.54660034179688,
"rewards/accuracies": 0.7515625357627869,
"rewards/chosen": 0.8103561401367188,
"rewards/margins": 0.7708074450492859,
"rewards/rejected": 0.039548713713884354,
"stability/repetition_rate_mean": 0.889843761920929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.2664062976837158,
"step": 360
},
{
"epoch": 0.3994062879503441,
"grad_norm": 14.872525215148926,
"learning_rate": 3.766275517436779e-07,
"logits/chosen": -1.3873792886734009,
"logits/rejected": -1.3985546827316284,
"logps/chosen": -319.5567321777344,
"logps/rejected": -254.91746520996094,
"loss": 0.4983,
"metrics/advantage_var": 75.8227767944336,
"rewards/accuracies": 0.729687511920929,
"rewards/chosen": 0.7894806861877441,
"rewards/margins": 0.7587611079216003,
"rewards/rejected": 0.03071962669491768,
"stability/repetition_rate_mean": 0.777148425579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.752539038658142,
"step": 370
},
{
"epoch": 0.4102010524895426,
"grad_norm": 14.203469276428223,
"learning_rate": 3.684197755419419e-07,
"logits/chosen": -1.4125322103500366,
"logits/rejected": -1.417395830154419,
"logps/chosen": -306.7962341308594,
"logps/rejected": -258.84234619140625,
"loss": 0.5001,
"metrics/advantage_var": 81.72932434082031,
"rewards/accuracies": 0.7593750357627869,
"rewards/chosen": 0.7916366457939148,
"rewards/margins": 0.7797738909721375,
"rewards/rejected": 0.01186272781342268,
"stability/repetition_rate_mean": 0.7945312261581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.844921827316284,
"step": 380
},
{
"epoch": 0.42099581702874106,
"grad_norm": 14.88988971710205,
"learning_rate": 3.60043987254384e-07,
"logits/chosen": -1.425724983215332,
"logits/rejected": -1.4372109174728394,
"logps/chosen": -316.85101318359375,
"logps/rejected": -264.2350769042969,
"loss": 0.4952,
"metrics/advantage_var": 92.4764404296875,
"rewards/accuracies": 0.765625,
"rewards/chosen": 0.8086903691291809,
"rewards/margins": 0.8516300320625305,
"rewards/rejected": -0.04293971136212349,
"stability/repetition_rate_mean": 0.74609375,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.869531273841858,
"step": 390
},
{
"epoch": 0.43179058156793954,
"grad_norm": 13.483888626098633,
"learning_rate": 3.5151207031562633e-07,
"logits/chosen": -1.3795498609542847,
"logits/rejected": -1.3812373876571655,
"logps/chosen": -310.2779235839844,
"logps/rejected": -268.259033203125,
"loss": 0.5137,
"metrics/advantage_var": 87.57612609863281,
"rewards/accuracies": 0.776562511920929,
"rewards/chosen": 0.7608602046966553,
"rewards/margins": 0.7845363020896912,
"rewards/rejected": -0.023676123470067978,
"stability/repetition_rate_mean": 0.7679687738418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.6525390148162842,
"step": 400
},
{
"epoch": 0.442585346107138,
"grad_norm": 15.127874374389648,
"learning_rate": 3.4283612967312687e-07,
"logits/chosen": -1.3496493101119995,
"logits/rejected": -1.3831802606582642,
"logps/chosen": -313.1993103027344,
"logps/rejected": -250.6515655517578,
"loss": 0.5069,
"metrics/advantage_var": 93.4025650024414,
"rewards/accuracies": 0.760937511920929,
"rewards/chosen": 0.7829875349998474,
"rewards/margins": 0.8043760657310486,
"rewards/rejected": -0.021388515830039978,
"stability/repetition_rate_mean": 0.854296863079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.6925780773162842,
"step": 410
},
{
"epoch": 0.45338011064633654,
"grad_norm": 13.931750297546387,
"learning_rate": 3.34028474612874e-07,
"logits/chosen": -1.3803904056549072,
"logits/rejected": -1.394149661064148,
"logps/chosen": -299.86065673828125,
"logps/rejected": -261.5897521972656,
"loss": 0.5122,
"metrics/advantage_var": 100.02517700195312,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": 0.7716613411903381,
"rewards/margins": 0.8435766100883484,
"rewards/rejected": -0.07191526889801025,
"stability/repetition_rate_mean": 0.822460949420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.1460938453674316,
"step": 420
},
{
"epoch": 0.464174875185535,
"grad_norm": 15.035143852233887,
"learning_rate": 3.2510160129516775e-07,
"logits/chosen": -1.3800678253173828,
"logits/rejected": -1.3930643796920776,
"logps/chosen": -319.5307922363281,
"logps/rejected": -265.67938232421875,
"loss": 0.4937,
"metrics/advantage_var": 104.3858413696289,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": 0.8203991055488586,
"rewards/margins": 0.8988785147666931,
"rewards/rejected": -0.07847942411899567,
"stability/repetition_rate_mean": 0.78125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.808789014816284,
"step": 430
},
{
"epoch": 0.4749696397247335,
"grad_norm": 14.138893127441406,
"learning_rate": 3.1606817502526736e-07,
"logits/chosen": -1.3767389059066772,
"logits/rejected": -1.4001781940460205,
"logps/chosen": -326.2301330566406,
"logps/rejected": -256.16412353515625,
"loss": 0.4891,
"metrics/advantage_var": 99.69866943359375,
"rewards/accuracies": 0.792187511920929,
"rewards/chosen": 0.8151880502700806,
"rewards/margins": 0.8713592886924744,
"rewards/rejected": -0.05617132410407066,
"stability/repetition_rate_mean": 0.8707031011581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.164990186691284,
"step": 440
},
{
"epoch": 0.48576440426393197,
"grad_norm": 11.794190406799316,
"learning_rate": 3.069410122840585e-07,
"logits/chosen": -1.372667670249939,
"logits/rejected": -1.4003467559814453,
"logps/chosen": -318.3290100097656,
"logps/rejected": -268.3873596191406,
"loss": 0.4817,
"metrics/advantage_var": 106.1712417602539,
"rewards/accuracies": 0.7890625,
"rewards/chosen": 0.8885055780410767,
"rewards/margins": 0.924716591835022,
"rewards/rejected": -0.03621084615588188,
"stability/repetition_rate_mean": 0.8236328363418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.7429687976837158,
"step": 450
},
{
"epoch": 0.4965591688031305,
"grad_norm": 12.17612361907959,
"learning_rate": 2.9773306254423513e-07,
"logits/chosen": -1.3681141138076782,
"logits/rejected": -1.3875712156295776,
"logps/chosen": -294.7362976074219,
"logps/rejected": -240.1558074951172,
"loss": 0.4832,
"metrics/advantage_var": 90.08233642578125,
"rewards/accuracies": 0.7718750238418579,
"rewards/chosen": 0.853641152381897,
"rewards/margins": 0.8702511191368103,
"rewards/rejected": -0.016610044986009598,
"stability/repetition_rate_mean": 0.8646484613418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.7371094226837158,
"step": 460
},
{
"epoch": 0.5073539333423289,
"grad_norm": 16.81270980834961,
"learning_rate": 2.884573898977941e-07,
"logits/chosen": -1.388026237487793,
"logits/rejected": -1.3998411893844604,
"logps/chosen": -308.7089538574219,
"logps/rejected": -268.0931091308594,
"loss": 0.4801,
"metrics/advantage_var": 102.28583526611328,
"rewards/accuracies": 0.7593750357627869,
"rewards/chosen": 0.8505555391311646,
"rewards/margins": 0.8920961618423462,
"rewards/rejected": -0.04154070094227791,
"stability/repetition_rate_mean": 0.8687499761581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.244531273841858,
"step": 470
},
{
"epoch": 0.5181486978815275,
"grad_norm": 17.988006591796875,
"learning_rate": 2.791271545209101e-07,
"logits/chosen": -1.3824865818023682,
"logits/rejected": -1.4162545204162598,
"logps/chosen": -294.66412353515625,
"logps/rejected": -248.6920928955078,
"loss": 0.4827,
"metrics/advantage_var": 95.26998138427734,
"rewards/accuracies": 0.7890625,
"rewards/chosen": 0.8823606371879578,
"rewards/margins": 0.8613284230232239,
"rewards/rejected": 0.021032243967056274,
"stability/repetition_rate_mean": 0.8519531488418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.1624999046325684,
"step": 480
},
{
"epoch": 0.528943462420726,
"grad_norm": 15.620615005493164,
"learning_rate": 2.697555940024887e-07,
"logits/chosen": -1.3622970581054688,
"logits/rejected": -1.3937467336654663,
"logps/chosen": -303.913330078125,
"logps/rejected": -242.33706665039062,
"loss": 0.4647,
"metrics/advantage_var": 105.72547912597656,
"rewards/accuracies": 0.792187511920929,
"rewards/chosen": 0.9351348280906677,
"rewards/margins": 0.9704492688179016,
"rewards/rejected": -0.035314515233039856,
"stability/repetition_rate_mean": 0.8125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.9038817882537842,
"step": 490
},
{
"epoch": 0.5397382269599245,
"grad_norm": 28.97076416015625,
"learning_rate": 2.603560045628857e-07,
"logits/chosen": -1.3485071659088135,
"logits/rejected": -1.3485997915267944,
"logps/chosen": -314.1101989746094,
"logps/rejected": -261.1115417480469,
"loss": 0.482,
"metrics/advantage_var": 112.3708267211914,
"rewards/accuracies": 0.7828125357627869,
"rewards/chosen": 0.9495781064033508,
"rewards/margins": 0.9663735628128052,
"rewards/rejected": -0.016795417293906212,
"stability/repetition_rate_mean": 0.7855468988418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.1546874046325684,
"step": 500
},
{
"epoch": 0.5505329914991229,
"grad_norm": 16.771753311157227,
"learning_rate": 2.509417221894427e-07,
"logits/chosen": -1.3849689960479736,
"logits/rejected": -1.3808684349060059,
"logps/chosen": -307.50494384765625,
"logps/rejected": -269.51959228515625,
"loss": 0.4861,
"metrics/advantage_var": 120.5189208984375,
"rewards/accuracies": 0.796875,
"rewards/chosen": 0.9477736353874207,
"rewards/margins": 0.9401771426200867,
"rewards/rejected": 0.007596464361995459,
"stability/repetition_rate_mean": 0.8619140386581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.330175757408142,
"step": 510
},
{
"epoch": 0.5613277560383214,
"grad_norm": 16.037240982055664,
"learning_rate": 2.4152610371560093e-07,
"logits/chosen": -1.4082396030426025,
"logits/rejected": -1.4073408842086792,
"logps/chosen": -308.14276123046875,
"logps/rejected": -257.72283935546875,
"loss": 0.4896,
"metrics/advantage_var": 103.19306945800781,
"rewards/accuracies": 0.765625,
"rewards/chosen": 0.9386131167411804,
"rewards/margins": 0.8744432330131531,
"rewards/rejected": 0.06416996568441391,
"stability/repetition_rate_mean": 0.856640636920929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.715429663658142,
"step": 520
},
{
"epoch": 0.5721225205775199,
"grad_norm": 20.248550415039062,
"learning_rate": 2.321225078704399e-07,
"logits/chosen": -1.4284799098968506,
"logits/rejected": -1.4444469213485718,
"logps/chosen": -317.8136291503906,
"logps/rejected": -260.39678955078125,
"loss": 0.4874,
"metrics/advantage_var": 103.1753921508789,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": 1.0152686834335327,
"rewards/margins": 0.8757182955741882,
"rewards/rejected": 0.1395503580570221,
"stability/repetition_rate_mean": 0.7701171636581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.365820288658142,
"step": 530
},
{
"epoch": 0.5829172851167184,
"grad_norm": 17.81089210510254,
"learning_rate": 2.2274427632552503e-07,
"logits/chosen": -1.4013619422912598,
"logits/rejected": -1.405015468597412,
"logps/chosen": -302.94537353515625,
"logps/rejected": -262.5538635253906,
"loss": 0.4931,
"metrics/advantage_var": 99.6611557006836,
"rewards/accuracies": 0.7734375,
"rewards/chosen": 0.9857921600341797,
"rewards/margins": 0.8826519250869751,
"rewards/rejected": 0.1031401976943016,
"stability/repetition_rate_mean": 0.8062499761581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.7644531726837158,
"step": 540
},
{
"epoch": 0.5937120496559168,
"grad_norm": 15.281890869140625,
"learning_rate": 2.134047147659583e-07,
"logits/chosen": -1.3965346813201904,
"logits/rejected": -1.3893383741378784,
"logps/chosen": -309.70013427734375,
"logps/rejected": -264.7682189941406,
"loss": 0.4744,
"metrics/advantage_var": 104.08345794677734,
"rewards/accuracies": 0.7671875357627869,
"rewards/chosen": 1.062072992324829,
"rewards/margins": 0.9215413928031921,
"rewards/rejected": 0.140531525015831,
"stability/repetition_rate_mean": 0.8472656011581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.9962890148162842,
"step": 550
},
{
"epoch": 0.6045068141951153,
"grad_norm": 19.41432762145996,
"learning_rate": 2.0411707401248403e-07,
"logits/chosen": -1.4111377000808716,
"logits/rejected": -1.4183025360107422,
"logps/chosen": -310.02789306640625,
"logps/rejected": -263.0741882324219,
"loss": 0.5088,
"metrics/advantage_var": 106.92705535888672,
"rewards/accuracies": 0.7328125238418579,
"rewards/chosen": 1.0473458766937256,
"rewards/margins": 0.841526210308075,
"rewards/rejected": 0.2058195173740387,
"stability/repetition_rate_mean": 0.784960925579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.4849610328674316,
"step": 560
},
{
"epoch": 0.6153015787343139,
"grad_norm": 15.733675956726074,
"learning_rate": 1.9489453122143603e-07,
"logits/chosen": -1.4051156044006348,
"logits/rejected": -1.4276247024536133,
"logps/chosen": -312.1006164550781,
"logps/rejected": -254.21788024902344,
"loss": 0.4437,
"metrics/advantage_var": 103.9989013671875,
"rewards/accuracies": 0.793749988079071,
"rewards/chosen": 1.1061866283416748,
"rewards/margins": 0.9930472373962402,
"rewards/rejected": 0.11313938349485397,
"stability/repetition_rate_mean": 0.828906238079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 0.817578136920929,
"step": 570
},
{
"epoch": 0.6260963432735124,
"grad_norm": 17.128223419189453,
"learning_rate": 1.8575017118919928e-07,
"logits/chosen": -1.4090861082077026,
"logits/rejected": -1.4279987812042236,
"logps/chosen": -319.3863830566406,
"logps/rejected": -252.8662109375,
"loss": 0.5064,
"metrics/advantage_var": 100.0505142211914,
"rewards/accuracies": 0.746874988079071,
"rewards/chosen": 0.9926168322563171,
"rewards/margins": 0.8279793858528137,
"rewards/rejected": 0.16463755071163177,
"stability/repetition_rate_mean": 0.7662109136581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.6736328601837158,
"step": 580
},
{
"epoch": 0.6368911078127109,
"grad_norm": 10.917092323303223,
"learning_rate": 1.7669696778770938e-07,
"logits/chosen": -1.4007326364517212,
"logits/rejected": -1.4263814687728882,
"logps/chosen": -316.9762268066406,
"logps/rejected": -245.96548461914062,
"loss": 0.4494,
"metrics/advantage_var": 108.5511245727539,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 1.0922644138336182,
"rewards/margins": 1.006994605064392,
"rewards/rejected": 0.08526992052793503,
"stability/repetition_rate_mean": 0.8466796875,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.351171851158142,
"step": 590
},
{
"epoch": 0.6476858723519093,
"grad_norm": 12.8851318359375,
"learning_rate": 1.6774776555733028e-07,
"logits/chosen": -1.382874846458435,
"logits/rejected": -1.3788057565689087,
"logps/chosen": -317.29132080078125,
"logps/rejected": -270.30157470703125,
"loss": 0.479,
"metrics/advantage_var": 106.93010711669922,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": 1.0857884883880615,
"rewards/margins": 0.9378126263618469,
"rewards/rejected": 0.14797575771808624,
"stability/repetition_rate_mean": 0.8042968511581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.7417969703674316,
"step": 600
},
{
"epoch": 0.6476858723519093,
"eval_logits/chosen": -1.3723105192184448,
"eval_logits/rejected": -1.3827242851257324,
"eval_logps/chosen": -316.3534240722656,
"eval_logps/rejected": -254.0533447265625,
"eval_loss": 0.47359293699264526,
"eval_metrics/advantage_var": 112.27565002441406,
"eval_rewards/accuracies": 0.7823232412338257,
"eval_rewards/chosen": 1.046919584274292,
"eval_rewards/margins": 0.9653631448745728,
"eval_rewards/rejected": 0.08155640214681625,
"eval_runtime": 271.831,
"eval_samples_per_second": 14.546,
"eval_stability/repetition_rate_mean": 0.8873046636581421,
"eval_stability/response_length_mean": 512.0,
"eval_stability/response_length_std": 0.0,
"eval_stability/response_length_var": 0.0,
"eval_stability/token_entropy_mean": 2.07421875,
"eval_steps_per_second": 1.821,
"step": 600
},
{
"epoch": 0.6584806368911078,
"grad_norm": 17.53192710876465,
"learning_rate": 1.5891526148322593e-07,
"logits/chosen": -1.3868850469589233,
"logits/rejected": -1.386324167251587,
"logps/chosen": -337.97906494140625,
"logps/rejected": -273.7126159667969,
"loss": 0.4827,
"metrics/advantage_var": 108.11744689941406,
"rewards/accuracies": 0.785937488079071,
"rewards/chosen": 1.0058691501617432,
"rewards/margins": 0.8874250650405884,
"rewards/rejected": 0.11844401806592941,
"stability/repetition_rate_mean": 0.744921863079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.9982421398162842,
"step": 610
},
{
"epoch": 0.6692754014303063,
"grad_norm": 12.167177200317383,
"learning_rate": 1.5021198698108036e-07,
"logits/chosen": -1.37397038936615,
"logits/rejected": -1.378446102142334,
"logps/chosen": -317.61688232421875,
"logps/rejected": -255.9473114013672,
"loss": 0.494,
"metrics/advantage_var": 114.5400390625,
"rewards/accuracies": 0.7734375,
"rewards/chosen": 1.0009812116622925,
"rewards/margins": 0.914712131023407,
"rewards/rejected": 0.0862690731883049,
"stability/repetition_rate_mean": 0.8783203363418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.2964844703674316,
"step": 620
},
{
"epoch": 0.6800701659695048,
"grad_norm": 17.123615264892578,
"learning_rate": 1.416502901177251e-07,
"logits/chosen": -1.3341145515441895,
"logits/rejected": -1.3377490043640137,
"logps/chosen": -311.6447448730469,
"logps/rejected": -254.44578552246094,
"loss": 0.4832,
"metrics/advantage_var": 98.87296295166016,
"rewards/accuracies": 0.7640625238418579,
"rewards/chosen": 0.9517079591751099,
"rewards/margins": 0.8569992184638977,
"rewards/rejected": 0.09470874816179276,
"stability/repetition_rate_mean": 0.8720703125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.377539038658142,
"step": 630
},
{
"epoch": 0.6908649305087032,
"grad_norm": 12.504801750183105,
"learning_rate": 1.3324231809189983e-07,
"logits/chosen": -1.3613817691802979,
"logits/rejected": -1.3694230318069458,
"logps/chosen": -319.6551818847656,
"logps/rejected": -258.7291564941406,
"loss": 0.4615,
"metrics/advantage_var": 120.3536376953125,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 1.0257498025894165,
"rewards/margins": 1.0272339582443237,
"rewards/rejected": -0.0014841229422017932,
"stability/repetition_rate_mean": 0.7818359136581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.7732422351837158,
"step": 640
},
{
"epoch": 0.7016596950479018,
"grad_norm": 14.149001121520996,
"learning_rate": 1.2500000000000005e-07,
"logits/chosen": -1.322688341140747,
"logits/rejected": -1.3375654220581055,
"logps/chosen": -292.4341735839844,
"logps/rejected": -252.3300323486328,
"loss": 0.4443,
"metrics/advantage_var": 113.12126922607422,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.9646196365356445,
"rewards/margins": 1.0090590715408325,
"rewards/rejected": -0.04443943500518799,
"stability/repetition_rate_mean": 0.850390613079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.5888671875,
"step": 650
},
{
"epoch": 0.7124544595871003,
"grad_norm": 14.957926750183105,
"learning_rate": 1.1693502991126608e-07,
"logits/chosen": -1.3551677465438843,
"logits/rejected": -1.361363410949707,
"logps/chosen": -333.7082214355469,
"logps/rejected": -250.5390625,
"loss": 0.461,
"metrics/advantage_var": 120.99027252197266,
"rewards/accuracies": 0.7890625,
"rewards/chosen": 1.0033434629440308,
"rewards/margins": 1.0358381271362305,
"rewards/rejected": -0.03249470144510269,
"stability/repetition_rate_mean": 0.775585949420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.3021483421325684,
"step": 660
},
{
"epoch": 0.7232492241262988,
"grad_norm": 19.125764846801758,
"learning_rate": 1.0905885027642483e-07,
"logits/chosen": -1.3491506576538086,
"logits/rejected": -1.3566621541976929,
"logps/chosen": -295.5746765136719,
"logps/rejected": -237.2896728515625,
"loss": 0.477,
"metrics/advantage_var": 120.28877258300781,
"rewards/accuracies": 0.78125,
"rewards/chosen": 0.9816705584526062,
"rewards/margins": 0.9865358471870422,
"rewards/rejected": -0.0048652635887265205,
"stability/repetition_rate_mean": 0.785937488079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.6103515625,
"step": 670
},
{
"epoch": 0.7340439886654972,
"grad_norm": 15.960895538330078,
"learning_rate": 1.0138263569332267e-07,
"logits/chosen": -1.372681975364685,
"logits/rejected": -1.3638715744018555,
"logps/chosen": -314.92364501953125,
"logps/rejected": -262.690673828125,
"loss": 0.4736,
"metrics/advantage_var": 117.98992156982422,
"rewards/accuracies": 0.7796875238418579,
"rewards/chosen": 1.0467315912246704,
"rewards/margins": 0.9876863360404968,
"rewards/rejected": 0.059045251458883286,
"stability/repetition_rate_mean": 0.7417968511581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.3708984851837158,
"step": 680
},
{
"epoch": 0.7448387532046957,
"grad_norm": 13.511040687561035,
"learning_rate": 9.391727705258502e-08,
"logits/chosen": -1.3565599918365479,
"logits/rejected": -1.3600114583969116,
"logps/chosen": -320.1853942871094,
"logps/rejected": -271.3731384277344,
"loss": 0.4613,
"metrics/advantage_var": 122.61201477050781,
"rewards/accuracies": 0.785937488079071,
"rewards/chosen": 1.0267409086227417,
"rewards/margins": 1.0403625965118408,
"rewards/rejected": -0.01362178660929203,
"stability/repetition_rate_mean": 0.8603515625,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.596289038658142,
"step": 690
},
{
"epoch": 0.7556335177438942,
"grad_norm": 10.739056587219238,
"learning_rate": 8.667336608579487e-08,
"logits/chosen": -1.3605684041976929,
"logits/rejected": -1.3892539739608765,
"logps/chosen": -318.6557312011719,
"logps/rejected": -261.914794921875,
"loss": 0.4775,
"metrics/advantage_var": 106.7211685180664,
"rewards/accuracies": 0.801562488079071,
"rewards/chosen": 0.9467880129814148,
"rewards/margins": 0.9349954724311829,
"rewards/rejected": 0.011792674660682678,
"stability/repetition_rate_mean": 0.8111327886581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.692675828933716,
"step": 700
},
{
"epoch": 0.7664282822830927,
"grad_norm": 15.086265563964844,
"learning_rate": 7.96611803381127e-08,
"logits/chosen": -1.3547624349594116,
"logits/rejected": -1.3488959074020386,
"logps/chosen": -303.3160095214844,
"logps/rejected": -258.77264404296875,
"loss": 0.4735,
"metrics/advantage_var": 115.3608627319336,
"rewards/accuracies": 0.796875,
"rewards/chosen": 0.9548945426940918,
"rewards/margins": 0.9802401661872864,
"rewards/rejected": -0.02534562349319458,
"stability/repetition_rate_mean": 0.840039074420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.209765672683716,
"step": 710
},
{
"epoch": 0.7772230468222912,
"grad_norm": 16.009767532348633,
"learning_rate": 7.28906685866599e-08,
"logits/chosen": -1.3702967166900635,
"logits/rejected": -1.3795828819274902,
"logps/chosen": -314.6918640136719,
"logps/rejected": -246.10391235351562,
"loss": 0.4632,
"metrics/advantage_var": 128.08262634277344,
"rewards/accuracies": 0.770312488079071,
"rewards/chosen": 1.0151904821395874,
"rewards/margins": 1.0434449911117554,
"rewards/rejected": -0.02825441025197506,
"stability/repetition_rate_mean": 0.8203125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 0.8656250238418579,
"step": 720
},
{
"epoch": 0.7880178113614896,
"grad_norm": 17.335712432861328,
"learning_rate": 6.637143672535281e-08,
"logits/chosen": -1.3211309909820557,
"logits/rejected": -1.3452644348144531,
"logps/chosen": -312.05279541015625,
"logps/rejected": -256.790283203125,
"loss": 0.4905,
"metrics/advantage_var": 118.5810317993164,
"rewards/accuracies": 0.78125,
"rewards/chosen": 1.0083658695220947,
"rewards/margins": 0.9445858001708984,
"rewards/rejected": 0.06377997249364853,
"stability/repetition_rate_mean": 0.7437499761581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.055468797683716,
"step": 730
},
{
"epoch": 0.7988125759006882,
"grad_norm": 10.28477668762207,
"learning_rate": 6.01127341362138e-08,
"logits/chosen": -1.339066505432129,
"logits/rejected": -1.343850016593933,
"logps/chosen": -314.6363220214844,
"logps/rejected": -265.0621032714844,
"loss": 0.4786,
"metrics/advantage_var": 117.64568328857422,
"rewards/accuracies": 0.7828125357627869,
"rewards/chosen": 0.9195186495780945,
"rewards/margins": 0.9779030680656433,
"rewards/rejected": -0.05838441848754883,
"stability/repetition_rate_mean": 0.775195300579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.513671875,
"step": 740
},
{
"epoch": 0.8096073404398867,
"grad_norm": 19.894166946411133,
"learning_rate": 5.412344056649526e-08,
"logits/chosen": -1.3286861181259155,
"logits/rejected": -1.3161166906356812,
"logps/chosen": -323.9461364746094,
"logps/rejected": -263.8832702636719,
"loss": 0.4702,
"metrics/advantage_var": 130.6468048095703,
"rewards/accuracies": 0.78125,
"rewards/chosen": 1.004286766052246,
"rewards/margins": 1.0564974546432495,
"rewards/rejected": -0.05221066623926163,
"stability/repetition_rate_mean": 0.825390636920929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.7333984375,
"step": 750
},
{
"epoch": 0.8204021049790852,
"grad_norm": 17.493206024169922,
"learning_rate": 4.841205353023714e-08,
"logits/chosen": -1.316092848777771,
"logits/rejected": -1.3103103637695312,
"logps/chosen": -308.4322814941406,
"logps/rejected": -251.6645965576172,
"loss": 0.4974,
"metrics/advantage_var": 116.90795135498047,
"rewards/accuracies": 0.7718750238418579,
"rewards/chosen": 0.985223114490509,
"rewards/margins": 0.9163877367973328,
"rewards/rejected": 0.0688353106379509,
"stability/repetition_rate_mean": 0.779101550579071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.017578125,
"step": 760
},
{
"epoch": 0.8311968695182836,
"grad_norm": 19.531171798706055,
"learning_rate": 4.298667625212904e-08,
"logits/chosen": -1.3329784870147705,
"logits/rejected": -1.3454676866531372,
"logps/chosen": -312.3675842285156,
"logps/rejected": -260.13189697265625,
"loss": 0.4839,
"metrics/advantage_var": 117.96380615234375,
"rewards/accuracies": 0.776562511920929,
"rewards/chosen": 0.9914433360099792,
"rewards/margins": 0.9715946316719055,
"rewards/rejected": 0.01984873227775097,
"stability/repetition_rate_mean": 0.8814452886581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.833984375,
"step": 770
},
{
"epoch": 0.8419916340574821,
"grad_norm": 13.06292724609375,
"learning_rate": 3.785500617078424e-08,
"logits/chosen": -1.3594963550567627,
"logits/rejected": -1.3589779138565063,
"logps/chosen": -339.4462585449219,
"logps/rejected": -283.0810546875,
"loss": 0.451,
"metrics/advantage_var": 135.08409118652344,
"rewards/accuracies": 0.8109375238418579,
"rewards/chosen": 1.0986813306808472,
"rewards/margins": 1.1291017532348633,
"rewards/rejected": -0.03042042814195156,
"stability/repetition_rate_mean": 0.795703113079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.575585961341858,
"step": 780
},
{
"epoch": 0.8527863985966806,
"grad_norm": 12.330864906311035,
"learning_rate": 3.3024324017737554e-08,
"logits/chosen": -1.341722846031189,
"logits/rejected": -1.332808256149292,
"logps/chosen": -314.4818115234375,
"logps/rejected": -266.3319396972656,
"loss": 0.4454,
"metrics/advantage_var": 123.73223876953125,
"rewards/accuracies": 0.809374988079071,
"rewards/chosen": 1.0409488677978516,
"rewards/margins": 1.057476282119751,
"rewards/rejected": -0.016527360305190086,
"stability/repetition_rate_mean": 0.8095703125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.439843773841858,
"step": 790
},
{
"epoch": 0.8635811631358791,
"grad_norm": 15.701562881469727,
"learning_rate": 2.850148348765921e-08,
"logits/chosen": -1.3520551919937134,
"logits/rejected": -1.355186939239502,
"logps/chosen": -304.0136413574219,
"logps/rejected": -258.58209228515625,
"loss": 0.4918,
"metrics/advantage_var": 115.4345703125,
"rewards/accuracies": 0.770312488079071,
"rewards/chosen": 0.9924200177192688,
"rewards/margins": 0.9020625948905945,
"rewards/rejected": 0.09035740047693253,
"stability/repetition_rate_mean": 0.9134765863418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.61376953125,
"step": 800
},
{
"epoch": 0.8743759276750775,
"grad_norm": 9.2330322265625,
"learning_rate": 2.4292901514442327e-08,
"logits/chosen": -1.3454469442367554,
"logits/rejected": -1.3386812210083008,
"logps/chosen": -321.5726013183594,
"logps/rejected": -258.95526123046875,
"loss": 0.4636,
"metrics/advantage_var": 119.41340637207031,
"rewards/accuracies": 0.7875000238418579,
"rewards/chosen": 1.0107624530792236,
"rewards/margins": 0.9988824725151062,
"rewards/rejected": 0.011880074627697468,
"stability/repetition_rate_mean": 0.8667968511581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.4040038585662842,
"step": 810
},
{
"epoch": 0.885170692214276,
"grad_norm": 10.787253379821777,
"learning_rate": 2.0404549166959718e-08,
"logits/chosen": -1.300979495048523,
"logits/rejected": -1.3161647319793701,
"logps/chosen": -308.5863037109375,
"logps/rejected": -254.77041625976562,
"loss": 0.4683,
"metrics/advantage_var": 121.4715347290039,
"rewards/accuracies": 0.7796875238418579,
"rewards/chosen": 1.0170072317123413,
"rewards/margins": 0.9836593866348267,
"rewards/rejected": 0.03334783762693405,
"stability/repetition_rate_mean": 0.9224609136581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.8826172351837158,
"step": 820
},
{
"epoch": 0.8959654567534746,
"grad_norm": 13.196191787719727,
"learning_rate": 1.6841943177406976e-08,
"logits/chosen": -1.3147996664047241,
"logits/rejected": -1.3255729675292969,
"logps/chosen": -319.1125793457031,
"logps/rejected": -274.5699462890625,
"loss": 0.4925,
"metrics/advantage_var": 126.98355865478516,
"rewards/accuracies": 0.7828125357627869,
"rewards/chosen": 0.9902805685997009,
"rewards/margins": 1.020569086074829,
"rewards/rejected": -0.030288374051451683,
"stability/repetition_rate_mean": 0.8330078125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.1563477516174316,
"step": 830
},
{
"epoch": 0.9067602212926731,
"grad_norm": 11.895269393920898,
"learning_rate": 1.3610138114250519e-08,
"logits/chosen": -1.327235221862793,
"logits/rejected": -1.343729853630066,
"logps/chosen": -327.689697265625,
"logps/rejected": -261.86480712890625,
"loss": 0.4735,
"metrics/advantage_var": 126.14949035644531,
"rewards/accuracies": 0.765625,
"rewards/chosen": 1.0660076141357422,
"rewards/margins": 1.0003070831298828,
"rewards/rejected": 0.06570061296224594,
"stability/repetition_rate_mean": 0.8158203363418579,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.4929687976837158,
"step": 840
},
{
"epoch": 0.9175549858318716,
"grad_norm": 15.15986156463623,
"learning_rate": 1.0713719210886928e-08,
"logits/chosen": -1.3229340314865112,
"logits/rejected": -1.3207443952560425,
"logps/chosen": -324.7861328125,
"logps/rejected": -261.70849609375,
"loss": 0.4859,
"metrics/advantage_var": 124.1085433959961,
"rewards/accuracies": 0.776562511920929,
"rewards/chosen": 1.053748369216919,
"rewards/margins": 0.9940328598022461,
"rewards/rejected": 0.05971544608473778,
"stability/repetition_rate_mean": 0.7544921636581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.3533203601837158,
"step": 850
},
{
"epoch": 0.92834975037107,
"grad_norm": 13.380340576171875,
"learning_rate": 8.156795860187027e-09,
"logits/chosen": -1.348737359046936,
"logits/rejected": -1.3422983884811401,
"logps/chosen": -336.0633850097656,
"logps/rejected": -262.0978088378906,
"loss": 0.4306,
"metrics/advantage_var": 125.40618133544922,
"rewards/accuracies": 0.8109375238418579,
"rewards/chosen": 1.0677509307861328,
"rewards/margins": 1.1280078887939453,
"rewards/rejected": -0.06025683507323265,
"stability/repetition_rate_mean": 0.805468738079071,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.556640625,
"step": 860
},
{
"epoch": 0.9391445149102685,
"grad_norm": 15.88653564453125,
"learning_rate": 5.942995784154692e-09,
"logits/chosen": -1.3338909149169922,
"logits/rejected": -1.3578466176986694,
"logps/chosen": -307.34771728515625,
"logps/rejected": -260.82586669921875,
"loss": 0.4598,
"metrics/advantage_var": 123.701904296875,
"rewards/accuracies": 0.7953125238418579,
"rewards/chosen": 1.0478211641311646,
"rewards/margins": 1.0490620136260986,
"rewards/rejected": -0.0012408018810674548,
"stability/repetition_rate_mean": 0.864453136920929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.4718749523162842,
"step": 870
},
{
"epoch": 0.949939279449467,
"grad_norm": 14.681868553161621,
"learning_rate": 4.075459886973082e-09,
"logits/chosen": -1.322386622428894,
"logits/rejected": -1.3134368658065796,
"logps/chosen": -288.3506774902344,
"logps/rejected": -259.43109130859375,
"loss": 0.4837,
"metrics/advantage_var": 120.60890197753906,
"rewards/accuracies": 0.7750000357627869,
"rewards/chosen": 0.9852436184883118,
"rewards/margins": 0.9725152850151062,
"rewards/rejected": 0.012728266417980194,
"stability/repetition_rate_mean": 0.8818359375,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.583984375,
"step": 880
},
{
"epoch": 0.9607340439886655,
"grad_norm": 16.591888427734375,
"learning_rate": 2.556837798739886e-09,
"logits/chosen": -1.3387622833251953,
"logits/rejected": -1.34397554397583,
"logps/chosen": -314.2873229980469,
"logps/rejected": -257.3379821777344,
"loss": 0.4647,
"metrics/advantage_var": 120.80268859863281,
"rewards/accuracies": 0.8109375238418579,
"rewards/chosen": 0.9938045740127563,
"rewards/margins": 0.9979391098022461,
"rewards/rejected": -0.0041345953941345215,
"stability/repetition_rate_mean": 0.8443359136581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.6566405296325684,
"step": 890
},
{
"epoch": 0.9715288085278639,
"grad_norm": 13.192713737487793,
"learning_rate": 1.3892841162143899e-09,
"logits/chosen": -1.3134535551071167,
"logits/rejected": -1.3294223546981812,
"logps/chosen": -304.279541015625,
"logps/rejected": -257.1634216308594,
"loss": 0.455,
"metrics/advantage_var": 119.05987548828125,
"rewards/accuracies": 0.785937488079071,
"rewards/chosen": 0.9897605776786804,
"rewards/margins": 1.0290673971176147,
"rewards/rejected": -0.039306994527578354,
"stability/repetition_rate_mean": 0.7369140386581421,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.3954589366912842,
"step": 900
},
{
"epoch": 0.9715288085278639,
"eval_logits/chosen": -1.3294035196304321,
"eval_logits/rejected": -1.3341506719589233,
"eval_logps/chosen": -316.4003601074219,
"eval_logps/rejected": -254.81178283691406,
"eval_loss": 0.46732690930366516,
"eval_metrics/advantage_var": 125.62010955810547,
"eval_rewards/accuracies": 0.7828283309936523,
"eval_rewards/chosen": 1.0422265529632568,
"eval_rewards/margins": 1.036514401435852,
"eval_rewards/rejected": 0.005712195765227079,
"eval_runtime": 269.7729,
"eval_samples_per_second": 14.657,
"eval_stability/repetition_rate_mean": 0.811718761920929,
"eval_stability/response_length_mean": 512.0,
"eval_stability/response_length_std": 0.0,
"eval_stability/response_length_var": 0.0,
"eval_stability/token_entropy_mean": 1.127832055091858,
"eval_steps_per_second": 1.835,
"step": 900
},
{
"epoch": 0.9823235730670625,
"grad_norm": 11.817096710205078,
"learning_rate": 5.744553459100243e-10,
"logits/chosen": -1.3120925426483154,
"logits/rejected": -1.3330440521240234,
"logps/chosen": -324.1401062011719,
"logps/rejected": -261.5168762207031,
"loss": 0.4964,
"metrics/advantage_var": 119.01836395263672,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": 0.9547117352485657,
"rewards/margins": 0.9206423163414001,
"rewards/rejected": 0.03406943380832672,
"stability/repetition_rate_mean": 0.875195324420929,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 1.468652367591858,
"step": 910
},
{
"epoch": 0.993118337606261,
"grad_norm": 19.269649505615234,
"learning_rate": 1.1350755386951849e-10,
"logits/chosen": -1.3584011793136597,
"logits/rejected": -1.3463481664657593,
"logps/chosen": -329.8288879394531,
"logps/rejected": -261.056640625,
"loss": 0.4515,
"metrics/advantage_var": 127.2206802368164,
"rewards/accuracies": 0.784375011920929,
"rewards/chosen": 1.044679045677185,
"rewards/margins": 1.0371487140655518,
"rewards/rejected": 0.007530394475907087,
"stability/repetition_rate_mean": 0.861328125,
"stability/response_length_mean": 512.0,
"stability/response_length_std": 0.0,
"stability/response_length_var": 0.0,
"stability/token_entropy_mean": 2.8033204078674316,
"step": 920
},
{
"epoch": 1.0,
"step": 927,
"total_flos": 1.761935585720664e+18,
"train_loss": 0.5203882457111775,
"train_runtime": 14354.8151,
"train_samples_per_second": 4.13,
"train_steps_per_second": 0.065
}
],
"logging_steps": 10,
"max_steps": 927,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1.761935585720664e+18,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
Reference in New Issue Copy Permalink