3239 lines
108 KiB
JSON
3239 lines
108 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2130,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.004694835680751174,
|
||
|
|
"grad_norm": 872.7149989993486,
|
||
|
|
"learning_rate": 2.1126760563380282e-08,
|
||
|
|
"logits/chosen": -0.5947265625,
|
||
|
|
"logits/rejected": -0.33906251192092896,
|
||
|
|
"logps/chosen": -453.3999938964844,
|
||
|
|
"logps/rejected": -552.2000122070312,
|
||
|
|
"loss": 2.6719,
|
||
|
|
"rewards/accuracies": 0.18046875298023224,
|
||
|
|
"rewards/chosen": 0.05380859225988388,
|
||
|
|
"rewards/margins": -0.4002929627895355,
|
||
|
|
"rewards/rejected": 0.45292967557907104,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.009389671361502348,
|
||
|
|
"grad_norm": 1040.3923772216594,
|
||
|
|
"learning_rate": 4.460093896713615e-08,
|
||
|
|
"logits/chosen": -0.661328136920929,
|
||
|
|
"logits/rejected": -0.3705078065395355,
|
||
|
|
"logps/chosen": -475.3999938964844,
|
||
|
|
"logps/rejected": -562.0,
|
||
|
|
"loss": 3.032,
|
||
|
|
"rewards/accuracies": 0.29296875,
|
||
|
|
"rewards/chosen": 0.06289062649011612,
|
||
|
|
"rewards/margins": 0.23325195908546448,
|
||
|
|
"rewards/rejected": -0.17041015625,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.014084507042253521,
|
||
|
|
"grad_norm": 821.0725520442512,
|
||
|
|
"learning_rate": 6.807511737089202e-08,
|
||
|
|
"logits/chosen": -0.5892578363418579,
|
||
|
|
"logits/rejected": -0.31704407930374146,
|
||
|
|
"logps/chosen": -511.6000061035156,
|
||
|
|
"logps/rejected": -615.0,
|
||
|
|
"loss": 3.46,
|
||
|
|
"rewards/accuracies": 0.2835937440395355,
|
||
|
|
"rewards/chosen": -0.3681640625,
|
||
|
|
"rewards/margins": -0.36103516817092896,
|
||
|
|
"rewards/rejected": -0.00820312462747097,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.018779342723004695,
|
||
|
|
"grad_norm": 839.936915817132,
|
||
|
|
"learning_rate": 9.154929577464789e-08,
|
||
|
|
"logits/chosen": -0.618359386920929,
|
||
|
|
"logits/rejected": -0.34697264432907104,
|
||
|
|
"logps/chosen": -497.0,
|
||
|
|
"logps/rejected": -628.5999755859375,
|
||
|
|
"loss": 3.4961,
|
||
|
|
"rewards/accuracies": 0.2750000059604645,
|
||
|
|
"rewards/chosen": -0.19160155951976776,
|
||
|
|
"rewards/margins": -0.2681640684604645,
|
||
|
|
"rewards/rejected": 0.07753906399011612,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.023474178403755867,
|
||
|
|
"grad_norm": 1016.2510549939029,
|
||
|
|
"learning_rate": 1.1502347417840374e-07,
|
||
|
|
"logits/chosen": -0.6236327886581421,
|
||
|
|
"logits/rejected": -0.3160156309604645,
|
||
|
|
"logps/chosen": -481.3999938964844,
|
||
|
|
"logps/rejected": -571.0,
|
||
|
|
"loss": 2.9598,
|
||
|
|
"rewards/accuracies": 0.3031249940395355,
|
||
|
|
"rewards/chosen": 0.18378905951976776,
|
||
|
|
"rewards/margins": 0.3031249940395355,
|
||
|
|
"rewards/rejected": -0.11796875298023224,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.028169014084507043,
|
||
|
|
"grad_norm": 1014.0340966177242,
|
||
|
|
"learning_rate": 1.384976525821596e-07,
|
||
|
|
"logits/chosen": -0.608593761920929,
|
||
|
|
"logits/rejected": -0.3189697265625,
|
||
|
|
"logps/chosen": -493.3999938964844,
|
||
|
|
"logps/rejected": -618.0,
|
||
|
|
"loss": 3.2755,
|
||
|
|
"rewards/accuracies": 0.27812498807907104,
|
||
|
|
"rewards/chosen": -0.1181640625,
|
||
|
|
"rewards/margins": -0.3111816346645355,
|
||
|
|
"rewards/rejected": 0.19282226264476776,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03286384976525822,
|
||
|
|
"grad_norm": 975.4384264175499,
|
||
|
|
"learning_rate": 1.619718309859155e-07,
|
||
|
|
"logits/chosen": -0.6507812738418579,
|
||
|
|
"logits/rejected": -0.3419433534145355,
|
||
|
|
"logps/chosen": -471.3999938964844,
|
||
|
|
"logps/rejected": -582.4000244140625,
|
||
|
|
"loss": 3.1017,
|
||
|
|
"rewards/accuracies": 0.28515625,
|
||
|
|
"rewards/chosen": -0.13139648735523224,
|
||
|
|
"rewards/margins": -0.10166015475988388,
|
||
|
|
"rewards/rejected": -0.03076171875,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03755868544600939,
|
||
|
|
"grad_norm": 778.9072904599768,
|
||
|
|
"learning_rate": 1.8544600938967138e-07,
|
||
|
|
"logits/chosen": -0.5478515625,
|
||
|
|
"logits/rejected": -0.2798828184604645,
|
||
|
|
"logps/chosen": -479.20001220703125,
|
||
|
|
"logps/rejected": -595.0,
|
||
|
|
"loss": 3.0709,
|
||
|
|
"rewards/accuracies": 0.3070312440395355,
|
||
|
|
"rewards/chosen": 0.3235839903354645,
|
||
|
|
"rewards/margins": 0.515332043170929,
|
||
|
|
"rewards/rejected": -0.19187012314796448,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04225352112676056,
|
||
|
|
"grad_norm": 823.2165864872086,
|
||
|
|
"learning_rate": 2.089201877934272e-07,
|
||
|
|
"logits/chosen": -0.5189453363418579,
|
||
|
|
"logits/rejected": -0.29389649629592896,
|
||
|
|
"logps/chosen": -466.20001220703125,
|
||
|
|
"logps/rejected": -551.2000122070312,
|
||
|
|
"loss": 3.4759,
|
||
|
|
"rewards/accuracies": 0.2984375059604645,
|
||
|
|
"rewards/chosen": 0.24560546875,
|
||
|
|
"rewards/margins": -0.0322265625,
|
||
|
|
"rewards/rejected": 0.2777343690395355,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.046948356807511735,
|
||
|
|
"grad_norm": 791.1365012572919,
|
||
|
|
"learning_rate": 2.323943661971831e-07,
|
||
|
|
"logits/chosen": -0.5580078363418579,
|
||
|
|
"logits/rejected": -0.3643554747104645,
|
||
|
|
"logps/chosen": -496.0,
|
||
|
|
"logps/rejected": -540.4000244140625,
|
||
|
|
"loss": 3.2352,
|
||
|
|
"rewards/accuracies": 0.31171876192092896,
|
||
|
|
"rewards/chosen": 0.6552734375,
|
||
|
|
"rewards/margins": 0.15117187798023224,
|
||
|
|
"rewards/rejected": 0.5042968988418579,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.051643192488262914,
|
||
|
|
"grad_norm": 871.0619948441355,
|
||
|
|
"learning_rate": 2.5586854460093895e-07,
|
||
|
|
"logits/chosen": -0.630078136920929,
|
||
|
|
"logits/rejected": -0.33916014432907104,
|
||
|
|
"logps/chosen": -478.6000061035156,
|
||
|
|
"logps/rejected": -599.5999755859375,
|
||
|
|
"loss": 3.3408,
|
||
|
|
"rewards/accuracies": 0.30390626192092896,
|
||
|
|
"rewards/chosen": 0.7401367425918579,
|
||
|
|
"rewards/margins": 0.14711913466453552,
|
||
|
|
"rewards/rejected": 0.59521484375,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.056338028169014086,
|
||
|
|
"grad_norm": 843.2886988958245,
|
||
|
|
"learning_rate": 2.7934272300469483e-07,
|
||
|
|
"logits/chosen": -0.603710949420929,
|
||
|
|
"logits/rejected": -0.43574219942092896,
|
||
|
|
"logps/chosen": -490.6000061035156,
|
||
|
|
"logps/rejected": -582.5999755859375,
|
||
|
|
"loss": 2.7437,
|
||
|
|
"rewards/accuracies": 0.3203125,
|
||
|
|
"rewards/chosen": 1.482812523841858,
|
||
|
|
"rewards/margins": 0.877246081829071,
|
||
|
|
"rewards/rejected": 0.6044921875,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06103286384976526,
|
||
|
|
"grad_norm": 952.288159877147,
|
||
|
|
"learning_rate": 3.0281690140845066e-07,
|
||
|
|
"logits/chosen": -0.6009765863418579,
|
||
|
|
"logits/rejected": -0.3548828065395355,
|
||
|
|
"logps/chosen": -495.20001220703125,
|
||
|
|
"logps/rejected": -597.5999755859375,
|
||
|
|
"loss": 2.785,
|
||
|
|
"rewards/accuracies": 0.33984375,
|
||
|
|
"rewards/chosen": 1.907617211341858,
|
||
|
|
"rewards/margins": 1.287011742591858,
|
||
|
|
"rewards/rejected": 0.6209961175918579,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06572769953051644,
|
||
|
|
"grad_norm": 861.5943619356324,
|
||
|
|
"learning_rate": 3.2629107981220654e-07,
|
||
|
|
"logits/chosen": -0.625,
|
||
|
|
"logits/rejected": -0.3183227479457855,
|
||
|
|
"logps/chosen": -485.79998779296875,
|
||
|
|
"logps/rejected": -603.0,
|
||
|
|
"loss": 3.3498,
|
||
|
|
"rewards/accuracies": 0.3515625,
|
||
|
|
"rewards/chosen": 2.0859375,
|
||
|
|
"rewards/margins": 0.5775390863418579,
|
||
|
|
"rewards/rejected": 1.508691430091858,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07042253521126761,
|
||
|
|
"grad_norm": 782.8569473535536,
|
||
|
|
"learning_rate": 3.497652582159624e-07,
|
||
|
|
"logits/chosen": -0.6156250238418579,
|
||
|
|
"logits/rejected": -0.328125,
|
||
|
|
"logps/chosen": -484.79998779296875,
|
||
|
|
"logps/rejected": -602.7999877929688,
|
||
|
|
"loss": 2.9995,
|
||
|
|
"rewards/accuracies": 0.33906251192092896,
|
||
|
|
"rewards/chosen": 3.19140625,
|
||
|
|
"rewards/margins": 1.111328125,
|
||
|
|
"rewards/rejected": 2.0765624046325684,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07511737089201878,
|
||
|
|
"grad_norm": 881.0291461116594,
|
||
|
|
"learning_rate": 3.732394366197183e-07,
|
||
|
|
"logits/chosen": -0.5777343511581421,
|
||
|
|
"logits/rejected": -0.2904296815395355,
|
||
|
|
"logps/chosen": -499.6000061035156,
|
||
|
|
"logps/rejected": -635.5999755859375,
|
||
|
|
"loss": 3.0294,
|
||
|
|
"rewards/accuracies": 0.37031251192092896,
|
||
|
|
"rewards/chosen": 2.854687452316284,
|
||
|
|
"rewards/margins": 1.842187523841858,
|
||
|
|
"rewards/rejected": 1.014990210533142,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07981220657276995,
|
||
|
|
"grad_norm": 739.9307651148558,
|
||
|
|
"learning_rate": 3.967136150234742e-07,
|
||
|
|
"logits/chosen": -0.5775390863418579,
|
||
|
|
"logits/rejected": -0.2758422791957855,
|
||
|
|
"logps/chosen": -459.20001220703125,
|
||
|
|
"logps/rejected": -593.2000122070312,
|
||
|
|
"loss": 2.9436,
|
||
|
|
"rewards/accuracies": 0.40312498807907104,
|
||
|
|
"rewards/chosen": 4.10546875,
|
||
|
|
"rewards/margins": 2.593505859375,
|
||
|
|
"rewards/rejected": 1.5078125,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08450704225352113,
|
||
|
|
"grad_norm": 785.9886031249839,
|
||
|
|
"learning_rate": 4.2018779342723e-07,
|
||
|
|
"logits/chosen": -0.6675781011581421,
|
||
|
|
"logits/rejected": -0.38544923067092896,
|
||
|
|
"logps/chosen": -445.20001220703125,
|
||
|
|
"logps/rejected": -567.0,
|
||
|
|
"loss": 3.1399,
|
||
|
|
"rewards/accuracies": 0.4453125,
|
||
|
|
"rewards/chosen": 4.3359375,
|
||
|
|
"rewards/margins": 2.619921922683716,
|
||
|
|
"rewards/rejected": 1.7184569835662842,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0892018779342723,
|
||
|
|
"grad_norm": 766.9666480869223,
|
||
|
|
"learning_rate": 4.436619718309859e-07,
|
||
|
|
"logits/chosen": -0.5816406011581421,
|
||
|
|
"logits/rejected": -0.32060545682907104,
|
||
|
|
"logps/chosen": -478.79998779296875,
|
||
|
|
"logps/rejected": -563.7999877929688,
|
||
|
|
"loss": 3.0276,
|
||
|
|
"rewards/accuracies": 0.41484373807907104,
|
||
|
|
"rewards/chosen": 4.615624904632568,
|
||
|
|
"rewards/margins": 2.627148389816284,
|
||
|
|
"rewards/rejected": 1.986914038658142,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09389671361502347,
|
||
|
|
"grad_norm": 697.3310407801386,
|
||
|
|
"learning_rate": 4.671361502347418e-07,
|
||
|
|
"logits/chosen": -0.6410156488418579,
|
||
|
|
"logits/rejected": -0.36445313692092896,
|
||
|
|
"logps/chosen": -466.6000061035156,
|
||
|
|
"logps/rejected": -571.4000244140625,
|
||
|
|
"loss": 3.208,
|
||
|
|
"rewards/accuracies": 0.44921875,
|
||
|
|
"rewards/chosen": 4.817187309265137,
|
||
|
|
"rewards/margins": 3.049999952316284,
|
||
|
|
"rewards/rejected": 1.762109398841858,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09859154929577464,
|
||
|
|
"grad_norm": 804.8462115887406,
|
||
|
|
"learning_rate": 4.906103286384976e-07,
|
||
|
|
"logits/chosen": -0.619140625,
|
||
|
|
"logits/rejected": -0.3290039002895355,
|
||
|
|
"logps/chosen": -480.20001220703125,
|
||
|
|
"logps/rejected": -603.5999755859375,
|
||
|
|
"loss": 3.1795,
|
||
|
|
"rewards/accuracies": 0.4281249940395355,
|
||
|
|
"rewards/chosen": 5.040625095367432,
|
||
|
|
"rewards/margins": 3.303906202316284,
|
||
|
|
"rewards/rejected": 1.731103539466858,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10328638497652583,
|
||
|
|
"grad_norm": 720.6896592892501,
|
||
|
|
"learning_rate": 4.984350547730829e-07,
|
||
|
|
"logits/chosen": -0.610156238079071,
|
||
|
|
"logits/rejected": -0.3102050721645355,
|
||
|
|
"logps/chosen": -491.6000061035156,
|
||
|
|
"logps/rejected": -582.0,
|
||
|
|
"loss": 2.7493,
|
||
|
|
"rewards/accuracies": 0.46015626192092896,
|
||
|
|
"rewards/chosen": 5.637499809265137,
|
||
|
|
"rewards/margins": 4.485937595367432,
|
||
|
|
"rewards/rejected": 1.149804711341858,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.107981220657277,
|
||
|
|
"grad_norm": 643.0765878361951,
|
||
|
|
"learning_rate": 4.958268127282212e-07,
|
||
|
|
"logits/chosen": -0.7027343511581421,
|
||
|
|
"logits/rejected": -0.4419921934604645,
|
||
|
|
"logps/chosen": -481.3999938964844,
|
||
|
|
"logps/rejected": -619.0,
|
||
|
|
"loss": 2.8096,
|
||
|
|
"rewards/accuracies": 0.47578126192092896,
|
||
|
|
"rewards/chosen": 5.731249809265137,
|
||
|
|
"rewards/margins": 4.317187309265137,
|
||
|
|
"rewards/rejected": 1.410302758216858,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11267605633802817,
|
||
|
|
"grad_norm": 715.6375388900655,
|
||
|
|
"learning_rate": 4.932185706833594e-07,
|
||
|
|
"logits/chosen": -0.656054675579071,
|
||
|
|
"logits/rejected": -0.43046873807907104,
|
||
|
|
"logps/chosen": -437.0,
|
||
|
|
"logps/rejected": -503.20001220703125,
|
||
|
|
"loss": 2.7177,
|
||
|
|
"rewards/accuracies": 0.47187501192092896,
|
||
|
|
"rewards/chosen": 6.243750095367432,
|
||
|
|
"rewards/margins": 4.46875,
|
||
|
|
"rewards/rejected": 1.7734375,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11737089201877934,
|
||
|
|
"grad_norm": 827.9977919444125,
|
||
|
|
"learning_rate": 4.906103286384976e-07,
|
||
|
|
"logits/chosen": -0.6390625238418579,
|
||
|
|
"logits/rejected": -0.3297363221645355,
|
||
|
|
"logps/chosen": -479.3999938964844,
|
||
|
|
"logps/rejected": -621.4000244140625,
|
||
|
|
"loss": 3.1442,
|
||
|
|
"rewards/accuracies": 0.4664062559604645,
|
||
|
|
"rewards/chosen": 6.040625095367432,
|
||
|
|
"rewards/margins": 5.340624809265137,
|
||
|
|
"rewards/rejected": 0.6978515386581421,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12206572769953052,
|
||
|
|
"grad_norm": 730.3263384722413,
|
||
|
|
"learning_rate": 4.880020865936358e-07,
|
||
|
|
"logits/chosen": -0.6376953125,
|
||
|
|
"logits/rejected": -0.345458984375,
|
||
|
|
"logps/chosen": -449.20001220703125,
|
||
|
|
"logps/rejected": -561.7999877929688,
|
||
|
|
"loss": 2.8628,
|
||
|
|
"rewards/accuracies": 0.49609375,
|
||
|
|
"rewards/chosen": 6.709374904632568,
|
||
|
|
"rewards/margins": 5.175000190734863,
|
||
|
|
"rewards/rejected": 1.53662109375,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1267605633802817,
|
||
|
|
"grad_norm": 1073.9835705489036,
|
||
|
|
"learning_rate": 4.853938445487741e-07,
|
||
|
|
"logits/chosen": -0.6292968988418579,
|
||
|
|
"logits/rejected": -0.3013244569301605,
|
||
|
|
"logps/chosen": -477.0,
|
||
|
|
"logps/rejected": -594.0,
|
||
|
|
"loss": 3.3852,
|
||
|
|
"rewards/accuracies": 0.4867187440395355,
|
||
|
|
"rewards/chosen": 6.953125,
|
||
|
|
"rewards/margins": 5.103125095367432,
|
||
|
|
"rewards/rejected": 1.853515625,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13145539906103287,
|
||
|
|
"grad_norm": 616.4450244340333,
|
||
|
|
"learning_rate": 4.827856025039123e-07,
|
||
|
|
"logits/chosen": -0.6548827886581421,
|
||
|
|
"logits/rejected": -0.34687501192092896,
|
||
|
|
"logps/chosen": -475.0,
|
||
|
|
"logps/rejected": -597.7999877929688,
|
||
|
|
"loss": 4.6967,
|
||
|
|
"rewards/accuracies": 0.49687498807907104,
|
||
|
|
"rewards/chosen": 7.324999809265137,
|
||
|
|
"rewards/margins": 4.553124904632568,
|
||
|
|
"rewards/rejected": 2.768359422683716,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13615023474178403,
|
||
|
|
"grad_norm": 586.6297592673741,
|
||
|
|
"learning_rate": 4.801773604590506e-07,
|
||
|
|
"logits/chosen": -0.6259765625,
|
||
|
|
"logits/rejected": -0.3440918028354645,
|
||
|
|
"logps/chosen": -512.4000244140625,
|
||
|
|
"logps/rejected": -594.7999877929688,
|
||
|
|
"loss": 3.1606,
|
||
|
|
"rewards/accuracies": 0.5218750238418579,
|
||
|
|
"rewards/chosen": 7.740624904632568,
|
||
|
|
"rewards/margins": 7.400000095367432,
|
||
|
|
"rewards/rejected": 0.34687501192092896,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14084507042253522,
|
||
|
|
"grad_norm": 616.6078643691559,
|
||
|
|
"learning_rate": 4.775691184141888e-07,
|
||
|
|
"logits/chosen": -0.689453125,
|
||
|
|
"logits/rejected": -0.34814453125,
|
||
|
|
"logps/chosen": -495.79998779296875,
|
||
|
|
"logps/rejected": -578.0,
|
||
|
|
"loss": 2.5858,
|
||
|
|
"rewards/accuracies": 0.5179687738418579,
|
||
|
|
"rewards/chosen": 8.743749618530273,
|
||
|
|
"rewards/margins": 7.167187690734863,
|
||
|
|
"rewards/rejected": 1.5810546875,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14553990610328638,
|
||
|
|
"grad_norm": 792.1331301357641,
|
||
|
|
"learning_rate": 4.749608763693271e-07,
|
||
|
|
"logits/chosen": -0.611132800579071,
|
||
|
|
"logits/rejected": -0.3414550721645355,
|
||
|
|
"logps/chosen": -471.3999938964844,
|
||
|
|
"logps/rejected": -576.2000122070312,
|
||
|
|
"loss": 3.1323,
|
||
|
|
"rewards/accuracies": 0.49687498807907104,
|
||
|
|
"rewards/chosen": 7.803124904632568,
|
||
|
|
"rewards/margins": 5.364062309265137,
|
||
|
|
"rewards/rejected": 2.4320311546325684,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15023474178403756,
|
||
|
|
"grad_norm": 753.0925191809758,
|
||
|
|
"learning_rate": 4.7235263432446533e-07,
|
||
|
|
"logits/chosen": -0.63671875,
|
||
|
|
"logits/rejected": -0.4302734434604645,
|
||
|
|
"logps/chosen": -470.79998779296875,
|
||
|
|
"logps/rejected": -600.7999877929688,
|
||
|
|
"loss": 2.8555,
|
||
|
|
"rewards/accuracies": 0.5249999761581421,
|
||
|
|
"rewards/chosen": 8.399999618530273,
|
||
|
|
"rewards/margins": 7.959374904632568,
|
||
|
|
"rewards/rejected": 0.44482421875,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15492957746478872,
|
||
|
|
"grad_norm": 692.6983641703249,
|
||
|
|
"learning_rate": 4.6974439227960353e-07,
|
||
|
|
"logits/chosen": -0.590039074420929,
|
||
|
|
"logits/rejected": -0.3519531190395355,
|
||
|
|
"logps/chosen": -486.6000061035156,
|
||
|
|
"logps/rejected": -593.7999877929688,
|
||
|
|
"loss": 3.2224,
|
||
|
|
"rewards/accuracies": 0.51171875,
|
||
|
|
"rewards/chosen": 8.181249618530273,
|
||
|
|
"rewards/margins": 7.006249904632568,
|
||
|
|
"rewards/rejected": 1.1728515625,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1596244131455399,
|
||
|
|
"grad_norm": 640.3534053430826,
|
||
|
|
"learning_rate": 4.671361502347418e-07,
|
||
|
|
"logits/chosen": -0.6216796636581421,
|
||
|
|
"logits/rejected": -0.34833985567092896,
|
||
|
|
"logps/chosen": -462.20001220703125,
|
||
|
|
"logps/rejected": -527.7999877929688,
|
||
|
|
"loss": 2.7153,
|
||
|
|
"rewards/accuracies": 0.5335937738418579,
|
||
|
|
"rewards/chosen": 8.774999618530273,
|
||
|
|
"rewards/margins": 6.715624809265137,
|
||
|
|
"rewards/rejected": 2.0667967796325684,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1643192488262911,
|
||
|
|
"grad_norm": 1163.4217516247206,
|
||
|
|
"learning_rate": 4.6452790818988004e-07,
|
||
|
|
"logits/chosen": -0.6539062261581421,
|
||
|
|
"logits/rejected": -0.34575194120407104,
|
||
|
|
"logps/chosen": -497.6000061035156,
|
||
|
|
"logps/rejected": -600.2000122070312,
|
||
|
|
"loss": 3.1416,
|
||
|
|
"rewards/accuracies": 0.5093749761581421,
|
||
|
|
"rewards/chosen": 7.703125,
|
||
|
|
"rewards/margins": 7.064062595367432,
|
||
|
|
"rewards/rejected": 0.635009765625,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16901408450704225,
|
||
|
|
"grad_norm": 910.1296329528788,
|
||
|
|
"learning_rate": 4.6191966614501824e-07,
|
||
|
|
"logits/chosen": -0.6441406011581421,
|
||
|
|
"logits/rejected": -0.4521484375,
|
||
|
|
"logps/chosen": -465.79998779296875,
|
||
|
|
"logps/rejected": -564.2000122070312,
|
||
|
|
"loss": 3.2758,
|
||
|
|
"rewards/accuracies": 0.526562511920929,
|
||
|
|
"rewards/chosen": 8.003125190734863,
|
||
|
|
"rewards/margins": 6.434374809265137,
|
||
|
|
"rewards/rejected": 1.561914086341858,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17370892018779344,
|
||
|
|
"grad_norm": 605.037254330656,
|
||
|
|
"learning_rate": 4.593114241001565e-07,
|
||
|
|
"logits/chosen": -0.655078113079071,
|
||
|
|
"logits/rejected": -0.3575195372104645,
|
||
|
|
"logps/chosen": -500.6000061035156,
|
||
|
|
"logps/rejected": -589.4000244140625,
|
||
|
|
"loss": 2.6754,
|
||
|
|
"rewards/accuracies": 0.5484374761581421,
|
||
|
|
"rewards/chosen": 8.446874618530273,
|
||
|
|
"rewards/margins": 7.946875095367432,
|
||
|
|
"rewards/rejected": 0.49897462129592896,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1784037558685446,
|
||
|
|
"grad_norm": 854.5733785236513,
|
||
|
|
"learning_rate": 4.5670318205529474e-07,
|
||
|
|
"logits/chosen": -0.6050781011581421,
|
||
|
|
"logits/rejected": -0.3639160096645355,
|
||
|
|
"logps/chosen": -478.20001220703125,
|
||
|
|
"logps/rejected": -590.4000244140625,
|
||
|
|
"loss": 3.4332,
|
||
|
|
"rewards/accuracies": 0.5054687261581421,
|
||
|
|
"rewards/chosen": 6.628125190734863,
|
||
|
|
"rewards/margins": 5.946875095367432,
|
||
|
|
"rewards/rejected": 0.686328113079071,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18309859154929578,
|
||
|
|
"grad_norm": 990.0888875524229,
|
||
|
|
"learning_rate": 4.54094940010433e-07,
|
||
|
|
"logits/chosen": -0.6166015863418579,
|
||
|
|
"logits/rejected": -0.30000001192092896,
|
||
|
|
"logps/chosen": -487.0,
|
||
|
|
"logps/rejected": -603.2000122070312,
|
||
|
|
"loss": 3.6306,
|
||
|
|
"rewards/accuracies": 0.5249999761581421,
|
||
|
|
"rewards/chosen": 7.671875,
|
||
|
|
"rewards/margins": 7.482812404632568,
|
||
|
|
"rewards/rejected": 0.19082030653953552,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18779342723004694,
|
||
|
|
"grad_norm": 613.3307436370222,
|
||
|
|
"learning_rate": 4.514866979655712e-07,
|
||
|
|
"logits/chosen": -0.58984375,
|
||
|
|
"logits/rejected": -0.35791015625,
|
||
|
|
"logps/chosen": -476.79998779296875,
|
||
|
|
"logps/rejected": -561.2000122070312,
|
||
|
|
"loss": 3.213,
|
||
|
|
"rewards/accuracies": 0.5375000238418579,
|
||
|
|
"rewards/chosen": 7.796875,
|
||
|
|
"rewards/margins": 7.675000190734863,
|
||
|
|
"rewards/rejected": 0.12324218451976776,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19248826291079812,
|
||
|
|
"grad_norm": 664.3746841962233,
|
||
|
|
"learning_rate": 4.4887845592070945e-07,
|
||
|
|
"logits/chosen": -0.6435546875,
|
||
|
|
"logits/rejected": -0.3348144590854645,
|
||
|
|
"logps/chosen": -463.3999938964844,
|
||
|
|
"logps/rejected": -615.2000122070312,
|
||
|
|
"loss": 3.1333,
|
||
|
|
"rewards/accuracies": 0.526562511920929,
|
||
|
|
"rewards/chosen": 7.574999809265137,
|
||
|
|
"rewards/margins": 7.464062690734863,
|
||
|
|
"rewards/rejected": 0.12148437649011612,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19718309859154928,
|
||
|
|
"grad_norm": 709.6086156398218,
|
||
|
|
"learning_rate": 4.462702138758477e-07,
|
||
|
|
"logits/chosen": -0.5648437738418579,
|
||
|
|
"logits/rejected": -0.35502928495407104,
|
||
|
|
"logps/chosen": -497.3999938964844,
|
||
|
|
"logps/rejected": -580.5999755859375,
|
||
|
|
"loss": 3.0143,
|
||
|
|
"rewards/accuracies": 0.50390625,
|
||
|
|
"rewards/chosen": 7.359375,
|
||
|
|
"rewards/margins": 6.731249809265137,
|
||
|
|
"rewards/rejected": 0.633984386920929,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20187793427230047,
|
||
|
|
"grad_norm": 764.2647223364141,
|
||
|
|
"learning_rate": 4.436619718309859e-07,
|
||
|
|
"logits/chosen": -0.6640625,
|
||
|
|
"logits/rejected": -0.37519532442092896,
|
||
|
|
"logps/chosen": -478.79998779296875,
|
||
|
|
"logps/rejected": -614.7999877929688,
|
||
|
|
"loss": 3.5353,
|
||
|
|
"rewards/accuracies": 0.530468761920929,
|
||
|
|
"rewards/chosen": 7.037499904632568,
|
||
|
|
"rewards/margins": 8.073437690734863,
|
||
|
|
"rewards/rejected": -1.0339844226837158,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20657276995305165,
|
||
|
|
"grad_norm": 796.7623727550922,
|
||
|
|
"learning_rate": 4.4105372978612415e-07,
|
||
|
|
"logits/chosen": -0.5611327886581421,
|
||
|
|
"logits/rejected": -0.3167968690395355,
|
||
|
|
"logps/chosen": -498.6000061035156,
|
||
|
|
"logps/rejected": -603.7999877929688,
|
||
|
|
"loss": 3.1253,
|
||
|
|
"rewards/accuracies": 0.5335937738418579,
|
||
|
|
"rewards/chosen": 6.903124809265137,
|
||
|
|
"rewards/margins": 7.978125095367432,
|
||
|
|
"rewards/rejected": -1.0832030773162842,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2112676056338028,
|
||
|
|
"grad_norm": 758.2943346307572,
|
||
|
|
"learning_rate": 4.384454877412624e-07,
|
||
|
|
"logits/chosen": -0.575976550579071,
|
||
|
|
"logits/rejected": -0.2857421934604645,
|
||
|
|
"logps/chosen": -488.0,
|
||
|
|
"logps/rejected": -663.0,
|
||
|
|
"loss": 3.0912,
|
||
|
|
"rewards/accuracies": 0.4976562559604645,
|
||
|
|
"rewards/chosen": 6.612500190734863,
|
||
|
|
"rewards/margins": 8.126562118530273,
|
||
|
|
"rewards/rejected": -1.5125000476837158,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.215962441314554,
|
||
|
|
"grad_norm": 614.0075268937729,
|
||
|
|
"learning_rate": 4.358372456964006e-07,
|
||
|
|
"logits/chosen": -0.6195312738418579,
|
||
|
|
"logits/rejected": -0.37089842557907104,
|
||
|
|
"logps/chosen": -477.6000061035156,
|
||
|
|
"logps/rejected": -589.2000122070312,
|
||
|
|
"loss": 3.3256,
|
||
|
|
"rewards/accuracies": 0.528124988079071,
|
||
|
|
"rewards/chosen": 7.050000190734863,
|
||
|
|
"rewards/margins": 7.612500190734863,
|
||
|
|
"rewards/rejected": -0.5589843988418579,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22065727699530516,
|
||
|
|
"grad_norm": 711.4072103687785,
|
||
|
|
"learning_rate": 4.3322900365153886e-07,
|
||
|
|
"logits/chosen": -0.556640625,
|
||
|
|
"logits/rejected": -0.38496094942092896,
|
||
|
|
"logps/chosen": -522.2000122070312,
|
||
|
|
"logps/rejected": -592.5999755859375,
|
||
|
|
"loss": 3.1879,
|
||
|
|
"rewards/accuracies": 0.5296875238418579,
|
||
|
|
"rewards/chosen": 6.599999904632568,
|
||
|
|
"rewards/margins": 7.34375,
|
||
|
|
"rewards/rejected": -0.74560546875,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22535211267605634,
|
||
|
|
"grad_norm": 602.6833162705392,
|
||
|
|
"learning_rate": 4.306207616066771e-07,
|
||
|
|
"logits/chosen": -0.615429699420929,
|
||
|
|
"logits/rejected": -0.36845701932907104,
|
||
|
|
"logps/chosen": -471.3999938964844,
|
||
|
|
"logps/rejected": -566.7999877929688,
|
||
|
|
"loss": 3.005,
|
||
|
|
"rewards/accuracies": 0.5249999761581421,
|
||
|
|
"rewards/chosen": 7.456250190734863,
|
||
|
|
"rewards/margins": 6.8046875,
|
||
|
|
"rewards/rejected": 0.653124988079071,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2300469483568075,
|
||
|
|
"grad_norm": 769.9216104809816,
|
||
|
|
"learning_rate": 4.280125195618153e-07,
|
||
|
|
"logits/chosen": -0.6519531011581421,
|
||
|
|
"logits/rejected": -0.39697265625,
|
||
|
|
"logps/chosen": -475.6000061035156,
|
||
|
|
"logps/rejected": -604.4000244140625,
|
||
|
|
"loss": 3.0814,
|
||
|
|
"rewards/accuracies": 0.52734375,
|
||
|
|
"rewards/chosen": 7.215624809265137,
|
||
|
|
"rewards/margins": 8.103124618530273,
|
||
|
|
"rewards/rejected": -0.890820324420929,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2347417840375587,
|
||
|
|
"grad_norm": 697.875587060794,
|
||
|
|
"learning_rate": 4.2540427751695357e-07,
|
||
|
|
"logits/chosen": -0.6177734136581421,
|
||
|
|
"logits/rejected": -0.3504882752895355,
|
||
|
|
"logps/chosen": -487.0,
|
||
|
|
"logps/rejected": -571.0,
|
||
|
|
"loss": 3.2251,
|
||
|
|
"rewards/accuracies": 0.535937488079071,
|
||
|
|
"rewards/chosen": 7.078125,
|
||
|
|
"rewards/margins": 7.78125,
|
||
|
|
"rewards/rejected": -0.712890625,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23943661971830985,
|
||
|
|
"grad_norm": 690.7723036829651,
|
||
|
|
"learning_rate": 4.227960354720918e-07,
|
||
|
|
"logits/chosen": -0.6328125,
|
||
|
|
"logits/rejected": -0.38525390625,
|
||
|
|
"logps/chosen": -471.3999938964844,
|
||
|
|
"logps/rejected": -604.7999877929688,
|
||
|
|
"loss": 2.9682,
|
||
|
|
"rewards/accuracies": 0.516406238079071,
|
||
|
|
"rewards/chosen": 7.715624809265137,
|
||
|
|
"rewards/margins": 8.248437881469727,
|
||
|
|
"rewards/rejected": -0.525390625,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24413145539906103,
|
||
|
|
"grad_norm": 724.8433594547769,
|
||
|
|
"learning_rate": 4.2018779342723e-07,
|
||
|
|
"logits/chosen": -0.650390625,
|
||
|
|
"logits/rejected": -0.39326173067092896,
|
||
|
|
"logps/chosen": -487.0,
|
||
|
|
"logps/rejected": -570.7999877929688,
|
||
|
|
"loss": 3.2936,
|
||
|
|
"rewards/accuracies": 0.5296875238418579,
|
||
|
|
"rewards/chosen": 6.775000095367432,
|
||
|
|
"rewards/margins": 6.9765625,
|
||
|
|
"rewards/rejected": -0.20476074516773224,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24882629107981222,
|
||
|
|
"grad_norm": 745.4876334505535,
|
||
|
|
"learning_rate": 4.1757955138236827e-07,
|
||
|
|
"logits/chosen": -0.553906261920929,
|
||
|
|
"logits/rejected": -0.30498045682907104,
|
||
|
|
"logps/chosen": -474.3999938964844,
|
||
|
|
"logps/rejected": -593.4000244140625,
|
||
|
|
"loss": 3.1194,
|
||
|
|
"rewards/accuracies": 0.53515625,
|
||
|
|
"rewards/chosen": 7.115624904632568,
|
||
|
|
"rewards/margins": 9.071874618530273,
|
||
|
|
"rewards/rejected": -1.9503905773162842,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2535211267605634,
|
||
|
|
"grad_norm": 761.9189082082895,
|
||
|
|
"learning_rate": 4.149713093375065e-07,
|
||
|
|
"logits/chosen": -0.645312488079071,
|
||
|
|
"logits/rejected": -0.41435545682907104,
|
||
|
|
"logps/chosen": -472.79998779296875,
|
||
|
|
"logps/rejected": -547.7999877929688,
|
||
|
|
"loss": 2.5263,
|
||
|
|
"rewards/accuracies": 0.5570312738418579,
|
||
|
|
"rewards/chosen": 7.571875095367432,
|
||
|
|
"rewards/margins": 10.071874618530273,
|
||
|
|
"rewards/rejected": -2.5082030296325684,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25821596244131456,
|
||
|
|
"grad_norm": 691.3755683218671,
|
||
|
|
"learning_rate": 4.123630672926447e-07,
|
||
|
|
"logits/chosen": -0.637499988079071,
|
||
|
|
"logits/rejected": -0.39091795682907104,
|
||
|
|
"logps/chosen": -458.20001220703125,
|
||
|
|
"logps/rejected": -577.2000122070312,
|
||
|
|
"loss": 2.8925,
|
||
|
|
"rewards/accuracies": 0.5179687738418579,
|
||
|
|
"rewards/chosen": 6.618750095367432,
|
||
|
|
"rewards/margins": 7.512499809265137,
|
||
|
|
"rewards/rejected": -0.8910156488418579,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.26291079812206575,
|
||
|
|
"grad_norm": 682.4871571592513,
|
||
|
|
"learning_rate": 4.09754825247783e-07,
|
||
|
|
"logits/chosen": -0.562695324420929,
|
||
|
|
"logits/rejected": -0.3373046815395355,
|
||
|
|
"logps/chosen": -482.79998779296875,
|
||
|
|
"logps/rejected": -579.5999755859375,
|
||
|
|
"loss": 3.0853,
|
||
|
|
"rewards/accuracies": 0.535937488079071,
|
||
|
|
"rewards/chosen": 7.518750190734863,
|
||
|
|
"rewards/margins": 9.790624618530273,
|
||
|
|
"rewards/rejected": -2.261523485183716,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2676056338028169,
|
||
|
|
"grad_norm": 520.1844542646321,
|
||
|
|
"learning_rate": 4.0714658320292123e-07,
|
||
|
|
"logits/chosen": -0.5775390863418579,
|
||
|
|
"logits/rejected": -0.33662110567092896,
|
||
|
|
"logps/chosen": -479.6000061035156,
|
||
|
|
"logps/rejected": -584.7999877929688,
|
||
|
|
"loss": 3.0361,
|
||
|
|
"rewards/accuracies": 0.5757812261581421,
|
||
|
|
"rewards/chosen": 7.574999809265137,
|
||
|
|
"rewards/margins": 10.137499809265137,
|
||
|
|
"rewards/rejected": -2.572460889816284,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27230046948356806,
|
||
|
|
"grad_norm": 669.5468412735042,
|
||
|
|
"learning_rate": 4.045383411580595e-07,
|
||
|
|
"logits/chosen": -0.6265624761581421,
|
||
|
|
"logits/rejected": -0.33491212129592896,
|
||
|
|
"logps/chosen": -460.3999938964844,
|
||
|
|
"logps/rejected": -587.7999877929688,
|
||
|
|
"loss": 2.9734,
|
||
|
|
"rewards/accuracies": 0.5328124761581421,
|
||
|
|
"rewards/chosen": 7.806250095367432,
|
||
|
|
"rewards/margins": 8.28125,
|
||
|
|
"rewards/rejected": -0.47138673067092896,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27699530516431925,
|
||
|
|
"grad_norm": 663.4127690326764,
|
||
|
|
"learning_rate": 4.019300991131977e-07,
|
||
|
|
"logits/chosen": -0.639453113079071,
|
||
|
|
"logits/rejected": -0.38408201932907104,
|
||
|
|
"logps/chosen": -465.6000061035156,
|
||
|
|
"logps/rejected": -578.5999755859375,
|
||
|
|
"loss": 2.9831,
|
||
|
|
"rewards/accuracies": 0.5406249761581421,
|
||
|
|
"rewards/chosen": 7.628125190734863,
|
||
|
|
"rewards/margins": 8.8671875,
|
||
|
|
"rewards/rejected": -1.240625023841858,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28169014084507044,
|
||
|
|
"grad_norm": 662.1782820940265,
|
||
|
|
"learning_rate": 3.9932185706833594e-07,
|
||
|
|
"logits/chosen": -0.5667968988418579,
|
||
|
|
"logits/rejected": -0.3550781309604645,
|
||
|
|
"logps/chosen": -487.79998779296875,
|
||
|
|
"logps/rejected": -573.2000122070312,
|
||
|
|
"loss": 3.216,
|
||
|
|
"rewards/accuracies": 0.5210937261581421,
|
||
|
|
"rewards/chosen": 7.615624904632568,
|
||
|
|
"rewards/margins": 8.921875,
|
||
|
|
"rewards/rejected": -1.3052246570587158,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2863849765258216,
|
||
|
|
"grad_norm": 674.321793533583,
|
||
|
|
"learning_rate": 3.967136150234742e-07,
|
||
|
|
"logits/chosen": -0.6019531488418579,
|
||
|
|
"logits/rejected": -0.3726562559604645,
|
||
|
|
"logps/chosen": -483.6000061035156,
|
||
|
|
"logps/rejected": -589.2000122070312,
|
||
|
|
"loss": 2.5835,
|
||
|
|
"rewards/accuracies": 0.524218738079071,
|
||
|
|
"rewards/chosen": 7.646874904632568,
|
||
|
|
"rewards/margins": 8.925000190734863,
|
||
|
|
"rewards/rejected": -1.2830078601837158,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.29107981220657275,
|
||
|
|
"grad_norm": 571.494610670194,
|
||
|
|
"learning_rate": 3.941053729786124e-07,
|
||
|
|
"logits/chosen": -0.6468750238418579,
|
||
|
|
"logits/rejected": -0.3509765565395355,
|
||
|
|
"logps/chosen": -488.20001220703125,
|
||
|
|
"logps/rejected": -587.0,
|
||
|
|
"loss": 2.8213,
|
||
|
|
"rewards/accuracies": 0.539843738079071,
|
||
|
|
"rewards/chosen": 7.265625,
|
||
|
|
"rewards/margins": 9.806249618530273,
|
||
|
|
"rewards/rejected": -2.5419921875,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.29577464788732394,
|
||
|
|
"grad_norm": 706.423001116219,
|
||
|
|
"learning_rate": 3.9149713093375064e-07,
|
||
|
|
"logits/chosen": -0.5621093511581421,
|
||
|
|
"logits/rejected": -0.2916015684604645,
|
||
|
|
"logps/chosen": -475.79998779296875,
|
||
|
|
"logps/rejected": -604.5999755859375,
|
||
|
|
"loss": 2.7885,
|
||
|
|
"rewards/accuracies": 0.53125,
|
||
|
|
"rewards/chosen": 6.8125,
|
||
|
|
"rewards/margins": 8.753125190734863,
|
||
|
|
"rewards/rejected": -1.937744140625,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3004694835680751,
|
||
|
|
"grad_norm": 744.7765715005459,
|
||
|
|
"learning_rate": 3.888888888888889e-07,
|
||
|
|
"logits/chosen": -0.6585937738418579,
|
||
|
|
"logits/rejected": -0.29438477754592896,
|
||
|
|
"logps/chosen": -470.79998779296875,
|
||
|
|
"logps/rejected": -618.7999877929688,
|
||
|
|
"loss": 2.6548,
|
||
|
|
"rewards/accuracies": 0.5531250238418579,
|
||
|
|
"rewards/chosen": 7.15625,
|
||
|
|
"rewards/margins": 9.871874809265137,
|
||
|
|
"rewards/rejected": -2.719531297683716,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3051643192488263,
|
||
|
|
"grad_norm": 664.3294351661672,
|
||
|
|
"learning_rate": 3.862806468440271e-07,
|
||
|
|
"logits/chosen": -0.610546886920929,
|
||
|
|
"logits/rejected": -0.36992186307907104,
|
||
|
|
"logps/chosen": -446.3999938964844,
|
||
|
|
"logps/rejected": -547.4000244140625,
|
||
|
|
"loss": 2.8336,
|
||
|
|
"rewards/accuracies": 0.5210937261581421,
|
||
|
|
"rewards/chosen": 6.949999809265137,
|
||
|
|
"rewards/margins": 7.934374809265137,
|
||
|
|
"rewards/rejected": -0.977343738079071,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30985915492957744,
|
||
|
|
"grad_norm": 666.3994060099428,
|
||
|
|
"learning_rate": 3.8367240479916535e-07,
|
||
|
|
"logits/chosen": -0.6683593988418579,
|
||
|
|
"logits/rejected": -0.34892576932907104,
|
||
|
|
"logps/chosen": -468.0,
|
||
|
|
"logps/rejected": -546.7999877929688,
|
||
|
|
"loss": 2.9922,
|
||
|
|
"rewards/accuracies": 0.538281261920929,
|
||
|
|
"rewards/chosen": 7.15625,
|
||
|
|
"rewards/margins": 8.618749618530273,
|
||
|
|
"rewards/rejected": -1.4640624523162842,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3145539906103286,
|
||
|
|
"grad_norm": 653.0555976276125,
|
||
|
|
"learning_rate": 3.810641627543036e-07,
|
||
|
|
"logits/chosen": -0.669140636920929,
|
||
|
|
"logits/rejected": -0.4400390684604645,
|
||
|
|
"logps/chosen": -478.79998779296875,
|
||
|
|
"logps/rejected": -574.5999755859375,
|
||
|
|
"loss": 2.8329,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 7.028124809265137,
|
||
|
|
"rewards/margins": 9.543749809265137,
|
||
|
|
"rewards/rejected": -2.518261671066284,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3192488262910798,
|
||
|
|
"grad_norm": 647.1326542511971,
|
||
|
|
"learning_rate": 3.784559207094418e-07,
|
||
|
|
"logits/chosen": -0.576171875,
|
||
|
|
"logits/rejected": -0.3640075623989105,
|
||
|
|
"logps/chosen": -495.79998779296875,
|
||
|
|
"logps/rejected": -572.4000244140625,
|
||
|
|
"loss": 3.1994,
|
||
|
|
"rewards/accuracies": 0.5367187261581421,
|
||
|
|
"rewards/chosen": 7.643750190734863,
|
||
|
|
"rewards/margins": 7.828125,
|
||
|
|
"rewards/rejected": -0.18266601860523224,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.323943661971831,
|
||
|
|
"grad_norm": 691.3349369131505,
|
||
|
|
"learning_rate": 3.7584767866458005e-07,
|
||
|
|
"logits/chosen": -0.606249988079071,
|
||
|
|
"logits/rejected": -0.37187498807907104,
|
||
|
|
"logps/chosen": -486.20001220703125,
|
||
|
|
"logps/rejected": -551.2000122070312,
|
||
|
|
"loss": 3.0231,
|
||
|
|
"rewards/accuracies": 0.526562511920929,
|
||
|
|
"rewards/chosen": 7.03125,
|
||
|
|
"rewards/margins": 8.475000381469727,
|
||
|
|
"rewards/rejected": -1.454687476158142,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3286384976525822,
|
||
|
|
"grad_norm": 923.6366452665168,
|
||
|
|
"learning_rate": 3.732394366197183e-07,
|
||
|
|
"logits/chosen": -0.6220703125,
|
||
|
|
"logits/rejected": -0.3653320372104645,
|
||
|
|
"logps/chosen": -510.20001220703125,
|
||
|
|
"logps/rejected": -557.0,
|
||
|
|
"loss": 3.6784,
|
||
|
|
"rewards/accuracies": 0.55859375,
|
||
|
|
"rewards/chosen": 7.150000095367432,
|
||
|
|
"rewards/margins": 8.496874809265137,
|
||
|
|
"rewards/rejected": -1.338281273841858,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3333333333333333,
|
||
|
|
"grad_norm": 834.7186443139221,
|
||
|
|
"learning_rate": 3.706311945748565e-07,
|
||
|
|
"logits/chosen": -0.587109386920929,
|
||
|
|
"logits/rejected": -0.37470704317092896,
|
||
|
|
"logps/chosen": -524.2000122070312,
|
||
|
|
"logps/rejected": -614.2000122070312,
|
||
|
|
"loss": 3.3786,
|
||
|
|
"rewards/accuracies": 0.5335937738418579,
|
||
|
|
"rewards/chosen": 6.596875190734863,
|
||
|
|
"rewards/margins": 9.293749809265137,
|
||
|
|
"rewards/rejected": -2.684765577316284,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3380281690140845,
|
||
|
|
"grad_norm": 677.1479561183868,
|
||
|
|
"learning_rate": 3.6802295252999476e-07,
|
||
|
|
"logits/chosen": -0.5986328125,
|
||
|
|
"logits/rejected": -0.35136717557907104,
|
||
|
|
"logps/chosen": -496.20001220703125,
|
||
|
|
"logps/rejected": -641.5999755859375,
|
||
|
|
"loss": 2.7306,
|
||
|
|
"rewards/accuracies": 0.553906261920929,
|
||
|
|
"rewards/chosen": 6.925000190734863,
|
||
|
|
"rewards/margins": 11.534375190734863,
|
||
|
|
"rewards/rejected": -4.60546875,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3427230046948357,
|
||
|
|
"grad_norm": 760.2075426838928,
|
||
|
|
"learning_rate": 3.65414710485133e-07,
|
||
|
|
"logits/chosen": -0.66015625,
|
||
|
|
"logits/rejected": -0.41289061307907104,
|
||
|
|
"logps/chosen": -491.0,
|
||
|
|
"logps/rejected": -559.4000244140625,
|
||
|
|
"loss": 3.6616,
|
||
|
|
"rewards/accuracies": 0.5367187261581421,
|
||
|
|
"rewards/chosen": 5.503125190734863,
|
||
|
|
"rewards/margins": 8.175000190734863,
|
||
|
|
"rewards/rejected": -2.6663575172424316,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3474178403755869,
|
||
|
|
"grad_norm": 609.5626874715092,
|
||
|
|
"learning_rate": 3.6280646844027127e-07,
|
||
|
|
"logits/chosen": -0.6441406011581421,
|
||
|
|
"logits/rejected": -0.31376951932907104,
|
||
|
|
"logps/chosen": -471.3999938964844,
|
||
|
|
"logps/rejected": -622.2000122070312,
|
||
|
|
"loss": 2.856,
|
||
|
|
"rewards/accuracies": 0.5523437261581421,
|
||
|
|
"rewards/chosen": 6.331250190734863,
|
||
|
|
"rewards/margins": 9.540624618530273,
|
||
|
|
"rewards/rejected": -3.2027344703674316,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.352112676056338,
|
||
|
|
"grad_norm": 779.7973984508154,
|
||
|
|
"learning_rate": 3.6019822639540947e-07,
|
||
|
|
"logits/chosen": -0.604687511920929,
|
||
|
|
"logits/rejected": -0.36083984375,
|
||
|
|
"logps/chosen": -497.3999938964844,
|
||
|
|
"logps/rejected": -550.7999877929688,
|
||
|
|
"loss": 3.0115,
|
||
|
|
"rewards/accuracies": 0.5335937738418579,
|
||
|
|
"rewards/chosen": 6.228125095367432,
|
||
|
|
"rewards/margins": 8.162500381469727,
|
||
|
|
"rewards/rejected": -1.941015601158142,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3568075117370892,
|
||
|
|
"grad_norm": 632.3405083446447,
|
||
|
|
"learning_rate": 3.575899843505477e-07,
|
||
|
|
"logits/chosen": -0.587890625,
|
||
|
|
"logits/rejected": -0.3851562440395355,
|
||
|
|
"logps/chosen": -480.79998779296875,
|
||
|
|
"logps/rejected": -545.7999877929688,
|
||
|
|
"loss": 3.2305,
|
||
|
|
"rewards/accuracies": 0.561718761920929,
|
||
|
|
"rewards/chosen": 6.199999809265137,
|
||
|
|
"rewards/margins": 8.856249809265137,
|
||
|
|
"rewards/rejected": -2.6507325172424316,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3615023474178404,
|
||
|
|
"grad_norm": 765.8758610632187,
|
||
|
|
"learning_rate": 3.5498174230568597e-07,
|
||
|
|
"logits/chosen": -0.612109363079071,
|
||
|
|
"logits/rejected": -0.36894530057907104,
|
||
|
|
"logps/chosen": -476.3999938964844,
|
||
|
|
"logps/rejected": -578.7999877929688,
|
||
|
|
"loss": 3.0881,
|
||
|
|
"rewards/accuracies": 0.5562499761581421,
|
||
|
|
"rewards/chosen": 6.34375,
|
||
|
|
"rewards/margins": 9.356249809265137,
|
||
|
|
"rewards/rejected": -3.0054688453674316,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36619718309859156,
|
||
|
|
"grad_norm": 1391.7602311903074,
|
||
|
|
"learning_rate": 3.5237350026082417e-07,
|
||
|
|
"logits/chosen": -0.650390625,
|
||
|
|
"logits/rejected": -0.4476562440395355,
|
||
|
|
"logps/chosen": -489.6000061035156,
|
||
|
|
"logps/rejected": -539.5999755859375,
|
||
|
|
"loss": 3.4916,
|
||
|
|
"rewards/accuracies": 0.5218750238418579,
|
||
|
|
"rewards/chosen": 5.817187309265137,
|
||
|
|
"rewards/margins": 7.59375,
|
||
|
|
"rewards/rejected": -1.7790038585662842,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.37089201877934275,
|
||
|
|
"grad_norm": 786.0516921335555,
|
||
|
|
"learning_rate": 3.497652582159624e-07,
|
||
|
|
"logits/chosen": -0.650585949420929,
|
||
|
|
"logits/rejected": -0.37128907442092896,
|
||
|
|
"logps/chosen": -481.0,
|
||
|
|
"logps/rejected": -606.7999877929688,
|
||
|
|
"loss": 3.0778,
|
||
|
|
"rewards/accuracies": 0.547656238079071,
|
||
|
|
"rewards/chosen": 6.846875190734863,
|
||
|
|
"rewards/margins": 9.975000381469727,
|
||
|
|
"rewards/rejected": -3.1343750953674316,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3755868544600939,
|
||
|
|
"grad_norm": 660.2411740845354,
|
||
|
|
"learning_rate": 3.471570161711007e-07,
|
||
|
|
"logits/chosen": -0.5726562738418579,
|
||
|
|
"logits/rejected": -0.3438476622104645,
|
||
|
|
"logps/chosen": -515.4000244140625,
|
||
|
|
"logps/rejected": -606.4000244140625,
|
||
|
|
"loss": 2.8788,
|
||
|
|
"rewards/accuracies": 0.546093761920929,
|
||
|
|
"rewards/chosen": 6.849999904632568,
|
||
|
|
"rewards/margins": 9.665624618530273,
|
||
|
|
"rewards/rejected": -2.8228516578674316,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38028169014084506,
|
||
|
|
"grad_norm": 573.5136569308766,
|
||
|
|
"learning_rate": 3.445487741262389e-07,
|
||
|
|
"logits/chosen": -0.5523437261581421,
|
||
|
|
"logits/rejected": -0.2630371153354645,
|
||
|
|
"logps/chosen": -470.3999938964844,
|
||
|
|
"logps/rejected": -608.4000244140625,
|
||
|
|
"loss": 2.4566,
|
||
|
|
"rewards/accuracies": 0.5765625238418579,
|
||
|
|
"rewards/chosen": 6.78125,
|
||
|
|
"rewards/margins": 10.090624809265137,
|
||
|
|
"rewards/rejected": -3.3089842796325684,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38497652582159625,
|
||
|
|
"grad_norm": 612.3294091924785,
|
||
|
|
"learning_rate": 3.4194053208137713e-07,
|
||
|
|
"logits/chosen": -0.615039050579071,
|
||
|
|
"logits/rejected": -0.36865234375,
|
||
|
|
"logps/chosen": -477.3999938964844,
|
||
|
|
"logps/rejected": -555.0,
|
||
|
|
"loss": 2.8644,
|
||
|
|
"rewards/accuracies": 0.535937488079071,
|
||
|
|
"rewards/chosen": 6.559374809265137,
|
||
|
|
"rewards/margins": 9.753125190734863,
|
||
|
|
"rewards/rejected": -3.206835985183716,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38967136150234744,
|
||
|
|
"grad_norm": 885.1124633612113,
|
||
|
|
"learning_rate": 3.393322900365154e-07,
|
||
|
|
"logits/chosen": -0.6421874761581421,
|
||
|
|
"logits/rejected": -0.38749998807907104,
|
||
|
|
"logps/chosen": -455.79998779296875,
|
||
|
|
"logps/rejected": -552.7999877929688,
|
||
|
|
"loss": 3.2437,
|
||
|
|
"rewards/accuracies": 0.543749988079071,
|
||
|
|
"rewards/chosen": 5.739062309265137,
|
||
|
|
"rewards/margins": 8.662500381469727,
|
||
|
|
"rewards/rejected": -2.923583984375,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39436619718309857,
|
||
|
|
"grad_norm": 758.5513679353955,
|
||
|
|
"learning_rate": 3.367240479916536e-07,
|
||
|
|
"logits/chosen": -0.659375011920929,
|
||
|
|
"logits/rejected": -0.3814453184604645,
|
||
|
|
"logps/chosen": -508.79998779296875,
|
||
|
|
"logps/rejected": -589.2000122070312,
|
||
|
|
"loss": 3.5719,
|
||
|
|
"rewards/accuracies": 0.542187511920929,
|
||
|
|
"rewards/chosen": 5.959374904632568,
|
||
|
|
"rewards/margins": 8.631250381469727,
|
||
|
|
"rewards/rejected": -2.661914110183716,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39906103286384975,
|
||
|
|
"grad_norm": 529.5551420613754,
|
||
|
|
"learning_rate": 3.3411580594679184e-07,
|
||
|
|
"logits/chosen": -0.663867175579071,
|
||
|
|
"logits/rejected": -0.3340820372104645,
|
||
|
|
"logps/chosen": -467.20001220703125,
|
||
|
|
"logps/rejected": -580.5999755859375,
|
||
|
|
"loss": 2.6792,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 6.137499809265137,
|
||
|
|
"rewards/margins": 10.584375381469727,
|
||
|
|
"rewards/rejected": -4.448437690734863,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40375586854460094,
|
||
|
|
"grad_norm": 797.605403599976,
|
||
|
|
"learning_rate": 3.315075639019301e-07,
|
||
|
|
"logits/chosen": -0.620312511920929,
|
||
|
|
"logits/rejected": -0.2682556211948395,
|
||
|
|
"logps/chosen": -464.20001220703125,
|
||
|
|
"logps/rejected": -576.2000122070312,
|
||
|
|
"loss": 2.7995,
|
||
|
|
"rewards/accuracies": 0.5609375238418579,
|
||
|
|
"rewards/chosen": 6.284375190734863,
|
||
|
|
"rewards/margins": 9.84375,
|
||
|
|
"rewards/rejected": -3.5625,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4084507042253521,
|
||
|
|
"grad_norm": 756.2297616084339,
|
||
|
|
"learning_rate": 3.288993218570683e-07,
|
||
|
|
"logits/chosen": -0.595898449420929,
|
||
|
|
"logits/rejected": -0.384765625,
|
||
|
|
"logps/chosen": -471.79998779296875,
|
||
|
|
"logps/rejected": -549.7999877929688,
|
||
|
|
"loss": 3.2089,
|
||
|
|
"rewards/accuracies": 0.5523437261581421,
|
||
|
|
"rewards/chosen": 6.628125190734863,
|
||
|
|
"rewards/margins": 8.706250190734863,
|
||
|
|
"rewards/rejected": -2.080859422683716,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4131455399061033,
|
||
|
|
"grad_norm": 674.821947206969,
|
||
|
|
"learning_rate": 3.2629107981220654e-07,
|
||
|
|
"logits/chosen": -0.6390625238418579,
|
||
|
|
"logits/rejected": -0.36689454317092896,
|
||
|
|
"logps/chosen": -486.6000061035156,
|
||
|
|
"logps/rejected": -565.4000244140625,
|
||
|
|
"loss": 3.1942,
|
||
|
|
"rewards/accuracies": 0.553906261920929,
|
||
|
|
"rewards/chosen": 5.776562690734863,
|
||
|
|
"rewards/margins": 9.621874809265137,
|
||
|
|
"rewards/rejected": -3.841796875,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41784037558685444,
|
||
|
|
"grad_norm": 817.7630708964249,
|
||
|
|
"learning_rate": 3.236828377673448e-07,
|
||
|
|
"logits/chosen": -0.658203125,
|
||
|
|
"logits/rejected": -0.3584960997104645,
|
||
|
|
"logps/chosen": -509.3999938964844,
|
||
|
|
"logps/rejected": -565.2000122070312,
|
||
|
|
"loss": 3.4399,
|
||
|
|
"rewards/accuracies": 0.547656238079071,
|
||
|
|
"rewards/chosen": 5.326562404632568,
|
||
|
|
"rewards/margins": 8.449999809265137,
|
||
|
|
"rewards/rejected": -3.134570360183716,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4225352112676056,
|
||
|
|
"grad_norm": 650.7992124693492,
|
||
|
|
"learning_rate": 3.2107459572248305e-07,
|
||
|
|
"logits/chosen": -0.62890625,
|
||
|
|
"logits/rejected": -0.376953125,
|
||
|
|
"logps/chosen": -490.0,
|
||
|
|
"logps/rejected": -577.5999755859375,
|
||
|
|
"loss": 2.7743,
|
||
|
|
"rewards/accuracies": 0.546875,
|
||
|
|
"rewards/chosen": 5.574999809265137,
|
||
|
|
"rewards/margins": 9.46875,
|
||
|
|
"rewards/rejected": -3.891796827316284,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4272300469483568,
|
||
|
|
"grad_norm": 796.3371955928629,
|
||
|
|
"learning_rate": 3.1846635367762125e-07,
|
||
|
|
"logits/chosen": -0.6371093988418579,
|
||
|
|
"logits/rejected": -0.3519531190395355,
|
||
|
|
"logps/chosen": -486.20001220703125,
|
||
|
|
"logps/rejected": -615.0,
|
||
|
|
"loss": 2.7031,
|
||
|
|
"rewards/accuracies": 0.5796874761581421,
|
||
|
|
"rewards/chosen": 6.035937309265137,
|
||
|
|
"rewards/margins": 11.403124809265137,
|
||
|
|
"rewards/rejected": -5.353906154632568,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.431924882629108,
|
||
|
|
"grad_norm": 702.4815438667744,
|
||
|
|
"learning_rate": 3.158581116327595e-07,
|
||
|
|
"logits/chosen": -0.6410156488418579,
|
||
|
|
"logits/rejected": -0.330078125,
|
||
|
|
"logps/chosen": -488.79998779296875,
|
||
|
|
"logps/rejected": -634.7999877929688,
|
||
|
|
"loss": 3.03,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 6.043749809265137,
|
||
|
|
"rewards/margins": 11.140625,
|
||
|
|
"rewards/rejected": -5.09375,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.43661971830985913,
|
||
|
|
"grad_norm": 577.1124058468274,
|
||
|
|
"learning_rate": 3.1324986958789775e-07,
|
||
|
|
"logits/chosen": -0.6929687261581421,
|
||
|
|
"logits/rejected": -0.4501953125,
|
||
|
|
"logps/chosen": -457.3999938964844,
|
||
|
|
"logps/rejected": -552.2000122070312,
|
||
|
|
"loss": 2.8393,
|
||
|
|
"rewards/accuracies": 0.5531250238418579,
|
||
|
|
"rewards/chosen": 5.909375190734863,
|
||
|
|
"rewards/margins": 9.103124618530273,
|
||
|
|
"rewards/rejected": -3.187695264816284,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4413145539906103,
|
||
|
|
"grad_norm": 694.068916573537,
|
||
|
|
"learning_rate": 3.1064162754303595e-07,
|
||
|
|
"logits/chosen": -0.600390613079071,
|
||
|
|
"logits/rejected": -0.3330078125,
|
||
|
|
"logps/chosen": -493.20001220703125,
|
||
|
|
"logps/rejected": -603.7999877929688,
|
||
|
|
"loss": 3.3045,
|
||
|
|
"rewards/accuracies": 0.5414062738418579,
|
||
|
|
"rewards/chosen": 5.469531059265137,
|
||
|
|
"rewards/margins": 9.396875381469727,
|
||
|
|
"rewards/rejected": -3.9136719703674316,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4460093896713615,
|
||
|
|
"grad_norm": 698.6912411634398,
|
||
|
|
"learning_rate": 3.080333854981742e-07,
|
||
|
|
"logits/chosen": -0.678906261920929,
|
||
|
|
"logits/rejected": -0.4161132872104645,
|
||
|
|
"logps/chosen": -476.6000061035156,
|
||
|
|
"logps/rejected": -552.0,
|
||
|
|
"loss": 3.3694,
|
||
|
|
"rewards/accuracies": 0.5218750238418579,
|
||
|
|
"rewards/chosen": 6.356249809265137,
|
||
|
|
"rewards/margins": 8.053125381469727,
|
||
|
|
"rewards/rejected": -1.696044921875,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4507042253521127,
|
||
|
|
"grad_norm": 772.7371275000845,
|
||
|
|
"learning_rate": 3.0542514345331246e-07,
|
||
|
|
"logits/chosen": -0.6703125238418579,
|
||
|
|
"logits/rejected": -0.3642578125,
|
||
|
|
"logps/chosen": -485.0,
|
||
|
|
"logps/rejected": -626.2000122070312,
|
||
|
|
"loss": 3.2646,
|
||
|
|
"rewards/accuracies": 0.5703125,
|
||
|
|
"rewards/chosen": 6.3828125,
|
||
|
|
"rewards/margins": 10.456250190734863,
|
||
|
|
"rewards/rejected": -4.068749904632568,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45539906103286387,
|
||
|
|
"grad_norm": 837.3958131664808,
|
||
|
|
"learning_rate": 3.0281690140845066e-07,
|
||
|
|
"logits/chosen": -0.6460937261581421,
|
||
|
|
"logits/rejected": -0.40898436307907104,
|
||
|
|
"logps/chosen": -514.0,
|
||
|
|
"logps/rejected": -592.7999877929688,
|
||
|
|
"loss": 3.3593,
|
||
|
|
"rewards/accuracies": 0.55859375,
|
||
|
|
"rewards/chosen": 5.668749809265137,
|
||
|
|
"rewards/margins": 8.456250190734863,
|
||
|
|
"rewards/rejected": -2.80078125,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.460093896713615,
|
||
|
|
"grad_norm": 844.6768886422768,
|
||
|
|
"learning_rate": 3.002086593635889e-07,
|
||
|
|
"logits/chosen": -0.601367175579071,
|
||
|
|
"logits/rejected": -0.28681641817092896,
|
||
|
|
"logps/chosen": -481.3999938964844,
|
||
|
|
"logps/rejected": -605.4000244140625,
|
||
|
|
"loss": 3.0049,
|
||
|
|
"rewards/accuracies": 0.5523437261581421,
|
||
|
|
"rewards/chosen": 6.682812690734863,
|
||
|
|
"rewards/margins": 9.90625,
|
||
|
|
"rewards/rejected": -3.2279295921325684,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4647887323943662,
|
||
|
|
"grad_norm": 644.4990262705668,
|
||
|
|
"learning_rate": 2.9760041731872716e-07,
|
||
|
|
"logits/chosen": -0.631640613079071,
|
||
|
|
"logits/rejected": -0.40703123807907104,
|
||
|
|
"logps/chosen": -445.0,
|
||
|
|
"logps/rejected": -566.5999755859375,
|
||
|
|
"loss": 2.6677,
|
||
|
|
"rewards/accuracies": 0.5859375,
|
||
|
|
"rewards/chosen": 6.959374904632568,
|
||
|
|
"rewards/margins": 10.399999618530273,
|
||
|
|
"rewards/rejected": -3.4359374046325684,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4694835680751174,
|
||
|
|
"grad_norm": 587.6218993381004,
|
||
|
|
"learning_rate": 2.9499217527386536e-07,
|
||
|
|
"logits/chosen": -0.6435546875,
|
||
|
|
"logits/rejected": -0.33759766817092896,
|
||
|
|
"logps/chosen": -471.79998779296875,
|
||
|
|
"logps/rejected": -580.0,
|
||
|
|
"loss": 2.8798,
|
||
|
|
"rewards/accuracies": 0.561718761920929,
|
||
|
|
"rewards/chosen": 7.040625095367432,
|
||
|
|
"rewards/margins": 10.09375,
|
||
|
|
"rewards/rejected": -3.0492186546325684,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47417840375586856,
|
||
|
|
"grad_norm": 604.656132365586,
|
||
|
|
"learning_rate": 2.923839332290036e-07,
|
||
|
|
"logits/chosen": -0.6742187738418579,
|
||
|
|
"logits/rejected": -0.40947264432907104,
|
||
|
|
"logps/chosen": -461.79998779296875,
|
||
|
|
"logps/rejected": -586.7999877929688,
|
||
|
|
"loss": 2.8671,
|
||
|
|
"rewards/accuracies": 0.5335937738418579,
|
||
|
|
"rewards/chosen": 5.746874809265137,
|
||
|
|
"rewards/margins": 9.678125381469727,
|
||
|
|
"rewards/rejected": -3.935986280441284,
|
||
|
|
"step": 1010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4788732394366197,
|
||
|
|
"grad_norm": 711.6564950943481,
|
||
|
|
"learning_rate": 2.8977569118414187e-07,
|
||
|
|
"logits/chosen": -0.632031261920929,
|
||
|
|
"logits/rejected": -0.37060546875,
|
||
|
|
"logps/chosen": -463.20001220703125,
|
||
|
|
"logps/rejected": -568.0,
|
||
|
|
"loss": 2.9622,
|
||
|
|
"rewards/accuracies": 0.5625,
|
||
|
|
"rewards/chosen": 6.287499904632568,
|
||
|
|
"rewards/margins": 9.824999809265137,
|
||
|
|
"rewards/rejected": -3.5374999046325684,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4835680751173709,
|
||
|
|
"grad_norm": 560.4356682511219,
|
||
|
|
"learning_rate": 2.8716744913928007e-07,
|
||
|
|
"logits/chosen": -0.6000000238418579,
|
||
|
|
"logits/rejected": -0.3783203065395355,
|
||
|
|
"logps/chosen": -481.3999938964844,
|
||
|
|
"logps/rejected": -560.2000122070312,
|
||
|
|
"loss": 2.6402,
|
||
|
|
"rewards/accuracies": 0.567187488079071,
|
||
|
|
"rewards/chosen": 6.759375095367432,
|
||
|
|
"rewards/margins": 10.053125381469727,
|
||
|
|
"rewards/rejected": -3.2886719703674316,
|
||
|
|
"step": 1030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48826291079812206,
|
||
|
|
"grad_norm": 530.7749762428775,
|
||
|
|
"learning_rate": 2.845592070944183e-07,
|
||
|
|
"logits/chosen": -0.5777343511581421,
|
||
|
|
"logits/rejected": -0.2887206971645355,
|
||
|
|
"logps/chosen": -477.3999938964844,
|
||
|
|
"logps/rejected": -606.0,
|
||
|
|
"loss": 3.0081,
|
||
|
|
"rewards/accuracies": 0.5562499761581421,
|
||
|
|
"rewards/chosen": 6.931250095367432,
|
||
|
|
"rewards/margins": 9.646875381469727,
|
||
|
|
"rewards/rejected": -2.7171874046325684,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49295774647887325,
|
||
|
|
"grad_norm": 576.6234877985139,
|
||
|
|
"learning_rate": 2.819509650495566e-07,
|
||
|
|
"logits/chosen": -0.5697265863418579,
|
||
|
|
"logits/rejected": -0.3250976502895355,
|
||
|
|
"logps/chosen": -471.0,
|
||
|
|
"logps/rejected": -559.2000122070312,
|
||
|
|
"loss": 2.8855,
|
||
|
|
"rewards/accuracies": 0.543749988079071,
|
||
|
|
"rewards/chosen": 6.574999809265137,
|
||
|
|
"rewards/margins": 8.762499809265137,
|
||
|
|
"rewards/rejected": -2.202929735183716,
|
||
|
|
"step": 1050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49765258215962443,
|
||
|
|
"grad_norm": 628.4447769443366,
|
||
|
|
"learning_rate": 2.7934272300469483e-07,
|
||
|
|
"logits/chosen": -0.623828113079071,
|
||
|
|
"logits/rejected": -0.46269530057907104,
|
||
|
|
"logps/chosen": -500.6000061035156,
|
||
|
|
"logps/rejected": -562.4000244140625,
|
||
|
|
"loss": 2.8685,
|
||
|
|
"rewards/accuracies": 0.5492187738418579,
|
||
|
|
"rewards/chosen": 7.318749904632568,
|
||
|
|
"rewards/margins": 11.056249618530273,
|
||
|
|
"rewards/rejected": -3.73046875,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5023474178403756,
|
||
|
|
"grad_norm": 751.0950849141105,
|
||
|
|
"learning_rate": 2.7673448095983303e-07,
|
||
|
|
"logits/chosen": -0.58203125,
|
||
|
|
"logits/rejected": -0.3833984434604645,
|
||
|
|
"logps/chosen": -483.0,
|
||
|
|
"logps/rejected": -563.7999877929688,
|
||
|
|
"loss": 3.3681,
|
||
|
|
"rewards/accuracies": 0.535937488079071,
|
||
|
|
"rewards/chosen": 5.303124904632568,
|
||
|
|
"rewards/margins": 7.478125095367432,
|
||
|
|
"rewards/rejected": -2.171093702316284,
|
||
|
|
"step": 1070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5070422535211268,
|
||
|
|
"grad_norm": 633.4639586102986,
|
||
|
|
"learning_rate": 2.741262389149713e-07,
|
||
|
|
"logits/chosen": -0.5966796875,
|
||
|
|
"logits/rejected": -0.3272460997104645,
|
||
|
|
"logps/chosen": -482.6000061035156,
|
||
|
|
"logps/rejected": -582.7999877929688,
|
||
|
|
"loss": 3.2828,
|
||
|
|
"rewards/accuracies": 0.5367187261581421,
|
||
|
|
"rewards/chosen": 5.4296875,
|
||
|
|
"rewards/margins": 9.087499618530273,
|
||
|
|
"rewards/rejected": -3.6548829078674316,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5117370892018779,
|
||
|
|
"grad_norm": 942.692866910734,
|
||
|
|
"learning_rate": 2.7151799687010953e-07,
|
||
|
|
"logits/chosen": -0.6050781011581421,
|
||
|
|
"logits/rejected": -0.3544921875,
|
||
|
|
"logps/chosen": -486.0,
|
||
|
|
"logps/rejected": -580.2000122070312,
|
||
|
|
"loss": 3.0279,
|
||
|
|
"rewards/accuracies": 0.5796874761581421,
|
||
|
|
"rewards/chosen": 5.90234375,
|
||
|
|
"rewards/margins": 9.506250381469727,
|
||
|
|
"rewards/rejected": -3.590625047683716,
|
||
|
|
"step": 1090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5164319248826291,
|
||
|
|
"grad_norm": 666.2430398580567,
|
||
|
|
"learning_rate": 2.6890975482524773e-07,
|
||
|
|
"logits/chosen": -0.591992199420929,
|
||
|
|
"logits/rejected": -0.3760009706020355,
|
||
|
|
"logps/chosen": -505.3999938964844,
|
||
|
|
"logps/rejected": -580.4000244140625,
|
||
|
|
"loss": 3.1832,
|
||
|
|
"rewards/accuracies": 0.55078125,
|
||
|
|
"rewards/chosen": 5.348437309265137,
|
||
|
|
"rewards/margins": 9.081250190734863,
|
||
|
|
"rewards/rejected": -3.734375,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5211267605633803,
|
||
|
|
"grad_norm": 828.2599082683874,
|
||
|
|
"learning_rate": 2.66301512780386e-07,
|
||
|
|
"logits/chosen": -0.60546875,
|
||
|
|
"logits/rejected": -0.3958984315395355,
|
||
|
|
"logps/chosen": -459.0,
|
||
|
|
"logps/rejected": -585.5999755859375,
|
||
|
|
"loss": 2.8575,
|
||
|
|
"rewards/accuracies": 0.5796874761581421,
|
||
|
|
"rewards/chosen": 6.168749809265137,
|
||
|
|
"rewards/margins": 10.578125,
|
||
|
|
"rewards/rejected": -4.423437595367432,
|
||
|
|
"step": 1110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5258215962441315,
|
||
|
|
"grad_norm": 599.1557121354074,
|
||
|
|
"learning_rate": 2.6369327073552424e-07,
|
||
|
|
"logits/chosen": -0.6109374761581421,
|
||
|
|
"logits/rejected": -0.3890624940395355,
|
||
|
|
"logps/chosen": -466.0,
|
||
|
|
"logps/rejected": -557.4000244140625,
|
||
|
|
"loss": 2.8374,
|
||
|
|
"rewards/accuracies": 0.5703125,
|
||
|
|
"rewards/chosen": 6.909375190734863,
|
||
|
|
"rewards/margins": 9.809374809265137,
|
||
|
|
"rewards/rejected": -2.9164061546325684,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5305164319248826,
|
||
|
|
"grad_norm": 654.8160519989639,
|
||
|
|
"learning_rate": 2.6108502869066244e-07,
|
||
|
|
"logits/chosen": -0.6343749761581421,
|
||
|
|
"logits/rejected": -0.3978515565395355,
|
||
|
|
"logps/chosen": -497.0,
|
||
|
|
"logps/rejected": -646.4000244140625,
|
||
|
|
"loss": 3.2428,
|
||
|
|
"rewards/accuracies": 0.54296875,
|
||
|
|
"rewards/chosen": 6.017187595367432,
|
||
|
|
"rewards/margins": 8.856249809265137,
|
||
|
|
"rewards/rejected": -2.837890625,
|
||
|
|
"step": 1130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5352112676056338,
|
||
|
|
"grad_norm": 642.1472027153841,
|
||
|
|
"learning_rate": 2.584767866458007e-07,
|
||
|
|
"logits/chosen": -0.5982421636581421,
|
||
|
|
"logits/rejected": -0.3926757872104645,
|
||
|
|
"logps/chosen": -475.20001220703125,
|
||
|
|
"logps/rejected": -573.4000244140625,
|
||
|
|
"loss": 2.5976,
|
||
|
|
"rewards/accuracies": 0.559374988079071,
|
||
|
|
"rewards/chosen": 6.453125,
|
||
|
|
"rewards/margins": 10.484375,
|
||
|
|
"rewards/rejected": -4.026953220367432,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.539906103286385,
|
||
|
|
"grad_norm": 594.0793999473858,
|
||
|
|
"learning_rate": 2.5586854460093895e-07,
|
||
|
|
"logits/chosen": -0.6302734613418579,
|
||
|
|
"logits/rejected": -0.32197266817092896,
|
||
|
|
"logps/chosen": -483.79998779296875,
|
||
|
|
"logps/rejected": -592.0,
|
||
|
|
"loss": 2.8242,
|
||
|
|
"rewards/accuracies": 0.5703125,
|
||
|
|
"rewards/chosen": 6.28125,
|
||
|
|
"rewards/margins": 9.771875381469727,
|
||
|
|
"rewards/rejected": -3.5042967796325684,
|
||
|
|
"step": 1150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5446009389671361,
|
||
|
|
"grad_norm": 729.4734085292238,
|
||
|
|
"learning_rate": 2.5326030255607715e-07,
|
||
|
|
"logits/chosen": -0.587109386920929,
|
||
|
|
"logits/rejected": -0.35791015625,
|
||
|
|
"logps/chosen": -485.3999938964844,
|
||
|
|
"logps/rejected": -580.7999877929688,
|
||
|
|
"loss": 3.4139,
|
||
|
|
"rewards/accuracies": 0.5406249761581421,
|
||
|
|
"rewards/chosen": 6.021874904632568,
|
||
|
|
"rewards/margins": 9.256250381469727,
|
||
|
|
"rewards/rejected": -3.2210936546325684,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5492957746478874,
|
||
|
|
"grad_norm": 659.0014812957725,
|
||
|
|
"learning_rate": 2.506520605112154e-07,
|
||
|
|
"logits/chosen": -0.584179699420929,
|
||
|
|
"logits/rejected": -0.3343749940395355,
|
||
|
|
"logps/chosen": -479.6000061035156,
|
||
|
|
"logps/rejected": -578.2000122070312,
|
||
|
|
"loss": 3.7044,
|
||
|
|
"rewards/accuracies": 0.532031238079071,
|
||
|
|
"rewards/chosen": 5.703125,
|
||
|
|
"rewards/margins": 8.806249618530273,
|
||
|
|
"rewards/rejected": -3.092968702316284,
|
||
|
|
"step": 1170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5539906103286385,
|
||
|
|
"grad_norm": 578.009401332291,
|
||
|
|
"learning_rate": 2.4804381846635365e-07,
|
||
|
|
"logits/chosen": -0.5667968988418579,
|
||
|
|
"logits/rejected": -0.34746092557907104,
|
||
|
|
"logps/chosen": -497.6000061035156,
|
||
|
|
"logps/rejected": -572.2000122070312,
|
||
|
|
"loss": 3.3956,
|
||
|
|
"rewards/accuracies": 0.5640624761581421,
|
||
|
|
"rewards/chosen": 5.417187690734863,
|
||
|
|
"rewards/margins": 9.28125,
|
||
|
|
"rewards/rejected": -3.867968797683716,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5586854460093896,
|
||
|
|
"grad_norm": 647.6075026271984,
|
||
|
|
"learning_rate": 2.454355764214919e-07,
|
||
|
|
"logits/chosen": -0.5855468511581421,
|
||
|
|
"logits/rejected": -0.37939453125,
|
||
|
|
"logps/chosen": -461.79998779296875,
|
||
|
|
"logps/rejected": -603.7999877929688,
|
||
|
|
"loss": 3.3616,
|
||
|
|
"rewards/accuracies": 0.5390625,
|
||
|
|
"rewards/chosen": 4.892187595367432,
|
||
|
|
"rewards/margins": 9.690625190734863,
|
||
|
|
"rewards/rejected": -4.796875,
|
||
|
|
"step": 1190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5633802816901409,
|
||
|
|
"grad_norm": 633.0926270193409,
|
||
|
|
"learning_rate": 2.4282733437663016e-07,
|
||
|
|
"logits/chosen": -0.6527343988418579,
|
||
|
|
"logits/rejected": -0.39765626192092896,
|
||
|
|
"logps/chosen": -470.20001220703125,
|
||
|
|
"logps/rejected": -561.5999755859375,
|
||
|
|
"loss": 2.4865,
|
||
|
|
"rewards/accuracies": 0.5390625,
|
||
|
|
"rewards/chosen": 6.2890625,
|
||
|
|
"rewards/margins": 9.665624618530273,
|
||
|
|
"rewards/rejected": -3.380859375,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.568075117370892,
|
||
|
|
"grad_norm": 630.7612061712699,
|
||
|
|
"learning_rate": 2.4021909233176836e-07,
|
||
|
|
"logits/chosen": -0.5707031488418579,
|
||
|
|
"logits/rejected": -0.29143065214157104,
|
||
|
|
"logps/chosen": -458.6000061035156,
|
||
|
|
"logps/rejected": -581.2000122070312,
|
||
|
|
"loss": 2.9389,
|
||
|
|
"rewards/accuracies": 0.55078125,
|
||
|
|
"rewards/chosen": 5.78125,
|
||
|
|
"rewards/margins": 10.940625190734863,
|
||
|
|
"rewards/rejected": -5.158984184265137,
|
||
|
|
"step": 1210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5727699530516432,
|
||
|
|
"grad_norm": 595.570969042014,
|
||
|
|
"learning_rate": 2.376108502869066e-07,
|
||
|
|
"logits/chosen": -0.622265636920929,
|
||
|
|
"logits/rejected": -0.33952635526657104,
|
||
|
|
"logps/chosen": -489.79998779296875,
|
||
|
|
"logps/rejected": -605.7999877929688,
|
||
|
|
"loss": 3.3003,
|
||
|
|
"rewards/accuracies": 0.5648437738418579,
|
||
|
|
"rewards/chosen": 5.360937595367432,
|
||
|
|
"rewards/margins": 9.878125190734863,
|
||
|
|
"rewards/rejected": -4.520312309265137,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5774647887323944,
|
||
|
|
"grad_norm": 648.9856060845121,
|
||
|
|
"learning_rate": 2.3500260824204484e-07,
|
||
|
|
"logits/chosen": -0.5869140625,
|
||
|
|
"logits/rejected": -0.37568360567092896,
|
||
|
|
"logps/chosen": -514.7999877929688,
|
||
|
|
"logps/rejected": -605.2000122070312,
|
||
|
|
"loss": 3.723,
|
||
|
|
"rewards/accuracies": 0.5484374761581421,
|
||
|
|
"rewards/chosen": 4.585156440734863,
|
||
|
|
"rewards/margins": 9.284375190734863,
|
||
|
|
"rewards/rejected": -4.696093559265137,
|
||
|
|
"step": 1230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5821596244131455,
|
||
|
|
"grad_norm": 690.5454986152695,
|
||
|
|
"learning_rate": 2.323943661971831e-07,
|
||
|
|
"logits/chosen": -0.6460937261581421,
|
||
|
|
"logits/rejected": -0.3495117127895355,
|
||
|
|
"logps/chosen": -479.0,
|
||
|
|
"logps/rejected": -578.5999755859375,
|
||
|
|
"loss": 3.5513,
|
||
|
|
"rewards/accuracies": 0.551562488079071,
|
||
|
|
"rewards/chosen": 5.856249809265137,
|
||
|
|
"rewards/margins": 9.259374618530273,
|
||
|
|
"rewards/rejected": -3.4007811546325684,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5868544600938967,
|
||
|
|
"grad_norm": 786.2182154081738,
|
||
|
|
"learning_rate": 2.2978612415232132e-07,
|
||
|
|
"logits/chosen": -0.609570324420929,
|
||
|
|
"logits/rejected": -0.3445800840854645,
|
||
|
|
"logps/chosen": -507.20001220703125,
|
||
|
|
"logps/rejected": -611.7999877929688,
|
||
|
|
"loss": 3.096,
|
||
|
|
"rewards/accuracies": 0.546093761920929,
|
||
|
|
"rewards/chosen": 6.315625190734863,
|
||
|
|
"rewards/margins": 11.021875381469727,
|
||
|
|
"rewards/rejected": -4.69921875,
|
||
|
|
"step": 1250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5915492957746479,
|
||
|
|
"grad_norm": 624.3423834974157,
|
||
|
|
"learning_rate": 2.2717788210745957e-07,
|
||
|
|
"logits/chosen": -0.5947265625,
|
||
|
|
"logits/rejected": -0.32038575410842896,
|
||
|
|
"logps/chosen": -478.79998779296875,
|
||
|
|
"logps/rejected": -580.2000122070312,
|
||
|
|
"loss": 3.6865,
|
||
|
|
"rewards/accuracies": 0.547656238079071,
|
||
|
|
"rewards/chosen": 6.01171875,
|
||
|
|
"rewards/margins": 9.689062118530273,
|
||
|
|
"rewards/rejected": -3.680859327316284,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.596244131455399,
|
||
|
|
"grad_norm": 660.1158863031696,
|
||
|
|
"learning_rate": 2.245696400625978e-07,
|
||
|
|
"logits/chosen": -0.6167968511581421,
|
||
|
|
"logits/rejected": -0.345101922750473,
|
||
|
|
"logps/chosen": -500.79998779296875,
|
||
|
|
"logps/rejected": -586.7999877929688,
|
||
|
|
"loss": 3.3953,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 6.060937404632568,
|
||
|
|
"rewards/margins": 9.821874618530273,
|
||
|
|
"rewards/rejected": -3.7734375,
|
||
|
|
"step": 1270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6009389671361502,
|
||
|
|
"grad_norm": 637.7082482104651,
|
||
|
|
"learning_rate": 2.2196139801773602e-07,
|
||
|
|
"logits/chosen": -0.6177734136581421,
|
||
|
|
"logits/rejected": -0.45136719942092896,
|
||
|
|
"logps/chosen": -469.6000061035156,
|
||
|
|
"logps/rejected": -550.4000244140625,
|
||
|
|
"loss": 2.7538,
|
||
|
|
"rewards/accuracies": 0.5625,
|
||
|
|
"rewards/chosen": 5.909375190734863,
|
||
|
|
"rewards/margins": 8.871874809265137,
|
||
|
|
"rewards/rejected": -2.962597608566284,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6056338028169014,
|
||
|
|
"grad_norm": 757.3363946159071,
|
||
|
|
"learning_rate": 2.1935315597287428e-07,
|
||
|
|
"logits/chosen": -0.664257824420929,
|
||
|
|
"logits/rejected": -0.4083496034145355,
|
||
|
|
"logps/chosen": -488.79998779296875,
|
||
|
|
"logps/rejected": -556.5999755859375,
|
||
|
|
"loss": 2.9542,
|
||
|
|
"rewards/accuracies": 0.5570312738418579,
|
||
|
|
"rewards/chosen": 6.03125,
|
||
|
|
"rewards/margins": 8.935937881469727,
|
||
|
|
"rewards/rejected": -2.9039063453674316,
|
||
|
|
"step": 1290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6103286384976526,
|
||
|
|
"grad_norm": 562.6983835936513,
|
||
|
|
"learning_rate": 2.167449139280125e-07,
|
||
|
|
"logits/chosen": -0.620898425579071,
|
||
|
|
"logits/rejected": -0.36357420682907104,
|
||
|
|
"logps/chosen": -487.3999938964844,
|
||
|
|
"logps/rejected": -587.2000122070312,
|
||
|
|
"loss": 2.5734,
|
||
|
|
"rewards/accuracies": 0.5531250238418579,
|
||
|
|
"rewards/chosen": 7.090624809265137,
|
||
|
|
"rewards/margins": 9.274999618530273,
|
||
|
|
"rewards/rejected": -2.190966844558716,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6150234741784038,
|
||
|
|
"grad_norm": 532.6423591583102,
|
||
|
|
"learning_rate": 2.1413667188315073e-07,
|
||
|
|
"logits/chosen": -0.6507812738418579,
|
||
|
|
"logits/rejected": -0.37895506620407104,
|
||
|
|
"logps/chosen": -462.79998779296875,
|
||
|
|
"logps/rejected": -548.5999755859375,
|
||
|
|
"loss": 2.3617,
|
||
|
|
"rewards/accuracies": 0.5757812261581421,
|
||
|
|
"rewards/chosen": 7.25,
|
||
|
|
"rewards/margins": 10.715624809265137,
|
||
|
|
"rewards/rejected": -3.4625000953674316,
|
||
|
|
"step": 1310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6197183098591549,
|
||
|
|
"grad_norm": 725.9027774994798,
|
||
|
|
"learning_rate": 2.1152842983828898e-07,
|
||
|
|
"logits/chosen": -0.639453113079071,
|
||
|
|
"logits/rejected": -0.4097656309604645,
|
||
|
|
"logps/chosen": -488.3999938964844,
|
||
|
|
"logps/rejected": -564.4000244140625,
|
||
|
|
"loss": 3.3347,
|
||
|
|
"rewards/accuracies": 0.5445312261581421,
|
||
|
|
"rewards/chosen": 6.453125,
|
||
|
|
"rewards/margins": 8.871874809265137,
|
||
|
|
"rewards/rejected": -2.417773485183716,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6244131455399061,
|
||
|
|
"grad_norm": 659.2077315489408,
|
||
|
|
"learning_rate": 2.089201877934272e-07,
|
||
|
|
"logits/chosen": -0.5494140386581421,
|
||
|
|
"logits/rejected": -0.36250001192092896,
|
||
|
|
"logps/chosen": -503.6000061035156,
|
||
|
|
"logps/rejected": -608.2000122070312,
|
||
|
|
"loss": 3.208,
|
||
|
|
"rewards/accuracies": 0.5796874761581421,
|
||
|
|
"rewards/chosen": 6.1015625,
|
||
|
|
"rewards/margins": 10.940625190734863,
|
||
|
|
"rewards/rejected": -4.835546970367432,
|
||
|
|
"step": 1330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6291079812206573,
|
||
|
|
"grad_norm": 621.018895021024,
|
||
|
|
"learning_rate": 2.0631194574856543e-07,
|
||
|
|
"logits/chosen": -0.65625,
|
||
|
|
"logits/rejected": -0.38994139432907104,
|
||
|
|
"logps/chosen": -466.20001220703125,
|
||
|
|
"logps/rejected": -528.4000244140625,
|
||
|
|
"loss": 2.9511,
|
||
|
|
"rewards/accuracies": 0.5609375238418579,
|
||
|
|
"rewards/chosen": 6.981249809265137,
|
||
|
|
"rewards/margins": 8.665624618530273,
|
||
|
|
"rewards/rejected": -1.681054711341858,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6338028169014085,
|
||
|
|
"grad_norm": 755.6999902720212,
|
||
|
|
"learning_rate": 2.0370370370370369e-07,
|
||
|
|
"logits/chosen": -0.594531238079071,
|
||
|
|
"logits/rejected": -0.3876953125,
|
||
|
|
"logps/chosen": -482.3999938964844,
|
||
|
|
"logps/rejected": -635.4000244140625,
|
||
|
|
"loss": 2.7193,
|
||
|
|
"rewards/accuracies": 0.55859375,
|
||
|
|
"rewards/chosen": 6.235937595367432,
|
||
|
|
"rewards/margins": 11.21875,
|
||
|
|
"rewards/rejected": -4.989453315734863,
|
||
|
|
"step": 1350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6384976525821596,
|
||
|
|
"grad_norm": 690.740837843504,
|
||
|
|
"learning_rate": 2.010954616588419e-07,
|
||
|
|
"logits/chosen": -0.6566406488418579,
|
||
|
|
"logits/rejected": -0.4322265684604645,
|
||
|
|
"logps/chosen": -473.3999938964844,
|
||
|
|
"logps/rejected": -554.7999877929688,
|
||
|
|
"loss": 2.8241,
|
||
|
|
"rewards/accuracies": 0.5375000238418579,
|
||
|
|
"rewards/chosen": 6.21875,
|
||
|
|
"rewards/margins": 8.662500381469727,
|
||
|
|
"rewards/rejected": -2.4496092796325684,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6431924882629108,
|
||
|
|
"grad_norm": 626.0572314123408,
|
||
|
|
"learning_rate": 1.9848721961398017e-07,
|
||
|
|
"logits/chosen": -0.611328125,
|
||
|
|
"logits/rejected": -0.37890625,
|
||
|
|
"logps/chosen": -484.6000061035156,
|
||
|
|
"logps/rejected": -553.5999755859375,
|
||
|
|
"loss": 2.8073,
|
||
|
|
"rewards/accuracies": 0.51953125,
|
||
|
|
"rewards/chosen": 6.121874809265137,
|
||
|
|
"rewards/margins": 8.221875190734863,
|
||
|
|
"rewards/rejected": -2.0960693359375,
|
||
|
|
"step": 1370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.647887323943662,
|
||
|
|
"grad_norm": 763.8994041058758,
|
||
|
|
"learning_rate": 1.958789775691184e-07,
|
||
|
|
"logits/chosen": -0.617968738079071,
|
||
|
|
"logits/rejected": -0.400146484375,
|
||
|
|
"logps/chosen": -465.20001220703125,
|
||
|
|
"logps/rejected": -564.4000244140625,
|
||
|
|
"loss": 2.492,
|
||
|
|
"rewards/accuracies": 0.557812511920929,
|
||
|
|
"rewards/chosen": 6.6875,
|
||
|
|
"rewards/margins": 9.9375,
|
||
|
|
"rewards/rejected": -3.249218702316284,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6525821596244131,
|
||
|
|
"grad_norm": 600.9134828496638,
|
||
|
|
"learning_rate": 1.9327073552425662e-07,
|
||
|
|
"logits/chosen": -0.5912109613418579,
|
||
|
|
"logits/rejected": -0.34746092557907104,
|
||
|
|
"logps/chosen": -461.3999938964844,
|
||
|
|
"logps/rejected": -554.5999755859375,
|
||
|
|
"loss": 2.6512,
|
||
|
|
"rewards/accuracies": 0.5765625238418579,
|
||
|
|
"rewards/chosen": 6.671875,
|
||
|
|
"rewards/margins": 10.396875381469727,
|
||
|
|
"rewards/rejected": -3.71875,
|
||
|
|
"step": 1390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6572769953051644,
|
||
|
|
"grad_norm": 661.0607407117031,
|
||
|
|
"learning_rate": 1.906624934793949e-07,
|
||
|
|
"logits/chosen": -0.624218761920929,
|
||
|
|
"logits/rejected": -0.316162109375,
|
||
|
|
"logps/chosen": -480.6000061035156,
|
||
|
|
"logps/rejected": -603.7999877929688,
|
||
|
|
"loss": 2.8555,
|
||
|
|
"rewards/accuracies": 0.557812511920929,
|
||
|
|
"rewards/chosen": 6.153124809265137,
|
||
|
|
"rewards/margins": 10.4375,
|
||
|
|
"rewards/rejected": -4.298437595367432,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6619718309859155,
|
||
|
|
"grad_norm": 665.5572661782013,
|
||
|
|
"learning_rate": 1.8805425143453312e-07,
|
||
|
|
"logits/chosen": -0.630078136920929,
|
||
|
|
"logits/rejected": -0.33637696504592896,
|
||
|
|
"logps/chosen": -460.3999938964844,
|
||
|
|
"logps/rejected": -626.0,
|
||
|
|
"loss": 2.8526,
|
||
|
|
"rewards/accuracies": 0.5445312261581421,
|
||
|
|
"rewards/chosen": 6.212500095367432,
|
||
|
|
"rewards/margins": 10.993749618530273,
|
||
|
|
"rewards/rejected": -4.782812595367432,
|
||
|
|
"step": 1410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6666666666666666,
|
||
|
|
"grad_norm": 684.2910679051905,
|
||
|
|
"learning_rate": 1.8544600938967138e-07,
|
||
|
|
"logits/chosen": -0.665234386920929,
|
||
|
|
"logits/rejected": -0.3689208924770355,
|
||
|
|
"logps/chosen": -472.3999938964844,
|
||
|
|
"logps/rejected": -556.5999755859375,
|
||
|
|
"loss": 2.4156,
|
||
|
|
"rewards/accuracies": 0.574999988079071,
|
||
|
|
"rewards/chosen": 6.515625,
|
||
|
|
"rewards/margins": 9.824999809265137,
|
||
|
|
"rewards/rejected": -3.310546875,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6713615023474179,
|
||
|
|
"grad_norm": 610.7560499645055,
|
||
|
|
"learning_rate": 1.828377673448096e-07,
|
||
|
|
"logits/chosen": -0.5941406488418579,
|
||
|
|
"logits/rejected": -0.3697265684604645,
|
||
|
|
"logps/chosen": -506.6000061035156,
|
||
|
|
"logps/rejected": -618.7999877929688,
|
||
|
|
"loss": 3.3454,
|
||
|
|
"rewards/accuracies": 0.565625011920929,
|
||
|
|
"rewards/chosen": 5.610547065734863,
|
||
|
|
"rewards/margins": 11.300000190734863,
|
||
|
|
"rewards/rejected": -5.693749904632568,
|
||
|
|
"step": 1430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.676056338028169,
|
||
|
|
"grad_norm": 539.3123257602061,
|
||
|
|
"learning_rate": 1.8022952529994783e-07,
|
||
|
|
"logits/chosen": -0.6126953363418579,
|
||
|
|
"logits/rejected": -0.39960938692092896,
|
||
|
|
"logps/chosen": -487.79998779296875,
|
||
|
|
"logps/rejected": -561.7999877929688,
|
||
|
|
"loss": 2.7382,
|
||
|
|
"rewards/accuracies": 0.5835937261581421,
|
||
|
|
"rewards/chosen": 6.653124809265137,
|
||
|
|
"rewards/margins": 11.943750381469727,
|
||
|
|
"rewards/rejected": -5.276562690734863,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6807511737089202,
|
||
|
|
"grad_norm": 742.7338940861721,
|
||
|
|
"learning_rate": 1.7762128325508608e-07,
|
||
|
|
"logits/chosen": -0.5542968511581421,
|
||
|
|
"logits/rejected": -0.28849488496780396,
|
||
|
|
"logps/chosen": -498.6000061035156,
|
||
|
|
"logps/rejected": -645.7999877929688,
|
||
|
|
"loss": 3.2944,
|
||
|
|
"rewards/accuracies": 0.5289062261581421,
|
||
|
|
"rewards/chosen": 6.196875095367432,
|
||
|
|
"rewards/margins": 10.21875,
|
||
|
|
"rewards/rejected": -4.017968654632568,
|
||
|
|
"step": 1450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6854460093896714,
|
||
|
|
"grad_norm": 936.2295004740513,
|
||
|
|
"learning_rate": 1.750130412102243e-07,
|
||
|
|
"logits/chosen": -0.621289074420929,
|
||
|
|
"logits/rejected": -0.3535400331020355,
|
||
|
|
"logps/chosen": -520.7999877929688,
|
||
|
|
"logps/rejected": -584.4000244140625,
|
||
|
|
"loss": 3.5784,
|
||
|
|
"rewards/accuracies": 0.5796874761581421,
|
||
|
|
"rewards/chosen": 5.396874904632568,
|
||
|
|
"rewards/margins": 9.306249618530273,
|
||
|
|
"rewards/rejected": -3.9124999046325684,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6901408450704225,
|
||
|
|
"grad_norm": 547.6432349300944,
|
||
|
|
"learning_rate": 1.7240479916536254e-07,
|
||
|
|
"logits/chosen": -0.61328125,
|
||
|
|
"logits/rejected": -0.3504882752895355,
|
||
|
|
"logps/chosen": -470.3999938964844,
|
||
|
|
"logps/rejected": -599.5999755859375,
|
||
|
|
"loss": 2.652,
|
||
|
|
"rewards/accuracies": 0.5718749761581421,
|
||
|
|
"rewards/chosen": 6.443749904632568,
|
||
|
|
"rewards/margins": 11.1875,
|
||
|
|
"rewards/rejected": -4.750781059265137,
|
||
|
|
"step": 1470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6948356807511737,
|
||
|
|
"grad_norm": 615.9834678070489,
|
||
|
|
"learning_rate": 1.697965571205008e-07,
|
||
|
|
"logits/chosen": -0.596484363079071,
|
||
|
|
"logits/rejected": -0.39208984375,
|
||
|
|
"logps/chosen": -473.6000061035156,
|
||
|
|
"logps/rejected": -550.5999755859375,
|
||
|
|
"loss": 3.3536,
|
||
|
|
"rewards/accuracies": 0.578906238079071,
|
||
|
|
"rewards/chosen": 5.821875095367432,
|
||
|
|
"rewards/margins": 9.524999618530273,
|
||
|
|
"rewards/rejected": -3.703125,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6995305164319249,
|
||
|
|
"grad_norm": 688.9391125783357,
|
||
|
|
"learning_rate": 1.6718831507563902e-07,
|
||
|
|
"logits/chosen": -0.616406261920929,
|
||
|
|
"logits/rejected": -0.3792968690395355,
|
||
|
|
"logps/chosen": -481.20001220703125,
|
||
|
|
"logps/rejected": -545.4000244140625,
|
||
|
|
"loss": 2.9182,
|
||
|
|
"rewards/accuracies": 0.578125,
|
||
|
|
"rewards/chosen": 6.625,
|
||
|
|
"rewards/margins": 9.134374618530273,
|
||
|
|
"rewards/rejected": -2.506640672683716,
|
||
|
|
"step": 1490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.704225352112676,
|
||
|
|
"grad_norm": 598.5238898421428,
|
||
|
|
"learning_rate": 1.6458007303077727e-07,
|
||
|
|
"logits/chosen": -0.6070312261581421,
|
||
|
|
"logits/rejected": -0.3792724609375,
|
||
|
|
"logps/chosen": -467.79998779296875,
|
||
|
|
"logps/rejected": -551.7999877929688,
|
||
|
|
"loss": 2.9362,
|
||
|
|
"rewards/accuracies": 0.5679687261581421,
|
||
|
|
"rewards/chosen": 6.478125095367432,
|
||
|
|
"rewards/margins": 9.015625,
|
||
|
|
"rewards/rejected": -2.536328077316284,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7089201877934272,
|
||
|
|
"grad_norm": 609.8267462600803,
|
||
|
|
"learning_rate": 1.619718309859155e-07,
|
||
|
|
"logits/chosen": -0.62109375,
|
||
|
|
"logits/rejected": -0.4292968809604645,
|
||
|
|
"logps/chosen": -485.3999938964844,
|
||
|
|
"logps/rejected": -597.5999755859375,
|
||
|
|
"loss": 3.3538,
|
||
|
|
"rewards/accuracies": 0.5523437261581421,
|
||
|
|
"rewards/chosen": 5.725390434265137,
|
||
|
|
"rewards/margins": 9.59375,
|
||
|
|
"rewards/rejected": -3.8712401390075684,
|
||
|
|
"step": 1510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7136150234741784,
|
||
|
|
"grad_norm": 647.4827893679629,
|
||
|
|
"learning_rate": 1.5936358894105372e-07,
|
||
|
|
"logits/chosen": -0.735546886920929,
|
||
|
|
"logits/rejected": -0.4537109434604645,
|
||
|
|
"logps/chosen": -483.79998779296875,
|
||
|
|
"logps/rejected": -578.2000122070312,
|
||
|
|
"loss": 3.1007,
|
||
|
|
"rewards/accuracies": 0.547656238079071,
|
||
|
|
"rewards/chosen": 5.709374904632568,
|
||
|
|
"rewards/margins": 9.506250381469727,
|
||
|
|
"rewards/rejected": -3.7933592796325684,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7183098591549296,
|
||
|
|
"grad_norm": 558.5048557470903,
|
||
|
|
"learning_rate": 1.5675534689619197e-07,
|
||
|
|
"logits/chosen": -0.620312511920929,
|
||
|
|
"logits/rejected": -0.3519531190395355,
|
||
|
|
"logps/chosen": -487.0,
|
||
|
|
"logps/rejected": -545.2000122070312,
|
||
|
|
"loss": 2.8184,
|
||
|
|
"rewards/accuracies": 0.5625,
|
||
|
|
"rewards/chosen": 5.6484375,
|
||
|
|
"rewards/margins": 9.384374618530273,
|
||
|
|
"rewards/rejected": -3.7328124046325684,
|
||
|
|
"step": 1530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7230046948356808,
|
||
|
|
"grad_norm": 633.2676731241825,
|
||
|
|
"learning_rate": 1.541471048513302e-07,
|
||
|
|
"logits/chosen": -0.594921886920929,
|
||
|
|
"logits/rejected": -0.34619140625,
|
||
|
|
"logps/chosen": -488.20001220703125,
|
||
|
|
"logps/rejected": -576.7999877929688,
|
||
|
|
"loss": 2.7321,
|
||
|
|
"rewards/accuracies": 0.5796874761581421,
|
||
|
|
"rewards/chosen": 6.1640625,
|
||
|
|
"rewards/margins": 10.475000381469727,
|
||
|
|
"rewards/rejected": -4.301953315734863,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7276995305164319,
|
||
|
|
"grad_norm": 552.4300912165096,
|
||
|
|
"learning_rate": 1.5153886280646843e-07,
|
||
|
|
"logits/chosen": -0.6246093511581421,
|
||
|
|
"logits/rejected": -0.3617187440395355,
|
||
|
|
"logps/chosen": -456.0,
|
||
|
|
"logps/rejected": -534.0,
|
||
|
|
"loss": 2.945,
|
||
|
|
"rewards/accuracies": 0.5453125238418579,
|
||
|
|
"rewards/chosen": 5.893750190734863,
|
||
|
|
"rewards/margins": 8.903124809265137,
|
||
|
|
"rewards/rejected": -3.0132813453674316,
|
||
|
|
"step": 1550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7323943661971831,
|
||
|
|
"grad_norm": 610.7039328858499,
|
||
|
|
"learning_rate": 1.4893062076160668e-07,
|
||
|
|
"logits/chosen": -0.6259765625,
|
||
|
|
"logits/rejected": -0.3524414002895355,
|
||
|
|
"logps/chosen": -479.6000061035156,
|
||
|
|
"logps/rejected": -590.0,
|
||
|
|
"loss": 2.8537,
|
||
|
|
"rewards/accuracies": 0.557812511920929,
|
||
|
|
"rewards/chosen": 6.09375,
|
||
|
|
"rewards/margins": 10.875,
|
||
|
|
"rewards/rejected": -4.783593654632568,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7370892018779343,
|
||
|
|
"grad_norm": 461.72809158045067,
|
||
|
|
"learning_rate": 1.463223787167449e-07,
|
||
|
|
"logits/chosen": -0.570507824420929,
|
||
|
|
"logits/rejected": -0.3739257752895355,
|
||
|
|
"logps/chosen": -486.0,
|
||
|
|
"logps/rejected": -565.7999877929688,
|
||
|
|
"loss": 2.9577,
|
||
|
|
"rewards/accuracies": 0.550000011920929,
|
||
|
|
"rewards/chosen": 5.737500190734863,
|
||
|
|
"rewards/margins": 10.40625,
|
||
|
|
"rewards/rejected": -4.663476467132568,
|
||
|
|
"step": 1570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7417840375586855,
|
||
|
|
"grad_norm": 760.3093459308201,
|
||
|
|
"learning_rate": 1.4371413667188313e-07,
|
||
|
|
"logits/chosen": -0.6371093988418579,
|
||
|
|
"logits/rejected": -0.33281248807907104,
|
||
|
|
"logps/chosen": -474.6000061035156,
|
||
|
|
"logps/rejected": -611.0,
|
||
|
|
"loss": 2.8873,
|
||
|
|
"rewards/accuracies": 0.5484374761581421,
|
||
|
|
"rewards/chosen": 5.775000095367432,
|
||
|
|
"rewards/margins": 9.596875190734863,
|
||
|
|
"rewards/rejected": -3.8052735328674316,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7464788732394366,
|
||
|
|
"grad_norm": 632.0384675473865,
|
||
|
|
"learning_rate": 1.4110589462702139e-07,
|
||
|
|
"logits/chosen": -0.5960937738418579,
|
||
|
|
"logits/rejected": -0.35566407442092896,
|
||
|
|
"logps/chosen": -474.0,
|
||
|
|
"logps/rejected": -550.0,
|
||
|
|
"loss": 2.8232,
|
||
|
|
"rewards/accuracies": 0.561718761920929,
|
||
|
|
"rewards/chosen": 6.4375,
|
||
|
|
"rewards/margins": 8.978124618530273,
|
||
|
|
"rewards/rejected": -2.5498046875,
|
||
|
|
"step": 1590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7511737089201878,
|
||
|
|
"grad_norm": 707.4509916878376,
|
||
|
|
"learning_rate": 1.384976525821596e-07,
|
||
|
|
"logits/chosen": -0.582226574420929,
|
||
|
|
"logits/rejected": -0.2969726622104645,
|
||
|
|
"logps/chosen": -490.20001220703125,
|
||
|
|
"logps/rejected": -641.7999877929688,
|
||
|
|
"loss": 3.0785,
|
||
|
|
"rewards/accuracies": 0.56640625,
|
||
|
|
"rewards/chosen": 6.025000095367432,
|
||
|
|
"rewards/margins": 12.993749618530273,
|
||
|
|
"rewards/rejected": -6.975683689117432,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.755868544600939,
|
||
|
|
"grad_norm": 723.94390452839,
|
||
|
|
"learning_rate": 1.3588941053729787e-07,
|
||
|
|
"logits/chosen": -0.572460949420929,
|
||
|
|
"logits/rejected": -0.2802734375,
|
||
|
|
"logps/chosen": -493.6000061035156,
|
||
|
|
"logps/rejected": -597.5999755859375,
|
||
|
|
"loss": 3.2445,
|
||
|
|
"rewards/accuracies": 0.546093761920929,
|
||
|
|
"rewards/chosen": 6.215624809265137,
|
||
|
|
"rewards/margins": 10.246874809265137,
|
||
|
|
"rewards/rejected": -4.030419826507568,
|
||
|
|
"step": 1610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7605633802816901,
|
||
|
|
"grad_norm": 641.6159967512289,
|
||
|
|
"learning_rate": 1.332811684924361e-07,
|
||
|
|
"logits/chosen": -0.5865234136581421,
|
||
|
|
"logits/rejected": -0.39179688692092896,
|
||
|
|
"logps/chosen": -498.20001220703125,
|
||
|
|
"logps/rejected": -578.0,
|
||
|
|
"loss": 3.4994,
|
||
|
|
"rewards/accuracies": 0.55078125,
|
||
|
|
"rewards/chosen": 6.206250190734863,
|
||
|
|
"rewards/margins": 9.040624618530273,
|
||
|
|
"rewards/rejected": -2.8343749046325684,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7652582159624414,
|
||
|
|
"grad_norm": 629.6004443267193,
|
||
|
|
"learning_rate": 1.3067292644757432e-07,
|
||
|
|
"logits/chosen": -0.623046875,
|
||
|
|
"logits/rejected": -0.3005127012729645,
|
||
|
|
"logps/chosen": -494.3999938964844,
|
||
|
|
"logps/rejected": -609.5999755859375,
|
||
|
|
"loss": 3.2432,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 5.759375095367432,
|
||
|
|
"rewards/margins": 10.084375381469727,
|
||
|
|
"rewards/rejected": -4.317187309265137,
|
||
|
|
"step": 1630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7699530516431925,
|
||
|
|
"grad_norm": 662.0708579800561,
|
||
|
|
"learning_rate": 1.2806468440271257e-07,
|
||
|
|
"logits/chosen": -0.6214843988418579,
|
||
|
|
"logits/rejected": -0.3638671934604645,
|
||
|
|
"logps/chosen": -467.3999938964844,
|
||
|
|
"logps/rejected": -568.4000244140625,
|
||
|
|
"loss": 2.7966,
|
||
|
|
"rewards/accuracies": 0.561718761920929,
|
||
|
|
"rewards/chosen": 6.46875,
|
||
|
|
"rewards/margins": 10.309374809265137,
|
||
|
|
"rewards/rejected": -3.846874952316284,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7746478873239436,
|
||
|
|
"grad_norm": 589.3514065014772,
|
||
|
|
"learning_rate": 1.254564423578508e-07,
|
||
|
|
"logits/chosen": -0.591601550579071,
|
||
|
|
"logits/rejected": -0.3807617127895355,
|
||
|
|
"logps/chosen": -474.3999938964844,
|
||
|
|
"logps/rejected": -573.2000122070312,
|
||
|
|
"loss": 2.7199,
|
||
|
|
"rewards/accuracies": 0.578906238079071,
|
||
|
|
"rewards/chosen": 6.296875,
|
||
|
|
"rewards/margins": 10.256250381469727,
|
||
|
|
"rewards/rejected": -3.9496092796325684,
|
||
|
|
"step": 1650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7793427230046949,
|
||
|
|
"grad_norm": 717.3645613865825,
|
||
|
|
"learning_rate": 1.2284820031298902e-07,
|
||
|
|
"logits/chosen": -0.5863281488418579,
|
||
|
|
"logits/rejected": -0.37871092557907104,
|
||
|
|
"logps/chosen": -509.6000061035156,
|
||
|
|
"logps/rejected": -600.2000122070312,
|
||
|
|
"loss": 2.84,
|
||
|
|
"rewards/accuracies": 0.5914062261581421,
|
||
|
|
"rewards/chosen": 6.359375,
|
||
|
|
"rewards/margins": 11.893750190734863,
|
||
|
|
"rewards/rejected": -5.534375190734863,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.784037558685446,
|
||
|
|
"grad_norm": 704.4962065749928,
|
||
|
|
"learning_rate": 1.2023995826812728e-07,
|
||
|
|
"logits/chosen": -0.6363281011581421,
|
||
|
|
"logits/rejected": -0.3787841796875,
|
||
|
|
"logps/chosen": -479.20001220703125,
|
||
|
|
"logps/rejected": -585.5999755859375,
|
||
|
|
"loss": 2.7515,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 6.275000095367432,
|
||
|
|
"rewards/margins": 9.556249618530273,
|
||
|
|
"rewards/rejected": -3.274609327316284,
|
||
|
|
"step": 1670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7887323943661971,
|
||
|
|
"grad_norm": 686.2605842038708,
|
||
|
|
"learning_rate": 1.176317162232655e-07,
|
||
|
|
"logits/chosen": -0.6783202886581421,
|
||
|
|
"logits/rejected": -0.39238280057907104,
|
||
|
|
"logps/chosen": -470.6000061035156,
|
||
|
|
"logps/rejected": -592.0,
|
||
|
|
"loss": 2.5811,
|
||
|
|
"rewards/accuracies": 0.56640625,
|
||
|
|
"rewards/chosen": 6.637499809265137,
|
||
|
|
"rewards/margins": 10.268750190734863,
|
||
|
|
"rewards/rejected": -3.647656202316284,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7934272300469484,
|
||
|
|
"grad_norm": 864.8367128799877,
|
||
|
|
"learning_rate": 1.1502347417840374e-07,
|
||
|
|
"logits/chosen": -0.612500011920929,
|
||
|
|
"logits/rejected": -0.31708985567092896,
|
||
|
|
"logps/chosen": -478.3999938964844,
|
||
|
|
"logps/rejected": -571.5999755859375,
|
||
|
|
"loss": 3.3918,
|
||
|
|
"rewards/accuracies": 0.5531250238418579,
|
||
|
|
"rewards/chosen": 6.121874809265137,
|
||
|
|
"rewards/margins": 9.328125,
|
||
|
|
"rewards/rejected": -3.2015624046325684,
|
||
|
|
"step": 1690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7981220657276995,
|
||
|
|
"grad_norm": 646.1239282839907,
|
||
|
|
"learning_rate": 1.1241523213354198e-07,
|
||
|
|
"logits/chosen": -0.639453113079071,
|
||
|
|
"logits/rejected": -0.41035157442092896,
|
||
|
|
"logps/chosen": -477.0,
|
||
|
|
"logps/rejected": -594.0,
|
||
|
|
"loss": 2.8632,
|
||
|
|
"rewards/accuracies": 0.5625,
|
||
|
|
"rewards/chosen": 6.574999809265137,
|
||
|
|
"rewards/margins": 9.625,
|
||
|
|
"rewards/rejected": -3.046093702316284,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8028169014084507,
|
||
|
|
"grad_norm": 889.7368057300482,
|
||
|
|
"learning_rate": 1.0980699008868022e-07,
|
||
|
|
"logits/chosen": -0.638476550579071,
|
||
|
|
"logits/rejected": -0.4117187559604645,
|
||
|
|
"logps/chosen": -487.3999938964844,
|
||
|
|
"logps/rejected": -562.0,
|
||
|
|
"loss": 3.2101,
|
||
|
|
"rewards/accuracies": 0.52734375,
|
||
|
|
"rewards/chosen": 5.903124809265137,
|
||
|
|
"rewards/margins": 8.943750381469727,
|
||
|
|
"rewards/rejected": -3.039013624191284,
|
||
|
|
"step": 1710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8075117370892019,
|
||
|
|
"grad_norm": 670.5945361341489,
|
||
|
|
"learning_rate": 1.0719874804381846e-07,
|
||
|
|
"logits/chosen": -0.71484375,
|
||
|
|
"logits/rejected": -0.4579101502895355,
|
||
|
|
"logps/chosen": -476.0,
|
||
|
|
"logps/rejected": -534.7999877929688,
|
||
|
|
"loss": 2.9623,
|
||
|
|
"rewards/accuracies": 0.559374988079071,
|
||
|
|
"rewards/chosen": 6.646874904632568,
|
||
|
|
"rewards/margins": 8.528124809265137,
|
||
|
|
"rewards/rejected": -1.8826172351837158,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.812206572769953,
|
||
|
|
"grad_norm": 557.8336384778751,
|
||
|
|
"learning_rate": 1.045905059989567e-07,
|
||
|
|
"logits/chosen": -0.6513671875,
|
||
|
|
"logits/rejected": -0.357443243265152,
|
||
|
|
"logps/chosen": -452.0,
|
||
|
|
"logps/rejected": -528.0,
|
||
|
|
"loss": 2.6844,
|
||
|
|
"rewards/accuracies": 0.561718761920929,
|
||
|
|
"rewards/chosen": 6.387499809265137,
|
||
|
|
"rewards/margins": 9.034375190734863,
|
||
|
|
"rewards/rejected": -2.6421875953674316,
|
||
|
|
"step": 1730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8169014084507042,
|
||
|
|
"grad_norm": 794.0049404166609,
|
||
|
|
"learning_rate": 1.0198226395409494e-07,
|
||
|
|
"logits/chosen": -0.59765625,
|
||
|
|
"logits/rejected": -0.3827148377895355,
|
||
|
|
"logps/chosen": -457.20001220703125,
|
||
|
|
"logps/rejected": -549.4000244140625,
|
||
|
|
"loss": 2.8934,
|
||
|
|
"rewards/accuracies": 0.577343761920929,
|
||
|
|
"rewards/chosen": 6.287499904632568,
|
||
|
|
"rewards/margins": 9.8125,
|
||
|
|
"rewards/rejected": -3.5179686546325684,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8215962441314554,
|
||
|
|
"grad_norm": 607.9960816355205,
|
||
|
|
"learning_rate": 9.937402190923318e-08,
|
||
|
|
"logits/chosen": -0.6332031488418579,
|
||
|
|
"logits/rejected": -0.349609375,
|
||
|
|
"logps/chosen": -448.20001220703125,
|
||
|
|
"logps/rejected": -587.5999755859375,
|
||
|
|
"loss": 2.5202,
|
||
|
|
"rewards/accuracies": 0.585156261920929,
|
||
|
|
"rewards/chosen": 6.271874904632568,
|
||
|
|
"rewards/margins": 11.278124809265137,
|
||
|
|
"rewards/rejected": -5.004687309265137,
|
||
|
|
"step": 1750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8262910798122066,
|
||
|
|
"grad_norm": 609.7008259001816,
|
||
|
|
"learning_rate": 9.676577986437141e-08,
|
||
|
|
"logits/chosen": -0.627148449420929,
|
||
|
|
"logits/rejected": -0.3775390684604645,
|
||
|
|
"logps/chosen": -497.6000061035156,
|
||
|
|
"logps/rejected": -564.0,
|
||
|
|
"loss": 2.8412,
|
||
|
|
"rewards/accuracies": 0.5703125,
|
||
|
|
"rewards/chosen": 5.910937309265137,
|
||
|
|
"rewards/margins": 10.662500381469727,
|
||
|
|
"rewards/rejected": -4.746069431304932,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8309859154929577,
|
||
|
|
"grad_norm": 713.4878146730244,
|
||
|
|
"learning_rate": 9.415753781950965e-08,
|
||
|
|
"logits/chosen": -0.630859375,
|
||
|
|
"logits/rejected": -0.3597656190395355,
|
||
|
|
"logps/chosen": -467.6000061035156,
|
||
|
|
"logps/rejected": -564.7999877929688,
|
||
|
|
"loss": 2.577,
|
||
|
|
"rewards/accuracies": 0.56640625,
|
||
|
|
"rewards/chosen": 6.0625,
|
||
|
|
"rewards/margins": 10.824999809265137,
|
||
|
|
"rewards/rejected": -4.754248142242432,
|
||
|
|
"step": 1770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8356807511737089,
|
||
|
|
"grad_norm": 608.1596683646704,
|
||
|
|
"learning_rate": 9.154929577464789e-08,
|
||
|
|
"logits/chosen": -0.6195312738418579,
|
||
|
|
"logits/rejected": -0.30791014432907104,
|
||
|
|
"logps/chosen": -477.0,
|
||
|
|
"logps/rejected": -616.5999755859375,
|
||
|
|
"loss": 2.4931,
|
||
|
|
"rewards/accuracies": 0.573437511920929,
|
||
|
|
"rewards/chosen": 6.243750095367432,
|
||
|
|
"rewards/margins": 11.003125190734863,
|
||
|
|
"rewards/rejected": -4.764062404632568,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8403755868544601,
|
||
|
|
"grad_norm": 540.0576495423178,
|
||
|
|
"learning_rate": 8.894105372978613e-08,
|
||
|
|
"logits/chosen": -0.6234375238418579,
|
||
|
|
"logits/rejected": -0.32646483182907104,
|
||
|
|
"logps/chosen": -489.0,
|
||
|
|
"logps/rejected": -577.4000244140625,
|
||
|
|
"loss": 3.102,
|
||
|
|
"rewards/accuracies": 0.5406249761581421,
|
||
|
|
"rewards/chosen": 5.956250190734863,
|
||
|
|
"rewards/margins": 10.209375381469727,
|
||
|
|
"rewards/rejected": -4.258984565734863,
|
||
|
|
"step": 1790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8450704225352113,
|
||
|
|
"grad_norm": 569.7692426945313,
|
||
|
|
"learning_rate": 8.633281168492435e-08,
|
||
|
|
"logits/chosen": -0.6371093988418579,
|
||
|
|
"logits/rejected": -0.3421386778354645,
|
||
|
|
"logps/chosen": -474.0,
|
||
|
|
"logps/rejected": -561.2000122070312,
|
||
|
|
"loss": 2.6476,
|
||
|
|
"rewards/accuracies": 0.589062511920929,
|
||
|
|
"rewards/chosen": 7.014062404632568,
|
||
|
|
"rewards/margins": 11.246874809265137,
|
||
|
|
"rewards/rejected": -4.239648342132568,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8497652582159625,
|
||
|
|
"grad_norm": 538.7402745726536,
|
||
|
|
"learning_rate": 8.372456964006259e-08,
|
||
|
|
"logits/chosen": -0.607617199420929,
|
||
|
|
"logits/rejected": -0.37529295682907104,
|
||
|
|
"logps/chosen": -475.20001220703125,
|
||
|
|
"logps/rejected": -568.0,
|
||
|
|
"loss": 3.0153,
|
||
|
|
"rewards/accuracies": 0.5835937261581421,
|
||
|
|
"rewards/chosen": 6.40625,
|
||
|
|
"rewards/margins": 11.699999809265137,
|
||
|
|
"rewards/rejected": -5.283593654632568,
|
||
|
|
"step": 1810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8544600938967136,
|
||
|
|
"grad_norm": 853.8099679820054,
|
||
|
|
"learning_rate": 8.111632759520083e-08,
|
||
|
|
"logits/chosen": -0.563281238079071,
|
||
|
|
"logits/rejected": -0.2962890565395355,
|
||
|
|
"logps/chosen": -512.4000244140625,
|
||
|
|
"logps/rejected": -578.7999877929688,
|
||
|
|
"loss": 3.4237,
|
||
|
|
"rewards/accuracies": 0.5648437738418579,
|
||
|
|
"rewards/chosen": 4.946875095367432,
|
||
|
|
"rewards/margins": 9.800000190734863,
|
||
|
|
"rewards/rejected": -4.865234375,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8591549295774648,
|
||
|
|
"grad_norm": 636.370411986038,
|
||
|
|
"learning_rate": 7.850808555033907e-08,
|
||
|
|
"logits/chosen": -0.56640625,
|
||
|
|
"logits/rejected": -0.3749023377895355,
|
||
|
|
"logps/chosen": -480.6000061035156,
|
||
|
|
"logps/rejected": -592.4000244140625,
|
||
|
|
"loss": 2.6429,
|
||
|
|
"rewards/accuracies": 0.5718749761581421,
|
||
|
|
"rewards/chosen": 5.9140625,
|
||
|
|
"rewards/margins": 12.556249618530273,
|
||
|
|
"rewards/rejected": -6.633496284484863,
|
||
|
|
"step": 1830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.863849765258216,
|
||
|
|
"grad_norm": 646.0438348411047,
|
||
|
|
"learning_rate": 7.58998435054773e-08,
|
||
|
|
"logits/chosen": -0.6488281488418579,
|
||
|
|
"logits/rejected": -0.3470703065395355,
|
||
|
|
"logps/chosen": -491.3999938964844,
|
||
|
|
"logps/rejected": -577.7999877929688,
|
||
|
|
"loss": 3.3568,
|
||
|
|
"rewards/accuracies": 0.5570312738418579,
|
||
|
|
"rewards/chosen": 5.465624809265137,
|
||
|
|
"rewards/margins": 9.2109375,
|
||
|
|
"rewards/rejected": -3.7535157203674316,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8685446009389671,
|
||
|
|
"grad_norm": 739.6708058022318,
|
||
|
|
"learning_rate": 7.329160146061554e-08,
|
||
|
|
"logits/chosen": -0.621874988079071,
|
||
|
|
"logits/rejected": -0.345703125,
|
||
|
|
"logps/chosen": -467.6000061035156,
|
||
|
|
"logps/rejected": -566.4000244140625,
|
||
|
|
"loss": 2.6541,
|
||
|
|
"rewards/accuracies": 0.5640624761581421,
|
||
|
|
"rewards/chosen": 6.421875,
|
||
|
|
"rewards/margins": 9.228124618530273,
|
||
|
|
"rewards/rejected": -2.808837890625,
|
||
|
|
"step": 1850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8732394366197183,
|
||
|
|
"grad_norm": 575.8530502463047,
|
||
|
|
"learning_rate": 7.068335941575378e-08,
|
||
|
|
"logits/chosen": -0.578906238079071,
|
||
|
|
"logits/rejected": -0.3954101502895355,
|
||
|
|
"logps/chosen": -471.20001220703125,
|
||
|
|
"logps/rejected": -558.0,
|
||
|
|
"loss": 3.0191,
|
||
|
|
"rewards/accuracies": 0.5648437738418579,
|
||
|
|
"rewards/chosen": 5.432812690734863,
|
||
|
|
"rewards/margins": 9.953125,
|
||
|
|
"rewards/rejected": -4.521484375,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8779342723004695,
|
||
|
|
"grad_norm": 774.4729546707873,
|
||
|
|
"learning_rate": 6.807511737089202e-08,
|
||
|
|
"logits/chosen": -0.66796875,
|
||
|
|
"logits/rejected": -0.31816405057907104,
|
||
|
|
"logps/chosen": -469.3999938964844,
|
||
|
|
"logps/rejected": -582.2000122070312,
|
||
|
|
"loss": 2.8176,
|
||
|
|
"rewards/accuracies": 0.5625,
|
||
|
|
"rewards/chosen": 5.903124809265137,
|
||
|
|
"rewards/margins": 10.046875,
|
||
|
|
"rewards/rejected": -4.142187595367432,
|
||
|
|
"step": 1870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8826291079812206,
|
||
|
|
"grad_norm": 658.659301878977,
|
||
|
|
"learning_rate": 6.546687532603024e-08,
|
||
|
|
"logits/chosen": -0.5562499761581421,
|
||
|
|
"logits/rejected": -0.28251951932907104,
|
||
|
|
"logps/chosen": -475.3999938964844,
|
||
|
|
"logps/rejected": -576.7999877929688,
|
||
|
|
"loss": 2.7453,
|
||
|
|
"rewards/accuracies": 0.5679687261581421,
|
||
|
|
"rewards/chosen": 6.348437309265137,
|
||
|
|
"rewards/margins": 9.787500381469727,
|
||
|
|
"rewards/rejected": -3.4374022483825684,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8873239436619719,
|
||
|
|
"grad_norm": 631.3050554145307,
|
||
|
|
"learning_rate": 6.285863328116848e-08,
|
||
|
|
"logits/chosen": -0.6167968511581421,
|
||
|
|
"logits/rejected": -0.30976563692092896,
|
||
|
|
"logps/chosen": -433.0,
|
||
|
|
"logps/rejected": -594.0,
|
||
|
|
"loss": 2.7215,
|
||
|
|
"rewards/accuracies": 0.571093738079071,
|
||
|
|
"rewards/chosen": 5.6328125,
|
||
|
|
"rewards/margins": 10.850000381469727,
|
||
|
|
"rewards/rejected": -5.217187404632568,
|
||
|
|
"step": 1890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.892018779342723,
|
||
|
|
"grad_norm": 656.2475220290227,
|
||
|
|
"learning_rate": 6.025039123630672e-08,
|
||
|
|
"logits/chosen": -0.6275390386581421,
|
||
|
|
"logits/rejected": -0.4159179627895355,
|
||
|
|
"logps/chosen": -487.79998779296875,
|
||
|
|
"logps/rejected": -543.4000244140625,
|
||
|
|
"loss": 3.5189,
|
||
|
|
"rewards/accuracies": 0.550000011920929,
|
||
|
|
"rewards/chosen": 5.778124809265137,
|
||
|
|
"rewards/margins": 8.756250381469727,
|
||
|
|
"rewards/rejected": -2.981250047683716,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8967136150234741,
|
||
|
|
"grad_norm": 582.3674155920564,
|
||
|
|
"learning_rate": 5.764214919144496e-08,
|
||
|
|
"logits/chosen": -0.623046875,
|
||
|
|
"logits/rejected": -0.40156251192092896,
|
||
|
|
"logps/chosen": -488.79998779296875,
|
||
|
|
"logps/rejected": -543.2000122070312,
|
||
|
|
"loss": 2.8874,
|
||
|
|
"rewards/accuracies": 0.565625011920929,
|
||
|
|
"rewards/chosen": 6.618750095367432,
|
||
|
|
"rewards/margins": 9.171875,
|
||
|
|
"rewards/rejected": -2.564746141433716,
|
||
|
|
"step": 1910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9014084507042254,
|
||
|
|
"grad_norm": 757.0199286423315,
|
||
|
|
"learning_rate": 5.50339071465832e-08,
|
||
|
|
"logits/chosen": -0.6234375238418579,
|
||
|
|
"logits/rejected": -0.3946289122104645,
|
||
|
|
"logps/chosen": -484.3999938964844,
|
||
|
|
"logps/rejected": -540.2000122070312,
|
||
|
|
"loss": 2.846,
|
||
|
|
"rewards/accuracies": 0.5726562738418579,
|
||
|
|
"rewards/chosen": 6.221875190734863,
|
||
|
|
"rewards/margins": 9.449999809265137,
|
||
|
|
"rewards/rejected": -3.235156297683716,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9061032863849765,
|
||
|
|
"grad_norm": 617.138612569707,
|
||
|
|
"learning_rate": 5.2425665101721436e-08,
|
||
|
|
"logits/chosen": -0.638671875,
|
||
|
|
"logits/rejected": -0.37578123807907104,
|
||
|
|
"logps/chosen": -488.3999938964844,
|
||
|
|
"logps/rejected": -611.2000122070312,
|
||
|
|
"loss": 2.6755,
|
||
|
|
"rewards/accuracies": 0.5687500238418579,
|
||
|
|
"rewards/chosen": 6.415625095367432,
|
||
|
|
"rewards/margins": 10.737500190734863,
|
||
|
|
"rewards/rejected": -4.317968845367432,
|
||
|
|
"step": 1930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9107981220657277,
|
||
|
|
"grad_norm": 593.3854666134151,
|
||
|
|
"learning_rate": 4.9817423056859675e-08,
|
||
|
|
"logits/chosen": -0.6416015625,
|
||
|
|
"logits/rejected": -0.39179688692092896,
|
||
|
|
"logps/chosen": -488.6000061035156,
|
||
|
|
"logps/rejected": -586.4000244140625,
|
||
|
|
"loss": 2.9049,
|
||
|
|
"rewards/accuracies": 0.5726562738418579,
|
||
|
|
"rewards/chosen": 6.278124809265137,
|
||
|
|
"rewards/margins": 10.350000381469727,
|
||
|
|
"rewards/rejected": -4.073046684265137,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9154929577464789,
|
||
|
|
"grad_norm": 657.7312826347563,
|
||
|
|
"learning_rate": 4.720918101199791e-08,
|
||
|
|
"logits/chosen": -0.631640613079071,
|
||
|
|
"logits/rejected": -0.3326171934604645,
|
||
|
|
"logps/chosen": -490.79998779296875,
|
||
|
|
"logps/rejected": -636.4000244140625,
|
||
|
|
"loss": 2.9807,
|
||
|
|
"rewards/accuracies": 0.5726562738418579,
|
||
|
|
"rewards/chosen": 5.762499809265137,
|
||
|
|
"rewards/margins": 12.371874809265137,
|
||
|
|
"rewards/rejected": -6.611718654632568,
|
||
|
|
"step": 1950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.92018779342723,
|
||
|
|
"grad_norm": 717.47310251892,
|
||
|
|
"learning_rate": 4.460093896713615e-08,
|
||
|
|
"logits/chosen": -0.639843761920929,
|
||
|
|
"logits/rejected": -0.3790039122104645,
|
||
|
|
"logps/chosen": -480.3999938964844,
|
||
|
|
"logps/rejected": -588.5999755859375,
|
||
|
|
"loss": 2.9656,
|
||
|
|
"rewards/accuracies": 0.55078125,
|
||
|
|
"rewards/chosen": 5.885937690734863,
|
||
|
|
"rewards/margins": 9.693750381469727,
|
||
|
|
"rewards/rejected": -3.8109374046325684,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9248826291079812,
|
||
|
|
"grad_norm": 717.1337569499667,
|
||
|
|
"learning_rate": 4.199269692227438e-08,
|
||
|
|
"logits/chosen": -0.643750011920929,
|
||
|
|
"logits/rejected": -0.38261717557907104,
|
||
|
|
"logps/chosen": -471.20001220703125,
|
||
|
|
"logps/rejected": -536.7999877929688,
|
||
|
|
"loss": 3.1999,
|
||
|
|
"rewards/accuracies": 0.54296875,
|
||
|
|
"rewards/chosen": 5.660937309265137,
|
||
|
|
"rewards/margins": 8.409375190734863,
|
||
|
|
"rewards/rejected": -2.746875047683716,
|
||
|
|
"step": 1970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9295774647887324,
|
||
|
|
"grad_norm": 603.8165703387633,
|
||
|
|
"learning_rate": 3.938445487741262e-08,
|
||
|
|
"logits/chosen": -0.651562511920929,
|
||
|
|
"logits/rejected": -0.43115234375,
|
||
|
|
"logps/chosen": -495.79998779296875,
|
||
|
|
"logps/rejected": -583.0,
|
||
|
|
"loss": 3.4693,
|
||
|
|
"rewards/accuracies": 0.5531250238418579,
|
||
|
|
"rewards/chosen": 5.540625095367432,
|
||
|
|
"rewards/margins": 10.165624618530273,
|
||
|
|
"rewards/rejected": -4.621289253234863,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9342723004694836,
|
||
|
|
"grad_norm": 587.6229524523164,
|
||
|
|
"learning_rate": 3.677621283255086e-08,
|
||
|
|
"logits/chosen": -0.6285156011581421,
|
||
|
|
"logits/rejected": -0.2652587890625,
|
||
|
|
"logps/chosen": -493.79998779296875,
|
||
|
|
"logps/rejected": -571.7999877929688,
|
||
|
|
"loss": 2.7933,
|
||
|
|
"rewards/accuracies": 0.57421875,
|
||
|
|
"rewards/chosen": 6.824999809265137,
|
||
|
|
"rewards/margins": 10.756250381469727,
|
||
|
|
"rewards/rejected": -3.9296875,
|
||
|
|
"step": 1990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9389671361502347,
|
||
|
|
"grad_norm": 502.95370350623455,
|
||
|
|
"learning_rate": 3.41679707876891e-08,
|
||
|
|
"logits/chosen": -0.6050781011581421,
|
||
|
|
"logits/rejected": -0.328125,
|
||
|
|
"logps/chosen": -481.20001220703125,
|
||
|
|
"logps/rejected": -628.7999877929688,
|
||
|
|
"loss": 3.1899,
|
||
|
|
"rewards/accuracies": 0.5679687261581421,
|
||
|
|
"rewards/chosen": 5.628125190734863,
|
||
|
|
"rewards/margins": 10.162500381469727,
|
||
|
|
"rewards/rejected": -4.537499904632568,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9436619718309859,
|
||
|
|
"grad_norm": 737.5624816540195,
|
||
|
|
"learning_rate": 3.155972874282733e-08,
|
||
|
|
"logits/chosen": -0.634765625,
|
||
|
|
"logits/rejected": -0.3101806640625,
|
||
|
|
"logps/chosen": -462.3999938964844,
|
||
|
|
"logps/rejected": -564.4000244140625,
|
||
|
|
"loss": 2.8283,
|
||
|
|
"rewards/accuracies": 0.578906238079071,
|
||
|
|
"rewards/chosen": 6.484375,
|
||
|
|
"rewards/margins": 9.459375381469727,
|
||
|
|
"rewards/rejected": -2.978320360183716,
|
||
|
|
"step": 2010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9483568075117371,
|
||
|
|
"grad_norm": 556.962868429337,
|
||
|
|
"learning_rate": 2.8951486697965573e-08,
|
||
|
|
"logits/chosen": -0.636523425579071,
|
||
|
|
"logits/rejected": -0.44902342557907104,
|
||
|
|
"logps/chosen": -490.0,
|
||
|
|
"logps/rejected": -576.4000244140625,
|
||
|
|
"loss": 2.6236,
|
||
|
|
"rewards/accuracies": 0.58984375,
|
||
|
|
"rewards/chosen": 6.449999809265137,
|
||
|
|
"rewards/margins": 10.487500190734863,
|
||
|
|
"rewards/rejected": -4.046093940734863,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9530516431924883,
|
||
|
|
"grad_norm": 650.423605569204,
|
||
|
|
"learning_rate": 2.634324465310381e-08,
|
||
|
|
"logits/chosen": -0.626953125,
|
||
|
|
"logits/rejected": -0.36210936307907104,
|
||
|
|
"logps/chosen": -467.6000061035156,
|
||
|
|
"logps/rejected": -576.2000122070312,
|
||
|
|
"loss": 2.5884,
|
||
|
|
"rewards/accuracies": 0.546875,
|
||
|
|
"rewards/chosen": 6.181250095367432,
|
||
|
|
"rewards/margins": 10.328125,
|
||
|
|
"rewards/rejected": -4.152734279632568,
|
||
|
|
"step": 2030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9577464788732394,
|
||
|
|
"grad_norm": 589.1687734243089,
|
||
|
|
"learning_rate": 2.3735002608242045e-08,
|
||
|
|
"logits/chosen": -0.593945324420929,
|
||
|
|
"logits/rejected": -0.31953126192092896,
|
||
|
|
"logps/chosen": -491.79998779296875,
|
||
|
|
"logps/rejected": -549.2000122070312,
|
||
|
|
"loss": 2.8419,
|
||
|
|
"rewards/accuracies": 0.5601562261581421,
|
||
|
|
"rewards/chosen": 5.631249904632568,
|
||
|
|
"rewards/margins": 9.918749809265137,
|
||
|
|
"rewards/rejected": -4.2890625,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9624413145539906,
|
||
|
|
"grad_norm": 655.8588208054493,
|
||
|
|
"learning_rate": 2.1126760563380282e-08,
|
||
|
|
"logits/chosen": -0.6376953125,
|
||
|
|
"logits/rejected": -0.3558593690395355,
|
||
|
|
"logps/chosen": -475.6000061035156,
|
||
|
|
"logps/rejected": -528.0,
|
||
|
|
"loss": 2.8583,
|
||
|
|
"rewards/accuracies": 0.555468738079071,
|
||
|
|
"rewards/chosen": 5.560937404632568,
|
||
|
|
"rewards/margins": 8.165624618530273,
|
||
|
|
"rewards/rejected": -2.608105421066284,
|
||
|
|
"step": 2050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9671361502347418,
|
||
|
|
"grad_norm": 588.6318902133594,
|
||
|
|
"learning_rate": 1.8518518518518518e-08,
|
||
|
|
"logits/chosen": -0.579296886920929,
|
||
|
|
"logits/rejected": -0.33935546875,
|
||
|
|
"logps/chosen": -493.6000061035156,
|
||
|
|
"logps/rejected": -570.0,
|
||
|
|
"loss": 3.0038,
|
||
|
|
"rewards/accuracies": 0.5625,
|
||
|
|
"rewards/chosen": 6.043749809265137,
|
||
|
|
"rewards/margins": 9.65625,
|
||
|
|
"rewards/rejected": -3.610156297683716,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.971830985915493,
|
||
|
|
"grad_norm": 656.4194303523547,
|
||
|
|
"learning_rate": 1.5910276473656755e-08,
|
||
|
|
"logits/chosen": -0.591601550579071,
|
||
|
|
"logits/rejected": -0.4166015684604645,
|
||
|
|
"logps/chosen": -472.0,
|
||
|
|
"logps/rejected": -568.4000244140625,
|
||
|
|
"loss": 2.4855,
|
||
|
|
"rewards/accuracies": 0.6000000238418579,
|
||
|
|
"rewards/chosen": 6.184374809265137,
|
||
|
|
"rewards/margins": 10.953125,
|
||
|
|
"rewards/rejected": -4.769140720367432,
|
||
|
|
"step": 2070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9765258215962441,
|
||
|
|
"grad_norm": 606.365640404609,
|
||
|
|
"learning_rate": 1.3302034428794991e-08,
|
||
|
|
"logits/chosen": -0.5853515863418579,
|
||
|
|
"logits/rejected": -0.3349609375,
|
||
|
|
"logps/chosen": -456.6000061035156,
|
||
|
|
"logps/rejected": -534.7999877929688,
|
||
|
|
"loss": 2.7839,
|
||
|
|
"rewards/accuracies": 0.578906238079071,
|
||
|
|
"rewards/chosen": 6.28125,
|
||
|
|
"rewards/margins": 10.143750190734863,
|
||
|
|
"rewards/rejected": -3.86328125,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9812206572769953,
|
||
|
|
"grad_norm": 557.2895281814657,
|
||
|
|
"learning_rate": 1.0693792383933229e-08,
|
||
|
|
"logits/chosen": -0.6597656011581421,
|
||
|
|
"logits/rejected": -0.34716796875,
|
||
|
|
"logps/chosen": -485.0,
|
||
|
|
"logps/rejected": -589.2000122070312,
|
||
|
|
"loss": 2.5817,
|
||
|
|
"rewards/accuracies": 0.5835937261581421,
|
||
|
|
"rewards/chosen": 6.028124809265137,
|
||
|
|
"rewards/margins": 10.15625,
|
||
|
|
"rewards/rejected": -4.1171875,
|
||
|
|
"step": 2090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9859154929577465,
|
||
|
|
"grad_norm": 739.9522930693506,
|
||
|
|
"learning_rate": 8.085550339071465e-09,
|
||
|
|
"logits/chosen": -0.66796875,
|
||
|
|
"logits/rejected": -0.3379882872104645,
|
||
|
|
"logps/chosen": -494.79998779296875,
|
||
|
|
"logps/rejected": -609.5999755859375,
|
||
|
|
"loss": 2.7688,
|
||
|
|
"rewards/accuracies": 0.55859375,
|
||
|
|
"rewards/chosen": 5.590624809265137,
|
||
|
|
"rewards/margins": 10.846875190734863,
|
||
|
|
"rewards/rejected": -5.259375095367432,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9906103286384976,
|
||
|
|
"grad_norm": 785.7303808751117,
|
||
|
|
"learning_rate": 5.4773082942097025e-09,
|
||
|
|
"logits/chosen": -0.624804675579071,
|
||
|
|
"logits/rejected": -0.40410155057907104,
|
||
|
|
"logps/chosen": -469.6000061035156,
|
||
|
|
"logps/rejected": -548.4000244140625,
|
||
|
|
"loss": 2.5245,
|
||
|
|
"rewards/accuracies": 0.5835937261581421,
|
||
|
|
"rewards/chosen": 6.145312309265137,
|
||
|
|
"rewards/margins": 10.068750381469727,
|
||
|
|
"rewards/rejected": -3.9312500953674316,
|
||
|
|
"step": 2110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9953051643192489,
|
||
|
|
"grad_norm": 623.430834691913,
|
||
|
|
"learning_rate": 2.8690662493479393e-09,
|
||
|
|
"logits/chosen": -0.6214843988418579,
|
||
|
|
"logits/rejected": -0.41796875,
|
||
|
|
"logps/chosen": -476.6000061035156,
|
||
|
|
"logps/rejected": -556.5999755859375,
|
||
|
|
"loss": 2.9216,
|
||
|
|
"rewards/accuracies": 0.563281238079071,
|
||
|
|
"rewards/chosen": 6.015625,
|
||
|
|
"rewards/margins": 10.578125,
|
||
|
|
"rewards/rejected": -4.564062595367432,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0,
|
||
|
|
"grad_norm": 518.3862097423236,
|
||
|
|
"learning_rate": 2.608242044861763e-10,
|
||
|
|
"logits/chosen": -0.633593738079071,
|
||
|
|
"logits/rejected": -0.33964842557907104,
|
||
|
|
"logps/chosen": -469.6000061035156,
|
||
|
|
"logps/rejected": -583.2000122070312,
|
||
|
|
"loss": 2.7931,
|
||
|
|
"rewards/accuracies": 0.5856806635856628,
|
||
|
|
"rewards/chosen": 6.064062595367432,
|
||
|
|
"rewards/margins": 11.362500190734863,
|
||
|
|
"rewards/rejected": -5.292187690734863,
|
||
|
|
"step": 2130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0,
|
||
|
|
"step": 2130,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 3.0147399812796865,
|
||
|
|
"train_runtime": 5108.3896,
|
||
|
|
"train_samples_per_second": 53.36,
|
||
|
|
"train_steps_per_second": 0.417
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 2130,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 200,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|