1cb70daf8b
Model: ali-elganzory/Qwen3-1.7B-Base-DPO-Tulu3-decontaminated-masked Source: Original Platform
3239 lines
108 KiB
JSON
3239 lines
108 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2130,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004694835680751174,
|
|
"grad_norm": 2530.8731816493337,
|
|
"learning_rate": 2.1126760563380282e-08,
|
|
"logits/chosen": 4.15625,
|
|
"logits/rejected": 5.196875095367432,
|
|
"logps/chosen": -410.3999938964844,
|
|
"logps/rejected": -568.2000122070312,
|
|
"loss": 2.8437,
|
|
"rewards/accuracies": 0.24921874701976776,
|
|
"rewards/chosen": -0.19355468451976776,
|
|
"rewards/margins": -0.03457031399011612,
|
|
"rewards/rejected": -0.158447265625,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.009389671361502348,
|
|
"grad_norm": 3185.13300486243,
|
|
"learning_rate": 4.460093896713615e-08,
|
|
"logits/chosen": 3.989062547683716,
|
|
"logits/rejected": 5.159375190734863,
|
|
"logps/chosen": -425.79998779296875,
|
|
"logps/rejected": -556.7999877929688,
|
|
"loss": 3.2779,
|
|
"rewards/accuracies": 0.31718748807907104,
|
|
"rewards/chosen": 0.141357421875,
|
|
"rewards/margins": 0.3154296875,
|
|
"rewards/rejected": -0.17451171576976776,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.014084507042253521,
|
|
"grad_norm": 3774.9267033294827,
|
|
"learning_rate": 6.807511737089202e-08,
|
|
"logits/chosen": 4.032812595367432,
|
|
"logits/rejected": 5.365624904632568,
|
|
"logps/chosen": -459.6000061035156,
|
|
"logps/rejected": -640.4000244140625,
|
|
"loss": 3.2142,
|
|
"rewards/accuracies": 0.32499998807907104,
|
|
"rewards/chosen": 0.435546875,
|
|
"rewards/margins": 0.6337890625,
|
|
"rewards/rejected": -0.19785156846046448,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.018779342723004695,
|
|
"grad_norm": 2674.7157567339127,
|
|
"learning_rate": 9.154929577464789e-08,
|
|
"logits/chosen": 3.965625047683716,
|
|
"logits/rejected": 5.300000190734863,
|
|
"logps/chosen": -446.3999938964844,
|
|
"logps/rejected": -647.5999755859375,
|
|
"loss": 3.5842,
|
|
"rewards/accuracies": 0.3265624940395355,
|
|
"rewards/chosen": -0.614208996295929,
|
|
"rewards/margins": 0.23330077528953552,
|
|
"rewards/rejected": -0.8477538824081421,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.023474178403755867,
|
|
"grad_norm": 2954.600798680306,
|
|
"learning_rate": 1.1502347417840374e-07,
|
|
"logits/chosen": 4.137499809265137,
|
|
"logits/rejected": 5.224999904632568,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -557.4000244140625,
|
|
"loss": 3.9306,
|
|
"rewards/accuracies": 0.328125,
|
|
"rewards/chosen": -0.5848144292831421,
|
|
"rewards/margins": -0.3150390684604645,
|
|
"rewards/rejected": -0.2694335877895355,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.028169014084507043,
|
|
"grad_norm": 2677.899522366395,
|
|
"learning_rate": 1.384976525821596e-07,
|
|
"logits/chosen": 3.9765625,
|
|
"logits/rejected": 5.212500095367432,
|
|
"logps/chosen": -448.6000061035156,
|
|
"logps/rejected": -616.4000244140625,
|
|
"loss": 3.5612,
|
|
"rewards/accuracies": 0.30078125,
|
|
"rewards/chosen": -0.7354491949081421,
|
|
"rewards/margins": -0.07685546576976776,
|
|
"rewards/rejected": -0.658496081829071,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.03286384976525822,
|
|
"grad_norm": 2097.762764518072,
|
|
"learning_rate": 1.619718309859155e-07,
|
|
"logits/chosen": 3.8890624046325684,
|
|
"logits/rejected": 5.103125095367432,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -577.4000244140625,
|
|
"loss": 3.2782,
|
|
"rewards/accuracies": 0.37421876192092896,
|
|
"rewards/chosen": 0.2908691465854645,
|
|
"rewards/margins": 1.471826195716858,
|
|
"rewards/rejected": -1.182519555091858,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.03755868544600939,
|
|
"grad_norm": 2115.2546699814293,
|
|
"learning_rate": 1.8544600938967138e-07,
|
|
"logits/chosen": 4.029687404632568,
|
|
"logits/rejected": 5.259375095367432,
|
|
"logps/chosen": -433.3999938964844,
|
|
"logps/rejected": -590.7999877929688,
|
|
"loss": 3.2107,
|
|
"rewards/accuracies": 0.375,
|
|
"rewards/chosen": -0.0712890625,
|
|
"rewards/margins": 1.6689453125,
|
|
"rewards/rejected": -1.743554711341858,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.04225352112676056,
|
|
"grad_norm": 2165.5849683891656,
|
|
"learning_rate": 2.089201877934272e-07,
|
|
"logits/chosen": 4.232812404632568,
|
|
"logits/rejected": 5.290625095367432,
|
|
"logps/chosen": -423.79998779296875,
|
|
"logps/rejected": -546.4000244140625,
|
|
"loss": 3.323,
|
|
"rewards/accuracies": 0.36328125,
|
|
"rewards/chosen": -0.38007813692092896,
|
|
"rewards/margins": 1.613134741783142,
|
|
"rewards/rejected": -1.990625023841858,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.046948356807511735,
|
|
"grad_norm": 1882.2171631105837,
|
|
"learning_rate": 2.323943661971831e-07,
|
|
"logits/chosen": 3.909374952316284,
|
|
"logits/rejected": 5.071875095367432,
|
|
"logps/chosen": -449.79998779296875,
|
|
"logps/rejected": -542.2000122070312,
|
|
"loss": 2.9656,
|
|
"rewards/accuracies": 0.41328126192092896,
|
|
"rewards/chosen": -0.041015625,
|
|
"rewards/margins": 2.0953125953674316,
|
|
"rewards/rejected": -2.133593797683716,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.051643192488262914,
|
|
"grad_norm": 2213.727980603247,
|
|
"learning_rate": 2.5586854460093895e-07,
|
|
"logits/chosen": 4.059374809265137,
|
|
"logits/rejected": 5.171875,
|
|
"logps/chosen": -434.20001220703125,
|
|
"logps/rejected": -589.0,
|
|
"loss": 3.6624,
|
|
"rewards/accuracies": 0.4390625059604645,
|
|
"rewards/chosen": -0.012500000186264515,
|
|
"rewards/margins": 3.171875,
|
|
"rewards/rejected": -3.186718702316284,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.056338028169014086,
|
|
"grad_norm": 4446.624188869667,
|
|
"learning_rate": 2.7934272300469483e-07,
|
|
"logits/chosen": 4.051562309265137,
|
|
"logits/rejected": 5.146874904632568,
|
|
"logps/chosen": -447.0,
|
|
"logps/rejected": -592.0,
|
|
"loss": 3.0254,
|
|
"rewards/accuracies": 0.4664062559604645,
|
|
"rewards/chosen": 0.302001953125,
|
|
"rewards/margins": 5.357812404632568,
|
|
"rewards/rejected": -5.05859375,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.06103286384976526,
|
|
"grad_norm": 4308.397577637661,
|
|
"learning_rate": 3.0281690140845066e-07,
|
|
"logits/chosen": 4.025000095367432,
|
|
"logits/rejected": 5.090624809265137,
|
|
"logps/chosen": -459.3999938964844,
|
|
"logps/rejected": -609.7999877929688,
|
|
"loss": 3.6787,
|
|
"rewards/accuracies": 0.47265625,
|
|
"rewards/chosen": 0.10200195014476776,
|
|
"rewards/margins": 3.9300780296325684,
|
|
"rewards/rejected": -3.828906297683716,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.06572769953051644,
|
|
"grad_norm": 4550.109121693464,
|
|
"learning_rate": 3.2629107981220654e-07,
|
|
"logits/chosen": 3.9390625953674316,
|
|
"logits/rejected": 5.099999904632568,
|
|
"logps/chosen": -437.0,
|
|
"logps/rejected": -616.0,
|
|
"loss": 3.0268,
|
|
"rewards/accuracies": 0.4906249940395355,
|
|
"rewards/chosen": 0.8116210699081421,
|
|
"rewards/margins": 6.415625095367432,
|
|
"rewards/rejected": -5.596875190734863,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.07042253521126761,
|
|
"grad_norm": 2658.5396683431854,
|
|
"learning_rate": 3.497652582159624e-07,
|
|
"logits/chosen": 4.017187595367432,
|
|
"logits/rejected": 5.387499809265137,
|
|
"logps/chosen": -443.79998779296875,
|
|
"logps/rejected": -608.5999755859375,
|
|
"loss": 2.9262,
|
|
"rewards/accuracies": 0.49531251192092896,
|
|
"rewards/chosen": 0.575878918170929,
|
|
"rewards/margins": 6.192187309265137,
|
|
"rewards/rejected": -5.6171875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.07511737089201878,
|
|
"grad_norm": 2006.3069045213883,
|
|
"learning_rate": 3.732394366197183e-07,
|
|
"logits/chosen": 4.006249904632568,
|
|
"logits/rejected": 5.228125095367432,
|
|
"logps/chosen": -455.79998779296875,
|
|
"logps/rejected": -614.2000122070312,
|
|
"loss": 3.2526,
|
|
"rewards/accuracies": 0.4828124940395355,
|
|
"rewards/chosen": 0.12607422471046448,
|
|
"rewards/margins": 6.1640625,
|
|
"rewards/rejected": -6.028124809265137,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.07981220657276995,
|
|
"grad_norm": 1685.64762699035,
|
|
"learning_rate": 3.967136150234742e-07,
|
|
"logits/chosen": 4.128125190734863,
|
|
"logits/rejected": 5.393750190734863,
|
|
"logps/chosen": -414.6000061035156,
|
|
"logps/rejected": -593.4000244140625,
|
|
"loss": 3.2183,
|
|
"rewards/accuracies": 0.5390625,
|
|
"rewards/chosen": 0.806445300579071,
|
|
"rewards/margins": 8.074999809265137,
|
|
"rewards/rejected": -7.275000095367432,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.08450704225352113,
|
|
"grad_norm": 1808.5166785043325,
|
|
"learning_rate": 4.2018779342723e-07,
|
|
"logits/chosen": 3.9359374046325684,
|
|
"logits/rejected": 5.253125190734863,
|
|
"logps/chosen": -400.0,
|
|
"logps/rejected": -579.2000122070312,
|
|
"loss": 2.6046,
|
|
"rewards/accuracies": 0.5531250238418579,
|
|
"rewards/chosen": 1.5114257335662842,
|
|
"rewards/margins": 8.4375,
|
|
"rewards/rejected": -6.921875,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.0892018779342723,
|
|
"grad_norm": 1942.922552493111,
|
|
"learning_rate": 4.436619718309859e-07,
|
|
"logits/chosen": 4.004687309265137,
|
|
"logits/rejected": 5.256249904632568,
|
|
"logps/chosen": -438.3999938964844,
|
|
"logps/rejected": -560.0,
|
|
"loss": 3.363,
|
|
"rewards/accuracies": 0.547656238079071,
|
|
"rewards/chosen": 1.26953125,
|
|
"rewards/margins": 7.650000095367432,
|
|
"rewards/rejected": -6.3828125,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.09389671361502347,
|
|
"grad_norm": 1901.2267762637775,
|
|
"learning_rate": 4.671361502347418e-07,
|
|
"logits/chosen": 3.8499999046325684,
|
|
"logits/rejected": 5.046875,
|
|
"logps/chosen": -417.79998779296875,
|
|
"logps/rejected": -576.2000122070312,
|
|
"loss": 3.0296,
|
|
"rewards/accuracies": 0.546093761920929,
|
|
"rewards/chosen": 2.178515672683716,
|
|
"rewards/margins": 9.143750190734863,
|
|
"rewards/rejected": -6.9765625,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.09859154929577464,
|
|
"grad_norm": 2310.111802868271,
|
|
"learning_rate": 4.906103286384976e-07,
|
|
"logits/chosen": 4.0703125,
|
|
"logits/rejected": 5.196875095367432,
|
|
"logps/chosen": -434.0,
|
|
"logps/rejected": -608.4000244140625,
|
|
"loss": 3.16,
|
|
"rewards/accuracies": 0.5726562738418579,
|
|
"rewards/chosen": 2.6851563453674316,
|
|
"rewards/margins": 10.215624809265137,
|
|
"rewards/rejected": -7.532812595367432,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.10328638497652583,
|
|
"grad_norm": 4481.351927015124,
|
|
"learning_rate": 4.984350547730829e-07,
|
|
"logits/chosen": 4.165625095367432,
|
|
"logits/rejected": 5.368750095367432,
|
|
"logps/chosen": -451.3999938964844,
|
|
"logps/rejected": -595.2000122070312,
|
|
"loss": 3.6463,
|
|
"rewards/accuracies": 0.5679687261581421,
|
|
"rewards/chosen": 2.1500000953674316,
|
|
"rewards/margins": 10.140625,
|
|
"rewards/rejected": -8.003125190734863,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.107981220657277,
|
|
"grad_norm": 1603.704385314072,
|
|
"learning_rate": 4.958268127282212e-07,
|
|
"logits/chosen": 3.918750047683716,
|
|
"logits/rejected": 5.012499809265137,
|
|
"logps/chosen": -432.20001220703125,
|
|
"logps/rejected": -606.4000244140625,
|
|
"loss": 2.9984,
|
|
"rewards/accuracies": 0.586718738079071,
|
|
"rewards/chosen": 2.0941405296325684,
|
|
"rewards/margins": 11.456250190734863,
|
|
"rewards/rejected": -9.359375,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.11267605633802817,
|
|
"grad_norm": 2515.426685544547,
|
|
"learning_rate": 4.932185706833594e-07,
|
|
"logits/chosen": 3.799999952316284,
|
|
"logits/rejected": 4.965624809265137,
|
|
"logps/chosen": -391.6000061035156,
|
|
"logps/rejected": -499.3999938964844,
|
|
"loss": 2.4876,
|
|
"rewards/accuracies": 0.5601562261581421,
|
|
"rewards/chosen": 2.3531250953674316,
|
|
"rewards/margins": 8.225000381469727,
|
|
"rewards/rejected": -5.884375095367432,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.11737089201877934,
|
|
"grad_norm": 2121.563609169002,
|
|
"learning_rate": 4.906103286384976e-07,
|
|
"logits/chosen": 3.9781250953674316,
|
|
"logits/rejected": 5.353125095367432,
|
|
"logps/chosen": -437.20001220703125,
|
|
"logps/rejected": -620.0,
|
|
"loss": 2.8495,
|
|
"rewards/accuracies": 0.567187488079071,
|
|
"rewards/chosen": 1.1037108898162842,
|
|
"rewards/margins": 11.821874618530273,
|
|
"rewards/rejected": -10.71875,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.12206572769953052,
|
|
"grad_norm": 1757.323421541841,
|
|
"learning_rate": 4.880020865936358e-07,
|
|
"logits/chosen": 3.9609375,
|
|
"logits/rejected": 5.050000190734863,
|
|
"logps/chosen": -407.79998779296875,
|
|
"logps/rejected": -568.5999755859375,
|
|
"loss": 2.9323,
|
|
"rewards/accuracies": 0.578906238079071,
|
|
"rewards/chosen": 1.2654297351837158,
|
|
"rewards/margins": 10.9375,
|
|
"rewards/rejected": -9.662500381469727,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.1267605633802817,
|
|
"grad_norm": 3024.6999596057794,
|
|
"learning_rate": 4.853938445487741e-07,
|
|
"logits/chosen": 4.010937690734863,
|
|
"logits/rejected": 5.165625095367432,
|
|
"logps/chosen": -433.20001220703125,
|
|
"logps/rejected": -599.7999877929688,
|
|
"loss": 3.1039,
|
|
"rewards/accuracies": 0.5882812738418579,
|
|
"rewards/chosen": 1.91162109375,
|
|
"rewards/margins": 12.203125,
|
|
"rewards/rejected": -10.303125381469727,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.13145539906103287,
|
|
"grad_norm": 1822.6778601429469,
|
|
"learning_rate": 4.827856025039123e-07,
|
|
"logits/chosen": 3.9765625,
|
|
"logits/rejected": 5.278124809265137,
|
|
"logps/chosen": -432.79998779296875,
|
|
"logps/rejected": -597.4000244140625,
|
|
"loss": 3.078,
|
|
"rewards/accuracies": 0.5726562738418579,
|
|
"rewards/chosen": 2.38671875,
|
|
"rewards/margins": 11.668749809265137,
|
|
"rewards/rejected": -9.28125,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.13615023474178403,
|
|
"grad_norm": 1938.4169944875766,
|
|
"learning_rate": 4.801773604590506e-07,
|
|
"logits/chosen": 4.090624809265137,
|
|
"logits/rejected": 5.240624904632568,
|
|
"logps/chosen": -474.3999938964844,
|
|
"logps/rejected": -592.4000244140625,
|
|
"loss": 4.5813,
|
|
"rewards/accuracies": 0.5726562738418579,
|
|
"rewards/chosen": 0.06777343899011612,
|
|
"rewards/margins": 9.128125190734863,
|
|
"rewards/rejected": -9.046875,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.14084507042253522,
|
|
"grad_norm": 1795.4700253310202,
|
|
"learning_rate": 4.775691184141888e-07,
|
|
"logits/chosen": 3.987499952316284,
|
|
"logits/rejected": 5.3125,
|
|
"logps/chosen": -445.20001220703125,
|
|
"logps/rejected": -592.0,
|
|
"loss": 3.5743,
|
|
"rewards/accuracies": 0.5648437738418579,
|
|
"rewards/chosen": 1.321533203125,
|
|
"rewards/margins": 10.828125,
|
|
"rewards/rejected": -9.509374618530273,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.14553990610328638,
|
|
"grad_norm": 1666.0895011897555,
|
|
"learning_rate": 4.749608763693271e-07,
|
|
"logits/chosen": 3.984375,
|
|
"logits/rejected": 5.206250190734863,
|
|
"logps/chosen": -428.20001220703125,
|
|
"logps/rejected": -577.5999755859375,
|
|
"loss": 2.9428,
|
|
"rewards/accuracies": 0.6015625,
|
|
"rewards/chosen": 1.66015625,
|
|
"rewards/margins": 11.315625190734863,
|
|
"rewards/rejected": -9.65625,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.15023474178403756,
|
|
"grad_norm": 3368.2745501859345,
|
|
"learning_rate": 4.7235263432446533e-07,
|
|
"logits/chosen": 4.045312404632568,
|
|
"logits/rejected": 4.918749809265137,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -566.4000244140625,
|
|
"loss": 2.8186,
|
|
"rewards/accuracies": 0.5882812738418579,
|
|
"rewards/chosen": 1.514062523841858,
|
|
"rewards/margins": 11.9375,
|
|
"rewards/rejected": -10.418749809265137,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.15492957746478872,
|
|
"grad_norm": 2175.920646151767,
|
|
"learning_rate": 4.6974439227960353e-07,
|
|
"logits/chosen": 4.059374809265137,
|
|
"logits/rejected": 5.224999904632568,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -604.0,
|
|
"loss": 4.0061,
|
|
"rewards/accuracies": 0.5835937261581421,
|
|
"rewards/chosen": 0.7040039300918579,
|
|
"rewards/margins": 11.512499809265137,
|
|
"rewards/rejected": -10.815625190734863,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.1596244131455399,
|
|
"grad_norm": 2693.0619407609274,
|
|
"learning_rate": 4.671361502347418e-07,
|
|
"logits/chosen": 3.926562547683716,
|
|
"logits/rejected": 5.034375190734863,
|
|
"logps/chosen": -414.3999938964844,
|
|
"logps/rejected": -530.0,
|
|
"loss": 2.7272,
|
|
"rewards/accuracies": 0.58984375,
|
|
"rewards/chosen": 2.766406297683716,
|
|
"rewards/margins": 11.381250381469727,
|
|
"rewards/rejected": -8.628125190734863,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.1643192488262911,
|
|
"grad_norm": 2032.645385729415,
|
|
"learning_rate": 4.6452790818988004e-07,
|
|
"logits/chosen": 3.9468750953674316,
|
|
"logits/rejected": 5.196875095367432,
|
|
"logps/chosen": -451.79998779296875,
|
|
"logps/rejected": -598.7999877929688,
|
|
"loss": 3.3658,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 2.1029295921325684,
|
|
"rewards/margins": 12.506250381469727,
|
|
"rewards/rejected": -10.403124809265137,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.16901408450704225,
|
|
"grad_norm": 1792.9198622286024,
|
|
"learning_rate": 4.6191966614501824e-07,
|
|
"logits/chosen": 3.9203124046325684,
|
|
"logits/rejected": 5.046875,
|
|
"logps/chosen": -422.6000061035156,
|
|
"logps/rejected": -576.4000244140625,
|
|
"loss": 3.0426,
|
|
"rewards/accuracies": 0.5921875238418579,
|
|
"rewards/chosen": 2.2476563453674316,
|
|
"rewards/margins": 11.578125,
|
|
"rewards/rejected": -9.323437690734863,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.17370892018779344,
|
|
"grad_norm": 1957.0580259087676,
|
|
"learning_rate": 4.593114241001565e-07,
|
|
"logits/chosen": 3.8968749046325684,
|
|
"logits/rejected": 5.134375095367432,
|
|
"logps/chosen": -454.0,
|
|
"logps/rejected": -601.2000122070312,
|
|
"loss": 3.6543,
|
|
"rewards/accuracies": 0.6109374761581421,
|
|
"rewards/chosen": 1.6574218273162842,
|
|
"rewards/margins": 12.637499809265137,
|
|
"rewards/rejected": -10.987500190734863,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.1784037558685446,
|
|
"grad_norm": 1773.632791600037,
|
|
"learning_rate": 4.5670318205529474e-07,
|
|
"logits/chosen": 3.9937500953674316,
|
|
"logits/rejected": 5.240624904632568,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -600.2000122070312,
|
|
"loss": 2.9405,
|
|
"rewards/accuracies": 0.573437511920929,
|
|
"rewards/chosen": 1.007421851158142,
|
|
"rewards/margins": 11.543749809265137,
|
|
"rewards/rejected": -10.537500381469727,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.18309859154929578,
|
|
"grad_norm": 2042.8004134535709,
|
|
"learning_rate": 4.54094940010433e-07,
|
|
"logits/chosen": 4.123437404632568,
|
|
"logits/rejected": 5.359375,
|
|
"logps/chosen": -440.20001220703125,
|
|
"logps/rejected": -613.0,
|
|
"loss": 3.0528,
|
|
"rewards/accuracies": 0.629687488079071,
|
|
"rewards/chosen": 3.208203077316284,
|
|
"rewards/margins": 12.625,
|
|
"rewards/rejected": -9.434374809265137,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.18779342723004694,
|
|
"grad_norm": 1956.6511737119936,
|
|
"learning_rate": 4.514866979655712e-07,
|
|
"logits/chosen": 4.032812595367432,
|
|
"logits/rejected": 5.199999809265137,
|
|
"logps/chosen": -428.3999938964844,
|
|
"logps/rejected": -585.4000244140625,
|
|
"loss": 2.5753,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": 3.8375000953674316,
|
|
"rewards/margins": 13.356249809265137,
|
|
"rewards/rejected": -9.506250381469727,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.19248826291079812,
|
|
"grad_norm": 2668.4538317788424,
|
|
"learning_rate": 4.4887845592070945e-07,
|
|
"logits/chosen": 3.917187452316284,
|
|
"logits/rejected": 5.159375190734863,
|
|
"logps/chosen": -420.3999938964844,
|
|
"logps/rejected": -616.4000244140625,
|
|
"loss": 3.1278,
|
|
"rewards/accuracies": 0.6070312261581421,
|
|
"rewards/chosen": 2.52734375,
|
|
"rewards/margins": 12.581250190734863,
|
|
"rewards/rejected": -10.050000190734863,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.19718309859154928,
|
|
"grad_norm": 1897.7945866304633,
|
|
"learning_rate": 4.462702138758477e-07,
|
|
"logits/chosen": 4.0234375,
|
|
"logits/rejected": 5.193749904632568,
|
|
"logps/chosen": -453.0,
|
|
"logps/rejected": -575.0,
|
|
"loss": 3.7912,
|
|
"rewards/accuracies": 0.57421875,
|
|
"rewards/chosen": 1.3203125,
|
|
"rewards/margins": 11.350000381469727,
|
|
"rewards/rejected": -10.024999618530273,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.20187793427230047,
|
|
"grad_norm": 1628.7110521343573,
|
|
"learning_rate": 4.436619718309859e-07,
|
|
"logits/chosen": 3.90625,
|
|
"logits/rejected": 5.09375,
|
|
"logps/chosen": -436.6000061035156,
|
|
"logps/rejected": -630.0,
|
|
"loss": 2.6684,
|
|
"rewards/accuracies": 0.585156261920929,
|
|
"rewards/chosen": 2.5279297828674316,
|
|
"rewards/margins": 13.5625,
|
|
"rewards/rejected": -11.03125,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.20657276995305165,
|
|
"grad_norm": 1642.5840566644301,
|
|
"learning_rate": 4.4105372978612415e-07,
|
|
"logits/chosen": 4.173437595367432,
|
|
"logits/rejected": 5.1875,
|
|
"logps/chosen": -458.6000061035156,
|
|
"logps/rejected": -609.7999877929688,
|
|
"loss": 3.5929,
|
|
"rewards/accuracies": 0.602343738079071,
|
|
"rewards/chosen": 2.9434571266174316,
|
|
"rewards/margins": 11.534375190734863,
|
|
"rewards/rejected": -8.596875190734863,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.2112676056338028,
|
|
"grad_norm": 1793.9626744031539,
|
|
"learning_rate": 4.384454877412624e-07,
|
|
"logits/chosen": 4.21875,
|
|
"logits/rejected": 5.496874809265137,
|
|
"logps/chosen": -449.79998779296875,
|
|
"logps/rejected": -682.0,
|
|
"loss": 3.3195,
|
|
"rewards/accuracies": 0.610156238079071,
|
|
"rewards/chosen": 4.328125,
|
|
"rewards/margins": 15.149999618530273,
|
|
"rewards/rejected": -10.815625190734863,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.215962441314554,
|
|
"grad_norm": 2155.420712230773,
|
|
"learning_rate": 4.358372456964006e-07,
|
|
"logits/chosen": 3.934375047683716,
|
|
"logits/rejected": 5.178124904632568,
|
|
"logps/chosen": -430.3999938964844,
|
|
"logps/rejected": -598.7999877929688,
|
|
"loss": 2.9333,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": 4.004687309265137,
|
|
"rewards/margins": 14.862500190734863,
|
|
"rewards/rejected": -10.859375,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.22065727699530516,
|
|
"grad_norm": 3357.038768004447,
|
|
"learning_rate": 4.3322900365153886e-07,
|
|
"logits/chosen": 4.203125,
|
|
"logits/rejected": 5.181250095367432,
|
|
"logps/chosen": -484.20001220703125,
|
|
"logps/rejected": -593.7999877929688,
|
|
"loss": 3.0805,
|
|
"rewards/accuracies": 0.6070312261581421,
|
|
"rewards/chosen": 1.4265625476837158,
|
|
"rewards/margins": 13.475000381469727,
|
|
"rewards/rejected": -12.046875,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.22535211267605634,
|
|
"grad_norm": 1615.6004326582888,
|
|
"learning_rate": 4.306207616066771e-07,
|
|
"logits/chosen": 4.115624904632568,
|
|
"logits/rejected": 5.150000095367432,
|
|
"logps/chosen": -424.3999938964844,
|
|
"logps/rejected": -564.0,
|
|
"loss": 2.9211,
|
|
"rewards/accuracies": 0.5859375,
|
|
"rewards/chosen": 2.5992188453674316,
|
|
"rewards/margins": 10.96875,
|
|
"rewards/rejected": -8.364062309265137,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.2300469483568075,
|
|
"grad_norm": 2374.117833928882,
|
|
"learning_rate": 4.280125195618153e-07,
|
|
"logits/chosen": 4.040625095367432,
|
|
"logits/rejected": 5.1875,
|
|
"logps/chosen": -433.0,
|
|
"logps/rejected": -610.4000244140625,
|
|
"loss": 2.6972,
|
|
"rewards/accuracies": 0.600781261920929,
|
|
"rewards/chosen": 3.024218797683716,
|
|
"rewards/margins": 13.893750190734863,
|
|
"rewards/rejected": -10.871874809265137,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.2347417840375587,
|
|
"grad_norm": 2756.7174367124217,
|
|
"learning_rate": 4.2540427751695357e-07,
|
|
"logits/chosen": 4.110937595367432,
|
|
"logits/rejected": 5.09375,
|
|
"logps/chosen": -441.0,
|
|
"logps/rejected": -560.2000122070312,
|
|
"loss": 3.7149,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": 1.8986327648162842,
|
|
"rewards/margins": 11.050000190734863,
|
|
"rewards/rejected": -9.146875381469727,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.23943661971830985,
|
|
"grad_norm": 2171.990857829901,
|
|
"learning_rate": 4.227960354720918e-07,
|
|
"logits/chosen": 4.071875095367432,
|
|
"logits/rejected": 5.387499809265137,
|
|
"logps/chosen": -427.0,
|
|
"logps/rejected": -622.7999877929688,
|
|
"loss": 2.8075,
|
|
"rewards/accuracies": 0.586718738079071,
|
|
"rewards/chosen": 2.0238280296325684,
|
|
"rewards/margins": 13.837499618530273,
|
|
"rewards/rejected": -11.793749809265137,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.24413145539906103,
|
|
"grad_norm": 1713.643591107746,
|
|
"learning_rate": 4.2018779342723e-07,
|
|
"logits/chosen": 4.0,
|
|
"logits/rejected": 5.162499904632568,
|
|
"logps/chosen": -445.6000061035156,
|
|
"logps/rejected": -564.0,
|
|
"loss": 3.0985,
|
|
"rewards/accuracies": 0.610156238079071,
|
|
"rewards/chosen": 1.295312523841858,
|
|
"rewards/margins": 12.112500190734863,
|
|
"rewards/rejected": -10.837499618530273,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.24882629107981222,
|
|
"grad_norm": 1886.7514925858072,
|
|
"learning_rate": 4.1757955138236827e-07,
|
|
"logits/chosen": 4.1015625,
|
|
"logits/rejected": 5.287499904632568,
|
|
"logps/chosen": -428.79998779296875,
|
|
"logps/rejected": -591.7999877929688,
|
|
"loss": 2.7949,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": 1.6066405773162842,
|
|
"rewards/margins": 14.925000190734863,
|
|
"rewards/rejected": -13.331250190734863,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.2535211267605634,
|
|
"grad_norm": 2084.6033369952656,
|
|
"learning_rate": 4.149713093375065e-07,
|
|
"logits/chosen": 3.9703125953674316,
|
|
"logits/rejected": 5.037499904632568,
|
|
"logps/chosen": -429.3999938964844,
|
|
"logps/rejected": -551.0,
|
|
"loss": 2.7701,
|
|
"rewards/accuracies": 0.5992187261581421,
|
|
"rewards/chosen": 2.6890625953674316,
|
|
"rewards/margins": 12.78125,
|
|
"rewards/rejected": -10.090624809265137,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.25821596244131456,
|
|
"grad_norm": 2328.7422800051213,
|
|
"learning_rate": 4.123630672926447e-07,
|
|
"logits/chosen": 3.996875047683716,
|
|
"logits/rejected": 5.090624809265137,
|
|
"logps/chosen": -414.0,
|
|
"logps/rejected": -586.7999877929688,
|
|
"loss": 3.1392,
|
|
"rewards/accuracies": 0.59765625,
|
|
"rewards/chosen": 3.2132811546325684,
|
|
"rewards/margins": 13.512499809265137,
|
|
"rewards/rejected": -10.300000190734863,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.26291079812206575,
|
|
"grad_norm": 1889.3095393093702,
|
|
"learning_rate": 4.09754825247783e-07,
|
|
"logits/chosen": 4.126562595367432,
|
|
"logits/rejected": 5.0625,
|
|
"logps/chosen": -435.6000061035156,
|
|
"logps/rejected": -580.2000122070312,
|
|
"loss": 2.6717,
|
|
"rewards/accuracies": 0.62109375,
|
|
"rewards/chosen": 2.010546922683716,
|
|
"rewards/margins": 14.387499809265137,
|
|
"rewards/rejected": -12.365625381469727,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.2676056338028169,
|
|
"grad_norm": 1472.6759173334137,
|
|
"learning_rate": 4.0714658320292123e-07,
|
|
"logits/chosen": 4.068749904632568,
|
|
"logits/rejected": 5.324999809265137,
|
|
"logps/chosen": -437.20001220703125,
|
|
"logps/rejected": -594.4000244140625,
|
|
"loss": 2.6766,
|
|
"rewards/accuracies": 0.6273437738418579,
|
|
"rewards/chosen": 3.1156249046325684,
|
|
"rewards/margins": 15.831250190734863,
|
|
"rewards/rejected": -12.709375381469727,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.27230046948356806,
|
|
"grad_norm": 1762.786704289365,
|
|
"learning_rate": 4.045383411580595e-07,
|
|
"logits/chosen": 3.9046874046325684,
|
|
"logits/rejected": 5.181250095367432,
|
|
"logps/chosen": -419.79998779296875,
|
|
"logps/rejected": -583.2000122070312,
|
|
"loss": 3.215,
|
|
"rewards/accuracies": 0.5914062261581421,
|
|
"rewards/chosen": 2.5667967796325684,
|
|
"rewards/margins": 13.096875190734863,
|
|
"rewards/rejected": -10.53125,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.27699530516431925,
|
|
"grad_norm": 1713.0682783502837,
|
|
"learning_rate": 4.019300991131977e-07,
|
|
"logits/chosen": 3.9000000953674316,
|
|
"logits/rejected": 5.162499904632568,
|
|
"logps/chosen": -422.79998779296875,
|
|
"logps/rejected": -577.0,
|
|
"loss": 2.6953,
|
|
"rewards/accuracies": 0.61328125,
|
|
"rewards/chosen": 2.1937499046325684,
|
|
"rewards/margins": 13.015625,
|
|
"rewards/rejected": -10.828125,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.28169014084507044,
|
|
"grad_norm": 1730.5721044594577,
|
|
"learning_rate": 3.9932185706833594e-07,
|
|
"logits/chosen": 4.110937595367432,
|
|
"logits/rejected": 5.240624904632568,
|
|
"logps/chosen": -444.6000061035156,
|
|
"logps/rejected": -586.2000122070312,
|
|
"loss": 2.7095,
|
|
"rewards/accuracies": 0.617968738079071,
|
|
"rewards/chosen": 2.81640625,
|
|
"rewards/margins": 13.793749809265137,
|
|
"rewards/rejected": -10.981249809265137,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.2863849765258216,
|
|
"grad_norm": 1940.4990480121226,
|
|
"learning_rate": 3.967136150234742e-07,
|
|
"logits/chosen": 4.004687309265137,
|
|
"logits/rejected": 5.103125095367432,
|
|
"logps/chosen": -435.6000061035156,
|
|
"logps/rejected": -579.5999755859375,
|
|
"loss": 2.5753,
|
|
"rewards/accuracies": 0.5953124761581421,
|
|
"rewards/chosen": 3.7203125953674316,
|
|
"rewards/margins": 14.231249809265137,
|
|
"rewards/rejected": -10.518750190734863,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.29107981220657275,
|
|
"grad_norm": 1790.7235872051747,
|
|
"learning_rate": 3.941053729786124e-07,
|
|
"logits/chosen": 3.9078125953674316,
|
|
"logits/rejected": 5.168749809265137,
|
|
"logps/chosen": -439.3999938964844,
|
|
"logps/rejected": -594.0,
|
|
"loss": 2.7942,
|
|
"rewards/accuracies": 0.6421874761581421,
|
|
"rewards/chosen": 3.4906249046325684,
|
|
"rewards/margins": 16.137500762939453,
|
|
"rewards/rejected": -12.653124809265137,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.29577464788732394,
|
|
"grad_norm": 1848.0380319893525,
|
|
"learning_rate": 3.9149713093375064e-07,
|
|
"logits/chosen": 3.9140625,
|
|
"logits/rejected": 5.137499809265137,
|
|
"logps/chosen": -437.3999938964844,
|
|
"logps/rejected": -614.4000244140625,
|
|
"loss": 2.7477,
|
|
"rewards/accuracies": 0.614062488079071,
|
|
"rewards/chosen": 3.538281202316284,
|
|
"rewards/margins": 15.0625,
|
|
"rewards/rejected": -11.503125190734863,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.3004694835680751,
|
|
"grad_norm": 1640.1596179375044,
|
|
"learning_rate": 3.888888888888889e-07,
|
|
"logits/chosen": 3.9156250953674316,
|
|
"logits/rejected": 5.306250095367432,
|
|
"logps/chosen": -422.0,
|
|
"logps/rejected": -626.0,
|
|
"loss": 2.8855,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 3.215625047683716,
|
|
"rewards/margins": 16.875,
|
|
"rewards/rejected": -13.65625,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.3051643192488263,
|
|
"grad_norm": 1769.8875677272242,
|
|
"learning_rate": 3.862806468440271e-07,
|
|
"logits/chosen": 4.115624904632568,
|
|
"logits/rejected": 5.193749904632568,
|
|
"logps/chosen": -405.0,
|
|
"logps/rejected": -547.7999877929688,
|
|
"loss": 2.6623,
|
|
"rewards/accuracies": 0.614062488079071,
|
|
"rewards/chosen": 3.58203125,
|
|
"rewards/margins": 14.181249618530273,
|
|
"rewards/rejected": -10.615625381469727,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.30985915492957744,
|
|
"grad_norm": 1562.0273533034501,
|
|
"learning_rate": 3.8367240479916535e-07,
|
|
"logits/chosen": 3.9781250953674316,
|
|
"logits/rejected": 5.118750095367432,
|
|
"logps/chosen": -424.20001220703125,
|
|
"logps/rejected": -552.7999877929688,
|
|
"loss": 2.87,
|
|
"rewards/accuracies": 0.6195312738418579,
|
|
"rewards/chosen": 3.4957032203674316,
|
|
"rewards/margins": 13.321874618530273,
|
|
"rewards/rejected": -9.837499618530273,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.3145539906103286,
|
|
"grad_norm": 1930.7380264796477,
|
|
"learning_rate": 3.810641627543036e-07,
|
|
"logits/chosen": 4.021874904632568,
|
|
"logits/rejected": 5.209374904632568,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -591.7999877929688,
|
|
"loss": 3.2982,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": 3.2728514671325684,
|
|
"rewards/margins": 13.887499809265137,
|
|
"rewards/rejected": -10.615625381469727,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.3192488262910798,
|
|
"grad_norm": 1900.8359358399823,
|
|
"learning_rate": 3.784559207094418e-07,
|
|
"logits/chosen": 4.087500095367432,
|
|
"logits/rejected": 5.059374809265137,
|
|
"logps/chosen": -453.20001220703125,
|
|
"logps/rejected": -563.7999877929688,
|
|
"loss": 4.1114,
|
|
"rewards/accuracies": 0.5859375,
|
|
"rewards/chosen": 3.41259765625,
|
|
"rewards/margins": 11.371874809265137,
|
|
"rewards/rejected": -7.962500095367432,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.323943661971831,
|
|
"grad_norm": 2436.8582607220233,
|
|
"learning_rate": 3.7584767866458005e-07,
|
|
"logits/chosen": 4.087500095367432,
|
|
"logits/rejected": 5.159375190734863,
|
|
"logps/chosen": -444.0,
|
|
"logps/rejected": -556.4000244140625,
|
|
"loss": 3.1233,
|
|
"rewards/accuracies": 0.620312511920929,
|
|
"rewards/chosen": 4.939062595367432,
|
|
"rewards/margins": 14.134374618530273,
|
|
"rewards/rejected": -9.189062118530273,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.3286384976525822,
|
|
"grad_norm": 2208.5757594729275,
|
|
"learning_rate": 3.732394366197183e-07,
|
|
"logits/chosen": 4.106249809265137,
|
|
"logits/rejected": 5.0625,
|
|
"logps/chosen": -466.79998779296875,
|
|
"logps/rejected": -565.5999755859375,
|
|
"loss": 3.2533,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 4.764062404632568,
|
|
"rewards/margins": 14.449999809265137,
|
|
"rewards/rejected": -9.693750381469727,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.3333333333333333,
|
|
"grad_norm": 2866.3464531079444,
|
|
"learning_rate": 3.706311945748565e-07,
|
|
"logits/chosen": 4.098437309265137,
|
|
"logits/rejected": 5.128125190734863,
|
|
"logps/chosen": -481.20001220703125,
|
|
"logps/rejected": -569.4000244140625,
|
|
"loss": 3.8406,
|
|
"rewards/accuracies": 0.6070312261581421,
|
|
"rewards/chosen": 3.267382860183716,
|
|
"rewards/margins": 12.774999618530273,
|
|
"rewards/rejected": -9.512499809265137,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.3380281690140845,
|
|
"grad_norm": 1627.9810240553547,
|
|
"learning_rate": 3.6802295252999476e-07,
|
|
"logits/chosen": 4.175000190734863,
|
|
"logits/rejected": 5.234375,
|
|
"logps/chosen": -450.79998779296875,
|
|
"logps/rejected": -619.4000244140625,
|
|
"loss": 3.3919,
|
|
"rewards/accuracies": 0.6070312261581421,
|
|
"rewards/chosen": 3.1380372047424316,
|
|
"rewards/margins": 15.462499618530273,
|
|
"rewards/rejected": -12.328125,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.3427230046948357,
|
|
"grad_norm": 2315.181300839969,
|
|
"learning_rate": 3.65414710485133e-07,
|
|
"logits/chosen": 3.926562547683716,
|
|
"logits/rejected": 4.893750190734863,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -549.0,
|
|
"loss": 3.5519,
|
|
"rewards/accuracies": 0.6234375238418579,
|
|
"rewards/chosen": 2.813281297683716,
|
|
"rewards/margins": 12.618749618530273,
|
|
"rewards/rejected": -9.803125381469727,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.3474178403755869,
|
|
"grad_norm": 1722.1822621307153,
|
|
"learning_rate": 3.6280646844027127e-07,
|
|
"logits/chosen": 3.9625000953674316,
|
|
"logits/rejected": 5.284375190734863,
|
|
"logps/chosen": -424.79998779296875,
|
|
"logps/rejected": -631.5999755859375,
|
|
"loss": 2.9033,
|
|
"rewards/accuracies": 0.62890625,
|
|
"rewards/chosen": 3.5289063453674316,
|
|
"rewards/margins": 15.399999618530273,
|
|
"rewards/rejected": -11.868749618530273,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.352112676056338,
|
|
"grad_norm": 2014.8092898664677,
|
|
"learning_rate": 3.6019822639540947e-07,
|
|
"logits/chosen": 4.0234375,
|
|
"logits/rejected": 5.028124809265137,
|
|
"logps/chosen": -455.3999938964844,
|
|
"logps/rejected": -554.7999877929688,
|
|
"loss": 3.0796,
|
|
"rewards/accuracies": 0.616406261920929,
|
|
"rewards/chosen": 3.3101563453674316,
|
|
"rewards/margins": 12.381250381469727,
|
|
"rewards/rejected": -9.0625,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.3568075117370892,
|
|
"grad_norm": 1886.7947532933256,
|
|
"learning_rate": 3.575899843505477e-07,
|
|
"logits/chosen": 4.060937404632568,
|
|
"logits/rejected": 5.09375,
|
|
"logps/chosen": -434.6000061035156,
|
|
"logps/rejected": -542.4000244140625,
|
|
"loss": 3.8373,
|
|
"rewards/accuracies": 0.635937511920929,
|
|
"rewards/chosen": 3.0835938453674316,
|
|
"rewards/margins": 13.337499618530273,
|
|
"rewards/rejected": -10.274999618530273,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.3615023474178404,
|
|
"grad_norm": 4546.668546265293,
|
|
"learning_rate": 3.5498174230568597e-07,
|
|
"logits/chosen": 3.9828124046325684,
|
|
"logits/rejected": 5.209374904632568,
|
|
"logps/chosen": -430.6000061035156,
|
|
"logps/rejected": -585.0,
|
|
"loss": 3.0535,
|
|
"rewards/accuracies": 0.61328125,
|
|
"rewards/chosen": 3.3265624046325684,
|
|
"rewards/margins": 14.149999618530273,
|
|
"rewards/rejected": -10.831250190734863,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.36619718309859156,
|
|
"grad_norm": 1812.8335753452184,
|
|
"learning_rate": 3.5237350026082417e-07,
|
|
"logits/chosen": 4.035937309265137,
|
|
"logits/rejected": 4.90625,
|
|
"logps/chosen": -440.79998779296875,
|
|
"logps/rejected": -531.0,
|
|
"loss": 3.0636,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": 2.8387694358825684,
|
|
"rewards/margins": 11.481249809265137,
|
|
"rewards/rejected": -8.634374618530273,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.37089201877934275,
|
|
"grad_norm": 1368.5606341791402,
|
|
"learning_rate": 3.497652582159624e-07,
|
|
"logits/chosen": 3.957812547683716,
|
|
"logits/rejected": 5.162499904632568,
|
|
"logps/chosen": -441.79998779296875,
|
|
"logps/rejected": -599.4000244140625,
|
|
"loss": 2.9049,
|
|
"rewards/accuracies": 0.610156238079071,
|
|
"rewards/chosen": 2.8446288108825684,
|
|
"rewards/margins": 12.868749618530273,
|
|
"rewards/rejected": -10.018750190734863,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.3755868544600939,
|
|
"grad_norm": 4830.950750187237,
|
|
"learning_rate": 3.471570161711007e-07,
|
|
"logits/chosen": 4.189062595367432,
|
|
"logits/rejected": 5.287499904632568,
|
|
"logps/chosen": -475.79998779296875,
|
|
"logps/rejected": -606.7999877929688,
|
|
"loss": 3.4453,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 1.602294921875,
|
|
"rewards/margins": 13.600000381469727,
|
|
"rewards/rejected": -11.984375,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.38028169014084506,
|
|
"grad_norm": 1848.9101247311496,
|
|
"learning_rate": 3.445487741262389e-07,
|
|
"logits/chosen": 4.074999809265137,
|
|
"logits/rejected": 5.290625095367432,
|
|
"logps/chosen": -426.6000061035156,
|
|
"logps/rejected": -574.2000122070312,
|
|
"loss": 4.2489,
|
|
"rewards/accuracies": 0.6265624761581421,
|
|
"rewards/chosen": 3.582812547683716,
|
|
"rewards/margins": 12.231249809265137,
|
|
"rewards/rejected": -8.652734756469727,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.38497652582159625,
|
|
"grad_norm": 1419.8849481704503,
|
|
"learning_rate": 3.4194053208137713e-07,
|
|
"logits/chosen": 4.021874904632568,
|
|
"logits/rejected": 5.068749904632568,
|
|
"logps/chosen": -437.3999938964844,
|
|
"logps/rejected": -547.5999755859375,
|
|
"loss": 2.8147,
|
|
"rewards/accuracies": 0.633593738079071,
|
|
"rewards/chosen": 4.8046875,
|
|
"rewards/margins": 13.737500190734863,
|
|
"rewards/rejected": -8.940625190734863,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.38967136150234744,
|
|
"grad_norm": 1882.6539648844353,
|
|
"learning_rate": 3.393322900365154e-07,
|
|
"logits/chosen": 3.9703125953674316,
|
|
"logits/rejected": 5.050000190734863,
|
|
"logps/chosen": -411.0,
|
|
"logps/rejected": -556.0,
|
|
"loss": 3.1191,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": 3.2945313453674316,
|
|
"rewards/margins": 13.318750381469727,
|
|
"rewards/rejected": -10.028124809265137,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.39436619718309857,
|
|
"grad_norm": 2070.1248169949204,
|
|
"learning_rate": 3.367240479916536e-07,
|
|
"logits/chosen": 4.1171875,
|
|
"logits/rejected": 5.215624809265137,
|
|
"logps/chosen": -467.3999938964844,
|
|
"logps/rejected": -597.5999755859375,
|
|
"loss": 3.2619,
|
|
"rewards/accuracies": 0.6382812261581421,
|
|
"rewards/chosen": 3.418164014816284,
|
|
"rewards/margins": 14.506250381469727,
|
|
"rewards/rejected": -11.109375,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.39906103286384975,
|
|
"grad_norm": 1617.7116761735354,
|
|
"learning_rate": 3.3411580594679184e-07,
|
|
"logits/chosen": 4.114062309265137,
|
|
"logits/rejected": 5.309374809265137,
|
|
"logps/chosen": -425.3999938964844,
|
|
"logps/rejected": -582.2000122070312,
|
|
"loss": 2.8404,
|
|
"rewards/accuracies": 0.633593738079071,
|
|
"rewards/chosen": 3.2992186546325684,
|
|
"rewards/margins": 14.503125190734863,
|
|
"rewards/rejected": -11.21875,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.40375586854460094,
|
|
"grad_norm": 2300.7299528679546,
|
|
"learning_rate": 3.315075639019301e-07,
|
|
"logits/chosen": 4.050000190734863,
|
|
"logits/rejected": 5.300000190734863,
|
|
"logps/chosen": -417.6000061035156,
|
|
"logps/rejected": -570.7999877929688,
|
|
"loss": 3.0707,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 2.8812499046325684,
|
|
"rewards/margins": 12.46875,
|
|
"rewards/rejected": -9.587499618530273,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.4084507042253521,
|
|
"grad_norm": 1891.8240314635384,
|
|
"learning_rate": 3.288993218570683e-07,
|
|
"logits/chosen": 3.981250047683716,
|
|
"logits/rejected": 4.96875,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -552.7999877929688,
|
|
"loss": 2.8769,
|
|
"rewards/accuracies": 0.617968738079071,
|
|
"rewards/chosen": 4.112500190734863,
|
|
"rewards/margins": 13.456250190734863,
|
|
"rewards/rejected": -9.356249809265137,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.4131455399061033,
|
|
"grad_norm": 1750.3601164563806,
|
|
"learning_rate": 3.2629107981220654e-07,
|
|
"logits/chosen": 4.03125,
|
|
"logits/rejected": 5.112500190734863,
|
|
"logps/chosen": -447.79998779296875,
|
|
"logps/rejected": -575.2000122070312,
|
|
"loss": 3.1016,
|
|
"rewards/accuracies": 0.6460937261581421,
|
|
"rewards/chosen": 3.93359375,
|
|
"rewards/margins": 14.449999809265137,
|
|
"rewards/rejected": -10.503125190734863,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.41784037558685444,
|
|
"grad_norm": 1825.611957912023,
|
|
"learning_rate": 3.236828377673448e-07,
|
|
"logits/chosen": 4.010937690734863,
|
|
"logits/rejected": 5.15625,
|
|
"logps/chosen": -470.0,
|
|
"logps/rejected": -566.4000244140625,
|
|
"loss": 3.4575,
|
|
"rewards/accuracies": 0.63671875,
|
|
"rewards/chosen": 4.103515625,
|
|
"rewards/margins": 12.509374618530273,
|
|
"rewards/rejected": -8.396875381469727,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.4225352112676056,
|
|
"grad_norm": 1826.3637978934096,
|
|
"learning_rate": 3.2107459572248305e-07,
|
|
"logits/chosen": 3.934375047683716,
|
|
"logits/rejected": 5.209374904632568,
|
|
"logps/chosen": -446.6000061035156,
|
|
"logps/rejected": -578.4000244140625,
|
|
"loss": 2.9483,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 4.361523628234863,
|
|
"rewards/margins": 12.262499809265137,
|
|
"rewards/rejected": -7.890625,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.4272300469483568,
|
|
"grad_norm": 2028.4456104230878,
|
|
"learning_rate": 3.1846635367762125e-07,
|
|
"logits/chosen": 3.9546875953674316,
|
|
"logits/rejected": 5.181250095367432,
|
|
"logps/chosen": -447.0,
|
|
"logps/rejected": -608.5999755859375,
|
|
"loss": 3.4513,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": 3.2289061546325684,
|
|
"rewards/margins": 13.318750381469727,
|
|
"rewards/rejected": -10.084375381469727,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.431924882629108,
|
|
"grad_norm": 1819.1422145382367,
|
|
"learning_rate": 3.158581116327595e-07,
|
|
"logits/chosen": 4.020312309265137,
|
|
"logits/rejected": 5.393750190734863,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -639.7999877929688,
|
|
"loss": 2.585,
|
|
"rewards/accuracies": 0.6273437738418579,
|
|
"rewards/chosen": 3.9859375953674316,
|
|
"rewards/margins": 15.518750190734863,
|
|
"rewards/rejected": -11.534375190734863,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.43661971830985913,
|
|
"grad_norm": 1786.7263139840866,
|
|
"learning_rate": 3.1324986958789775e-07,
|
|
"logits/chosen": 3.840625047683716,
|
|
"logits/rejected": 4.993750095367432,
|
|
"logps/chosen": -416.3999938964844,
|
|
"logps/rejected": -551.5999755859375,
|
|
"loss": 3.0033,
|
|
"rewards/accuracies": 0.6000000238418579,
|
|
"rewards/chosen": 3.104199171066284,
|
|
"rewards/margins": 13.09375,
|
|
"rewards/rejected": -9.981249809265137,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.4413145539906103,
|
|
"grad_norm": 2714.440135036024,
|
|
"learning_rate": 3.1064162754303595e-07,
|
|
"logits/chosen": 4.099999904632568,
|
|
"logits/rejected": 5.356249809265137,
|
|
"logps/chosen": -444.6000061035156,
|
|
"logps/rejected": -616.4000244140625,
|
|
"loss": 3.4449,
|
|
"rewards/accuracies": 0.6148437261581421,
|
|
"rewards/chosen": 2.635546922683716,
|
|
"rewards/margins": 13.625,
|
|
"rewards/rejected": -10.987500190734863,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.4460093896713615,
|
|
"grad_norm": 1807.1582481524367,
|
|
"learning_rate": 3.080333854981742e-07,
|
|
"logits/chosen": 4.051562309265137,
|
|
"logits/rejected": 5.050000190734863,
|
|
"logps/chosen": -434.20001220703125,
|
|
"logps/rejected": -546.7999877929688,
|
|
"loss": 2.8828,
|
|
"rewards/accuracies": 0.6234375238418579,
|
|
"rewards/chosen": 3.1070313453674316,
|
|
"rewards/margins": 12.774999618530273,
|
|
"rewards/rejected": -9.65625,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 0.4507042253521127,
|
|
"grad_norm": 2588.7700440358844,
|
|
"learning_rate": 3.0542514345331246e-07,
|
|
"logits/chosen": 4.103125095367432,
|
|
"logits/rejected": 5.1875,
|
|
"logps/chosen": -444.3999938964844,
|
|
"logps/rejected": -640.0,
|
|
"loss": 3.6158,
|
|
"rewards/accuracies": 0.600781261920929,
|
|
"rewards/chosen": 3.1015625,
|
|
"rewards/margins": 14.515625,
|
|
"rewards/rejected": -11.396875381469727,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 0.45539906103286387,
|
|
"grad_norm": 2107.666597802804,
|
|
"learning_rate": 3.0281690140845066e-07,
|
|
"logits/chosen": 3.934375047683716,
|
|
"logits/rejected": 5.109375,
|
|
"logps/chosen": -465.20001220703125,
|
|
"logps/rejected": -595.2000122070312,
|
|
"loss": 3.0307,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 2.866406202316284,
|
|
"rewards/margins": 13.887499809265137,
|
|
"rewards/rejected": -11.015625,
|
|
"step": 970
|
|
},
|
|
{
|
|
"epoch": 0.460093896713615,
|
|
"grad_norm": 2251.9460059907237,
|
|
"learning_rate": 3.002086593635889e-07,
|
|
"logits/chosen": 4.098437309265137,
|
|
"logits/rejected": 5.315625190734863,
|
|
"logps/chosen": -435.79998779296875,
|
|
"logps/rejected": -611.7999877929688,
|
|
"loss": 2.8283,
|
|
"rewards/accuracies": 0.620312511920929,
|
|
"rewards/chosen": 3.3265624046325684,
|
|
"rewards/margins": 14.59375,
|
|
"rewards/rejected": -11.265625,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 0.4647887323943662,
|
|
"grad_norm": 1477.0563674383548,
|
|
"learning_rate": 2.9760041731872716e-07,
|
|
"logits/chosen": 3.979687452316284,
|
|
"logits/rejected": 5.03125,
|
|
"logps/chosen": -399.3999938964844,
|
|
"logps/rejected": -570.4000244140625,
|
|
"loss": 2.557,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 3.7249999046325684,
|
|
"rewards/margins": 13.899999618530273,
|
|
"rewards/rejected": -10.193750381469727,
|
|
"step": 990
|
|
},
|
|
{
|
|
"epoch": 0.4694835680751174,
|
|
"grad_norm": 1937.9293732492592,
|
|
"learning_rate": 2.9499217527386536e-07,
|
|
"logits/chosen": 4.035937309265137,
|
|
"logits/rejected": 5.221875190734863,
|
|
"logps/chosen": -422.6000061035156,
|
|
"logps/rejected": -585.7999877929688,
|
|
"loss": 2.9096,
|
|
"rewards/accuracies": 0.58203125,
|
|
"rewards/chosen": 2.9000000953674316,
|
|
"rewards/margins": 13.006250381469727,
|
|
"rewards/rejected": -10.096875190734863,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 0.47417840375586856,
|
|
"grad_norm": 1573.4620095148887,
|
|
"learning_rate": 2.923839332290036e-07,
|
|
"logits/chosen": 3.909374952316284,
|
|
"logits/rejected": 5.065625190734863,
|
|
"logps/chosen": -417.6000061035156,
|
|
"logps/rejected": -607.4000244140625,
|
|
"loss": 3.4428,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 2.758984327316284,
|
|
"rewards/margins": 14.931249618530273,
|
|
"rewards/rejected": -12.15625,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"epoch": 0.4788732394366197,
|
|
"grad_norm": 2083.5554455522847,
|
|
"learning_rate": 2.8977569118414187e-07,
|
|
"logits/chosen": 4.114062309265137,
|
|
"logits/rejected": 5.340624809265137,
|
|
"logps/chosen": -418.0,
|
|
"logps/rejected": -573.7999877929688,
|
|
"loss": 2.8386,
|
|
"rewards/accuracies": 0.6234375238418579,
|
|
"rewards/chosen": 3.9820313453674316,
|
|
"rewards/margins": 14.787500381469727,
|
|
"rewards/rejected": -10.8125,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 0.4835680751173709,
|
|
"grad_norm": 1676.9346098064448,
|
|
"learning_rate": 2.8716744913928007e-07,
|
|
"logits/chosen": 3.96875,
|
|
"logits/rejected": 5.040625095367432,
|
|
"logps/chosen": -433.6000061035156,
|
|
"logps/rejected": -556.7999877929688,
|
|
"loss": 2.8414,
|
|
"rewards/accuracies": 0.617968738079071,
|
|
"rewards/chosen": 4.900000095367432,
|
|
"rewards/margins": 13.765625,
|
|
"rewards/rejected": -8.865625381469727,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"epoch": 0.48826291079812206,
|
|
"grad_norm": 1409.2418189442037,
|
|
"learning_rate": 2.845592070944183e-07,
|
|
"logits/chosen": 4.037499904632568,
|
|
"logits/rejected": 5.353125095367432,
|
|
"logps/chosen": -431.79998779296875,
|
|
"logps/rejected": -610.4000244140625,
|
|
"loss": 2.8352,
|
|
"rewards/accuracies": 0.6460937261581421,
|
|
"rewards/chosen": 4.779687404632568,
|
|
"rewards/margins": 14.9375,
|
|
"rewards/rejected": -10.159375190734863,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 0.49295774647887325,
|
|
"grad_norm": 1848.9212475116044,
|
|
"learning_rate": 2.819509650495566e-07,
|
|
"logits/chosen": 4.134375095367432,
|
|
"logits/rejected": 5.206250190734863,
|
|
"logps/chosen": -429.3999938964844,
|
|
"logps/rejected": -564.7999877929688,
|
|
"loss": 3.2723,
|
|
"rewards/accuracies": 0.608593761920929,
|
|
"rewards/chosen": 3.3011717796325684,
|
|
"rewards/margins": 12.778124809265137,
|
|
"rewards/rejected": -9.481249809265137,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"epoch": 0.49765258215962443,
|
|
"grad_norm": 1711.283712422891,
|
|
"learning_rate": 2.7934272300469483e-07,
|
|
"logits/chosen": 4.114062309265137,
|
|
"logits/rejected": 5.203125,
|
|
"logps/chosen": -451.3999938964844,
|
|
"logps/rejected": -574.2000122070312,
|
|
"loss": 2.9536,
|
|
"rewards/accuracies": 0.6117187738418579,
|
|
"rewards/chosen": 3.395312547683716,
|
|
"rewards/margins": 14.265625,
|
|
"rewards/rejected": -10.881250381469727,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 0.5023474178403756,
|
|
"grad_norm": 1679.434619247341,
|
|
"learning_rate": 2.7673448095983303e-07,
|
|
"logits/chosen": 4.1796875,
|
|
"logits/rejected": 5.184374809265137,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -573.7999877929688,
|
|
"loss": 3.1645,
|
|
"rewards/accuracies": 0.6078125238418579,
|
|
"rewards/chosen": 2.4749999046325684,
|
|
"rewards/margins": 14.15625,
|
|
"rewards/rejected": -11.681249618530273,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"epoch": 0.5070422535211268,
|
|
"grad_norm": 1866.1794491034852,
|
|
"learning_rate": 2.741262389149713e-07,
|
|
"logits/chosen": 4.084374904632568,
|
|
"logits/rejected": 5.206250190734863,
|
|
"logps/chosen": -441.3999938964844,
|
|
"logps/rejected": -590.4000244140625,
|
|
"loss": 3.1803,
|
|
"rewards/accuracies": 0.624218761920929,
|
|
"rewards/chosen": 2.2945313453674316,
|
|
"rewards/margins": 15.168749809265137,
|
|
"rewards/rejected": -12.884374618530273,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 0.5117370892018779,
|
|
"grad_norm": 2496.8646392621235,
|
|
"learning_rate": 2.7151799687010953e-07,
|
|
"logits/chosen": 3.9468750953674316,
|
|
"logits/rejected": 5.128125190734863,
|
|
"logps/chosen": -447.79998779296875,
|
|
"logps/rejected": -599.5999755859375,
|
|
"loss": 3.3709,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 2.8539061546325684,
|
|
"rewards/margins": 14.518750190734863,
|
|
"rewards/rejected": -11.659375190734863,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"epoch": 0.5164319248826291,
|
|
"grad_norm": 1606.604989817021,
|
|
"learning_rate": 2.6890975482524773e-07,
|
|
"logits/chosen": 4.043749809265137,
|
|
"logits/rejected": 5.096875190734863,
|
|
"logps/chosen": -457.6000061035156,
|
|
"logps/rejected": -594.0,
|
|
"loss": 3.0805,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": 2.3257813453674316,
|
|
"rewards/margins": 14.96875,
|
|
"rewards/rejected": -12.637499809265137,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 0.5211267605633803,
|
|
"grad_norm": 2855.862027863952,
|
|
"learning_rate": 2.66301512780386e-07,
|
|
"logits/chosen": 3.918750047683716,
|
|
"logits/rejected": 5.112500190734863,
|
|
"logps/chosen": -418.3999938964844,
|
|
"logps/rejected": -579.2000122070312,
|
|
"loss": 3.1276,
|
|
"rewards/accuracies": 0.6265624761581421,
|
|
"rewards/chosen": 3.2515625953674316,
|
|
"rewards/margins": 14.0,
|
|
"rewards/rejected": -10.762499809265137,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"epoch": 0.5258215962441315,
|
|
"grad_norm": 1602.896120340799,
|
|
"learning_rate": 2.6369327073552424e-07,
|
|
"logits/chosen": 4.090624809265137,
|
|
"logits/rejected": 5.171875,
|
|
"logps/chosen": -423.6000061035156,
|
|
"logps/rejected": -549.5999755859375,
|
|
"loss": 3.5014,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": 3.676562547683716,
|
|
"rewards/margins": 13.268750190734863,
|
|
"rewards/rejected": -9.5869140625,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 0.5305164319248826,
|
|
"grad_norm": 2319.2992369948406,
|
|
"learning_rate": 2.6108502869066244e-07,
|
|
"logits/chosen": 3.965625047683716,
|
|
"logits/rejected": 5.259375095367432,
|
|
"logps/chosen": -452.6000061035156,
|
|
"logps/rejected": -612.7999877929688,
|
|
"loss": 3.1899,
|
|
"rewards/accuracies": 0.6382812261581421,
|
|
"rewards/chosen": 3.6015625,
|
|
"rewards/margins": 14.6875,
|
|
"rewards/rejected": -11.065625190734863,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"epoch": 0.5352112676056338,
|
|
"grad_norm": 1662.7896468029244,
|
|
"learning_rate": 2.584767866458007e-07,
|
|
"logits/chosen": 4.0234375,
|
|
"logits/rejected": 5.221875190734863,
|
|
"logps/chosen": -432.3999938964844,
|
|
"logps/rejected": -568.2000122070312,
|
|
"loss": 2.5352,
|
|
"rewards/accuracies": 0.6273437738418579,
|
|
"rewards/chosen": 3.9828124046325684,
|
|
"rewards/margins": 14.274999618530273,
|
|
"rewards/rejected": -10.309374809265137,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 0.539906103286385,
|
|
"grad_norm": 1981.6782754183025,
|
|
"learning_rate": 2.5586854460093895e-07,
|
|
"logits/chosen": 4.043749809265137,
|
|
"logits/rejected": 5.3125,
|
|
"logps/chosen": -449.3999938964844,
|
|
"logps/rejected": -598.0,
|
|
"loss": 3.0639,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": 2.849414110183716,
|
|
"rewards/margins": 14.081250190734863,
|
|
"rewards/rejected": -11.243749618530273,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"epoch": 0.5446009389671361,
|
|
"grad_norm": 1877.3305878460353,
|
|
"learning_rate": 2.5326030255607715e-07,
|
|
"logits/chosen": 4.037499904632568,
|
|
"logits/rejected": 5.109375,
|
|
"logps/chosen": -443.3999938964844,
|
|
"logps/rejected": -576.7999877929688,
|
|
"loss": 3.4593,
|
|
"rewards/accuracies": 0.614062488079071,
|
|
"rewards/chosen": 3.520312547683716,
|
|
"rewards/margins": 14.5,
|
|
"rewards/rejected": -10.987500190734863,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 0.5492957746478874,
|
|
"grad_norm": 1455.786451484247,
|
|
"learning_rate": 2.506520605112154e-07,
|
|
"logits/chosen": 4.057812690734863,
|
|
"logits/rejected": 5.171875,
|
|
"logps/chosen": -432.0,
|
|
"logps/rejected": -581.7999877929688,
|
|
"loss": 2.9562,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": 4.205468654632568,
|
|
"rewards/margins": 13.737500190734863,
|
|
"rewards/rejected": -9.540624618530273,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"epoch": 0.5539906103286385,
|
|
"grad_norm": 2112.8193697249417,
|
|
"learning_rate": 2.4804381846635365e-07,
|
|
"logits/chosen": 4.178124904632568,
|
|
"logits/rejected": 5.259375095367432,
|
|
"logps/chosen": -459.79998779296875,
|
|
"logps/rejected": -578.0,
|
|
"loss": 2.9921,
|
|
"rewards/accuracies": 0.639843761920929,
|
|
"rewards/chosen": 4.014062404632568,
|
|
"rewards/margins": 15.056249618530273,
|
|
"rewards/rejected": -11.040624618530273,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 0.5586854460093896,
|
|
"grad_norm": 1807.549690641133,
|
|
"learning_rate": 2.454355764214919e-07,
|
|
"logits/chosen": 4.035937309265137,
|
|
"logits/rejected": 5.215624809265137,
|
|
"logps/chosen": -423.3999938964844,
|
|
"logps/rejected": -606.2000122070312,
|
|
"loss": 3.1127,
|
|
"rewards/accuracies": 0.610156238079071,
|
|
"rewards/chosen": 2.383593797683716,
|
|
"rewards/margins": 14.34375,
|
|
"rewards/rejected": -11.962499618530273,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"epoch": 0.5633802816901409,
|
|
"grad_norm": 1810.0629812144884,
|
|
"learning_rate": 2.4282733437663016e-07,
|
|
"logits/chosen": 3.9671874046325684,
|
|
"logits/rejected": 5.099999904632568,
|
|
"logps/chosen": -428.3999938964844,
|
|
"logps/rejected": -555.0,
|
|
"loss": 2.9406,
|
|
"rewards/accuracies": 0.608593761920929,
|
|
"rewards/chosen": 3.659374952316284,
|
|
"rewards/margins": 12.84375,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 0.568075117370892,
|
|
"grad_norm": 1671.4694973999788,
|
|
"learning_rate": 2.4021909233176836e-07,
|
|
"logits/chosen": 4.050000190734863,
|
|
"logits/rejected": 5.234375,
|
|
"logps/chosen": -418.79998779296875,
|
|
"logps/rejected": -589.2000122070312,
|
|
"loss": 3.1084,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": 3.7925782203674316,
|
|
"rewards/margins": 13.984375,
|
|
"rewards/rejected": -10.196874618530273,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"epoch": 0.5727699530516432,
|
|
"grad_norm": 1580.617734768072,
|
|
"learning_rate": 2.376108502869066e-07,
|
|
"logits/chosen": 4.0625,
|
|
"logits/rejected": 5.362500190734863,
|
|
"logps/chosen": -446.3999938964844,
|
|
"logps/rejected": -611.2000122070312,
|
|
"loss": 3.3963,
|
|
"rewards/accuracies": 0.6031249761581421,
|
|
"rewards/chosen": 4.170312404632568,
|
|
"rewards/margins": 14.542187690734863,
|
|
"rewards/rejected": -10.365625381469727,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 0.5774647887323944,
|
|
"grad_norm": 3811.950746334152,
|
|
"learning_rate": 2.3500260824204484e-07,
|
|
"logits/chosen": 4.090624809265137,
|
|
"logits/rejected": 5.1875,
|
|
"logps/chosen": -472.0,
|
|
"logps/rejected": -615.4000244140625,
|
|
"loss": 3.051,
|
|
"rewards/accuracies": 0.6226562261581421,
|
|
"rewards/chosen": 3.55859375,
|
|
"rewards/margins": 14.418749809265137,
|
|
"rewards/rejected": -10.884374618530273,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"epoch": 0.5821596244131455,
|
|
"grad_norm": 1586.6778779228716,
|
|
"learning_rate": 2.323943661971831e-07,
|
|
"logits/chosen": 3.9984374046325684,
|
|
"logits/rejected": 5.196875095367432,
|
|
"logps/chosen": -431.3999938964844,
|
|
"logps/rejected": -573.5999755859375,
|
|
"loss": 3.154,
|
|
"rewards/accuracies": 0.624218761920929,
|
|
"rewards/chosen": 4.482812404632568,
|
|
"rewards/margins": 14.112500190734863,
|
|
"rewards/rejected": -9.634374618530273,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 0.5868544600938967,
|
|
"grad_norm": 1870.5950562357916,
|
|
"learning_rate": 2.2978612415232132e-07,
|
|
"logits/chosen": 4.110937595367432,
|
|
"logits/rejected": 5.318749904632568,
|
|
"logps/chosen": -460.0,
|
|
"logps/rejected": -605.7999877929688,
|
|
"loss": 3.1517,
|
|
"rewards/accuracies": 0.629687488079071,
|
|
"rewards/chosen": 4.379687309265137,
|
|
"rewards/margins": 13.899999618530273,
|
|
"rewards/rejected": -9.521875381469727,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"epoch": 0.5915492957746479,
|
|
"grad_norm": 1409.5579342326466,
|
|
"learning_rate": 2.2717788210745957e-07,
|
|
"logits/chosen": 4.037499904632568,
|
|
"logits/rejected": 5.181250095367432,
|
|
"logps/chosen": -436.0,
|
|
"logps/rejected": -598.5999755859375,
|
|
"loss": 2.8344,
|
|
"rewards/accuracies": 0.617968738079071,
|
|
"rewards/chosen": 4.7265625,
|
|
"rewards/margins": 14.931249618530273,
|
|
"rewards/rejected": -10.203125,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 0.596244131455399,
|
|
"grad_norm": 3354.1087339713085,
|
|
"learning_rate": 2.245696400625978e-07,
|
|
"logits/chosen": 4.096875190734863,
|
|
"logits/rejected": 5.068749904632568,
|
|
"logps/chosen": -461.0,
|
|
"logps/rejected": -607.7999877929688,
|
|
"loss": 3.5973,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": 3.6005859375,
|
|
"rewards/margins": 12.368749618530273,
|
|
"rewards/rejected": -8.759374618530273,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"epoch": 0.6009389671361502,
|
|
"grad_norm": 1777.5134258310454,
|
|
"learning_rate": 2.2196139801773602e-07,
|
|
"logits/chosen": 3.9515624046325684,
|
|
"logits/rejected": 5.137499809265137,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -549.5999755859375,
|
|
"loss": 2.7861,
|
|
"rewards/accuracies": 0.6109374761581421,
|
|
"rewards/chosen": 3.471874952316284,
|
|
"rewards/margins": 13.028124809265137,
|
|
"rewards/rejected": -9.550000190734863,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 0.6056338028169014,
|
|
"grad_norm": 2291.757926124569,
|
|
"learning_rate": 2.1935315597287428e-07,
|
|
"logits/chosen": 3.989062547683716,
|
|
"logits/rejected": 5.021874904632568,
|
|
"logps/chosen": -442.0,
|
|
"logps/rejected": -555.4000244140625,
|
|
"loss": 3.1403,
|
|
"rewards/accuracies": 0.614062488079071,
|
|
"rewards/chosen": 3.4268555641174316,
|
|
"rewards/margins": 13.393750190734863,
|
|
"rewards/rejected": -9.965624809265137,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"epoch": 0.6103286384976526,
|
|
"grad_norm": 2602.1792939739025,
|
|
"learning_rate": 2.167449139280125e-07,
|
|
"logits/chosen": 4.037499904632568,
|
|
"logits/rejected": 5.096875190734863,
|
|
"logps/chosen": -443.79998779296875,
|
|
"logps/rejected": -593.2000122070312,
|
|
"loss": 2.8462,
|
|
"rewards/accuracies": 0.6226562261581421,
|
|
"rewards/chosen": 4.567968845367432,
|
|
"rewards/margins": 14.824999809265137,
|
|
"rewards/rejected": -10.262499809265137,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 0.6150234741784038,
|
|
"grad_norm": 1618.5126673916352,
|
|
"learning_rate": 2.1413667188315073e-07,
|
|
"logits/chosen": 4.051562309265137,
|
|
"logits/rejected": 5.25,
|
|
"logps/chosen": -421.3999938964844,
|
|
"logps/rejected": -551.2000122070312,
|
|
"loss": 3.3124,
|
|
"rewards/accuracies": 0.617968738079071,
|
|
"rewards/chosen": 3.4775390625,
|
|
"rewards/margins": 13.987500190734863,
|
|
"rewards/rejected": -10.5,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"epoch": 0.6197183098591549,
|
|
"grad_norm": 1906.5809887514088,
|
|
"learning_rate": 2.1152842983828898e-07,
|
|
"logits/chosen": 4.010937690734863,
|
|
"logits/rejected": 4.9375,
|
|
"logps/chosen": -444.79998779296875,
|
|
"logps/rejected": -572.7999877929688,
|
|
"loss": 4.2894,
|
|
"rewards/accuracies": 0.6070312261581421,
|
|
"rewards/chosen": 3.051562547683716,
|
|
"rewards/margins": 12.512499809265137,
|
|
"rewards/rejected": -9.462499618530273,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 0.6244131455399061,
|
|
"grad_norm": 2126.2653210876406,
|
|
"learning_rate": 2.089201877934272e-07,
|
|
"logits/chosen": 4.246874809265137,
|
|
"logits/rejected": 5.199999809265137,
|
|
"logps/chosen": -465.3999938964844,
|
|
"logps/rejected": -640.7999877929688,
|
|
"loss": 3.4108,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": 2.7699217796325684,
|
|
"rewards/margins": 16.100000381469727,
|
|
"rewards/rejected": -13.340624809265137,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"epoch": 0.6291079812206573,
|
|
"grad_norm": 2093.8610542868937,
|
|
"learning_rate": 2.0631194574856543e-07,
|
|
"logits/chosen": 4.0625,
|
|
"logits/rejected": 5.118750095367432,
|
|
"logps/chosen": -426.6000061035156,
|
|
"logps/rejected": -520.4000244140625,
|
|
"loss": 3.1035,
|
|
"rewards/accuracies": 0.608593761920929,
|
|
"rewards/chosen": 3.9671874046325684,
|
|
"rewards/margins": 12.706250190734863,
|
|
"rewards/rejected": -8.731249809265137,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 0.6338028169014085,
|
|
"grad_norm": 2179.7138213708427,
|
|
"learning_rate": 2.0370370370370369e-07,
|
|
"logits/chosen": 4.0234375,
|
|
"logits/rejected": 5.293749809265137,
|
|
"logps/chosen": -438.0,
|
|
"logps/rejected": -620.7999877929688,
|
|
"loss": 3.5285,
|
|
"rewards/accuracies": 0.617968738079071,
|
|
"rewards/chosen": 3.354687452316284,
|
|
"rewards/margins": 14.165624618530273,
|
|
"rewards/rejected": -10.806249618530273,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"epoch": 0.6384976525821596,
|
|
"grad_norm": 1910.5330291483394,
|
|
"learning_rate": 2.010954616588419e-07,
|
|
"logits/chosen": 3.917187452316284,
|
|
"logits/rejected": 4.96875,
|
|
"logps/chosen": -431.20001220703125,
|
|
"logps/rejected": -552.5999755859375,
|
|
"loss": 3.1286,
|
|
"rewards/accuracies": 0.61328125,
|
|
"rewards/chosen": 3.2093749046325684,
|
|
"rewards/margins": 12.899999618530273,
|
|
"rewards/rejected": -9.684374809265137,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 0.6431924882629108,
|
|
"grad_norm": 1686.6058679872501,
|
|
"learning_rate": 1.9848721961398017e-07,
|
|
"logits/chosen": 4.207812309265137,
|
|
"logits/rejected": 5.243750095367432,
|
|
"logps/chosen": -438.6000061035156,
|
|
"logps/rejected": -566.0,
|
|
"loss": 2.9503,
|
|
"rewards/accuracies": 0.6195312738418579,
|
|
"rewards/chosen": 3.5171875953674316,
|
|
"rewards/margins": 13.806249618530273,
|
|
"rewards/rejected": -10.284375190734863,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"epoch": 0.647887323943662,
|
|
"grad_norm": 1971.6567766458825,
|
|
"learning_rate": 1.958789775691184e-07,
|
|
"logits/chosen": 3.926562547683716,
|
|
"logits/rejected": 5.006249904632568,
|
|
"logps/chosen": -418.6000061035156,
|
|
"logps/rejected": -580.0,
|
|
"loss": 2.8484,
|
|
"rewards/accuracies": 0.6015625,
|
|
"rewards/chosen": 2.971874952316284,
|
|
"rewards/margins": 14.162500381469727,
|
|
"rewards/rejected": -11.175000190734863,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 0.6525821596244131,
|
|
"grad_norm": 1487.6044651302464,
|
|
"learning_rate": 1.9327073552425662e-07,
|
|
"logits/chosen": 4.103125095367432,
|
|
"logits/rejected": 5.243750095367432,
|
|
"logps/chosen": -417.20001220703125,
|
|
"logps/rejected": -553.0,
|
|
"loss": 2.4576,
|
|
"rewards/accuracies": 0.616406261920929,
|
|
"rewards/chosen": 3.7261719703674316,
|
|
"rewards/margins": 14.537500381469727,
|
|
"rewards/rejected": -10.815625190734863,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"epoch": 0.6572769953051644,
|
|
"grad_norm": 1840.1508300417279,
|
|
"learning_rate": 1.906624934793949e-07,
|
|
"logits/chosen": 3.878124952316284,
|
|
"logits/rejected": 5.159375190734863,
|
|
"logps/chosen": -440.6000061035156,
|
|
"logps/rejected": -608.5999755859375,
|
|
"loss": 3.3019,
|
|
"rewards/accuracies": 0.6382812261581421,
|
|
"rewards/chosen": 3.57421875,
|
|
"rewards/margins": 15.475000381469727,
|
|
"rewards/rejected": -11.918749809265137,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 0.6619718309859155,
|
|
"grad_norm": 1938.5520262018872,
|
|
"learning_rate": 1.8805425143453312e-07,
|
|
"logits/chosen": 4.037499904632568,
|
|
"logits/rejected": 5.15625,
|
|
"logps/chosen": -416.0,
|
|
"logps/rejected": -639.7999877929688,
|
|
"loss": 2.9842,
|
|
"rewards/accuracies": 0.6195312738418579,
|
|
"rewards/chosen": 3.496875047683716,
|
|
"rewards/margins": 15.5,
|
|
"rewards/rejected": -12.006250381469727,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"epoch": 0.6666666666666666,
|
|
"grad_norm": 1960.9368496724724,
|
|
"learning_rate": 1.8544600938967138e-07,
|
|
"logits/chosen": 3.957812547683716,
|
|
"logits/rejected": 5.209374904632568,
|
|
"logps/chosen": -428.79998779296875,
|
|
"logps/rejected": -555.7999877929688,
|
|
"loss": 2.8504,
|
|
"rewards/accuracies": 0.641406238079071,
|
|
"rewards/chosen": 3.8109374046325684,
|
|
"rewards/margins": 14.568750381469727,
|
|
"rewards/rejected": -10.753125190734863,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 0.6713615023474179,
|
|
"grad_norm": 1950.0893467985395,
|
|
"learning_rate": 1.828377673448096e-07,
|
|
"logits/chosen": 4.034375190734863,
|
|
"logits/rejected": 5.234375,
|
|
"logps/chosen": -469.3999938964844,
|
|
"logps/rejected": -585.2000122070312,
|
|
"loss": 3.3429,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": 2.577343702316284,
|
|
"rewards/margins": 13.887499809265137,
|
|
"rewards/rejected": -11.3125,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"epoch": 0.676056338028169,
|
|
"grad_norm": 1641.995639414283,
|
|
"learning_rate": 1.8022952529994783e-07,
|
|
"logits/chosen": 4.076562404632568,
|
|
"logits/rejected": 5.246874809265137,
|
|
"logps/chosen": -441.3999938964844,
|
|
"logps/rejected": -572.5999755859375,
|
|
"loss": 2.6721,
|
|
"rewards/accuracies": 0.6148437261581421,
|
|
"rewards/chosen": 3.4117188453674316,
|
|
"rewards/margins": 16.318750381469727,
|
|
"rewards/rejected": -12.887499809265137,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 0.6807511737089202,
|
|
"grad_norm": 1913.8064878399764,
|
|
"learning_rate": 1.7762128325508608e-07,
|
|
"logits/chosen": 4.2109375,
|
|
"logits/rejected": 5.315625190734863,
|
|
"logps/chosen": -461.0,
|
|
"logps/rejected": -614.5999755859375,
|
|
"loss": 3.5959,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 3.1343750953674316,
|
|
"rewards/margins": 14.550000190734863,
|
|
"rewards/rejected": -11.421875,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"epoch": 0.6854460093896714,
|
|
"grad_norm": 1643.0456710968572,
|
|
"learning_rate": 1.750130412102243e-07,
|
|
"logits/chosen": 4.051562309265137,
|
|
"logits/rejected": 5.356249809265137,
|
|
"logps/chosen": -477.79998779296875,
|
|
"logps/rejected": -600.2000122070312,
|
|
"loss": 2.8498,
|
|
"rewards/accuracies": 0.653124988079071,
|
|
"rewards/chosen": 4.756249904632568,
|
|
"rewards/margins": 16.393749237060547,
|
|
"rewards/rejected": -11.631250381469727,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 0.6901408450704225,
|
|
"grad_norm": 2861.967595740867,
|
|
"learning_rate": 1.7240479916536254e-07,
|
|
"logits/chosen": 3.996875047683716,
|
|
"logits/rejected": 5.209374904632568,
|
|
"logps/chosen": -424.20001220703125,
|
|
"logps/rejected": -615.7999877929688,
|
|
"loss": 2.7667,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": 4.102343559265137,
|
|
"rewards/margins": 15.381250381469727,
|
|
"rewards/rejected": -11.284375190734863,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"epoch": 0.6948356807511737,
|
|
"grad_norm": 1539.3679508540467,
|
|
"learning_rate": 1.697965571205008e-07,
|
|
"logits/chosen": 4.087500095367432,
|
|
"logits/rejected": 5.128125190734863,
|
|
"logps/chosen": -431.79998779296875,
|
|
"logps/rejected": -556.4000244140625,
|
|
"loss": 2.8762,
|
|
"rewards/accuracies": 0.6460937261581421,
|
|
"rewards/chosen": 3.7406249046325684,
|
|
"rewards/margins": 14.956250190734863,
|
|
"rewards/rejected": -11.231249809265137,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 0.6995305164319249,
|
|
"grad_norm": 2919.4079268210103,
|
|
"learning_rate": 1.6718831507563902e-07,
|
|
"logits/chosen": 3.9546875953674316,
|
|
"logits/rejected": 5.071875095367432,
|
|
"logps/chosen": -436.3999938964844,
|
|
"logps/rejected": -536.4000244140625,
|
|
"loss": 2.4317,
|
|
"rewards/accuracies": 0.635937511920929,
|
|
"rewards/chosen": 4.206250190734863,
|
|
"rewards/margins": 13.518750190734863,
|
|
"rewards/rejected": -9.309374809265137,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"epoch": 0.704225352112676,
|
|
"grad_norm": 1885.6760688773297,
|
|
"learning_rate": 1.6458007303077727e-07,
|
|
"logits/chosen": 3.9984374046325684,
|
|
"logits/rejected": 5.009375095367432,
|
|
"logps/chosen": -416.79998779296875,
|
|
"logps/rejected": -549.4000244140625,
|
|
"loss": 3.0932,
|
|
"rewards/accuracies": 0.6226562261581421,
|
|
"rewards/chosen": 3.5859375,
|
|
"rewards/margins": 13.175000190734863,
|
|
"rewards/rejected": -9.581250190734863,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 0.7089201877934272,
|
|
"grad_norm": 1725.7154717804146,
|
|
"learning_rate": 1.619718309859155e-07,
|
|
"logits/chosen": 3.971874952316284,
|
|
"logits/rejected": 4.953125,
|
|
"logps/chosen": -444.20001220703125,
|
|
"logps/rejected": -596.5999755859375,
|
|
"loss": 3.3031,
|
|
"rewards/accuracies": 0.6382812261581421,
|
|
"rewards/chosen": 3.109375,
|
|
"rewards/margins": 14.90625,
|
|
"rewards/rejected": -11.793749809265137,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"epoch": 0.7136150234741784,
|
|
"grad_norm": 1999.9137436941403,
|
|
"learning_rate": 1.5936358894105372e-07,
|
|
"logits/chosen": 3.910937547683716,
|
|
"logits/rejected": 5.050000190734863,
|
|
"logps/chosen": -437.20001220703125,
|
|
"logps/rejected": -572.4000244140625,
|
|
"loss": 2.8758,
|
|
"rewards/accuracies": 0.602343738079071,
|
|
"rewards/chosen": 3.21875,
|
|
"rewards/margins": 13.024999618530273,
|
|
"rewards/rejected": -9.803125381469727,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 0.7183098591549296,
|
|
"grad_norm": 1877.6609039823759,
|
|
"learning_rate": 1.5675534689619197e-07,
|
|
"logits/chosen": 4.089062690734863,
|
|
"logits/rejected": 5.128125190734863,
|
|
"logps/chosen": -446.0,
|
|
"logps/rejected": -548.0,
|
|
"loss": 3.1026,
|
|
"rewards/accuracies": 0.600781261920929,
|
|
"rewards/chosen": 2.27490234375,
|
|
"rewards/margins": 13.112500190734863,
|
|
"rewards/rejected": -10.834375381469727,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"epoch": 0.7230046948356808,
|
|
"grad_norm": 1893.7367068041842,
|
|
"learning_rate": 1.541471048513302e-07,
|
|
"logits/chosen": 3.973437547683716,
|
|
"logits/rejected": 5.171875,
|
|
"logps/chosen": -443.79998779296875,
|
|
"logps/rejected": -587.4000244140625,
|
|
"loss": 2.7559,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 3.116015672683716,
|
|
"rewards/margins": 15.274999618530273,
|
|
"rewards/rejected": -12.175000190734863,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 0.7276995305164319,
|
|
"grad_norm": 1591.1514678685764,
|
|
"learning_rate": 1.5153886280646843e-07,
|
|
"logits/chosen": 4.076562404632568,
|
|
"logits/rejected": 5.137499809265137,
|
|
"logps/chosen": -408.0,
|
|
"logps/rejected": -537.5999755859375,
|
|
"loss": 2.9053,
|
|
"rewards/accuracies": 0.625781238079071,
|
|
"rewards/chosen": 2.9312500953674316,
|
|
"rewards/margins": 13.28125,
|
|
"rewards/rejected": -10.34375,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"epoch": 0.7323943661971831,
|
|
"grad_norm": 1657.2825091908285,
|
|
"learning_rate": 1.4893062076160668e-07,
|
|
"logits/chosen": 4.079687595367432,
|
|
"logits/rejected": 5.384375095367432,
|
|
"logps/chosen": -439.0,
|
|
"logps/rejected": -592.5999755859375,
|
|
"loss": 2.6206,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": 3.969531297683716,
|
|
"rewards/margins": 15.393750190734863,
|
|
"rewards/rejected": -11.421875,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 0.7370892018779343,
|
|
"grad_norm": 1140.3960593075892,
|
|
"learning_rate": 1.463223787167449e-07,
|
|
"logits/chosen": 3.9859375953674316,
|
|
"logits/rejected": 5.143750190734863,
|
|
"logps/chosen": -443.0,
|
|
"logps/rejected": -565.0,
|
|
"loss": 2.9896,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": 4.3203125,
|
|
"rewards/margins": 13.15625,
|
|
"rewards/rejected": -8.840624809265137,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"epoch": 0.7417840375586855,
|
|
"grad_norm": 2846.881466180518,
|
|
"learning_rate": 1.4371413667188313e-07,
|
|
"logits/chosen": 4.046875,
|
|
"logits/rejected": 5.193749904632568,
|
|
"logps/chosen": -428.6000061035156,
|
|
"logps/rejected": -608.7999877929688,
|
|
"loss": 2.7376,
|
|
"rewards/accuracies": 0.625781238079071,
|
|
"rewards/chosen": 4.09375,
|
|
"rewards/margins": 14.412500381469727,
|
|
"rewards/rejected": -10.328125,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 0.7464788732394366,
|
|
"grad_norm": 1950.7404102135329,
|
|
"learning_rate": 1.4110589462702139e-07,
|
|
"logits/chosen": 4.145312309265137,
|
|
"logits/rejected": 5.196875095367432,
|
|
"logps/chosen": -432.20001220703125,
|
|
"logps/rejected": -549.5999755859375,
|
|
"loss": 2.9168,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": 3.76953125,
|
|
"rewards/margins": 13.425000190734863,
|
|
"rewards/rejected": -9.659375190734863,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"epoch": 0.7511737089201878,
|
|
"grad_norm": 1782.6783556436744,
|
|
"learning_rate": 1.384976525821596e-07,
|
|
"logits/chosen": 4.076562404632568,
|
|
"logits/rejected": 5.265625,
|
|
"logps/chosen": -445.0,
|
|
"logps/rejected": -598.4000244140625,
|
|
"loss": 2.7436,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 3.715625047683716,
|
|
"rewards/margins": 13.675000190734863,
|
|
"rewards/rejected": -9.959375381469727,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 0.755868544600939,
|
|
"grad_norm": 2031.3684264934145,
|
|
"learning_rate": 1.3588941053729787e-07,
|
|
"logits/chosen": 4.025000095367432,
|
|
"logits/rejected": 5.321875095367432,
|
|
"logps/chosen": -450.79998779296875,
|
|
"logps/rejected": -604.4000244140625,
|
|
"loss": 3.0584,
|
|
"rewards/accuracies": 0.620312511920929,
|
|
"rewards/chosen": 3.429980516433716,
|
|
"rewards/margins": 14.256250381469727,
|
|
"rewards/rejected": -10.818750381469727,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"epoch": 0.7605633802816901,
|
|
"grad_norm": 1590.6997008766277,
|
|
"learning_rate": 1.332811684924361e-07,
|
|
"logits/chosen": 4.121874809265137,
|
|
"logits/rejected": 5.162499904632568,
|
|
"logps/chosen": -453.0,
|
|
"logps/rejected": -589.5999755859375,
|
|
"loss": 3.3092,
|
|
"rewards/accuracies": 0.6265624761581421,
|
|
"rewards/chosen": 3.6343750953674316,
|
|
"rewards/margins": 15.068750381469727,
|
|
"rewards/rejected": -11.446874618530273,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 0.7652582159624414,
|
|
"grad_norm": 1821.4455048096704,
|
|
"learning_rate": 1.3067292644757432e-07,
|
|
"logits/chosen": 3.903125047683716,
|
|
"logits/rejected": 5.137499809265137,
|
|
"logps/chosen": -450.3999938964844,
|
|
"logps/rejected": -628.7999877929688,
|
|
"loss": 2.3551,
|
|
"rewards/accuracies": 0.629687488079071,
|
|
"rewards/chosen": 3.966796875,
|
|
"rewards/margins": 15.875,
|
|
"rewards/rejected": -11.899999618530273,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"epoch": 0.7699530516431925,
|
|
"grad_norm": 1902.4431207695484,
|
|
"learning_rate": 1.2806468440271257e-07,
|
|
"logits/chosen": 4.131249904632568,
|
|
"logits/rejected": 5.256249904632568,
|
|
"logps/chosen": -424.0,
|
|
"logps/rejected": -574.2000122070312,
|
|
"loss": 2.6772,
|
|
"rewards/accuracies": 0.6148437261581421,
|
|
"rewards/chosen": 4.098437309265137,
|
|
"rewards/margins": 14.350000381469727,
|
|
"rewards/rejected": -10.246874809265137,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 0.7746478873239436,
|
|
"grad_norm": 1727.3370449591312,
|
|
"learning_rate": 1.254564423578508e-07,
|
|
"logits/chosen": 4.192187309265137,
|
|
"logits/rejected": 5.21875,
|
|
"logps/chosen": -429.3999938964844,
|
|
"logps/rejected": -580.2000122070312,
|
|
"loss": 2.5673,
|
|
"rewards/accuracies": 0.632031261920929,
|
|
"rewards/chosen": 3.659374952316284,
|
|
"rewards/margins": 15.149999618530273,
|
|
"rewards/rejected": -11.481249809265137,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"epoch": 0.7793427230046949,
|
|
"grad_norm": 1747.1462397726139,
|
|
"learning_rate": 1.2284820031298902e-07,
|
|
"logits/chosen": 4.157812595367432,
|
|
"logits/rejected": 5.131249904632568,
|
|
"logps/chosen": -465.20001220703125,
|
|
"logps/rejected": -612.0,
|
|
"loss": 2.6703,
|
|
"rewards/accuracies": 0.645312488079071,
|
|
"rewards/chosen": 2.910937547683716,
|
|
"rewards/margins": 15.53125,
|
|
"rewards/rejected": -12.612500190734863,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 0.784037558685446,
|
|
"grad_norm": 2108.3919506389298,
|
|
"learning_rate": 1.2023995826812728e-07,
|
|
"logits/chosen": 4.109375,
|
|
"logits/rejected": 5.25,
|
|
"logps/chosen": -435.79998779296875,
|
|
"logps/rejected": -582.4000244140625,
|
|
"loss": 3.0872,
|
|
"rewards/accuracies": 0.62109375,
|
|
"rewards/chosen": 3.219531297683716,
|
|
"rewards/margins": 14.271875381469727,
|
|
"rewards/rejected": -11.057812690734863,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"epoch": 0.7887323943661971,
|
|
"grad_norm": 2131.971708366251,
|
|
"learning_rate": 1.176317162232655e-07,
|
|
"logits/chosen": 3.9921875,
|
|
"logits/rejected": 5.165625095367432,
|
|
"logps/chosen": -429.79998779296875,
|
|
"logps/rejected": -591.4000244140625,
|
|
"loss": 2.5143,
|
|
"rewards/accuracies": 0.624218761920929,
|
|
"rewards/chosen": 3.5179686546325684,
|
|
"rewards/margins": 14.431249618530273,
|
|
"rewards/rejected": -10.918749809265137,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 0.7934272300469484,
|
|
"grad_norm": 2320.914190527593,
|
|
"learning_rate": 1.1502347417840374e-07,
|
|
"logits/chosen": 3.890625,
|
|
"logits/rejected": 5.056250095367432,
|
|
"logps/chosen": -430.79998779296875,
|
|
"logps/rejected": -581.0,
|
|
"loss": 3.19,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": 4.009375095367432,
|
|
"rewards/margins": 14.1875,
|
|
"rewards/rejected": -10.181249618530273,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"epoch": 0.7981220657276995,
|
|
"grad_norm": 2123.4341292231416,
|
|
"learning_rate": 1.1241523213354198e-07,
|
|
"logits/chosen": 3.8578124046325684,
|
|
"logits/rejected": 5.025000095367432,
|
|
"logps/chosen": -433.3999938964844,
|
|
"logps/rejected": -600.5999755859375,
|
|
"loss": 2.7804,
|
|
"rewards/accuracies": 0.62890625,
|
|
"rewards/chosen": 4.407812595367432,
|
|
"rewards/margins": 15.487500190734863,
|
|
"rewards/rejected": -11.100000381469727,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 0.8028169014084507,
|
|
"grad_norm": 2256.025098059084,
|
|
"learning_rate": 1.0980699008868022e-07,
|
|
"logits/chosen": 3.9312500953674316,
|
|
"logits/rejected": 5.056250095367432,
|
|
"logps/chosen": -451.0,
|
|
"logps/rejected": -559.5999755859375,
|
|
"loss": 3.6071,
|
|
"rewards/accuracies": 0.6070312261581421,
|
|
"rewards/chosen": 3.060546875,
|
|
"rewards/margins": 13.778124809265137,
|
|
"rewards/rejected": -10.712499618530273,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"epoch": 0.8075117370892019,
|
|
"grad_norm": 1872.600712325682,
|
|
"learning_rate": 1.0719874804381846e-07,
|
|
"logits/chosen": 3.8109374046325684,
|
|
"logits/rejected": 4.912499904632568,
|
|
"logps/chosen": -426.0,
|
|
"logps/rejected": -524.7999877929688,
|
|
"loss": 2.7741,
|
|
"rewards/accuracies": 0.629687488079071,
|
|
"rewards/chosen": 4.0859375,
|
|
"rewards/margins": 12.65625,
|
|
"rewards/rejected": -8.5625,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 0.812206572769953,
|
|
"grad_norm": 1650.5620250523568,
|
|
"learning_rate": 1.045905059989567e-07,
|
|
"logits/chosen": 3.984375,
|
|
"logits/rejected": 5.074999809265137,
|
|
"logps/chosen": -409.79998779296875,
|
|
"logps/rejected": -518.2000122070312,
|
|
"loss": 2.4639,
|
|
"rewards/accuracies": 0.616406261920929,
|
|
"rewards/chosen": 4.393750190734863,
|
|
"rewards/margins": 13.96875,
|
|
"rewards/rejected": -9.578125,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"epoch": 0.8169014084507042,
|
|
"grad_norm": 1781.6361243581262,
|
|
"learning_rate": 1.0198226395409494e-07,
|
|
"logits/chosen": 3.932812452316284,
|
|
"logits/rejected": 5.137499809265137,
|
|
"logps/chosen": -416.6000061035156,
|
|
"logps/rejected": -559.4000244140625,
|
|
"loss": 2.7034,
|
|
"rewards/accuracies": 0.6382812261581421,
|
|
"rewards/chosen": 4.296875,
|
|
"rewards/margins": 15.03125,
|
|
"rewards/rejected": -10.712499618530273,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 0.8215962441314554,
|
|
"grad_norm": 1511.440729739189,
|
|
"learning_rate": 9.937402190923318e-08,
|
|
"logits/chosen": 3.9468750953674316,
|
|
"logits/rejected": 5.262499809265137,
|
|
"logps/chosen": -405.6000061035156,
|
|
"logps/rejected": -605.0,
|
|
"loss": 2.9172,
|
|
"rewards/accuracies": 0.625781238079071,
|
|
"rewards/chosen": 3.6078124046325684,
|
|
"rewards/margins": 15.493749618530273,
|
|
"rewards/rejected": -11.896875381469727,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"epoch": 0.8262910798122066,
|
|
"grad_norm": 2054.4841535018077,
|
|
"learning_rate": 9.676577986437141e-08,
|
|
"logits/chosen": 4.067187309265137,
|
|
"logits/rejected": 5.143750190734863,
|
|
"logps/chosen": -451.0,
|
|
"logps/rejected": -576.5999755859375,
|
|
"loss": 3.4167,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": 3.1566405296325684,
|
|
"rewards/margins": 15.0,
|
|
"rewards/rejected": -11.837499618530273,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 0.8309859154929577,
|
|
"grad_norm": 1569.7043460387988,
|
|
"learning_rate": 9.415753781950965e-08,
|
|
"logits/chosen": 4.060937404632568,
|
|
"logits/rejected": 5.165625095367432,
|
|
"logps/chosen": -422.6000061035156,
|
|
"logps/rejected": -576.4000244140625,
|
|
"loss": 2.6968,
|
|
"rewards/accuracies": 0.6390625238418579,
|
|
"rewards/chosen": 3.1468749046325684,
|
|
"rewards/margins": 16.125,
|
|
"rewards/rejected": -12.975000381469727,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"epoch": 0.8356807511737089,
|
|
"grad_norm": 1404.8595701320046,
|
|
"learning_rate": 9.154929577464789e-08,
|
|
"logits/chosen": 4.098437309265137,
|
|
"logits/rejected": 5.246874809265137,
|
|
"logps/chosen": -429.3999938964844,
|
|
"logps/rejected": -570.5999755859375,
|
|
"loss": 2.517,
|
|
"rewards/accuracies": 0.6265624761581421,
|
|
"rewards/chosen": 3.147656202316284,
|
|
"rewards/margins": 14.225000381469727,
|
|
"rewards/rejected": -11.068750381469727,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 0.8403755868544601,
|
|
"grad_norm": 1839.072310036757,
|
|
"learning_rate": 8.894105372978613e-08,
|
|
"logits/chosen": 4.0546875,
|
|
"logits/rejected": 5.275000095367432,
|
|
"logps/chosen": -440.79998779296875,
|
|
"logps/rejected": -572.2000122070312,
|
|
"loss": 2.5776,
|
|
"rewards/accuracies": 0.632031261920929,
|
|
"rewards/chosen": 3.7249999046325684,
|
|
"rewards/margins": 14.196874618530273,
|
|
"rewards/rejected": -10.471875190734863,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"epoch": 0.8450704225352113,
|
|
"grad_norm": 1467.1086502122791,
|
|
"learning_rate": 8.633281168492435e-08,
|
|
"logits/chosen": 4.026562690734863,
|
|
"logits/rejected": 5.228125095367432,
|
|
"logps/chosen": -429.0,
|
|
"logps/rejected": -570.0,
|
|
"loss": 2.5018,
|
|
"rewards/accuracies": 0.663281261920929,
|
|
"rewards/chosen": 3.559375047683716,
|
|
"rewards/margins": 15.837499618530273,
|
|
"rewards/rejected": -12.290624618530273,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 0.8497652582159625,
|
|
"grad_norm": 1728.8768867452925,
|
|
"learning_rate": 8.372456964006259e-08,
|
|
"logits/chosen": 4.0078125,
|
|
"logits/rejected": 5.125,
|
|
"logps/chosen": -427.79998779296875,
|
|
"logps/rejected": -578.2000122070312,
|
|
"loss": 3.0821,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 2.88671875,
|
|
"rewards/margins": 14.543749809265137,
|
|
"rewards/rejected": -11.675000190734863,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"epoch": 0.8544600938967136,
|
|
"grad_norm": 3052.0012936711237,
|
|
"learning_rate": 8.111632759520083e-08,
|
|
"logits/chosen": 3.9828124046325684,
|
|
"logits/rejected": 5.021874904632568,
|
|
"logps/chosen": -470.3999938964844,
|
|
"logps/rejected": -593.5999755859375,
|
|
"loss": 3.8531,
|
|
"rewards/accuracies": 0.6234375238418579,
|
|
"rewards/chosen": 2.9009766578674316,
|
|
"rewards/margins": 13.637499809265137,
|
|
"rewards/rejected": -10.746874809265137,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 0.8591549295774648,
|
|
"grad_norm": 1846.967321474956,
|
|
"learning_rate": 7.850808555033907e-08,
|
|
"logits/chosen": 4.284375190734863,
|
|
"logits/rejected": 5.378125190734863,
|
|
"logps/chosen": -435.3999938964844,
|
|
"logps/rejected": -602.4000244140625,
|
|
"loss": 2.8382,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": 3.589062452316284,
|
|
"rewards/margins": 17.643749237060547,
|
|
"rewards/rejected": -14.046875,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"epoch": 0.863849765258216,
|
|
"grad_norm": 1683.8382386620724,
|
|
"learning_rate": 7.58998435054773e-08,
|
|
"logits/chosen": 3.9156250953674316,
|
|
"logits/rejected": 4.974999904632568,
|
|
"logps/chosen": -441.3999938964844,
|
|
"logps/rejected": -573.4000244140625,
|
|
"loss": 3.0404,
|
|
"rewards/accuracies": 0.62109375,
|
|
"rewards/chosen": 4.303124904632568,
|
|
"rewards/margins": 13.78125,
|
|
"rewards/rejected": -9.481249809265137,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 0.8685446009389671,
|
|
"grad_norm": 2032.0127850425126,
|
|
"learning_rate": 7.329160146061554e-08,
|
|
"logits/chosen": 4.004687309265137,
|
|
"logits/rejected": 5.184374809265137,
|
|
"logps/chosen": -418.20001220703125,
|
|
"logps/rejected": -569.5999755859375,
|
|
"loss": 2.7466,
|
|
"rewards/accuracies": 0.635937511920929,
|
|
"rewards/chosen": 4.710156440734863,
|
|
"rewards/margins": 13.175000190734863,
|
|
"rewards/rejected": -8.465624809265137,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"epoch": 0.8732394366197183,
|
|
"grad_norm": 2201.95751324191,
|
|
"learning_rate": 7.068335941575378e-08,
|
|
"logits/chosen": 3.989062547683716,
|
|
"logits/rejected": 5.159375190734863,
|
|
"logps/chosen": -431.79998779296875,
|
|
"logps/rejected": -565.7999877929688,
|
|
"loss": 3.0091,
|
|
"rewards/accuracies": 0.6273437738418579,
|
|
"rewards/chosen": 3.65234375,
|
|
"rewards/margins": 14.996874809265137,
|
|
"rewards/rejected": -11.337499618530273,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 0.8779342723004695,
|
|
"grad_norm": 1882.6677013408898,
|
|
"learning_rate": 6.807511737089202e-08,
|
|
"logits/chosen": 4.092187404632568,
|
|
"logits/rejected": 5.231249809265137,
|
|
"logps/chosen": -422.3999938964844,
|
|
"logps/rejected": -586.2000122070312,
|
|
"loss": 3.138,
|
|
"rewards/accuracies": 0.614062488079071,
|
|
"rewards/chosen": 3.1070313453674316,
|
|
"rewards/margins": 14.081250190734863,
|
|
"rewards/rejected": -10.978124618530273,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"epoch": 0.8826291079812206,
|
|
"grad_norm": 1827.4360654688828,
|
|
"learning_rate": 6.546687532603024e-08,
|
|
"logits/chosen": 4.092187404632568,
|
|
"logits/rejected": 5.243750095367432,
|
|
"logps/chosen": -430.0,
|
|
"logps/rejected": -577.7999877929688,
|
|
"loss": 2.8782,
|
|
"rewards/accuracies": 0.6195312738418579,
|
|
"rewards/chosen": 3.819140672683716,
|
|
"rewards/margins": 14.081250190734863,
|
|
"rewards/rejected": -10.268750190734863,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 0.8873239436619719,
|
|
"grad_norm": 1580.9098365462808,
|
|
"learning_rate": 6.285863328116848e-08,
|
|
"logits/chosen": 3.910937547683716,
|
|
"logits/rejected": 5.221875190734863,
|
|
"logps/chosen": -392.20001220703125,
|
|
"logps/rejected": -592.4000244140625,
|
|
"loss": 2.577,
|
|
"rewards/accuracies": 0.6421874761581421,
|
|
"rewards/chosen": 4.568749904632568,
|
|
"rewards/margins": 16.081249237060547,
|
|
"rewards/rejected": -11.5,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"epoch": 0.892018779342723,
|
|
"grad_norm": 1736.5095225173611,
|
|
"learning_rate": 6.025039123630672e-08,
|
|
"logits/chosen": 3.9515624046325684,
|
|
"logits/rejected": 5.034375190734863,
|
|
"logps/chosen": -438.3999938964844,
|
|
"logps/rejected": -550.0,
|
|
"loss": 2.8292,
|
|
"rewards/accuracies": 0.6015625,
|
|
"rewards/chosen": 4.068749904632568,
|
|
"rewards/margins": 13.856249809265137,
|
|
"rewards/rejected": -9.784375190734863,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 0.8967136150234741,
|
|
"grad_norm": 1770.4992829590042,
|
|
"learning_rate": 5.764214919144496e-08,
|
|
"logits/chosen": 3.9984374046325684,
|
|
"logits/rejected": 5.121874809265137,
|
|
"logps/chosen": -443.20001220703125,
|
|
"logps/rejected": -542.5999755859375,
|
|
"loss": 3.4348,
|
|
"rewards/accuracies": 0.606249988079071,
|
|
"rewards/chosen": 3.592968702316284,
|
|
"rewards/margins": 12.262499809265137,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"epoch": 0.9014084507042254,
|
|
"grad_norm": 5243.250384499608,
|
|
"learning_rate": 5.50339071465832e-08,
|
|
"logits/chosen": 4.056250095367432,
|
|
"logits/rejected": 5.103125095367432,
|
|
"logps/chosen": -442.79998779296875,
|
|
"logps/rejected": -526.4000244140625,
|
|
"loss": 3.1452,
|
|
"rewards/accuracies": 0.629687488079071,
|
|
"rewards/chosen": 3.7191405296325684,
|
|
"rewards/margins": 12.34375,
|
|
"rewards/rejected": -8.637499809265137,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 0.9061032863849765,
|
|
"grad_norm": 1943.9022808354252,
|
|
"learning_rate": 5.2425665101721436e-08,
|
|
"logits/chosen": 3.964062452316284,
|
|
"logits/rejected": 5.259375095367432,
|
|
"logps/chosen": -443.0,
|
|
"logps/rejected": -617.2000122070312,
|
|
"loss": 3.0007,
|
|
"rewards/accuracies": 0.6148437261581421,
|
|
"rewards/chosen": 4.065625190734863,
|
|
"rewards/margins": 15.012499809265137,
|
|
"rewards/rejected": -10.934374809265137,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"epoch": 0.9107981220657277,
|
|
"grad_norm": 1709.5315030983054,
|
|
"learning_rate": 4.9817423056859675e-08,
|
|
"logits/chosen": 4.040625095367432,
|
|
"logits/rejected": 5.140625,
|
|
"logps/chosen": -446.79998779296875,
|
|
"logps/rejected": -598.4000244140625,
|
|
"loss": 2.7818,
|
|
"rewards/accuracies": 0.6390625238418579,
|
|
"rewards/chosen": 4.564453125,
|
|
"rewards/margins": 16.131250381469727,
|
|
"rewards/rejected": -11.571874618530273,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 0.9154929577464789,
|
|
"grad_norm": 1564.50275207027,
|
|
"learning_rate": 4.720918101199791e-08,
|
|
"logits/chosen": 3.893749952316284,
|
|
"logits/rejected": 5.215624809265137,
|
|
"logps/chosen": -444.20001220703125,
|
|
"logps/rejected": -654.2000122070312,
|
|
"loss": 2.7761,
|
|
"rewards/accuracies": 0.610156238079071,
|
|
"rewards/chosen": 3.4554686546325684,
|
|
"rewards/margins": 15.925000190734863,
|
|
"rewards/rejected": -12.475000381469727,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"epoch": 0.92018779342723,
|
|
"grad_norm": 1800.8015424542536,
|
|
"learning_rate": 4.460093896713615e-08,
|
|
"logits/chosen": 4.051562309265137,
|
|
"logits/rejected": 5.150000095367432,
|
|
"logps/chosen": -433.20001220703125,
|
|
"logps/rejected": -594.7999877929688,
|
|
"loss": 2.9427,
|
|
"rewards/accuracies": 0.6328125,
|
|
"rewards/chosen": 3.768749952316284,
|
|
"rewards/margins": 13.649999618530273,
|
|
"rewards/rejected": -9.887499809265137,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 0.9248826291079812,
|
|
"grad_norm": 1667.204497462417,
|
|
"learning_rate": 4.199269692227438e-08,
|
|
"logits/chosen": 4.056250095367432,
|
|
"logits/rejected": 5.253125190734863,
|
|
"logps/chosen": -428.0,
|
|
"logps/rejected": -534.0,
|
|
"loss": 3.2498,
|
|
"rewards/accuracies": 0.620312511920929,
|
|
"rewards/chosen": 3.9691405296325684,
|
|
"rewards/margins": 12.665624618530273,
|
|
"rewards/rejected": -8.6875,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"epoch": 0.9295774647887324,
|
|
"grad_norm": 1553.1786629396934,
|
|
"learning_rate": 3.938445487741262e-08,
|
|
"logits/chosen": 3.995312452316284,
|
|
"logits/rejected": 5.0625,
|
|
"logps/chosen": -444.79998779296875,
|
|
"logps/rejected": -577.2000122070312,
|
|
"loss": 2.9175,
|
|
"rewards/accuracies": 0.645312488079071,
|
|
"rewards/chosen": 3.442578077316284,
|
|
"rewards/margins": 15.912500381469727,
|
|
"rewards/rejected": -12.465624809265137,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 0.9342723004694836,
|
|
"grad_norm": 1606.5011509674803,
|
|
"learning_rate": 3.677621283255086e-08,
|
|
"logits/chosen": 3.924999952316284,
|
|
"logits/rejected": 5.184374809265137,
|
|
"logps/chosen": -448.0,
|
|
"logps/rejected": -577.7999877929688,
|
|
"loss": 2.507,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": 4.420312404632568,
|
|
"rewards/margins": 16.487499237060547,
|
|
"rewards/rejected": -12.065625190734863,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"epoch": 0.9389671361502347,
|
|
"grad_norm": 1732.008449865892,
|
|
"learning_rate": 3.41679707876891e-08,
|
|
"logits/chosen": 4.043749809265137,
|
|
"logits/rejected": 5.421875,
|
|
"logps/chosen": -433.6000061035156,
|
|
"logps/rejected": -637.2000122070312,
|
|
"loss": 2.9583,
|
|
"rewards/accuracies": 0.6351562738418579,
|
|
"rewards/chosen": 3.8765625953674316,
|
|
"rewards/margins": 16.299999237060547,
|
|
"rewards/rejected": -12.425000190734863,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 0.9436619718309859,
|
|
"grad_norm": 2013.3939293736494,
|
|
"learning_rate": 3.155972874282733e-08,
|
|
"logits/chosen": 4.0625,
|
|
"logits/rejected": 5.196875095367432,
|
|
"logps/chosen": -414.0,
|
|
"logps/rejected": -565.2000122070312,
|
|
"loss": 3.0528,
|
|
"rewards/accuracies": 0.6265624761581421,
|
|
"rewards/chosen": 4.345312595367432,
|
|
"rewards/margins": 13.981249809265137,
|
|
"rewards/rejected": -9.653124809265137,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"epoch": 0.9483568075117371,
|
|
"grad_norm": 1969.2194730488125,
|
|
"learning_rate": 2.8951486697965573e-08,
|
|
"logits/chosen": 4.0234375,
|
|
"logits/rejected": 5.071875095367432,
|
|
"logps/chosen": -437.20001220703125,
|
|
"logps/rejected": -582.5999755859375,
|
|
"loss": 3.0634,
|
|
"rewards/accuracies": 0.6234375238418579,
|
|
"rewards/chosen": 4.254687309265137,
|
|
"rewards/margins": 14.425000190734863,
|
|
"rewards/rejected": -10.171875,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 0.9530516431924883,
|
|
"grad_norm": 1589.5486835177276,
|
|
"learning_rate": 2.634324465310381e-08,
|
|
"logits/chosen": 4.057812690734863,
|
|
"logits/rejected": 5.0,
|
|
"logps/chosen": -421.20001220703125,
|
|
"logps/rejected": -572.4000244140625,
|
|
"loss": 2.8994,
|
|
"rewards/accuracies": 0.62109375,
|
|
"rewards/chosen": 3.7406249046325684,
|
|
"rewards/margins": 13.762499809265137,
|
|
"rewards/rejected": -10.006250381469727,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"epoch": 0.9577464788732394,
|
|
"grad_norm": 1637.3722814536288,
|
|
"learning_rate": 2.3735002608242045e-08,
|
|
"logits/chosen": 4.134375095367432,
|
|
"logits/rejected": 5.128125190734863,
|
|
"logps/chosen": -445.3999938964844,
|
|
"logps/rejected": -553.7999877929688,
|
|
"loss": 3.3368,
|
|
"rewards/accuracies": 0.63671875,
|
|
"rewards/chosen": 3.998828172683716,
|
|
"rewards/margins": 14.731249809265137,
|
|
"rewards/rejected": -10.721875190734863,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 0.9624413145539906,
|
|
"grad_norm": 2107.827929539929,
|
|
"learning_rate": 2.1126760563380282e-08,
|
|
"logits/chosen": 4.010937690734863,
|
|
"logits/rejected": 5.106249809265137,
|
|
"logps/chosen": -431.3999938964844,
|
|
"logps/rejected": -522.4000244140625,
|
|
"loss": 2.6644,
|
|
"rewards/accuracies": 0.639843761920929,
|
|
"rewards/chosen": 4.6015625,
|
|
"rewards/margins": 12.956250190734863,
|
|
"rewards/rejected": -8.346875190734863,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"epoch": 0.9671361502347418,
|
|
"grad_norm": 1490.1883097357231,
|
|
"learning_rate": 1.8518518518518518e-08,
|
|
"logits/chosen": 4.026562690734863,
|
|
"logits/rejected": 5.15625,
|
|
"logps/chosen": -454.6000061035156,
|
|
"logps/rejected": -563.4000244140625,
|
|
"loss": 2.8899,
|
|
"rewards/accuracies": 0.635937511920929,
|
|
"rewards/chosen": 4.146874904632568,
|
|
"rewards/margins": 13.331250190734863,
|
|
"rewards/rejected": -9.1875,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 0.971830985915493,
|
|
"grad_norm": 1870.6248491987546,
|
|
"learning_rate": 1.5910276473656755e-08,
|
|
"logits/chosen": 4.056250095367432,
|
|
"logits/rejected": 5.125,
|
|
"logps/chosen": -428.3999938964844,
|
|
"logps/rejected": -571.4000244140625,
|
|
"loss": 2.9026,
|
|
"rewards/accuracies": 0.6304687261581421,
|
|
"rewards/chosen": 4.493750095367432,
|
|
"rewards/margins": 15.984375,
|
|
"rewards/rejected": -11.484375,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"epoch": 0.9765258215962441,
|
|
"grad_norm": 1523.5841220847221,
|
|
"learning_rate": 1.3302034428794991e-08,
|
|
"logits/chosen": 4.048437595367432,
|
|
"logits/rejected": 5.25,
|
|
"logps/chosen": -417.3999938964844,
|
|
"logps/rejected": -533.7999877929688,
|
|
"loss": 2.4106,
|
|
"rewards/accuracies": 0.645312488079071,
|
|
"rewards/chosen": 4.6015625,
|
|
"rewards/margins": 14.28125,
|
|
"rewards/rejected": -9.6875,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 0.9812206572769953,
|
|
"grad_norm": 1576.4291411674672,
|
|
"learning_rate": 1.0693792383933229e-08,
|
|
"logits/chosen": 4.034375190734863,
|
|
"logits/rejected": 5.359375,
|
|
"logps/chosen": -442.6000061035156,
|
|
"logps/rejected": -589.5999755859375,
|
|
"loss": 2.8332,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 4.396093845367432,
|
|
"rewards/margins": 15.850000381469727,
|
|
"rewards/rejected": -11.456250190734863,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"epoch": 0.9859154929577465,
|
|
"grad_norm": 2823.3933700344646,
|
|
"learning_rate": 8.085550339071465e-09,
|
|
"logits/chosen": 3.9609375,
|
|
"logits/rejected": 5.137499809265137,
|
|
"logps/chosen": -449.0,
|
|
"logps/rejected": -619.7999877929688,
|
|
"loss": 3.0757,
|
|
"rewards/accuracies": 0.6273437738418579,
|
|
"rewards/chosen": 4.375,
|
|
"rewards/margins": 15.381250381469727,
|
|
"rewards/rejected": -10.993749618530273,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 0.9906103286384976,
|
|
"grad_norm": 2160.1932977561687,
|
|
"learning_rate": 5.4773082942097025e-09,
|
|
"logits/chosen": 3.924999952316284,
|
|
"logits/rejected": 5.199999809265137,
|
|
"logps/chosen": -427.6000061035156,
|
|
"logps/rejected": -553.0,
|
|
"loss": 2.6112,
|
|
"rewards/accuracies": 0.6578124761581421,
|
|
"rewards/chosen": 4.699999809265137,
|
|
"rewards/margins": 15.731249809265137,
|
|
"rewards/rejected": -11.03125,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"epoch": 0.9953051643192489,
|
|
"grad_norm": 1700.38917248292,
|
|
"learning_rate": 2.8690662493479393e-09,
|
|
"logits/chosen": 4.081250190734863,
|
|
"logits/rejected": 5.140625,
|
|
"logps/chosen": -433.79998779296875,
|
|
"logps/rejected": -565.2000122070312,
|
|
"loss": 2.8104,
|
|
"rewards/accuracies": 0.6382812261581421,
|
|
"rewards/chosen": 3.848437547683716,
|
|
"rewards/margins": 16.149999618530273,
|
|
"rewards/rejected": -12.315625190734863,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"grad_norm": 1373.3607617115563,
|
|
"learning_rate": 2.608242044861763e-10,
|
|
"logits/chosen": 3.8968749046325684,
|
|
"logits/rejected": 5.206250190734863,
|
|
"logps/chosen": -419.3999938964844,
|
|
"logps/rejected": -590.5999755859375,
|
|
"loss": 2.754,
|
|
"rewards/accuracies": 0.6493685841560364,
|
|
"rewards/chosen": 5.018750190734863,
|
|
"rewards/margins": 16.768749237060547,
|
|
"rewards/rejected": -11.728124618530273,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 2130,
|
|
"total_flos": 0.0,
|
|
"train_loss": 3.058436307772784,
|
|
"train_runtime": 6320.6844,
|
|
"train_samples_per_second": 43.126,
|
|
"train_steps_per_second": 0.337
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 2130,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|