3644 lines
135 KiB
JSON
3644 lines
135 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.3419838547706604,
|
|
"drift/rejected_abs": 0.14085355401039124,
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 7168.0,
|
|
"learning_rate": 1.111111111111111e-08,
|
|
"logits/chosen": -2.274840831756592,
|
|
"logits/rejected": -2.238874912261963,
|
|
"logps/chosen": -0.30753669142723083,
|
|
"logps/rejected": -0.3066186010837555,
|
|
"loss": 4418.6552734375,
|
|
"loss/core": 4418.6337890625,
|
|
"loss/preference_sft": 0.30753669142723083,
|
|
"loss/reference_anchor": 1.068562388420105,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4167866706848145,
|
|
"rewards/margins": 2.012580156326294,
|
|
"rewards/rejected": 1.4042068719863892,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29194900393486023,
|
|
"drift/rejected_abs": 0.13348758220672607,
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 1528.0,
|
|
"learning_rate": 2.5e-08,
|
|
"logits/chosen": -2.2050859928131104,
|
|
"logits/rejected": -2.1969170570373535,
|
|
"logps/chosen": -0.29071494936943054,
|
|
"logps/rejected": -0.2801753282546997,
|
|
"loss": 4423.58544921875,
|
|
"loss/core": 4423.5791015625,
|
|
"loss/preference_sft": 0.29071494936943054,
|
|
"loss/reference_anchor": 0.28804466128349304,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.919328212738037,
|
|
"rewards/margins": 1.5846813917160034,
|
|
"rewards/rejected": 1.3346469402313232,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1922224462032318,
|
|
"drift/rejected_abs": 0.08871154487133026,
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 700.0,
|
|
"learning_rate": 3.888888888888889e-08,
|
|
"logits/chosen": -1.8447561264038086,
|
|
"logits/rejected": -1.9001595973968506,
|
|
"logps/chosen": -0.29011744260787964,
|
|
"logps/rejected": -0.28691497445106506,
|
|
"loss": 4430.5283203125,
|
|
"loss/core": 4430.525390625,
|
|
"loss/preference_sft": 0.29011744260787964,
|
|
"loss/reference_anchor": 0.11491169780492783,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.9212669134140015,
|
|
"rewards/margins": 1.0505340099334717,
|
|
"rewards/rejected": 0.8707327842712402,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16317619383335114,
|
|
"drift/rejected_abs": 0.05642751604318619,
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 676.0,
|
|
"learning_rate": 5.2777777777777776e-08,
|
|
"logits/chosen": -2.1695523262023926,
|
|
"logits/rejected": -2.097202777862549,
|
|
"logps/chosen": -0.2871994376182556,
|
|
"logps/rejected": -0.28375035524368286,
|
|
"loss": 4430.1630859375,
|
|
"loss/core": 4430.16064453125,
|
|
"loss/preference_sft": 0.2871994376182556,
|
|
"loss/reference_anchor": 0.07742591947317123,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6311792135238647,
|
|
"rewards/margins": 1.0771574974060059,
|
|
"rewards/rejected": 0.5540218353271484,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18135179579257965,
|
|
"drift/rejected_abs": 0.11895482242107391,
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 1336.0,
|
|
"learning_rate": 6.666666666666667e-08,
|
|
"logits/chosen": -2.2033162117004395,
|
|
"logits/rejected": -2.218423366546631,
|
|
"logps/chosen": -0.27016180753707886,
|
|
"logps/rejected": -0.2789779007434845,
|
|
"loss": 4436.3212890625,
|
|
"loss/core": 4436.31787109375,
|
|
"loss/preference_sft": 0.27016180753707886,
|
|
"loss/reference_anchor": 0.13152866065502167,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8121395111083984,
|
|
"rewards/margins": 0.6226418018341064,
|
|
"rewards/rejected": 1.1894975900650024,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26979273557662964,
|
|
"drift/rejected_abs": 0.18020370602607727,
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 3728.0,
|
|
"learning_rate": 8.055555555555555e-08,
|
|
"logits/chosen": -2.1017913818359375,
|
|
"logits/rejected": -2.063615083694458,
|
|
"logps/chosen": -0.3164452612400055,
|
|
"logps/rejected": -0.2725190818309784,
|
|
"loss": 4433.0703125,
|
|
"loss/core": 4433.05859375,
|
|
"loss/preference_sft": 0.3164452612400055,
|
|
"loss/reference_anchor": 0.557770848274231,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6958069801330566,
|
|
"rewards/margins": 0.8953614234924316,
|
|
"rewards/rejected": 1.800445318222046,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18841537833213806,
|
|
"drift/rejected_abs": 0.08009415119886398,
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 912.0,
|
|
"learning_rate": 9.444444444444444e-08,
|
|
"logits/chosen": -2.1271777153015137,
|
|
"logits/rejected": -2.22860050201416,
|
|
"logps/chosen": -0.2820509970188141,
|
|
"logps/rejected": -0.2648252844810486,
|
|
"loss": 4429.8935546875,
|
|
"loss/core": 4429.8916015625,
|
|
"loss/preference_sft": 0.2820509970188141,
|
|
"loss/reference_anchor": 0.08949244022369385,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.883462905883789,
|
|
"rewards/margins": 1.0993850231170654,
|
|
"rewards/rejected": 0.7840781211853027,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2343977689743042,
|
|
"drift/rejected_abs": 0.09806372225284576,
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 1048.0,
|
|
"learning_rate": 1.0833333333333334e-07,
|
|
"logits/chosen": -2.1154208183288574,
|
|
"logits/rejected": -2.1933438777923584,
|
|
"logps/chosen": -0.29859572649002075,
|
|
"logps/rejected": -0.29969868063926697,
|
|
"loss": 4426.380859375,
|
|
"loss/core": 4426.3779296875,
|
|
"loss/preference_sft": 0.29859572649002075,
|
|
"loss/reference_anchor": 0.15946529805660248,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3425917625427246,
|
|
"rewards/margins": 1.366955280303955,
|
|
"rewards/rejected": 0.9756364822387695,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28305283188819885,
|
|
"drift/rejected_abs": 0.1835290640592575,
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 4864.0,
|
|
"learning_rate": 1.2222222222222222e-07,
|
|
"logits/chosen": -2.038555383682251,
|
|
"logits/rejected": -2.0794358253479004,
|
|
"logps/chosen": -0.28872132301330566,
|
|
"logps/rejected": -0.29276594519615173,
|
|
"loss": 4432.30322265625,
|
|
"loss/core": 4432.2890625,
|
|
"loss/preference_sft": 0.28872132301330566,
|
|
"loss/reference_anchor": 0.6807764172554016,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8277623653411865,
|
|
"rewards/margins": 0.9989973306655884,
|
|
"rewards/rejected": 1.8287651538848877,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18194088339805603,
|
|
"drift/rejected_abs": 0.13804134726524353,
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 430.0,
|
|
"learning_rate": 1.3611111111111108e-07,
|
|
"logits/chosen": -2.107499122619629,
|
|
"logits/rejected": -2.1948068141937256,
|
|
"logps/chosen": -0.28583410382270813,
|
|
"logps/rejected": -0.289579838514328,
|
|
"loss": 4438.6875,
|
|
"loss/core": 4438.68359375,
|
|
"loss/preference_sft": 0.28583410382270813,
|
|
"loss/reference_anchor": 0.17273597419261932,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.818098783493042,
|
|
"rewards/margins": 0.4417288303375244,
|
|
"rewards/rejected": 1.3763700723648071,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16760578751564026,
|
|
"drift/rejected_abs": 0.09244702756404877,
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 660.0,
|
|
"learning_rate": 1.5e-07,
|
|
"logits/chosen": -2.1269633769989014,
|
|
"logits/rejected": -2.2465131282806396,
|
|
"logps/chosen": -0.2968710958957672,
|
|
"logps/rejected": -0.29279255867004395,
|
|
"loss": 4434.4453125,
|
|
"loss/core": 4434.4443359375,
|
|
"loss/preference_sft": 0.2968710958957672,
|
|
"loss/reference_anchor": 0.05782536417245865,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6760451793670654,
|
|
"rewards/margins": 0.7522833347320557,
|
|
"rewards/rejected": 0.9237619638442993,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14607155323028564,
|
|
"drift/rejected_abs": 0.10737241804599762,
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 2272.0,
|
|
"learning_rate": 1.6388888888888888e-07,
|
|
"logits/chosen": -2.11145281791687,
|
|
"logits/rejected": -2.1462674140930176,
|
|
"logps/chosen": -0.29008573293685913,
|
|
"logps/rejected": -0.2873132824897766,
|
|
"loss": 4439.5068359375,
|
|
"loss/core": 4439.5029296875,
|
|
"loss/preference_sft": 0.29008573293685913,
|
|
"loss/reference_anchor": 0.1456277072429657,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.459886908531189,
|
|
"rewards/margins": 0.387167751789093,
|
|
"rewards/rejected": 1.0727192163467407,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24463272094726562,
|
|
"drift/rejected_abs": 0.09055524319410324,
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 2912.0,
|
|
"learning_rate": 1.7777777777777776e-07,
|
|
"logits/chosen": -2.143834114074707,
|
|
"logits/rejected": -2.2185416221618652,
|
|
"logps/chosen": -0.3164866268634796,
|
|
"logps/rejected": -0.3099859356880188,
|
|
"loss": 4424.44091796875,
|
|
"loss/core": 4424.4326171875,
|
|
"loss/preference_sft": 0.3164866268634796,
|
|
"loss/reference_anchor": 0.37436532974243164,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4456965923309326,
|
|
"rewards/margins": 1.5431140661239624,
|
|
"rewards/rejected": 0.9025824666023254,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23293833434581757,
|
|
"drift/rejected_abs": 0.10600552707910538,
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 8192.0,
|
|
"learning_rate": 1.9166666666666668e-07,
|
|
"logits/chosen": -2.1075563430786133,
|
|
"logits/rejected": -2.165278434753418,
|
|
"logps/chosen": -0.3008618950843811,
|
|
"logps/rejected": -0.30584263801574707,
|
|
"loss": 4426.07080078125,
|
|
"loss/core": 4426.06494140625,
|
|
"loss/preference_sft": 0.3008618950843811,
|
|
"loss/reference_anchor": 0.24336305260658264,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.329383373260498,
|
|
"rewards/margins": 1.4130064249038696,
|
|
"rewards/rejected": 0.9163768887519836,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1538292020559311,
|
|
"drift/rejected_abs": 0.10185450315475464,
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 804.0,
|
|
"learning_rate": 2.0555555555555553e-07,
|
|
"logits/chosen": -2.0664281845092773,
|
|
"logits/rejected": -2.1418495178222656,
|
|
"logps/chosen": -0.2909933924674988,
|
|
"logps/rejected": -0.2969188988208771,
|
|
"loss": 4437.65087890625,
|
|
"loss/core": 4437.6474609375,
|
|
"loss/preference_sft": 0.2909933924674988,
|
|
"loss/reference_anchor": 0.10792021453380585,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5364402532577515,
|
|
"rewards/margins": 0.5191212296485901,
|
|
"rewards/rejected": 1.0173190832138062,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12916477024555206,
|
|
"drift/rejected_abs": 0.07108369469642639,
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 892.0,
|
|
"learning_rate": 2.1944444444444442e-07,
|
|
"logits/chosen": -2.2707951068878174,
|
|
"logits/rejected": -2.3466694355010986,
|
|
"logps/chosen": -0.30908656120300293,
|
|
"logps/rejected": -0.30672696232795715,
|
|
"loss": 4436.71728515625,
|
|
"loss/core": 4436.7158203125,
|
|
"loss/preference_sft": 0.30908656120300293,
|
|
"loss/reference_anchor": 0.04678858071565628,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.291474461555481,
|
|
"rewards/margins": 0.5815840363502502,
|
|
"rewards/rejected": 0.7098905444145203,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24453523755073547,
|
|
"drift/rejected_abs": 0.1695018708705902,
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 556.0,
|
|
"learning_rate": 2.3333333333333333e-07,
|
|
"logits/chosen": -2.1169114112854004,
|
|
"logits/rejected": -2.1277921199798584,
|
|
"logps/chosen": -0.2921332120895386,
|
|
"logps/rejected": -0.28643548488616943,
|
|
"loss": 4435.0263671875,
|
|
"loss/core": 4435.017578125,
|
|
"loss/preference_sft": 0.2921332120895386,
|
|
"loss/reference_anchor": 0.4582666754722595,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.4450902938842773,
|
|
"rewards/margins": 0.7517923712730408,
|
|
"rewards/rejected": 1.6932977437973022,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28909215331077576,
|
|
"drift/rejected_abs": 0.06680949032306671,
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 1080.0,
|
|
"learning_rate": 2.4722222222222224e-07,
|
|
"logits/chosen": -2.001685380935669,
|
|
"logits/rejected": -2.1864590644836426,
|
|
"logps/chosen": -0.3068524897098541,
|
|
"logps/rejected": -0.30106669664382935,
|
|
"loss": 4415.2734375,
|
|
"loss/core": 4415.2666015625,
|
|
"loss/preference_sft": 0.3068524897098541,
|
|
"loss/reference_anchor": 0.31496793031692505,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.889475107192993,
|
|
"rewards/margins": 2.222256898880005,
|
|
"rewards/rejected": 0.6672185659408569,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2167942225933075,
|
|
"drift/rejected_abs": 0.13755588233470917,
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 2912.0,
|
|
"learning_rate": 2.4998495746616845e-07,
|
|
"logits/chosen": -2.0368285179138184,
|
|
"logits/rejected": -1.9507942199707031,
|
|
"logps/chosen": -0.28156915307044983,
|
|
"logps/rejected": -0.30304011702537537,
|
|
"loss": 4434.26171875,
|
|
"loss/core": 4434.25439453125,
|
|
"loss/preference_sft": 0.28156915307044983,
|
|
"loss/reference_anchor": 0.33130401372909546,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.167041063308716,
|
|
"rewards/margins": 0.7925023436546326,
|
|
"rewards/rejected": 1.3745384216308594,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29789382219314575,
|
|
"drift/rejected_abs": 0.1836269050836563,
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 600.0,
|
|
"learning_rate": 2.4992385337738696e-07,
|
|
"logits/chosen": -2.0101962089538574,
|
|
"logits/rejected": -2.0421595573425293,
|
|
"logps/chosen": -0.2820836007595062,
|
|
"logps/rejected": -0.31330883502960205,
|
|
"loss": 4426.01904296875,
|
|
"loss/core": 4426.0087890625,
|
|
"loss/preference_sft": 0.2820836007595062,
|
|
"loss/reference_anchor": 0.4864688813686371,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9787633419036865,
|
|
"rewards/margins": 1.4464952945709229,
|
|
"rewards/rejected": 1.5322680473327637,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3174459934234619,
|
|
"drift/rejected_abs": 0.13966134190559387,
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 812.0,
|
|
"learning_rate": 2.4981577053636144e-07,
|
|
"logits/chosen": -1.9345197677612305,
|
|
"logits/rejected": -1.9873930215835571,
|
|
"logps/chosen": -0.2772332727909088,
|
|
"logps/rejected": -0.2635360360145569,
|
|
"loss": 4421.1513671875,
|
|
"loss/core": 4421.1435546875,
|
|
"loss/preference_sft": 0.2772332727909088,
|
|
"loss/reference_anchor": 0.37949052453041077,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.1736035346984863,
|
|
"rewards/margins": 1.7784216403961182,
|
|
"rewards/rejected": 1.3951818943023682,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21933257579803467,
|
|
"drift/rejected_abs": 0.10228520631790161,
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 556.0,
|
|
"learning_rate": 2.496607495886281e-07,
|
|
"logits/chosen": -2.123267412185669,
|
|
"logits/rejected": -2.0964272022247314,
|
|
"logps/chosen": -0.2958363890647888,
|
|
"logps/rejected": -0.30722880363464355,
|
|
"loss": 4429.02294921875,
|
|
"loss/core": 4429.0166015625,
|
|
"loss/preference_sft": 0.2958363890647888,
|
|
"loss/reference_anchor": 0.31792789697647095,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1888623237609863,
|
|
"rewards/margins": 1.1705862283706665,
|
|
"rewards/rejected": 1.0182762145996094,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3494819402694702,
|
|
"drift/rejected_abs": 0.1267239898443222,
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 5792.0,
|
|
"learning_rate": 2.4945884883122553e-07,
|
|
"logits/chosen": -2.022082805633545,
|
|
"logits/rejected": -2.070870876312256,
|
|
"logps/chosen": -0.2908271253108978,
|
|
"logps/rejected": -0.28468307852745056,
|
|
"loss": 4415.51806640625,
|
|
"loss/core": 4415.5029296875,
|
|
"loss/preference_sft": 0.2908271253108978,
|
|
"loss/reference_anchor": 0.6909618377685547,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.49409556388855,
|
|
"rewards/margins": 2.2313833236694336,
|
|
"rewards/rejected": 1.2627122402191162,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18104346096515656,
|
|
"drift/rejected_abs": 0.140670508146286,
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 9856.0,
|
|
"learning_rate": 2.492101441907714e-07,
|
|
"logits/chosen": -2.241240978240967,
|
|
"logits/rejected": -2.1032841205596924,
|
|
"logps/chosen": -0.2601907551288605,
|
|
"logps/rejected": -0.28034669160842896,
|
|
"loss": 4439.17724609375,
|
|
"loss/core": 4439.17431640625,
|
|
"loss/preference_sft": 0.2601907551288605,
|
|
"loss/reference_anchor": 0.14795775711536407,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8099110126495361,
|
|
"rewards/margins": 0.4032059609889984,
|
|
"rewards/rejected": 1.4067050218582153,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.408181756734848,
|
|
"drift/rejected_abs": 0.10417692363262177,
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 422.0,
|
|
"learning_rate": 2.4891472919490977e-07,
|
|
"logits/chosen": -1.9139236211776733,
|
|
"logits/rejected": -1.9596039056777954,
|
|
"logps/chosen": -0.2813839018344879,
|
|
"logps/rejected": -0.2793610095977783,
|
|
"loss": 4404.7919921875,
|
|
"loss/core": 4404.77392578125,
|
|
"loss/preference_sft": 0.2813839018344879,
|
|
"loss/reference_anchor": 0.8949528932571411,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.081476211547852,
|
|
"rewards/margins": 3.0719616413116455,
|
|
"rewards/rejected": 1.0095146894454956,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2684691548347473,
|
|
"drift/rejected_abs": 0.1741405725479126,
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 8576.0,
|
|
"learning_rate": 2.4857271493713894e-07,
|
|
"logits/chosen": -2.075075626373291,
|
|
"logits/rejected": -2.073489189147949,
|
|
"logps/chosen": -0.29706940054893494,
|
|
"logps/rejected": -0.2922501266002655,
|
|
"loss": 4431.94091796875,
|
|
"loss/core": 4431.92919921875,
|
|
"loss/preference_sft": 0.29706940054893494,
|
|
"loss/reference_anchor": 0.525195300579071,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6840920448303223,
|
|
"rewards/margins": 0.9443261027336121,
|
|
"rewards/rejected": 1.7397658824920654,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17242693901062012,
|
|
"drift/rejected_abs": 0.0899103656411171,
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 1224.0,
|
|
"learning_rate": 2.481842300350339e-07,
|
|
"logits/chosen": -2.03088641166687,
|
|
"logits/rejected": -2.094935894012451,
|
|
"logps/chosen": -0.27687835693359375,
|
|
"logps/rejected": -0.26349860429763794,
|
|
"loss": 4433.5263671875,
|
|
"loss/core": 4433.5234375,
|
|
"loss/preference_sft": 0.27687835693359375,
|
|
"loss/reference_anchor": 0.09716491401195526,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.7209184169769287,
|
|
"rewards/margins": 0.8248888850212097,
|
|
"rewards/rejected": 0.8960295915603638,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22251293063163757,
|
|
"drift/rejected_abs": 0.09145991504192352,
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 500.0,
|
|
"learning_rate": 2.4774942058187854e-07,
|
|
"logits/chosen": -2.1904947757720947,
|
|
"logits/rejected": -2.24971079826355,
|
|
"logps/chosen": -0.2990642189979553,
|
|
"logps/rejected": -0.2923958897590637,
|
|
"loss": 4427.3486328125,
|
|
"loss/core": 4427.3408203125,
|
|
"loss/preference_sft": 0.2990642189979553,
|
|
"loss/reference_anchor": 0.39207085967063904,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.223792314529419,
|
|
"rewards/margins": 1.3103091716766357,
|
|
"rewards/rejected": 0.9134831428527832,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18590529263019562,
|
|
"drift/rejected_abs": 0.10894735902547836,
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 1584.0,
|
|
"learning_rate": 2.472684500917257e-07,
|
|
"logits/chosen": -1.9672470092773438,
|
|
"logits/rejected": -1.9949979782104492,
|
|
"logps/chosen": -0.33192163705825806,
|
|
"logps/rejected": -0.3151020109653473,
|
|
"loss": 4434.21435546875,
|
|
"loss/core": 4434.2119140625,
|
|
"loss/preference_sft": 0.33192163705825806,
|
|
"loss/reference_anchor": 0.10941161960363388,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8581035137176514,
|
|
"rewards/margins": 0.773113489151001,
|
|
"rewards/rejected": 1.0849900245666504,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15106205642223358,
|
|
"drift/rejected_abs": 0.05738229304552078,
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 624.0,
|
|
"learning_rate": 2.467414994379065e-07,
|
|
"logits/chosen": -2.0708765983581543,
|
|
"logits/rejected": -2.1375513076782227,
|
|
"logps/chosen": -0.30995962023735046,
|
|
"logps/rejected": -0.31707248091697693,
|
|
"loss": 4432.21533203125,
|
|
"loss/core": 4432.2119140625,
|
|
"loss/preference_sft": 0.30995962023735046,
|
|
"loss/reference_anchor": 0.15558083355426788,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5100862979888916,
|
|
"rewards/margins": 0.9373763799667358,
|
|
"rewards/rejected": 0.5727099776268005,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15896198153495789,
|
|
"drift/rejected_abs": 0.08986209332942963,
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 1496.0,
|
|
"learning_rate": 2.4616876678501114e-07,
|
|
"logits/chosen": -2.2561638355255127,
|
|
"logits/rejected": -2.3034205436706543,
|
|
"logps/chosen": -0.31711262464523315,
|
|
"logps/rejected": -0.302999347448349,
|
|
"loss": 4435.2392578125,
|
|
"loss/core": 4435.23681640625,
|
|
"loss/preference_sft": 0.31711262464523315,
|
|
"loss/reference_anchor": 0.06949084997177124,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.588337779045105,
|
|
"rewards/margins": 0.6926823258399963,
|
|
"rewards/rejected": 0.8956555128097534,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3718644082546234,
|
|
"drift/rejected_abs": 0.11488336324691772,
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 13056.0,
|
|
"learning_rate": 2.4555046751436735e-07,
|
|
"logits/chosen": -2.1315560340881348,
|
|
"logits/rejected": -2.117034912109375,
|
|
"logps/chosen": -0.29637980461120605,
|
|
"logps/rejected": -0.282162606716156,
|
|
"loss": 4410.93359375,
|
|
"loss/core": 4410.9189453125,
|
|
"loss/preference_sft": 0.29637980461120605,
|
|
"loss/reference_anchor": 0.6615738868713379,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.717616558074951,
|
|
"rewards/margins": 2.570777416229248,
|
|
"rewards/rejected": 1.1468391418457031,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18394318222999573,
|
|
"drift/rejected_abs": 0.09417982399463654,
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 944.0,
|
|
"learning_rate": 2.4488683414304425e-07,
|
|
"logits/chosen": -2.132768392562866,
|
|
"logits/rejected": -2.2345738410949707,
|
|
"logps/chosen": -0.31837791204452515,
|
|
"logps/rejected": -0.31375187635421753,
|
|
"loss": 4432.53857421875,
|
|
"loss/core": 4432.53564453125,
|
|
"loss/preference_sft": 0.31837791204452515,
|
|
"loss/reference_anchor": 0.1169905811548233,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.8377997875213623,
|
|
"rewards/margins": 0.896827220916748,
|
|
"rewards/rejected": 0.940972626209259,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15466679632663727,
|
|
"drift/rejected_abs": 0.09628058969974518,
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 1496.0,
|
|
"learning_rate": 2.4417811623641203e-07,
|
|
"logits/chosen": -2.274855136871338,
|
|
"logits/rejected": -2.3352713584899902,
|
|
"logps/chosen": -0.27421072125434875,
|
|
"logps/rejected": -0.2842170298099518,
|
|
"loss": 4436.6865234375,
|
|
"loss/core": 4436.68359375,
|
|
"loss/preference_sft": 0.27421072125434875,
|
|
"loss/reference_anchor": 0.11561869084835052,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.5460879802703857,
|
|
"rewards/margins": 0.5835286378860474,
|
|
"rewards/rejected": 0.9625593423843384,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5257049798965454,
|
|
"drift/rejected_abs": 0.19930550456047058,
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 1896.0,
|
|
"learning_rate": 2.4342458031429113e-07,
|
|
"logits/chosen": -1.8325287103652954,
|
|
"logits/rejected": -1.8387473821640015,
|
|
"logps/chosen": -0.2876504957675934,
|
|
"logps/rejected": -0.29608818888664246,
|
|
"loss": 4401.90185546875,
|
|
"loss/core": 4401.876953125,
|
|
"loss/preference_sft": 0.2876504957675934,
|
|
"loss/reference_anchor": 1.251502275466919,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 5.2567338943481445,
|
|
"rewards/margins": 3.286668300628662,
|
|
"rewards/rejected": 1.9700653553009033,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3301013112068176,
|
|
"drift/rejected_abs": 0.15288423001766205,
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 458.0,
|
|
"learning_rate": 2.4262650975072444e-07,
|
|
"logits/chosen": -2.189413070678711,
|
|
"logits/rejected": -2.3054258823394775,
|
|
"logps/chosen": -0.2844337224960327,
|
|
"logps/rejected": -0.2936963140964508,
|
|
"loss": 4420.90087890625,
|
|
"loss/core": 4420.8896484375,
|
|
"loss/preference_sft": 0.2844337224960327,
|
|
"loss/reference_anchor": 0.5586980581283569,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.300053119659424,
|
|
"rewards/margins": 1.8004553318023682,
|
|
"rewards/rejected": 1.4995979070663452,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1918279230594635,
|
|
"drift/rejected_abs": 0.07734932750463486,
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 704.0,
|
|
"learning_rate": 2.417842046674122e-07,
|
|
"logits/chosen": -2.3335790634155273,
|
|
"logits/rejected": -2.3553173542022705,
|
|
"logps/chosen": -0.2591295838356018,
|
|
"logps/rejected": -0.26806166768074036,
|
|
"loss": 4429.29296875,
|
|
"loss/core": 4429.2900390625,
|
|
"loss/preference_sft": 0.2591295838356018,
|
|
"loss/reference_anchor": 0.09960640221834183,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9182792901992798,
|
|
"rewards/margins": 1.1459853649139404,
|
|
"rewards/rejected": 0.7722936868667603,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19113460183143616,
|
|
"drift/rejected_abs": 0.09281430393457413,
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 564.0,
|
|
"learning_rate": 2.4089798182084843e-07,
|
|
"logits/chosen": -2.0862205028533936,
|
|
"logits/rejected": -2.1600217819213867,
|
|
"logps/chosen": -0.30776557326316833,
|
|
"logps/rejected": -0.30735090374946594,
|
|
"loss": 4431.3701171875,
|
|
"loss/core": 4431.36669921875,
|
|
"loss/preference_sft": 0.30776557326316833,
|
|
"loss/reference_anchor": 0.1401655375957489,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9096482992172241,
|
|
"rewards/margins": 0.9876093864440918,
|
|
"rewards/rejected": 0.9220390319824219,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21678169071674347,
|
|
"drift/rejected_abs": 0.13430018723011017,
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 704.0,
|
|
"learning_rate": 2.3996817448320195e-07,
|
|
"logits/chosen": -2.158468246459961,
|
|
"logits/rejected": -2.112973213195801,
|
|
"logps/chosen": -0.28618916869163513,
|
|
"logps/rejected": -0.29950863122940063,
|
|
"loss": 4432.5107421875,
|
|
"loss/core": 4432.50537109375,
|
|
"loss/preference_sft": 0.28618916869163513,
|
|
"loss/reference_anchor": 0.24302752315998077,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1672415733337402,
|
|
"rewards/margins": 0.914328932762146,
|
|
"rewards/rejected": 1.2529126405715942,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28183844685554504,
|
|
"drift/rejected_abs": 0.15761585533618927,
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 1928.0,
|
|
"learning_rate": 2.3899513231698607e-07,
|
|
"logits/chosen": -2.1264212131500244,
|
|
"logits/rejected": -2.0478241443634033,
|
|
"logps/chosen": -0.292626291513443,
|
|
"logps/rejected": -0.28839462995529175,
|
|
"loss": 4428.09912109375,
|
|
"loss/core": 4428.0927734375,
|
|
"loss/preference_sft": 0.292626291513443,
|
|
"loss/reference_anchor": 0.28291893005371094,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.8163602352142334,
|
|
"rewards/margins": 1.2429499626159668,
|
|
"rewards/rejected": 1.5734103918075562,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26241064071655273,
|
|
"drift/rejected_abs": 0.14250826835632324,
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 2240.0,
|
|
"learning_rate": 2.3797922124356506e-07,
|
|
"logits/chosen": -2.029865264892578,
|
|
"logits/rejected": -2.0273633003234863,
|
|
"logps/chosen": -0.39078933000564575,
|
|
"logps/rejected": -0.30789244174957275,
|
|
"loss": 4427.798828125,
|
|
"loss/core": 4427.78955078125,
|
|
"loss/preference_sft": 0.39078933000564575,
|
|
"loss/reference_anchor": 0.41654643416404724,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6241068840026855,
|
|
"rewards/margins": 1.2998729944229126,
|
|
"rewards/rejected": 1.3242336511611938,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17145346105098724,
|
|
"drift/rejected_abs": 0.07196025550365448,
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 532.0,
|
|
"learning_rate": 2.3692082330554585e-07,
|
|
"logits/chosen": -2.2569234371185303,
|
|
"logits/rejected": -2.352635145187378,
|
|
"logps/chosen": -0.25879400968551636,
|
|
"logps/rejected": -0.2546376585960388,
|
|
"loss": 4431.224609375,
|
|
"loss/core": 4431.22265625,
|
|
"loss/preference_sft": 0.25879400968551636,
|
|
"loss/reference_anchor": 0.05695471167564392,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7141616344451904,
|
|
"rewards/margins": 0.9953814744949341,
|
|
"rewards/rejected": 0.7187802791595459,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2754186987876892,
|
|
"drift/rejected_abs": 0.17294225096702576,
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 6752.0,
|
|
"learning_rate": 2.3582033652310765e-07,
|
|
"logits/chosen": -1.9878219366073608,
|
|
"logits/rejected": -1.9628711938858032,
|
|
"logps/chosen": -0.2865174412727356,
|
|
"logps/rejected": -0.27714353799819946,
|
|
"loss": 4431.0849609375,
|
|
"loss/core": 4431.0791015625,
|
|
"loss/preference_sft": 0.2865174412727356,
|
|
"loss/reference_anchor": 0.26327061653137207,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7541871070861816,
|
|
"rewards/margins": 1.025450587272644,
|
|
"rewards/rejected": 1.7287365198135376,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3495247960090637,
|
|
"drift/rejected_abs": 0.07642688602209091,
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 544.0,
|
|
"learning_rate": 2.346781747443227e-07,
|
|
"logits/chosen": -2.042937755584717,
|
|
"logits/rejected": -2.1352648735046387,
|
|
"logps/chosen": -0.29404938220977783,
|
|
"logps/rejected": -0.28113672137260437,
|
|
"loss": 4409.341796875,
|
|
"loss/core": 4409.32666015625,
|
|
"loss/preference_sft": 0.29404938220977783,
|
|
"loss/reference_anchor": 0.7147288918495178,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.494548797607422,
|
|
"rewards/margins": 2.73149037361145,
|
|
"rewards/rejected": 0.7630582451820374,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3245381712913513,
|
|
"drift/rejected_abs": 0.10127030313014984,
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 3024.0,
|
|
"learning_rate": 2.3349476748952508e-07,
|
|
"logits/chosen": -2.1589510440826416,
|
|
"logits/rejected": -2.2461776733398438,
|
|
"logps/chosen": -0.27290159463882446,
|
|
"logps/rejected": -0.27267274260520935,
|
|
"loss": 4415.03759765625,
|
|
"loss/core": 4415.03125,
|
|
"loss/preference_sft": 0.27290159463882446,
|
|
"loss/reference_anchor": 0.2985621392726898,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.2441680431365967,
|
|
"rewards/margins": 2.2339510917663574,
|
|
"rewards/rejected": 1.0102171897888184,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36413469910621643,
|
|
"drift/rejected_abs": 0.13577529788017273,
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 9600.0,
|
|
"learning_rate": 2.3227055978978545e-07,
|
|
"logits/chosen": -2.0139591693878174,
|
|
"logits/rejected": -2.053098678588867,
|
|
"logps/chosen": -0.2517145872116089,
|
|
"logps/rejected": -0.25124287605285645,
|
|
"loss": 4415.037109375,
|
|
"loss/core": 4415.02294921875,
|
|
"loss/preference_sft": 0.2517145872116089,
|
|
"loss/reference_anchor": 0.6684554815292358,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.639683246612549,
|
|
"rewards/margins": 2.2833621501922607,
|
|
"rewards/rejected": 1.3563209772109985,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2075643092393875,
|
|
"drift/rejected_abs": 0.12083862721920013,
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 616.0,
|
|
"learning_rate": 2.3100601201955321e-07,
|
|
"logits/chosen": -2.329841136932373,
|
|
"logits/rejected": -2.312140464782715,
|
|
"logps/chosen": -0.25918227434158325,
|
|
"logps/rejected": -0.2679867744445801,
|
|
"loss": 4433.1220703125,
|
|
"loss/core": 4433.1142578125,
|
|
"loss/preference_sft": 0.25918227434158325,
|
|
"loss/reference_anchor": 0.35554802417755127,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.0753586292266846,
|
|
"rewards/margins": 0.868099570274353,
|
|
"rewards/rejected": 1.207258939743042,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3381517231464386,
|
|
"drift/rejected_abs": 0.19172975420951843,
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 1488.0,
|
|
"learning_rate": 2.2970159972352864e-07,
|
|
"logits/chosen": -2.123835325241089,
|
|
"logits/rejected": -2.1424546241760254,
|
|
"logps/chosen": -0.2800813317298889,
|
|
"logps/rejected": -0.2956727147102356,
|
|
"loss": 4425.13037109375,
|
|
"loss/core": 4425.1181640625,
|
|
"loss/preference_sft": 0.2800813317298889,
|
|
"loss/reference_anchor": 0.6074963808059692,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.381516695022583,
|
|
"rewards/margins": 1.4686692953109741,
|
|
"rewards/rejected": 1.9128472805023193,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3294285535812378,
|
|
"drift/rejected_abs": 0.16067305207252502,
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 5056.0,
|
|
"learning_rate": 2.2835781343782966e-07,
|
|
"logits/chosen": -1.8609027862548828,
|
|
"logits/rejected": -1.8787086009979248,
|
|
"logps/chosen": -0.2937183976173401,
|
|
"logps/rejected": -0.2992231547832489,
|
|
"loss": 4422.87890625,
|
|
"loss/core": 4422.8642578125,
|
|
"loss/preference_sft": 0.2937183976173401,
|
|
"loss/reference_anchor": 0.6773186922073364,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.292163848876953,
|
|
"rewards/margins": 1.6896095275878906,
|
|
"rewards/rejected": 1.6025537252426147,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15980662405490875,
|
|
"drift/rejected_abs": 0.10799809545278549,
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 418.0,
|
|
"learning_rate": 2.2697515850552152e-07,
|
|
"logits/chosen": -2.347092866897583,
|
|
"logits/rejected": -2.329366683959961,
|
|
"logps/chosen": -0.288370281457901,
|
|
"logps/rejected": -0.29249584674835205,
|
|
"loss": 4437.6669921875,
|
|
"loss/core": 4437.66357421875,
|
|
"loss/preference_sft": 0.288370281457901,
|
|
"loss/reference_anchor": 0.1452062875032425,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5972728729248047,
|
|
"rewards/margins": 0.5199095606803894,
|
|
"rewards/rejected": 1.07736337184906,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1499588042497635,
|
|
"drift/rejected_abs": 0.0562698058784008,
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 376.0,
|
|
"learning_rate": 2.2555415488657775e-07,
|
|
"logits/chosen": -2.120940685272217,
|
|
"logits/rejected": -2.184211254119873,
|
|
"logps/chosen": -0.30225870013237,
|
|
"logps/rejected": -0.3060735762119293,
|
|
"loss": 4431.9951171875,
|
|
"loss/core": 4431.9931640625,
|
|
"loss/preference_sft": 0.30225870013237,
|
|
"loss/reference_anchor": 0.07390942424535751,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.499276876449585,
|
|
"rewards/margins": 0.9399444460868835,
|
|
"rewards/rejected": 0.5593323707580566,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23949389159679413,
|
|
"drift/rejected_abs": 0.10860034078359604,
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 482.0,
|
|
"learning_rate": 2.240953369623447e-07,
|
|
"logits/chosen": -2.2790729999542236,
|
|
"logits/rejected": -2.3324427604675293,
|
|
"logps/chosen": -0.27358928322792053,
|
|
"logps/rejected": -0.2760041356086731,
|
|
"loss": 4427.1962890625,
|
|
"loss/core": 4427.19140625,
|
|
"loss/preference_sft": 0.27358928322792053,
|
|
"loss/reference_anchor": 0.19914691150188446,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3933863639831543,
|
|
"rewards/margins": 1.307618260383606,
|
|
"rewards/rejected": 1.0857679843902588,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14223968982696533,
|
|
"drift/rejected_abs": 0.07098479568958282,
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 624.0,
|
|
"learning_rate": 2.225992533345824e-07,
|
|
"logits/chosen": -2.369666337966919,
|
|
"logits/rejected": -2.405613422393799,
|
|
"logps/chosen": -0.28374093770980835,
|
|
"logps/rejected": -0.2866232693195343,
|
|
"loss": 4434.9873046875,
|
|
"loss/core": 4434.9853515625,
|
|
"loss/preference_sft": 0.28374093770980835,
|
|
"loss/reference_anchor": 0.07014183700084686,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.4223629236221313,
|
|
"rewards/margins": 0.7129005193710327,
|
|
"rewards/rejected": 0.7094623446464539,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18329541385173798,
|
|
"drift/rejected_abs": 0.06777513027191162,
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 1004.0,
|
|
"learning_rate": 2.210664666191579e-07,
|
|
"logits/chosen": -2.1638669967651367,
|
|
"logits/rejected": -2.2433056831359863,
|
|
"logps/chosen": -0.3162863850593567,
|
|
"logps/rejected": -0.2930854558944702,
|
|
"loss": 4429.1513671875,
|
|
"loss/core": 4429.1494140625,
|
|
"loss/preference_sft": 0.3162863850593567,
|
|
"loss/reference_anchor": 0.09944357722997665,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8323523998260498,
|
|
"rewards/margins": 1.1556055545806885,
|
|
"rewards/rejected": 0.6767469644546509,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24756188690662384,
|
|
"drift/rejected_abs": 0.1202814131975174,
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 1320.0,
|
|
"learning_rate": 2.1949755323446848e-07,
|
|
"logits/chosen": -2.2548816204071045,
|
|
"logits/rejected": -2.2354800701141357,
|
|
"logps/chosen": -0.2556923031806946,
|
|
"logps/rejected": -0.2606434226036072,
|
|
"loss": 4427.748046875,
|
|
"loss/core": 4427.7431640625,
|
|
"loss/preference_sft": 0.2556923031806946,
|
|
"loss/reference_anchor": 0.2529018521308899,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.473615884780884,
|
|
"rewards/margins": 1.273666501045227,
|
|
"rewards/rejected": 1.1999495029449463,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2426796853542328,
|
|
"drift/rejected_abs": 0.13973277807235718,
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 2256.0,
|
|
"learning_rate": 2.1789310318467426e-07,
|
|
"logits/chosen": -1.8090507984161377,
|
|
"logits/rejected": -1.8918311595916748,
|
|
"logps/chosen": -0.3173496425151825,
|
|
"logps/rejected": -0.32060688734054565,
|
|
"loss": 4430.7841796875,
|
|
"loss/core": 4430.7802734375,
|
|
"loss/preference_sft": 0.3173496425151825,
|
|
"loss/reference_anchor": 0.15054082870483398,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4233853816986084,
|
|
"rewards/margins": 1.029157280921936,
|
|
"rewards/rejected": 1.3942279815673828,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28747934103012085,
|
|
"drift/rejected_abs": 0.16268548369407654,
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 2704.0,
|
|
"learning_rate": 2.1625371983782182e-07,
|
|
"logits/chosen": -2.0867972373962402,
|
|
"logits/rejected": -2.0485246181488037,
|
|
"logps/chosen": -0.2663497030735016,
|
|
"logps/rejected": -0.27806463837623596,
|
|
"loss": 4427.39501953125,
|
|
"loss/core": 4427.3876953125,
|
|
"loss/preference_sft": 0.2663497030735016,
|
|
"loss/reference_anchor": 0.32496556639671326,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.874793291091919,
|
|
"rewards/margins": 1.3126866817474365,
|
|
"rewards/rejected": 1.562106728553772,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2037903517484665,
|
|
"drift/rejected_abs": 0.09013260900974274,
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 1144.0,
|
|
"learning_rate": 2.14580019698942e-07,
|
|
"logits/chosen": -2.1753249168395996,
|
|
"logits/rejected": -2.2431302070617676,
|
|
"logps/chosen": -0.294014573097229,
|
|
"logps/rejected": -0.292147696018219,
|
|
"loss": 4429.5263671875,
|
|
"loss/core": 4429.5224609375,
|
|
"loss/preference_sft": 0.294014573097229,
|
|
"loss/reference_anchor": 0.18461060523986816,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0374693870544434,
|
|
"rewards/margins": 1.1413319110870361,
|
|
"rewards/rejected": 0.8961375951766968,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2267070710659027,
|
|
"drift/rejected_abs": 0.16824612021446228,
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 14912.0,
|
|
"learning_rate": 2.1287263217820773e-07,
|
|
"logits/chosen": -2.1485249996185303,
|
|
"logits/rejected": -2.1281566619873047,
|
|
"logps/chosen": -0.27404850721359253,
|
|
"logps/rejected": -0.2835575342178345,
|
|
"loss": 4437.10986328125,
|
|
"loss/core": 4437.1025390625,
|
|
"loss/preference_sft": 0.27404850721359253,
|
|
"loss/reference_anchor": 0.32569584250450134,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.265812397003174,
|
|
"rewards/margins": 0.5857616662979126,
|
|
"rewards/rejected": 1.6800508499145508,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27692022919654846,
|
|
"drift/rejected_abs": 0.1342446506023407,
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 1496.0,
|
|
"learning_rate": 2.111321993542385e-07,
|
|
"logits/chosen": -1.9965057373046875,
|
|
"logits/rejected": -1.9842058420181274,
|
|
"logps/chosen": -0.2693387567996979,
|
|
"logps/rejected": -0.28916555643081665,
|
|
"loss": 4425.4814453125,
|
|
"loss/core": 4425.4775390625,
|
|
"loss/preference_sft": 0.2693387567996979,
|
|
"loss/reference_anchor": 0.1915903389453888,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7670462131500244,
|
|
"rewards/margins": 1.4375035762786865,
|
|
"rewards/rejected": 1.3295425176620483,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18260391056537628,
|
|
"drift/rejected_abs": 0.13512252271175385,
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 4416.0,
|
|
"learning_rate": 2.0935937573264096e-07,
|
|
"logits/chosen": -2.097672700881958,
|
|
"logits/rejected": -2.110541820526123,
|
|
"logps/chosen": -0.28904464840888977,
|
|
"logps/rejected": -0.29786160588264465,
|
|
"loss": 4438.23974609375,
|
|
"loss/core": 4438.23681640625,
|
|
"loss/preference_sft": 0.28904464840888977,
|
|
"loss/reference_anchor": 0.13433417677879333,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8258289098739624,
|
|
"rewards/margins": 0.4759580194950104,
|
|
"rewards/rejected": 1.3498706817626953,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20982956886291504,
|
|
"drift/rejected_abs": 0.137477308511734,
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 3392.0,
|
|
"learning_rate": 2.0755482799987596e-07,
|
|
"logits/chosen": -2.025709629058838,
|
|
"logits/rejected": -1.9913642406463623,
|
|
"logps/chosen": -0.2986164689064026,
|
|
"logps/rejected": -0.3017011284828186,
|
|
"loss": 4434.96875,
|
|
"loss/core": 4434.96337890625,
|
|
"loss/preference_sft": 0.2986164689064026,
|
|
"loss/reference_anchor": 0.26511436700820923,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0975375175476074,
|
|
"rewards/margins": 0.7258044481277466,
|
|
"rewards/rejected": 1.3717331886291504,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15806427597999573,
|
|
"drift/rejected_abs": 0.09864726662635803,
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 640.0,
|
|
"learning_rate": 2.0571923477254526e-07,
|
|
"logits/chosen": -2.2739768028259277,
|
|
"logits/rejected": -2.2487998008728027,
|
|
"logps/chosen": -0.2909921109676361,
|
|
"logps/rejected": -0.2905241847038269,
|
|
"loss": 4436.4794921875,
|
|
"loss/core": 4436.4765625,
|
|
"loss/preference_sft": 0.2909921109676361,
|
|
"loss/reference_anchor": 0.10613216459751129,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5796961784362793,
|
|
"rewards/margins": 0.600797176361084,
|
|
"rewards/rejected": 0.9788990020751953,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28660792112350464,
|
|
"drift/rejected_abs": 0.19245363771915436,
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 668.0,
|
|
"learning_rate": 2.038532863421914e-07,
|
|
"logits/chosen": -2.1396350860595703,
|
|
"logits/rejected": -2.0602240562438965,
|
|
"logps/chosen": -0.28930574655532837,
|
|
"logps/rejected": -0.2744773030281067,
|
|
"loss": 4432.1318359375,
|
|
"loss/core": 4432.1240234375,
|
|
"loss/preference_sft": 0.28930574655532837,
|
|
"loss/reference_anchor": 0.3356773257255554,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.8660788536071777,
|
|
"rewards/margins": 0.9429539442062378,
|
|
"rewards/rejected": 1.9231246709823608,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15086838603019714,
|
|
"drift/rejected_abs": 0.1520775854587555,
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 856.0,
|
|
"learning_rate": 2.0195768441570726e-07,
|
|
"logits/chosen": -1.9964319467544556,
|
|
"logits/rejected": -2.05317759513855,
|
|
"logps/chosen": -0.2916659116744995,
|
|
"logps/rejected": -0.2865615487098694,
|
|
"loss": 4445.12451171875,
|
|
"loss/core": 4445.11767578125,
|
|
"loss/preference_sft": 0.2916659116744995,
|
|
"loss/reference_anchor": 0.3259647488594055,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5083897113800049,
|
|
"rewards/margins": -0.008684945292770863,
|
|
"rewards/rejected": 1.5170745849609375,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.259127140045166,
|
|
"drift/rejected_abs": 0.11504222452640533,
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 1272.0,
|
|
"learning_rate": 2.0003314185145307e-07,
|
|
"logits/chosen": -2.14955735206604,
|
|
"logits/rejected": -2.0943520069122314,
|
|
"logps/chosen": -0.28039321303367615,
|
|
"logps/rejected": -0.283574640750885,
|
|
"loss": 4425.5029296875,
|
|
"loss/core": 4425.498046875,
|
|
"loss/preference_sft": 0.28039321303367615,
|
|
"loss/reference_anchor": 0.229940265417099,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5912716388702393,
|
|
"rewards/margins": 1.4409186840057373,
|
|
"rewards/rejected": 1.150353193283081,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27030646800994873,
|
|
"drift/rejected_abs": 0.14940232038497925,
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 5376.0,
|
|
"learning_rate": 1.9808038239117913e-07,
|
|
"logits/chosen": -1.9976972341537476,
|
|
"logits/rejected": -1.965847373008728,
|
|
"logps/chosen": -0.28784728050231934,
|
|
"logps/rejected": -0.2873719334602356,
|
|
"loss": 4427.10205078125,
|
|
"loss/core": 4427.0966796875,
|
|
"loss/preference_sft": 0.28784728050231934,
|
|
"loss/reference_anchor": 0.2397988736629486,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.7028987407684326,
|
|
"rewards/margins": 1.3128286600112915,
|
|
"rewards/rejected": 1.3900703191757202,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16661614179611206,
|
|
"drift/rejected_abs": 0.08331886678934097,
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 1120.0,
|
|
"learning_rate": 1.9610014038785646e-07,
|
|
"logits/chosen": -2.2249255180358887,
|
|
"logits/rejected": -2.211129665374756,
|
|
"logps/chosen": -0.2958478629589081,
|
|
"logps/rejected": -0.3064306080341339,
|
|
"loss": 4433.41796875,
|
|
"loss/core": 4433.41552734375,
|
|
"loss/preference_sft": 0.2958478629589081,
|
|
"loss/reference_anchor": 0.08543743938207626,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.664698600769043,
|
|
"rewards/margins": 0.83290034532547,
|
|
"rewards/rejected": 0.8317981958389282,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28869912028312683,
|
|
"drift/rejected_abs": 0.23126927018165588,
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 1304.0,
|
|
"learning_rate": 1.9409316052951657e-07,
|
|
"logits/chosen": -1.9891431331634521,
|
|
"logits/rejected": -2.0764071941375732,
|
|
"logps/chosen": -0.2763379216194153,
|
|
"logps/rejected": -0.2923387885093689,
|
|
"loss": 4436.98974609375,
|
|
"loss/core": 4436.9755859375,
|
|
"loss/preference_sft": 0.2763379216194153,
|
|
"loss/reference_anchor": 0.6921347975730896,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.886991500854492,
|
|
"rewards/margins": 0.5755015015602112,
|
|
"rewards/rejected": 2.311490058898926,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37554502487182617,
|
|
"drift/rejected_abs": 0.12323877960443497,
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 2496.0,
|
|
"learning_rate": 1.920601975592047e-07,
|
|
"logits/chosen": -2.176884174346924,
|
|
"logits/rejected": -2.1506075859069824,
|
|
"logps/chosen": -0.2848382294178009,
|
|
"logps/rejected": -0.29111620783805847,
|
|
"loss": 4411.89306640625,
|
|
"loss/core": 4411.876953125,
|
|
"loss/preference_sft": 0.2848382294178009,
|
|
"loss/reference_anchor": 0.7674547433853149,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.7542431354522705,
|
|
"rewards/margins": 2.534789562225342,
|
|
"rewards/rejected": 1.2194538116455078,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1966264843940735,
|
|
"drift/rejected_abs": 0.07787960767745972,
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 470.0,
|
|
"learning_rate": 1.900020159911519e-07,
|
|
"logits/chosen": -2.094768524169922,
|
|
"logits/rejected": -2.162729263305664,
|
|
"logps/chosen": -0.3112557828426361,
|
|
"logps/rejected": -0.31276172399520874,
|
|
"loss": 4428.56884765625,
|
|
"loss/core": 4428.56494140625,
|
|
"loss/preference_sft": 0.3112557828426361,
|
|
"loss/reference_anchor": 0.14194537699222565,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9648243188858032,
|
|
"rewards/margins": 1.2004649639129639,
|
|
"rewards/rejected": 0.7643591165542603,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17585548758506775,
|
|
"drift/rejected_abs": 0.1803542524576187,
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 2096.0,
|
|
"learning_rate": 1.879193898232725e-07,
|
|
"logits/chosen": -2.1437807083129883,
|
|
"logits/rejected": -2.1195027828216553,
|
|
"logps/chosen": -0.31370845437049866,
|
|
"logps/rejected": -0.3198932111263275,
|
|
"loss": 4445.20849609375,
|
|
"loss/core": 4445.2041015625,
|
|
"loss/preference_sft": 0.31370845437049866,
|
|
"loss/reference_anchor": 0.19741208851337433,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.757554292678833,
|
|
"rewards/margins": -0.0393296480178833,
|
|
"rewards/rejected": 1.7968838214874268,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3701009154319763,
|
|
"drift/rejected_abs": 0.08817116916179657,
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 2512.0,
|
|
"learning_rate": 1.8581310224609496e-07,
|
|
"logits/chosen": -2.1306817531585693,
|
|
"logits/rejected": -2.202624797821045,
|
|
"logps/chosen": -0.2909499704837799,
|
|
"logps/rejected": -0.2839373052120209,
|
|
"loss": 4408.3310546875,
|
|
"loss/core": 4408.3134765625,
|
|
"loss/preference_sft": 0.2909499704837799,
|
|
"loss/reference_anchor": 0.8197543025016785,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.6982064247131348,
|
|
"rewards/margins": 2.817063808441162,
|
|
"rewards/rejected": 0.8811424970626831,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2949160039424896,
|
|
"drift/rejected_abs": 0.12354662269353867,
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 1032.0,
|
|
"learning_rate": 1.8368394534823635e-07,
|
|
"logits/chosen": -2.03263258934021,
|
|
"logits/rejected": -2.0422098636627197,
|
|
"logps/chosen": -0.268283873796463,
|
|
"logps/rejected": -0.2804756760597229,
|
|
"loss": 4421.8681640625,
|
|
"loss/core": 4421.8623046875,
|
|
"loss/preference_sft": 0.268283873796463,
|
|
"loss/reference_anchor": 0.2536497116088867,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.947744846343994,
|
|
"rewards/margins": 1.7133744955062866,
|
|
"rewards/rejected": 1.234370231628418,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.49125900864601135,
|
|
"drift/rejected_abs": 0.23263196647167206,
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 1624.0,
|
|
"learning_rate": 1.8153271981852968e-07,
|
|
"logits/chosen": -2.010446310043335,
|
|
"logits/rejected": -1.984309196472168,
|
|
"logps/chosen": -0.30579593777656555,
|
|
"logps/rejected": -0.29049938917160034,
|
|
"loss": 4411.03564453125,
|
|
"loss/core": 4411.00390625,
|
|
"loss/preference_sft": 0.30579593777656555,
|
|
"loss/reference_anchor": 1.5279619693756104,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.912415504455566,
|
|
"rewards/margins": 2.5893051624298096,
|
|
"rewards/rejected": 2.323111057281494,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2286815345287323,
|
|
"drift/rejected_abs": 0.1312122792005539,
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 3968.0,
|
|
"learning_rate": 1.7936023464491812e-07,
|
|
"logits/chosen": -2.0856316089630127,
|
|
"logits/rejected": -2.1967413425445557,
|
|
"logps/chosen": -0.27951109409332275,
|
|
"logps/rejected": -0.2807064950466156,
|
|
"loss": 4431.7783203125,
|
|
"loss/core": 4431.7705078125,
|
|
"loss/preference_sft": 0.27951109409332275,
|
|
"loss/reference_anchor": 0.35531511902809143,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.286098003387451,
|
|
"rewards/margins": 0.9744806289672852,
|
|
"rewards/rejected": 1.311617136001587,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1671498715877533,
|
|
"drift/rejected_abs": 0.1056649461388588,
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 1776.0,
|
|
"learning_rate": 1.7716730681022723e-07,
|
|
"logits/chosen": -2.121617078781128,
|
|
"logits/rejected": -2.081577777862549,
|
|
"logps/chosen": -0.31088462471961975,
|
|
"logps/rejected": -0.28644299507141113,
|
|
"loss": 4436.3212890625,
|
|
"loss/core": 4436.31787109375,
|
|
"loss/preference_sft": 0.31088462471961975,
|
|
"loss/reference_anchor": 0.11547583341598511,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6688594818115234,
|
|
"rewards/margins": 0.6149738430976868,
|
|
"rewards/rejected": 1.0538856983184814,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22545960545539856,
|
|
"drift/rejected_abs": 0.12498001754283905,
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 580.0,
|
|
"learning_rate": 1.7495476098493152e-07,
|
|
"logits/chosen": -2.199582099914551,
|
|
"logits/rejected": -2.216892957687378,
|
|
"logps/chosen": -0.27879446744918823,
|
|
"logps/rejected": -0.28513437509536743,
|
|
"loss": 4430.99560546875,
|
|
"loss/core": 4430.9912109375,
|
|
"loss/preference_sft": 0.27879446744918823,
|
|
"loss/reference_anchor": 0.1869107335805893,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2526803016662598,
|
|
"rewards/margins": 1.0196499824523926,
|
|
"rewards/rejected": 1.2330303192138672,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3004913330078125,
|
|
"drift/rejected_abs": 0.16865012049674988,
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 704.0,
|
|
"learning_rate": 1.7272342921702937e-07,
|
|
"logits/chosen": -2.0919270515441895,
|
|
"logits/rejected": -2.11561918258667,
|
|
"logps/chosen": -0.30641791224479675,
|
|
"logps/rejected": -0.303880900144577,
|
|
"loss": 4427.22021484375,
|
|
"loss/core": 4427.20849609375,
|
|
"loss/preference_sft": 0.30641791224479675,
|
|
"loss/reference_anchor": 0.5793026089668274,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0033681392669678,
|
|
"rewards/margins": 1.31758713722229,
|
|
"rewards/rejected": 1.6857811212539673,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2755824029445648,
|
|
"drift/rejected_abs": 0.2149803191423416,
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 432.0,
|
|
"learning_rate": 1.7047415061914393e-07,
|
|
"logits/chosen": -2.0630311965942383,
|
|
"logits/rejected": -2.2178916931152344,
|
|
"logps/chosen": -0.27443453669548035,
|
|
"logps/rejected": -0.2907470166683197,
|
|
"loss": 4437.09130859375,
|
|
"loss/core": 4437.0791015625,
|
|
"loss/preference_sft": 0.27443453669548035,
|
|
"loss/reference_anchor": 0.5900745987892151,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.7551357746124268,
|
|
"rewards/margins": 0.606482982635498,
|
|
"rewards/rejected": 2.1486525535583496,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26228779554367065,
|
|
"drift/rejected_abs": 0.12948082387447357,
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 4704.0,
|
|
"learning_rate": 1.6820777105296707e-07,
|
|
"logits/chosen": -2.1135382652282715,
|
|
"logits/rejected": -2.212303876876831,
|
|
"logps/chosen": -0.3208262026309967,
|
|
"logps/rejected": -0.2868503928184509,
|
|
"loss": 4427.3076171875,
|
|
"loss/core": 4427.296875,
|
|
"loss/preference_sft": 0.3208262026309967,
|
|
"loss/reference_anchor": 0.47984394431114197,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6186184883117676,
|
|
"rewards/margins": 1.3284586668014526,
|
|
"rewards/rejected": 1.2901595830917358,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2780519723892212,
|
|
"drift/rejected_abs": 0.13061827421188354,
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 540.0,
|
|
"learning_rate": 1.6592514281116553e-07,
|
|
"logits/chosen": -2.0848355293273926,
|
|
"logits/rejected": -2.101752519607544,
|
|
"logps/chosen": -0.26733091473579407,
|
|
"logps/rejected": -0.2704174518585205,
|
|
"loss": 4425.0966796875,
|
|
"loss/core": 4425.09033203125,
|
|
"loss/preference_sft": 0.26733091473579407,
|
|
"loss/reference_anchor": 0.29540133476257324,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.780134439468384,
|
|
"rewards/margins": 1.4746670722961426,
|
|
"rewards/rejected": 1.3054672479629517,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23144730925559998,
|
|
"drift/rejected_abs": 0.07132745534181595,
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 7296.0,
|
|
"learning_rate": 1.636271242968684e-07,
|
|
"logits/chosen": -2.039712429046631,
|
|
"logits/rejected": -2.0561647415161133,
|
|
"logps/chosen": -0.2998116612434387,
|
|
"logps/rejected": -0.28820380568504333,
|
|
"loss": 4422.5302734375,
|
|
"loss/core": 4422.5244140625,
|
|
"loss/preference_sft": 0.2998116612434387,
|
|
"loss/reference_anchor": 0.21635302901268005,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.3144731521606445,
|
|
"rewards/margins": 1.6700124740600586,
|
|
"rewards/rejected": 0.644460916519165,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32540231943130493,
|
|
"drift/rejected_abs": 0.18714219331741333,
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 900.0,
|
|
"learning_rate": 1.6131457970085684e-07,
|
|
"logits/chosen": -1.9318962097167969,
|
|
"logits/rejected": -1.9521112442016602,
|
|
"logps/chosen": -0.284771591424942,
|
|
"logps/rejected": -0.2870418131351471,
|
|
"loss": 4426.5439453125,
|
|
"loss/core": 4426.5341796875,
|
|
"loss/preference_sft": 0.284771591424942,
|
|
"loss/reference_anchor": 0.46427035331726074,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2529983520507812,
|
|
"rewards/margins": 1.3839852809906006,
|
|
"rewards/rejected": 1.8690131902694702,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2929674983024597,
|
|
"drift/rejected_abs": 0.1980661153793335,
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 1904.0,
|
|
"learning_rate": 1.5898837867657727e-07,
|
|
"logits/chosen": -1.9516912698745728,
|
|
"logits/rejected": -1.9994312524795532,
|
|
"logps/chosen": -0.28913792967796326,
|
|
"logps/rejected": -0.27645057439804077,
|
|
"loss": 4432.4873046875,
|
|
"loss/core": 4432.4775390625,
|
|
"loss/preference_sft": 0.28913792967796326,
|
|
"loss/reference_anchor": 0.4840005040168762,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9288175106048584,
|
|
"rewards/margins": 0.9494360685348511,
|
|
"rewards/rejected": 1.9793813228607178,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25099247694015503,
|
|
"drift/rejected_abs": 0.0831591859459877,
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 668.0,
|
|
"learning_rate": 1.5664939601310023e-07,
|
|
"logits/chosen": -2.182654857635498,
|
|
"logits/rejected": -2.2524609565734863,
|
|
"logps/chosen": -0.3082042336463928,
|
|
"logps/rejected": -0.30552542209625244,
|
|
"loss": 4422.7587890625,
|
|
"loss/core": 4422.75,
|
|
"loss/preference_sft": 0.3082042336463928,
|
|
"loss/reference_anchor": 0.41477394104003906,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.5072789192199707,
|
|
"rewards/margins": 1.676422357559204,
|
|
"rewards/rejected": 0.830856442451477,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24665072560310364,
|
|
"drift/rejected_abs": 0.12369408458471298,
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 1112.0,
|
|
"learning_rate": 1.5429851130614807e-07,
|
|
"logits/chosen": -2.1079397201538086,
|
|
"logits/rejected": -2.0382118225097656,
|
|
"logps/chosen": -0.28440576791763306,
|
|
"logps/rejected": -0.26352208852767944,
|
|
"loss": 4428.2412109375,
|
|
"loss/core": 4428.23681640625,
|
|
"loss/preference_sft": 0.28440576791763306,
|
|
"loss/reference_anchor": 0.22342030704021454,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4649205207824707,
|
|
"rewards/margins": 1.229503870010376,
|
|
"rewards/rejected": 1.2354166507720947,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22085194289684296,
|
|
"drift/rejected_abs": 0.08876828849315643,
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 4608.0,
|
|
"learning_rate": 1.51936608627315e-07,
|
|
"logits/chosen": -2.0002598762512207,
|
|
"logits/rejected": -2.0259335041046143,
|
|
"logps/chosen": -0.3116574287414551,
|
|
"logps/rejected": -0.31580719351768494,
|
|
"loss": 4426.97900390625,
|
|
"loss/core": 4426.974609375,
|
|
"loss/preference_sft": 0.3116574287414551,
|
|
"loss/reference_anchor": 0.183195561170578,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.205994129180908,
|
|
"rewards/margins": 1.3205196857452393,
|
|
"rewards/rejected": 0.8854740858078003,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37103694677352905,
|
|
"drift/rejected_abs": 0.18114325404167175,
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 7104.0,
|
|
"learning_rate": 1.4956457619160375e-07,
|
|
"logits/chosen": -2.050297975540161,
|
|
"logits/rejected": -2.0345191955566406,
|
|
"logps/chosen": -0.27982237935066223,
|
|
"logps/rejected": -0.25793343782424927,
|
|
"loss": 4420.06103515625,
|
|
"loss/core": 4420.046875,
|
|
"loss/preference_sft": 0.27982237935066223,
|
|
"loss/reference_anchor": 0.6739336252212524,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.709260940551758,
|
|
"rewards/margins": 1.9022051095962524,
|
|
"rewards/rejected": 1.8070557117462158,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19236549735069275,
|
|
"drift/rejected_abs": 0.08619394898414612,
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 768.0,
|
|
"learning_rate": 1.471833060234044e-07,
|
|
"logits/chosen": -2.10903263092041,
|
|
"logits/rejected": -2.172616481781006,
|
|
"logps/chosen": -0.30059614777565,
|
|
"logps/rejected": -0.2986820936203003,
|
|
"loss": 4430.38623046875,
|
|
"loss/core": 4430.3837890625,
|
|
"loss/preference_sft": 0.30059614777565,
|
|
"loss/reference_anchor": 0.12340258061885834,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9236551523208618,
|
|
"rewards/margins": 1.0623431205749512,
|
|
"rewards/rejected": 0.8613119125366211,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21780920028686523,
|
|
"drift/rejected_abs": 0.13248507678508759,
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 1728.0,
|
|
"learning_rate": 1.4479369362104037e-07,
|
|
"logits/chosen": -2.1391212940216064,
|
|
"logits/rejected": -2.252551555633545,
|
|
"logps/chosen": -0.26534712314605713,
|
|
"logps/rejected": -0.27983564138412476,
|
|
"loss": 4433.15234375,
|
|
"loss/core": 4433.1484375,
|
|
"loss/preference_sft": 0.26534712314605713,
|
|
"loss/reference_anchor": 0.1820131540298462,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.1771018505096436,
|
|
"rewards/margins": 0.8640018701553345,
|
|
"rewards/rejected": 1.313099980354309,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20761752128601074,
|
|
"drift/rejected_abs": 0.09282220900058746,
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 652.0,
|
|
"learning_rate": 1.4239663762000817e-07,
|
|
"logits/chosen": -2.139101505279541,
|
|
"logits/rejected": -2.070023536682129,
|
|
"logps/chosen": -0.2922465205192566,
|
|
"logps/rejected": -0.29120540618896484,
|
|
"loss": 4429.29296875,
|
|
"loss/core": 4429.28857421875,
|
|
"loss/preference_sft": 0.2922465205192566,
|
|
"loss/reference_anchor": 0.17264321446418762,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.0735580921173096,
|
|
"rewards/margins": 1.1463443040847778,
|
|
"rewards/rejected": 0.9272136688232422,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2610882520675659,
|
|
"drift/rejected_abs": 0.143789604306221,
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 10240.0,
|
|
"learning_rate": 1.3999303945503747e-07,
|
|
"logits/chosen": -2.1653521060943604,
|
|
"logits/rejected": -2.247696876525879,
|
|
"logps/chosen": -0.31048905849456787,
|
|
"logps/rejected": -0.31169334053993225,
|
|
"loss": 4429.36865234375,
|
|
"loss/core": 4429.359375,
|
|
"loss/preference_sft": 0.31048905849456787,
|
|
"loss/reference_anchor": 0.44403213262557983,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.610751152038574,
|
|
"rewards/margins": 1.1738497018814087,
|
|
"rewards/rejected": 1.4369014501571655,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15377414226531982,
|
|
"drift/rejected_abs": 0.10233066231012344,
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 884.0,
|
|
"learning_rate": 1.3758380302109808e-07,
|
|
"logits/chosen": -2.145686626434326,
|
|
"logits/rejected": -2.1528048515319824,
|
|
"logps/chosen": -0.2958281636238098,
|
|
"logps/rejected": -0.26482534408569336,
|
|
"loss": 4437.5927734375,
|
|
"loss/core": 4437.5908203125,
|
|
"loss/preference_sft": 0.2958281636238098,
|
|
"loss/reference_anchor": 0.08905575424432755,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.5374128818511963,
|
|
"rewards/margins": 0.5175694227218628,
|
|
"rewards/rejected": 1.019843578338623,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19338826835155487,
|
|
"drift/rejected_abs": 0.05476611852645874,
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 520.0,
|
|
"learning_rate": 1.351698343334823e-07,
|
|
"logits/chosen": -2.10408091545105,
|
|
"logits/rejected": -2.1622872352600098,
|
|
"logps/chosen": -0.3007555603981018,
|
|
"logps/rejected": -0.29687556624412537,
|
|
"loss": 4426.18212890625,
|
|
"loss/core": 4426.17724609375,
|
|
"loss/preference_sft": 0.3007555603981018,
|
|
"loss/reference_anchor": 0.22863420844078064,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9303607940673828,
|
|
"rewards/margins": 1.399163842201233,
|
|
"rewards/rejected": 0.5311970114707947,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24555771052837372,
|
|
"drift/rejected_abs": 0.133039191365242,
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 5504.0,
|
|
"learning_rate": 1.3275204118708942e-07,
|
|
"logits/chosen": -2.0921359062194824,
|
|
"logits/rejected": -1.9713423252105713,
|
|
"logps/chosen": -0.301283597946167,
|
|
"logps/rejected": -0.31479784846305847,
|
|
"loss": 4430.07861328125,
|
|
"loss/core": 4430.0654296875,
|
|
"loss/preference_sft": 0.301283597946167,
|
|
"loss/reference_anchor": 0.6136071681976318,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4555182456970215,
|
|
"rewards/margins": 1.125307559967041,
|
|
"rewards/rejected": 1.33021080493927,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20899991691112518,
|
|
"drift/rejected_abs": 0.10918338596820831,
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 2944.0,
|
|
"learning_rate": 1.3033133281504132e-07,
|
|
"logits/chosen": -1.9529765844345093,
|
|
"logits/rejected": -1.9704357385635376,
|
|
"logps/chosen": -0.29765480756759644,
|
|
"logps/rejected": -0.296121209859848,
|
|
"loss": 4431.419921875,
|
|
"loss/core": 4431.4150390625,
|
|
"loss/preference_sft": 0.29765480756759644,
|
|
"loss/reference_anchor": 0.23028752207756042,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.089999198913574,
|
|
"rewards/margins": 1.000409483909607,
|
|
"rewards/rejected": 1.0895897150039673,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23929134011268616,
|
|
"drift/rejected_abs": 0.11025607585906982,
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 980.0,
|
|
"learning_rate": 1.2790861954675702e-07,
|
|
"logits/chosen": -2.0607128143310547,
|
|
"logits/rejected": -2.0974857807159424,
|
|
"logps/chosen": -0.3084251582622528,
|
|
"logps/rejected": -0.30655646324157715,
|
|
"loss": 4427.3017578125,
|
|
"loss/core": 4427.2978515625,
|
|
"loss/preference_sft": 0.3084251582622528,
|
|
"loss/reference_anchor": 0.14670784771442413,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3920226097106934,
|
|
"rewards/margins": 1.2922639846801758,
|
|
"rewards/rejected": 1.0997586250305176,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19296832382678986,
|
|
"drift/rejected_abs": 0.09996967762708664,
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 1216.0,
|
|
"learning_rate": 1.2548481246561504e-07,
|
|
"logits/chosen": -2.0633440017700195,
|
|
"logits/rejected": -2.0872926712036133,
|
|
"logps/chosen": -0.2833230495452881,
|
|
"logps/rejected": -0.28482872247695923,
|
|
"loss": 4432.07861328125,
|
|
"loss/core": 4432.0751953125,
|
|
"loss/preference_sft": 0.2833230495452881,
|
|
"loss/reference_anchor": 0.14006902277469635,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.929682970046997,
|
|
"rewards/margins": 0.9321301579475403,
|
|
"rewards/rejected": 0.9975531697273254,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36558204889297485,
|
|
"drift/rejected_abs": 0.1663735955953598,
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 5088.0,
|
|
"learning_rate": 1.230608230663321e-07,
|
|
"logits/chosen": -2.070899486541748,
|
|
"logits/rejected": -2.0423977375030518,
|
|
"logps/chosen": -0.2811706066131592,
|
|
"logps/rejected": -0.28334346413612366,
|
|
"loss": 4418.2373046875,
|
|
"loss/core": 4418.22265625,
|
|
"loss/preference_sft": 0.2811706066131592,
|
|
"loss/reference_anchor": 0.7044563889503479,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.652653455734253,
|
|
"rewards/margins": 1.9970003366470337,
|
|
"rewards/rejected": 1.6556532382965088,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34318313002586365,
|
|
"drift/rejected_abs": 0.13852709531784058,
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 816.0,
|
|
"learning_rate": 1.206375629121874e-07,
|
|
"logits/chosen": -2.0183844566345215,
|
|
"logits/rejected": -2.008913278579712,
|
|
"logps/chosen": -0.3097713589668274,
|
|
"logps/rejected": -0.31553733348846436,
|
|
"loss": 4417.1552734375,
|
|
"loss/core": 4417.146484375,
|
|
"loss/preference_sft": 0.3097713589668274,
|
|
"loss/reference_anchor": 0.4379523694515228,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4314303398132324,
|
|
"rewards/margins": 2.0832791328430176,
|
|
"rewards/rejected": 1.348151683807373,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2345653474330902,
|
|
"drift/rejected_abs": 0.12095403671264648,
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 1072.0,
|
|
"learning_rate": 1.1821594329222079e-07,
|
|
"logits/chosen": -2.2127575874328613,
|
|
"logits/rejected": -2.1563878059387207,
|
|
"logps/chosen": -0.2902802526950836,
|
|
"logps/rejected": -0.28386133909225464,
|
|
"loss": 4429.57666015625,
|
|
"loss/core": 4429.5654296875,
|
|
"loss/preference_sft": 0.2902802526950836,
|
|
"loss/reference_anchor": 0.5237426161766052,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3436882495880127,
|
|
"rewards/margins": 1.1352647542953491,
|
|
"rewards/rejected": 1.208423376083374,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18333016335964203,
|
|
"drift/rejected_abs": 0.1809997260570526,
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 2576.0,
|
|
"learning_rate": 1.157968748785344e-07,
|
|
"logits/chosen": -1.914223074913025,
|
|
"logits/rejected": -1.981766939163208,
|
|
"logps/chosen": -0.2812362015247345,
|
|
"logps/rejected": -0.27950748801231384,
|
|
"loss": 4444.3720703125,
|
|
"loss/core": 4444.36767578125,
|
|
"loss/preference_sft": 0.2812362015247345,
|
|
"loss/reference_anchor": 0.19075706601142883,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.8306372165679932,
|
|
"rewards/margins": 0.022197848185896873,
|
|
"rewards/rejected": 1.8084392547607422,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3202487826347351,
|
|
"drift/rejected_abs": 0.13263753056526184,
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 2064.0,
|
|
"learning_rate": 1.1338126738382597e-07,
|
|
"logits/chosen": -1.9274823665618896,
|
|
"logits/rejected": -1.931096076965332,
|
|
"logps/chosen": -0.2886351943016052,
|
|
"logps/rejected": -0.2917082607746124,
|
|
"loss": 4419.962890625,
|
|
"loss/core": 4419.9501953125,
|
|
"loss/preference_sft": 0.2886351943016052,
|
|
"loss/reference_anchor": 0.6030197143554688,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.202364444732666,
|
|
"rewards/margins": 1.8788830041885376,
|
|
"rewards/rejected": 1.3234814405441284,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21438315510749817,
|
|
"drift/rejected_abs": 0.07398609071969986,
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 604.0,
|
|
"learning_rate": 1.1097002921928316e-07,
|
|
"logits/chosen": -2.086273670196533,
|
|
"logits/rejected": -2.169018268585205,
|
|
"logps/chosen": -0.2757102847099304,
|
|
"logps/rejected": -0.27384692430496216,
|
|
"loss": 4425.99462890625,
|
|
"loss/core": 4425.9892578125,
|
|
"loss/preference_sft": 0.2757102847099304,
|
|
"loss/reference_anchor": 0.19271975755691528,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.142575740814209,
|
|
"rewards/margins": 1.4045182466506958,
|
|
"rewards/rejected": 0.7380577921867371,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16398575901985168,
|
|
"drift/rejected_abs": 0.08183001726865768,
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 556.0,
|
|
"learning_rate": 1.0856406715296717e-07,
|
|
"logits/chosen": -2.1869404315948486,
|
|
"logits/rejected": -2.284536361694336,
|
|
"logps/chosen": -0.2831934988498688,
|
|
"logps/rejected": -0.2927997410297394,
|
|
"loss": 4433.50634765625,
|
|
"loss/core": 4433.50341796875,
|
|
"loss/preference_sft": 0.2831934988498688,
|
|
"loss/reference_anchor": 0.07691037654876709,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6397440433502197,
|
|
"rewards/margins": 0.822593092918396,
|
|
"rewards/rejected": 0.817150890827179,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3339373767375946,
|
|
"drift/rejected_abs": 0.12080500274896622,
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 564.0,
|
|
"learning_rate": 1.061642859688146e-07,
|
|
"logits/chosen": -2.11379075050354,
|
|
"logits/rejected": -2.042389392852783,
|
|
"logps/chosen": -0.2667238712310791,
|
|
"logps/rejected": -0.27604439854621887,
|
|
"loss": 4416.22265625,
|
|
"loss/core": 4416.216796875,
|
|
"loss/preference_sft": 0.2667238712310791,
|
|
"loss/reference_anchor": 0.2798369526863098,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.339317798614502,
|
|
"rewards/margins": 2.1484532356262207,
|
|
"rewards/rejected": 1.1908643245697021,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2929610311985016,
|
|
"drift/rejected_abs": 0.08645624667406082,
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 2688.0,
|
|
"learning_rate": 1.0377158812638491e-07,
|
|
"logits/chosen": -2.1425912380218506,
|
|
"logits/rejected": -2.234184980392456,
|
|
"logps/chosen": -0.29007601737976074,
|
|
"logps/rejected": -0.280531108379364,
|
|
"loss": 4416.90625,
|
|
"loss/core": 4416.8984375,
|
|
"loss/preference_sft": 0.29007601737976074,
|
|
"loss/reference_anchor": 0.3554505705833435,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9293251037597656,
|
|
"rewards/margins": 2.1080482006073,
|
|
"rewards/rejected": 0.821276843547821,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2901972234249115,
|
|
"drift/rejected_abs": 0.14684346318244934,
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 3792.0,
|
|
"learning_rate": 1.0138687342148249e-07,
|
|
"logits/chosen": -2.062620162963867,
|
|
"logits/rejected": -2.1082162857055664,
|
|
"logps/chosen": -0.30190232396125793,
|
|
"logps/rejected": -0.32236361503601074,
|
|
"loss": 4426.2060546875,
|
|
"loss/core": 4426.193359375,
|
|
"loss/preference_sft": 0.30190232396125793,
|
|
"loss/reference_anchor": 0.6151673197746277,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.90141224861145,
|
|
"rewards/margins": 1.4344381093978882,
|
|
"rewards/rejected": 1.4669743776321411,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24899080395698547,
|
|
"drift/rejected_abs": 0.20708198845386505,
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 592.0,
|
|
"learning_rate": 9.90110386477801e-08,
|
|
"logits/chosen": -2.1318652629852295,
|
|
"logits/rejected": -2.2062549591064453,
|
|
"logps/chosen": -0.25858795642852783,
|
|
"logps/rejected": -0.253892183303833,
|
|
"loss": 4438.9169921875,
|
|
"loss/core": 4438.9072265625,
|
|
"loss/preference_sft": 0.25858795642852783,
|
|
"loss/reference_anchor": 0.47126907110214233,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.489907741546631,
|
|
"rewards/margins": 0.4215244650840759,
|
|
"rewards/rejected": 2.0683834552764893,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21308758854866028,
|
|
"drift/rejected_abs": 0.15445470809936523,
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 1928.0,
|
|
"learning_rate": 9.664497725957164e-08,
|
|
"logits/chosen": -2.1934444904327393,
|
|
"logits/rejected": -2.2147269248962402,
|
|
"logps/chosen": -0.2912050783634186,
|
|
"logps/rejected": -0.2784259617328644,
|
|
"loss": 4436.732421875,
|
|
"loss/core": 4436.72607421875,
|
|
"loss/preference_sft": 0.2912050783634186,
|
|
"loss/reference_anchor": 0.23212206363677979,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.127945899963379,
|
|
"rewards/margins": 0.5856080651283264,
|
|
"rewards/rejected": 1.5423381328582764,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2658155560493469,
|
|
"drift/rejected_abs": 0.19533106684684753,
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 7616.0,
|
|
"learning_rate": 9.428957903578045e-08,
|
|
"logits/chosen": -1.9657647609710693,
|
|
"logits/rejected": -1.980133295059204,
|
|
"logps/chosen": -0.2851273715496063,
|
|
"logps/rejected": -0.28451281785964966,
|
|
"loss": 4435.39208984375,
|
|
"loss/core": 4435.38037109375,
|
|
"loss/preference_sft": 0.2851273715496063,
|
|
"loss/reference_anchor": 0.557953417301178,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.6540966033935547,
|
|
"rewards/margins": 0.7040950655937195,
|
|
"rewards/rejected": 1.9500014781951904,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34854352474212646,
|
|
"drift/rejected_abs": 0.19613364338874817,
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 696.0,
|
|
"learning_rate": 9.194572974534976e-08,
|
|
"logits/chosen": -2.024336338043213,
|
|
"logits/rejected": -2.093052864074707,
|
|
"logps/chosen": -0.27176398038864136,
|
|
"logps/rejected": -0.26332786679267883,
|
|
"loss": 4424.57861328125,
|
|
"loss/core": 4424.56201171875,
|
|
"loss/preference_sft": 0.27176398038864136,
|
|
"loss/reference_anchor": 0.8150361776351929,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4846279621124268,
|
|
"rewards/margins": 1.5311681032180786,
|
|
"rewards/rejected": 1.9534600973129272,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.227237731218338,
|
|
"drift/rejected_abs": 0.16712255775928497,
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 4416.0,
|
|
"learning_rate": 8.96143108141412e-08,
|
|
"logits/chosen": -1.9618107080459595,
|
|
"logits/rejected": -1.954664945602417,
|
|
"logps/chosen": -0.2768324613571167,
|
|
"logps/rejected": -0.27417004108428955,
|
|
"loss": 4436.580078125,
|
|
"loss/core": 4436.5751953125,
|
|
"loss/preference_sft": 0.2768324613571167,
|
|
"loss/reference_anchor": 0.21785183250904083,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.271784543991089,
|
|
"rewards/margins": 0.6015557050704956,
|
|
"rewards/rejected": 1.670229196548462,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22515693306922913,
|
|
"drift/rejected_abs": 0.08775409311056137,
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 360.0,
|
|
"learning_rate": 8.72961989934668e-08,
|
|
"logits/chosen": -2.231755495071411,
|
|
"logits/rejected": -2.245164632797241,
|
|
"logps/chosen": -0.2810257077217102,
|
|
"logps/rejected": -0.28133219480514526,
|
|
"loss": 4426.43994140625,
|
|
"loss/core": 4426.4345703125,
|
|
"loss/preference_sft": 0.2810257077217102,
|
|
"loss/reference_anchor": 0.2272057831287384,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2512524127960205,
|
|
"rewards/margins": 1.3741371631622314,
|
|
"rewards/rejected": 0.8771151304244995,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14085161685943604,
|
|
"drift/rejected_abs": 0.1031990498304367,
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 720.0,
|
|
"learning_rate": 8.499226603037854e-08,
|
|
"logits/chosen": -2.2430803775787354,
|
|
"logits/rejected": -2.2786197662353516,
|
|
"logps/chosen": -0.2992137372493744,
|
|
"logps/rejected": -0.29944515228271484,
|
|
"loss": 4439.31689453125,
|
|
"loss/core": 4439.31396484375,
|
|
"loss/preference_sft": 0.2992137372493744,
|
|
"loss/reference_anchor": 0.10587255656719208,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.407078742980957,
|
|
"rewards/margins": 0.39435142278671265,
|
|
"rewards/rejected": 1.0127273797988892,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22692131996154785,
|
|
"drift/rejected_abs": 0.10220160335302353,
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 608.0,
|
|
"learning_rate": 8.270337833983987e-08,
|
|
"logits/chosen": -2.0355045795440674,
|
|
"logits/rejected": -2.1195898056030273,
|
|
"logps/chosen": -0.30346351861953735,
|
|
"logps/rejected": -0.2966625392436981,
|
|
"loss": 4427.97900390625,
|
|
"loss/core": 4427.97216796875,
|
|
"loss/preference_sft": 0.30346351861953735,
|
|
"loss/reference_anchor": 0.29685312509536743,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2688138484954834,
|
|
"rewards/margins": 1.247382402420044,
|
|
"rewards/rejected": 1.0214314460754395,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21154217422008514,
|
|
"drift/rejected_abs": 0.13458245992660522,
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 1672.0,
|
|
"learning_rate": 8.04303966789025e-08,
|
|
"logits/chosen": -2.1316933631896973,
|
|
"logits/rejected": -2.150923252105713,
|
|
"logps/chosen": -0.27440136671066284,
|
|
"logps/rejected": -0.27802544832229614,
|
|
"loss": 4434.2373046875,
|
|
"loss/core": 4434.2314453125,
|
|
"loss/preference_sft": 0.27440136671066284,
|
|
"loss/reference_anchor": 0.25449132919311523,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.114384889602661,
|
|
"rewards/margins": 0.7690680623054504,
|
|
"rewards/rejected": 1.3453166484832764,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21570686995983124,
|
|
"drift/rejected_abs": 0.11859677731990814,
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 1072.0,
|
|
"learning_rate": 7.817417582301098e-08,
|
|
"logits/chosen": -1.8743267059326172,
|
|
"logits/rejected": -1.8942207098007202,
|
|
"logps/chosen": -0.284064382314682,
|
|
"logps/rejected": -0.28866657614707947,
|
|
"loss": 4431.5322265625,
|
|
"loss/core": 4431.529296875,
|
|
"loss/preference_sft": 0.284064382314682,
|
|
"loss/reference_anchor": 0.1300676316022873,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1570687294006348,
|
|
"rewards/margins": 0.9721587896347046,
|
|
"rewards/rejected": 1.1849100589752197,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19431225955486298,
|
|
"drift/rejected_abs": 0.1243596076965332,
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 1800.0,
|
|
"learning_rate": 7.59355642445566e-08,
|
|
"logits/chosen": -2.2408175468444824,
|
|
"logits/rejected": -2.15144419670105,
|
|
"logps/chosen": -0.26184308528900146,
|
|
"logps/rejected": -0.2716737985610962,
|
|
"loss": 4434.80078125,
|
|
"loss/core": 4434.79736328125,
|
|
"loss/preference_sft": 0.26184308528900146,
|
|
"loss/reference_anchor": 0.16086336970329285,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9430396556854248,
|
|
"rewards/margins": 0.7340011596679688,
|
|
"rewards/rejected": 1.2090386152267456,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21506313979625702,
|
|
"drift/rejected_abs": 0.09374111890792847,
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 7328.0,
|
|
"learning_rate": 7.37154037938015e-08,
|
|
"logits/chosen": -2.073078155517578,
|
|
"logits/rejected": -2.174865245819092,
|
|
"logps/chosen": -0.3253878951072693,
|
|
"logps/rejected": -0.33154183626174927,
|
|
"loss": 4428.4287109375,
|
|
"loss/core": 4428.4248046875,
|
|
"loss/preference_sft": 0.3253878951072693,
|
|
"loss/reference_anchor": 0.17495615780353546,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.148521900177002,
|
|
"rewards/margins": 1.2160072326660156,
|
|
"rewards/rejected": 0.9325143694877625,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21458843350410461,
|
|
"drift/rejected_abs": 0.11676479876041412,
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 1976.0,
|
|
"learning_rate": 7.151452938229325e-08,
|
|
"logits/chosen": -2.0549139976501465,
|
|
"logits/rejected": -2.0558762550354004,
|
|
"logps/chosen": -0.2830585837364197,
|
|
"logps/rejected": -0.29241445660591125,
|
|
"loss": 4431.64208984375,
|
|
"loss/core": 4431.63623046875,
|
|
"loss/preference_sft": 0.2830585837364197,
|
|
"loss/reference_anchor": 0.25421541929244995,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.145282745361328,
|
|
"rewards/margins": 0.9789865612983704,
|
|
"rewards/rejected": 1.166296362876892,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3049691319465637,
|
|
"drift/rejected_abs": 0.16369685530662537,
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 11968.0,
|
|
"learning_rate": 6.933376866888883e-08,
|
|
"logits/chosen": -2.038628339767456,
|
|
"logits/rejected": -2.1392931938171387,
|
|
"logps/chosen": -0.2770799994468689,
|
|
"logps/rejected": -0.3140313923358917,
|
|
"loss": 4426.2294921875,
|
|
"loss/core": 4426.2177734375,
|
|
"loss/preference_sft": 0.2770799994468689,
|
|
"loss/reference_anchor": 0.5493339896202087,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0482850074768066,
|
|
"rewards/margins": 1.4278751611709595,
|
|
"rewards/rejected": 1.6204099655151367,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26089203357696533,
|
|
"drift/rejected_abs": 0.1435510367155075,
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 15232.0,
|
|
"learning_rate": 6.717394174850605e-08,
|
|
"logits/chosen": -2.150646924972534,
|
|
"logits/rejected": -2.166919708251953,
|
|
"logps/chosen": -0.2822549045085907,
|
|
"logps/rejected": -0.29912179708480835,
|
|
"loss": 4429.2724609375,
|
|
"loss/core": 4429.2646484375,
|
|
"loss/preference_sft": 0.2822549045085907,
|
|
"loss/reference_anchor": 0.33313748240470886,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6065964698791504,
|
|
"rewards/margins": 1.1729382276535034,
|
|
"rewards/rejected": 1.433658480644226,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19429583847522736,
|
|
"drift/rejected_abs": 0.1452254056930542,
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 1168.0,
|
|
"learning_rate": 6.503586084371926e-08,
|
|
"logits/chosen": -2.172908306121826,
|
|
"logits/rejected": -2.1675751209259033,
|
|
"logps/chosen": -0.2829391360282898,
|
|
"logps/rejected": -0.2997308373451233,
|
|
"loss": 4438.06396484375,
|
|
"loss/core": 4438.0595703125,
|
|
"loss/preference_sft": 0.2829391360282898,
|
|
"loss/reference_anchor": 0.17451608180999756,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9426425695419312,
|
|
"rewards/margins": 0.4928255081176758,
|
|
"rewards/rejected": 1.4498172998428345,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2902236878871918,
|
|
"drift/rejected_abs": 0.13140039145946503,
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 2480.0,
|
|
"learning_rate": 6.29203299993155e-08,
|
|
"logits/chosen": -2.1176817417144775,
|
|
"logits/rejected": -2.0981597900390625,
|
|
"logps/chosen": -0.30078643560409546,
|
|
"logps/rejected": -0.3083663880825043,
|
|
"loss": 4419.7548828125,
|
|
"loss/core": 4419.74609375,
|
|
"loss/preference_sft": 0.30078643560409546,
|
|
"loss/reference_anchor": 0.43593829870224,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.901242256164551,
|
|
"rewards/margins": 1.8964433670043945,
|
|
"rewards/rejected": 1.0047987699508667,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17258086800575256,
|
|
"drift/rejected_abs": 0.1321617066860199,
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 3680.0,
|
|
"learning_rate": 6.082814477992656e-08,
|
|
"logits/chosen": -2.1916544437408447,
|
|
"logits/rejected": -2.11995530128479,
|
|
"logps/chosen": -0.30291134119033813,
|
|
"logps/rejected": -0.306274950504303,
|
|
"loss": 4439.31982421875,
|
|
"loss/core": 4439.31494140625,
|
|
"loss/preference_sft": 0.30291134119033813,
|
|
"loss/reference_anchor": 0.23412831127643585,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.723780632019043,
|
|
"rewards/margins": 0.4066292643547058,
|
|
"rewards/rejected": 1.317151427268982,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1660202294588089,
|
|
"drift/rejected_abs": 0.06376107037067413,
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 1200.0,
|
|
"learning_rate": 5.87600919708494e-08,
|
|
"logits/chosen": -2.0012378692626953,
|
|
"logits/rejected": -2.0457763671875,
|
|
"logps/chosen": -0.28748735785484314,
|
|
"logps/rejected": -0.2896059453487396,
|
|
"loss": 4430.99169921875,
|
|
"loss/core": 4430.98876953125,
|
|
"loss/preference_sft": 0.28748735785484314,
|
|
"loss/reference_anchor": 0.1316143423318863,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.6601464748382568,
|
|
"rewards/margins": 1.0247886180877686,
|
|
"rewards/rejected": 0.6353579163551331,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.203060582280159,
|
|
"drift/rejected_abs": 0.18572744727134705,
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 5792.0,
|
|
"learning_rate": 5.671694928216829e-08,
|
|
"logits/chosen": -2.225250244140625,
|
|
"logits/rejected": -2.2663300037384033,
|
|
"logps/chosen": -0.28407204151153564,
|
|
"logps/rejected": -0.27350106835365295,
|
|
"loss": 4442.513671875,
|
|
"loss/core": 4442.50830078125,
|
|
"loss/preference_sft": 0.28407204151153564,
|
|
"loss/reference_anchor": 0.25616979598999023,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0303478240966797,
|
|
"rewards/margins": 0.17582125961780548,
|
|
"rewards/rejected": 1.8545265197753906,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3154805302619934,
|
|
"drift/rejected_abs": 0.2100723534822464,
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 6400.0,
|
|
"learning_rate": 5.469948505629e-08,
|
|
"logits/chosen": -2.017364263534546,
|
|
"logits/rejected": -2.079219341278076,
|
|
"logps/chosen": -0.28993043303489685,
|
|
"logps/rejected": -0.28738412261009216,
|
|
"loss": 4430.4951171875,
|
|
"loss/core": 4430.47998046875,
|
|
"loss/preference_sft": 0.28993043303489685,
|
|
"loss/reference_anchor": 0.7488113641738892,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1542510986328125,
|
|
"rewards/margins": 1.0541503429412842,
|
|
"rewards/rejected": 2.1001007556915283,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1768004447221756,
|
|
"drift/rejected_abs": 0.12505300343036652,
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 8960.0,
|
|
"learning_rate": 5.270845797900168e-08,
|
|
"logits/chosen": -2.223820209503174,
|
|
"logits/rejected": -2.153284788131714,
|
|
"logps/chosen": -0.28208497166633606,
|
|
"logps/rejected": -0.29163843393325806,
|
|
"loss": 4437.68603515625,
|
|
"loss/core": 4437.68017578125,
|
|
"loss/preference_sft": 0.28208497166633606,
|
|
"loss/reference_anchor": 0.26417121291160583,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.765069603919983,
|
|
"rewards/margins": 0.5174070596694946,
|
|
"rewards/rejected": 1.2476627826690674,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18167448043823242,
|
|
"drift/rejected_abs": 0.14043021202087402,
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 524.0,
|
|
"learning_rate": 5.0744616794160104e-08,
|
|
"logits/chosen": -2.0674781799316406,
|
|
"logits/rejected": -2.1492257118225098,
|
|
"logps/chosen": -0.27626532316207886,
|
|
"logps/rejected": -0.28123727440834045,
|
|
"loss": 4439.0361328125,
|
|
"loss/core": 4439.03369140625,
|
|
"loss/preference_sft": 0.27626532316207886,
|
|
"loss/reference_anchor": 0.11728350073099136,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8160436153411865,
|
|
"rewards/margins": 0.41351088881492615,
|
|
"rewards/rejected": 1.402532935142517,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22600972652435303,
|
|
"drift/rejected_abs": 0.1264456957578659,
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 796.0,
|
|
"learning_rate": 4.880870002211963e-08,
|
|
"logits/chosen": -2.2491302490234375,
|
|
"logits/rejected": -2.25697660446167,
|
|
"logps/chosen": -0.27793681621551514,
|
|
"logps/rejected": -0.25179094076156616,
|
|
"loss": 4431.15283203125,
|
|
"loss/core": 4431.146484375,
|
|
"loss/preference_sft": 0.27793681621551514,
|
|
"loss/reference_anchor": 0.2595617473125458,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.259474754333496,
|
|
"rewards/margins": 1.0019110441207886,
|
|
"rewards/rejected": 1.2575637102127075,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13364611566066742,
|
|
"drift/rejected_abs": 0.09645704180002213,
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 540.0,
|
|
"learning_rate": 4.6901435682004996e-08,
|
|
"logits/chosen": -2.2417209148406982,
|
|
"logits/rejected": -2.2763659954071045,
|
|
"logps/chosen": -0.29147717356681824,
|
|
"logps/rejected": -0.2934878468513489,
|
|
"loss": 4439.51708984375,
|
|
"loss/core": 4439.515625,
|
|
"loss/preference_sft": 0.29147717356681824,
|
|
"loss/reference_anchor": 0.06112232059240341,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.3360573053359985,
|
|
"rewards/margins": 0.3732479214668274,
|
|
"rewards/rejected": 0.9628093838691711,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1628817468881607,
|
|
"drift/rejected_abs": 0.13016179203987122,
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 2032.0,
|
|
"learning_rate": 4.502354101793314e-08,
|
|
"logits/chosen": -2.3644509315490723,
|
|
"logits/rejected": -2.2733235359191895,
|
|
"logps/chosen": -0.27977296710014343,
|
|
"logps/rejected": -0.29269108176231384,
|
|
"loss": 4440.22900390625,
|
|
"loss/core": 4440.2255859375,
|
|
"loss/preference_sft": 0.27977296710014343,
|
|
"loss/reference_anchor": 0.17271290719509125,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6281238794326782,
|
|
"rewards/margins": 0.33540040254592896,
|
|
"rewards/rejected": 1.2927234172821045,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3136964738368988,
|
|
"drift/rejected_abs": 0.12950031459331512,
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 1472.0,
|
|
"learning_rate": 4.3175722229287034e-08,
|
|
"logits/chosen": -2.1214194297790527,
|
|
"logits/rejected": -2.114104747772217,
|
|
"logps/chosen": -0.27199044823646545,
|
|
"logps/rejected": -0.27481597661972046,
|
|
"loss": 4419.1142578125,
|
|
"loss/core": 4419.107421875,
|
|
"loss/preference_sft": 0.27199044823646545,
|
|
"loss/reference_anchor": 0.31056779623031616,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.136345624923706,
|
|
"rewards/margins": 1.9250768423080444,
|
|
"rewards/rejected": 1.2112689018249512,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23225578665733337,
|
|
"drift/rejected_abs": 0.23389002680778503,
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 2960.0,
|
|
"learning_rate": 4.135867420514276e-08,
|
|
"logits/chosen": -2.2221508026123047,
|
|
"logits/rejected": -2.2248616218566895,
|
|
"logps/chosen": -0.2901459038257599,
|
|
"logps/rejected": -0.26247456669807434,
|
|
"loss": 4445.1943359375,
|
|
"loss/core": 4445.1845703125,
|
|
"loss/preference_sft": 0.2901459038257599,
|
|
"loss/reference_anchor": 0.46210724115371704,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.3225579261779785,
|
|
"rewards/margins": -0.016050493344664574,
|
|
"rewards/rejected": 2.3386082649230957,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2951485216617584,
|
|
"drift/rejected_abs": 0.2140798568725586,
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 9984.0,
|
|
"learning_rate": 3.957308026295035e-08,
|
|
"logits/chosen": -2.093303680419922,
|
|
"logits/rejected": -2.1656064987182617,
|
|
"logps/chosen": -0.31711024045944214,
|
|
"logps/rejected": -0.32263869047164917,
|
|
"loss": 4435.220703125,
|
|
"loss/core": 4435.2021484375,
|
|
"loss/preference_sft": 0.31711024045944214,
|
|
"loss/reference_anchor": 0.8856789469718933,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.9496922492980957,
|
|
"rewards/margins": 0.8108631372451782,
|
|
"rewards/rejected": 2.138829469680786,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23619715869426727,
|
|
"drift/rejected_abs": 0.11902375519275665,
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 1752.0,
|
|
"learning_rate": 3.7819611891566084e-08,
|
|
"logits/chosen": -1.9933494329452515,
|
|
"logits/rejected": -1.9954357147216797,
|
|
"logps/chosen": -0.268182635307312,
|
|
"logps/rejected": -0.28130608797073364,
|
|
"loss": 4428.77734375,
|
|
"loss/core": 4428.77392578125,
|
|
"loss/preference_sft": 0.268182635307312,
|
|
"loss/reference_anchor": 0.13460581004619598,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3605830669403076,
|
|
"rewards/margins": 1.1809606552124023,
|
|
"rewards/rejected": 1.1796222925186157,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3273255228996277,
|
|
"drift/rejected_abs": 0.16635169088840485,
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 3360.0,
|
|
"learning_rate": 3.609892849873286e-08,
|
|
"logits/chosen": -2.2435264587402344,
|
|
"logits/rejected": -2.1759636402130127,
|
|
"logps/chosen": -0.2879250645637512,
|
|
"logps/rejected": -0.2684784531593323,
|
|
"loss": 4424.0283203125,
|
|
"loss/core": 4424.0146484375,
|
|
"loss/preference_sft": 0.2879250645637512,
|
|
"loss/reference_anchor": 0.6359601616859436,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2727675437927246,
|
|
"rewards/margins": 1.61063551902771,
|
|
"rewards/rejected": 1.6621326208114624,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13324177265167236,
|
|
"drift/rejected_abs": 0.052085865288972855,
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 450.0,
|
|
"learning_rate": 3.4411677163103894e-08,
|
|
"logits/chosen": -2.1316683292388916,
|
|
"logits/rejected": -2.1392605304718018,
|
|
"logps/chosen": -0.3257368206977844,
|
|
"logps/rejected": -0.31321820616722107,
|
|
"loss": 4433.72265625,
|
|
"loss/core": 4433.72119140625,
|
|
"loss/preference_sft": 0.3257368206977844,
|
|
"loss/reference_anchor": 0.06571613252162933,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.3298277854919434,
|
|
"rewards/margins": 0.8104680180549622,
|
|
"rewards/rejected": 0.5193597078323364,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24692943692207336,
|
|
"drift/rejected_abs": 0.08578932285308838,
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 1712.0,
|
|
"learning_rate": 3.2758492390902945e-08,
|
|
"logits/chosen": -1.9040149450302124,
|
|
"logits/rejected": -1.964320421218872,
|
|
"logps/chosen": -0.278387188911438,
|
|
"logps/rejected": -0.28676044940948486,
|
|
"loss": 4422.9931640625,
|
|
"loss/core": 4422.98876953125,
|
|
"loss/preference_sft": 0.278387188911438,
|
|
"loss/reference_anchor": 0.17939075827598572,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4657251834869385,
|
|
"rewards/margins": 1.624982237815857,
|
|
"rewards/rejected": 0.8407428860664368,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2357044667005539,
|
|
"drift/rejected_abs": 0.09313996881246567,
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 884.0,
|
|
"learning_rate": 3.11399958773126e-08,
|
|
"logits/chosen": -2.0695366859436035,
|
|
"logits/rejected": -2.1179842948913574,
|
|
"logps/chosen": -0.28330761194229126,
|
|
"logps/rejected": -0.2819029688835144,
|
|
"loss": 4425.52294921875,
|
|
"loss/core": 4425.52001953125,
|
|
"loss/preference_sft": 0.28330761194229126,
|
|
"loss/reference_anchor": 0.11744773387908936,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3560948371887207,
|
|
"rewards/margins": 1.4276307821273804,
|
|
"rewards/rejected": 0.9284639358520508,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2549819350242615,
|
|
"drift/rejected_abs": 0.11347774416208267,
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 704.0,
|
|
"learning_rate": 2.9556796272679972e-08,
|
|
"logits/chosen": -2.1076555252075195,
|
|
"logits/rejected": -2.143085479736328,
|
|
"logps/chosen": -0.28872913122177124,
|
|
"logps/rejected": -0.3133319020271301,
|
|
"loss": 4425.2392578125,
|
|
"loss/core": 4425.23291015625,
|
|
"loss/preference_sft": 0.28872913122177124,
|
|
"loss/reference_anchor": 0.2932053208351135,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.549466133117676,
|
|
"rewards/margins": 1.4717979431152344,
|
|
"rewards/rejected": 1.0776679515838623,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11858288198709488,
|
|
"drift/rejected_abs": 0.1146119013428688,
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 396.0,
|
|
"learning_rate": 2.8009488953628215e-08,
|
|
"logits/chosen": -2.0915627479553223,
|
|
"logits/rejected": -2.0317933559417725,
|
|
"logps/chosen": -0.2838587760925293,
|
|
"logps/rejected": -0.2749176621437073,
|
|
"loss": 4444.0947265625,
|
|
"loss/core": 4444.091796875,
|
|
"loss/preference_sft": 0.2838587760925293,
|
|
"loss/reference_anchor": 0.11706352233886719,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.1828110218048096,
|
|
"rewards/margins": 0.038224995136260986,
|
|
"rewards/rejected": 1.1445858478546143,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28038161993026733,
|
|
"drift/rejected_abs": 0.10303550958633423,
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 2032.0,
|
|
"learning_rate": 2.649865579915976e-08,
|
|
"logits/chosen": -2.084571123123169,
|
|
"logits/rejected": -2.0591697692871094,
|
|
"logps/chosen": -0.3031243681907654,
|
|
"logps/rejected": -0.3205743432044983,
|
|
"loss": 4420.45458984375,
|
|
"loss/core": 4420.4453125,
|
|
"loss/preference_sft": 0.3031243681907654,
|
|
"loss/reference_anchor": 0.4167904257774353,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.803816556930542,
|
|
"rewards/margins": 1.8526108264923096,
|
|
"rewards/rejected": 0.9512056112289429,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28037887811660767,
|
|
"drift/rejected_abs": 0.11086313426494598,
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 7264.0,
|
|
"learning_rate": 2.5024864971835507e-08,
|
|
"logits/chosen": -2.197540760040283,
|
|
"logits/rejected": -2.1911709308624268,
|
|
"logps/chosen": -0.2654857039451599,
|
|
"logps/rejected": -0.27312350273132324,
|
|
"loss": 4422.1875,
|
|
"loss/core": 4422.1806640625,
|
|
"loss/preference_sft": 0.2654857039451599,
|
|
"loss/reference_anchor": 0.29703524708747864,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.802237033843994,
|
|
"rewards/margins": 1.697741150856018,
|
|
"rewards/rejected": 1.1044957637786865,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21863920986652374,
|
|
"drift/rejected_abs": 0.165724977850914,
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 3120.0,
|
|
"learning_rate": 2.3588670704111997e-08,
|
|
"logits/chosen": -1.931133508682251,
|
|
"logits/rejected": -1.8537521362304688,
|
|
"logps/chosen": -0.2914375364780426,
|
|
"logps/rejected": -0.3034089207649231,
|
|
"loss": 4437.83544921875,
|
|
"loss/core": 4437.828125,
|
|
"loss/preference_sft": 0.2914375364780426,
|
|
"loss/reference_anchor": 0.32963576912879944,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.18485689163208,
|
|
"rewards/margins": 0.5304257869720459,
|
|
"rewards/rejected": 1.6544309854507446,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22953541576862335,
|
|
"drift/rejected_abs": 0.12510935962200165,
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 636.0,
|
|
"learning_rate": 2.219061308991721e-08,
|
|
"logits/chosen": -2.2116262912750244,
|
|
"logits/rejected": -2.234771251678467,
|
|
"logps/chosen": -0.2957614064216614,
|
|
"logps/rejected": -0.28059297800064087,
|
|
"loss": 4430.69677734375,
|
|
"loss/core": 4430.68994140625,
|
|
"loss/preference_sft": 0.2957614064216614,
|
|
"loss/reference_anchor": 0.3111433684825897,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.295354127883911,
|
|
"rewards/margins": 1.0461337566375732,
|
|
"rewards/rejected": 1.2492201328277588,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22382231056690216,
|
|
"drift/rejected_abs": 0.1400088369846344,
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 7872.0,
|
|
"learning_rate": 2.0831217881543557e-08,
|
|
"logits/chosen": -2.2578673362731934,
|
|
"logits/rejected": -2.194847583770752,
|
|
"logps/chosen": -0.27832141518592834,
|
|
"logps/rejected": -0.30063265562057495,
|
|
"loss": 4433.3857421875,
|
|
"loss/core": 4433.3818359375,
|
|
"loss/preference_sft": 0.27832141518592834,
|
|
"loss/reference_anchor": 0.1458483338356018,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2344486713409424,
|
|
"rewards/margins": 0.8346776962280273,
|
|
"rewards/rejected": 1.3997710943222046,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3602936863899231,
|
|
"drift/rejected_abs": 0.19181835651397705,
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 560.0,
|
|
"learning_rate": 1.951099629193366e-08,
|
|
"logits/chosen": -2.169501781463623,
|
|
"logits/rejected": -2.14741587638855,
|
|
"logps/chosen": -0.2755396068096161,
|
|
"logps/rejected": -0.2857043743133545,
|
|
"loss": 4422.30126953125,
|
|
"loss/core": 4422.2890625,
|
|
"loss/preference_sft": 0.2755396068096161,
|
|
"loss/reference_anchor": 0.6152486801147461,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.602374315261841,
|
|
"rewards/margins": 1.6893268823623657,
|
|
"rewards/rejected": 1.9130470752716064,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3550851047039032,
|
|
"drift/rejected_abs": 0.20769810676574707,
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 8768.0,
|
|
"learning_rate": 1.823044480243431e-08,
|
|
"logits/chosen": -2.0105481147766113,
|
|
"logits/rejected": -2.006004810333252,
|
|
"logps/chosen": -0.3070371150970459,
|
|
"logps/rejected": -0.3485947549343109,
|
|
"loss": 4425.2255859375,
|
|
"loss/core": 4425.2138671875,
|
|
"loss/preference_sft": 0.3070371150970459,
|
|
"loss/reference_anchor": 0.5617225170135498,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.5491435527801514,
|
|
"rewards/margins": 1.4741640090942383,
|
|
"rewards/rejected": 2.074979305267334,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16385112702846527,
|
|
"drift/rejected_abs": 0.08402819186449051,
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 844.0,
|
|
"learning_rate": 1.699004497608994e-08,
|
|
"logits/chosen": -2.3256075382232666,
|
|
"logits/rejected": -2.332451105117798,
|
|
"logps/chosen": -0.27758467197418213,
|
|
"logps/rejected": -0.26881900429725647,
|
|
"loss": 4433.8310546875,
|
|
"loss/core": 4433.8291015625,
|
|
"loss/preference_sft": 0.27758467197418213,
|
|
"loss/reference_anchor": 0.07551290094852448,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.637289047241211,
|
|
"rewards/margins": 0.7998763918876648,
|
|
"rewards/rejected": 0.8374127149581909,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2287888526916504,
|
|
"drift/rejected_abs": 0.07531949877738953,
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 11264.0,
|
|
"learning_rate": 1.5790263276546658e-08,
|
|
"logits/chosen": -2.220702648162842,
|
|
"logits/rejected": -2.295063018798828,
|
|
"logps/chosen": -0.286711722612381,
|
|
"logps/rejected": -0.2794603705406189,
|
|
"loss": 4424.37353515625,
|
|
"loss/core": 4424.3681640625,
|
|
"loss/preference_sft": 0.286711722612381,
|
|
"loss/reference_anchor": 0.25186899304389954,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.287888526916504,
|
|
"rewards/margins": 1.5352530479431152,
|
|
"rewards/rejected": 0.7526354789733887,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18583203852176666,
|
|
"drift/rejected_abs": 0.1396988481283188,
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 528.0,
|
|
"learning_rate": 1.4631550892634126e-08,
|
|
"logits/chosen": -2.1028919219970703,
|
|
"logits/rejected": -1.9989187717437744,
|
|
"logps/chosen": -0.2728358209133148,
|
|
"logps/rejected": -0.2678207755088806,
|
|
"loss": 4438.3876953125,
|
|
"loss/core": 4438.38330078125,
|
|
"loss/preference_sft": 0.2728358209133148,
|
|
"loss/reference_anchor": 0.1961844116449356,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.8562612533569336,
|
|
"rewards/margins": 0.4593871533870697,
|
|
"rewards/rejected": 1.3968740701675415,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2970121502876282,
|
|
"drift/rejected_abs": 0.19716815650463104,
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 29184.0,
|
|
"learning_rate": 1.3514343568692132e-08,
|
|
"logits/chosen": -2.24552845954895,
|
|
"logits/rejected": -2.210707426071167,
|
|
"logps/chosen": -0.29537197947502136,
|
|
"logps/rejected": -0.2902478277683258,
|
|
"loss": 4431.3583984375,
|
|
"loss/core": 4431.33984375,
|
|
"loss/preference_sft": 0.29537197947502136,
|
|
"loss/reference_anchor": 0.8959586024284363,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9684689044952393,
|
|
"rewards/margins": 0.9983173608779907,
|
|
"rewards/rejected": 1.9701515436172485,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19076788425445557,
|
|
"drift/rejected_abs": 0.07851524651050568,
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 944.0,
|
|
"learning_rate": 1.2439061440704724e-08,
|
|
"logits/chosen": -2.2142930030822754,
|
|
"logits/rejected": -2.226604700088501,
|
|
"logps/chosen": -0.2813388407230377,
|
|
"logps/rejected": -0.281820148229599,
|
|
"loss": 4430.03515625,
|
|
"loss/core": 4430.0283203125,
|
|
"loss/preference_sft": 0.2813388407230377,
|
|
"loss/reference_anchor": 0.3390761911869049,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9072763919830322,
|
|
"rewards/margins": 1.1225610971450806,
|
|
"rewards/rejected": 0.7847151756286621,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33144018054008484,
|
|
"drift/rejected_abs": 0.13096961379051208,
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 3488.0,
|
|
"learning_rate": 1.1406108878304468e-08,
|
|
"logits/chosen": -2.039895534515381,
|
|
"logits/rejected": -2.0326807498931885,
|
|
"logps/chosen": -0.2768157422542572,
|
|
"logps/rejected": -0.27031582593917847,
|
|
"loss": 4417.72802734375,
|
|
"loss/core": 4417.72021484375,
|
|
"loss/preference_sft": 0.2768157422542572,
|
|
"loss/reference_anchor": 0.3354681134223938,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.3139610290527344,
|
|
"rewards/margins": 2.033888339996338,
|
|
"rewards/rejected": 1.2800720930099487,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3740524649620056,
|
|
"drift/rejected_abs": 0.09586691856384277,
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 8256.0,
|
|
"learning_rate": 1.0415874332705381e-08,
|
|
"logits/chosen": -2.1573445796966553,
|
|
"logits/rejected": -2.1065096855163574,
|
|
"logps/chosen": -0.2745102345943451,
|
|
"logps/rejected": -0.2804674804210663,
|
|
"loss": 4408.5126953125,
|
|
"loss/core": 4408.4970703125,
|
|
"loss/preference_sft": 0.2745102345943451,
|
|
"loss/reference_anchor": 0.7442028522491455,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.7369015216827393,
|
|
"rewards/margins": 2.7862377166748047,
|
|
"rewards/rejected": 0.9506635665893555,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2516661584377289,
|
|
"drift/rejected_abs": 0.14492499828338623,
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 1168.0,
|
|
"learning_rate": 9.468730190622484e-09,
|
|
"logits/chosen": -2.473392963409424,
|
|
"logits/rejected": -2.458265781402588,
|
|
"logps/chosen": -0.2965652346611023,
|
|
"logps/rejected": -0.3087556064128876,
|
|
"loss": 4430.37841796875,
|
|
"loss/core": 4430.365234375,
|
|
"loss/preference_sft": 0.2965652346611023,
|
|
"loss/reference_anchor": 0.6160001158714294,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5161986351013184,
|
|
"rewards/margins": 1.0671333074569702,
|
|
"rewards/rejected": 1.4490649700164795,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.09642498195171356,
|
|
"drift/rejected_abs": 0.06358600407838821,
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 772.0,
|
|
"learning_rate": 8.565032634232194e-09,
|
|
"logits/chosen": -2.1253273487091064,
|
|
"logits/rejected": -2.0759212970733643,
|
|
"logps/chosen": -0.3012627065181732,
|
|
"logps/rejected": -0.3006402850151062,
|
|
"loss": 4440.068359375,
|
|
"loss/core": 4440.0673828125,
|
|
"loss/preference_sft": 0.3012627065181732,
|
|
"loss/reference_anchor": 0.021201241761446,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 0.962838351726532,
|
|
"rewards/margins": 0.3286899924278259,
|
|
"rewards/rejected": 0.634148359298706,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1828320473432541,
|
|
"drift/rejected_abs": 0.1643926501274109,
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 3008.0,
|
|
"learning_rate": 7.70512150722702e-09,
|
|
"logits/chosen": -2.1040894985198975,
|
|
"logits/rejected": -2.042335033416748,
|
|
"logps/chosen": -0.2662808299064636,
|
|
"logps/rejected": -0.26375871896743774,
|
|
"loss": 4442.34619140625,
|
|
"loss/core": 4442.3408203125,
|
|
"loss/preference_sft": 0.2662808299064636,
|
|
"loss/reference_anchor": 0.245298832654953,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8268150091171265,
|
|
"rewards/margins": 0.18445713818073273,
|
|
"rewards/rejected": 1.6423580646514893,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24483975768089294,
|
|
"drift/rejected_abs": 0.10785166919231415,
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 588.0,
|
|
"learning_rate": 6.8893201870139915e-09,
|
|
"logits/chosen": -2.097686290740967,
|
|
"logits/rejected": -2.2861804962158203,
|
|
"logps/chosen": -0.31777235865592957,
|
|
"logps/rejected": -0.3054574131965637,
|
|
"loss": 4426.5830078125,
|
|
"loss/core": 4426.57666015625,
|
|
"loss/preference_sft": 0.31777235865592957,
|
|
"loss/reference_anchor": 0.284587562084198,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.445316791534424,
|
|
"rewards/margins": 1.3685089349746704,
|
|
"rewards/rejected": 1.0768078565597534,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26204735040664673,
|
|
"drift/rejected_abs": 0.20677991211414337,
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 2288.0,
|
|
"learning_rate": 6.117935463105808e-09,
|
|
"logits/chosen": -1.823381781578064,
|
|
"logits/rejected": -1.8026411533355713,
|
|
"logps/chosen": -0.2800712585449219,
|
|
"logps/rejected": -0.3262674808502197,
|
|
"loss": 4437.53466796875,
|
|
"loss/core": 4437.5244140625,
|
|
"loss/preference_sft": 0.2800712585449219,
|
|
"loss/reference_anchor": 0.490020215511322,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.620026111602783,
|
|
"rewards/margins": 0.566874086856842,
|
|
"rewards/rejected": 2.053152084350586,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35397663712501526,
|
|
"drift/rejected_abs": 0.16934973001480103,
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 1288.0,
|
|
"learning_rate": 5.391257421749826e-09,
|
|
"logits/chosen": -2.1389248371124268,
|
|
"logits/rejected": -2.1224210262298584,
|
|
"logps/chosen": -0.29372864961624146,
|
|
"logps/rejected": -0.2752332091331482,
|
|
"loss": 4420.16455078125,
|
|
"loss/core": 4420.1484375,
|
|
"loss/preference_sft": 0.29372864961624146,
|
|
"loss/reference_anchor": 0.7682027816772461,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.5389609336853027,
|
|
"rewards/margins": 1.84674072265625,
|
|
"rewards/rejected": 1.6922200918197632,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28098636865615845,
|
|
"drift/rejected_abs": 0.12216347455978394,
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 7328.0,
|
|
"learning_rate": 4.709559336838462e-09,
|
|
"logits/chosen": -1.9067840576171875,
|
|
"logits/rejected": -1.9582593441009521,
|
|
"logps/chosen": -0.3030111491680145,
|
|
"logps/rejected": -0.3043147027492523,
|
|
"loss": 4423.7431640625,
|
|
"loss/core": 4423.7333984375,
|
|
"loss/preference_sft": 0.3030111491680145,
|
|
"loss/reference_anchor": 0.462859570980072,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.809863567352295,
|
|
"rewards/margins": 1.590742588043213,
|
|
"rewards/rejected": 1.2191212177276611,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16951501369476318,
|
|
"drift/rejected_abs": 0.10462959110736847,
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 596.0,
|
|
"learning_rate": 4.073097567142025e-09,
|
|
"logits/chosen": -2.2416045665740967,
|
|
"logits/rejected": -2.323671817779541,
|
|
"logps/chosen": -0.29937443137168884,
|
|
"logps/rejected": -0.28850868344306946,
|
|
"loss": 4435.8212890625,
|
|
"loss/core": 4435.818359375,
|
|
"loss/preference_sft": 0.29937443137168884,
|
|
"loss/reference_anchor": 0.1115211695432663,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6946293115615845,
|
|
"rewards/margins": 0.649151086807251,
|
|
"rewards/rejected": 1.0454782247543335,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20362646877765656,
|
|
"drift/rejected_abs": 0.15938568115234375,
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 1040.0,
|
|
"learning_rate": 3.482111459902695e-09,
|
|
"logits/chosen": -2.0392162799835205,
|
|
"logits/rejected": -2.133927822113037,
|
|
"logps/chosen": -0.2889602780342102,
|
|
"logps/rejected": -0.3033998906612396,
|
|
"loss": 4439.00927734375,
|
|
"loss/core": 4439.0029296875,
|
|
"loss/preference_sft": 0.2889602780342102,
|
|
"loss/reference_anchor": 0.33250418305397034,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.036264657974243,
|
|
"rewards/margins": 0.44385433197021484,
|
|
"rewards/rejected": 1.5924100875854492,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.46066784858703613,
|
|
"drift/rejected_abs": 0.22359898686408997,
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 1544.0,
|
|
"learning_rate": 2.9368232608258797e-09,
|
|
"logits/chosen": -2.173532485961914,
|
|
"logits/rejected": -2.1856465339660645,
|
|
"logps/chosen": -0.2729622721672058,
|
|
"logps/rejected": -0.27251607179641724,
|
|
"loss": 4414.25390625,
|
|
"loss/core": 4414.23193359375,
|
|
"loss/preference_sft": 0.2729622721672058,
|
|
"loss/reference_anchor": 1.0718512535095215,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.606678009033203,
|
|
"rewards/margins": 2.375056028366089,
|
|
"rewards/rejected": 2.2316224575042725,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17466862499713898,
|
|
"drift/rejected_abs": 0.06862124055624008,
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 424.0,
|
|
"learning_rate": 2.4374380305025866e-09,
|
|
"logits/chosen": -2.2522521018981934,
|
|
"logits/rejected": -2.309323787689209,
|
|
"logps/chosen": -0.2542715072631836,
|
|
"logps/rejected": -0.2512281537055969,
|
|
"loss": 4430.23974609375,
|
|
"loss/core": 4430.2373046875,
|
|
"loss/preference_sft": 0.2542715072631836,
|
|
"loss/reference_anchor": 0.06204066798090935,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7456880807876587,
|
|
"rewards/margins": 1.0702584981918335,
|
|
"rewards/rejected": 0.6754294633865356,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23169496655464172,
|
|
"drift/rejected_abs": 0.19141283631324768,
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 2560.0,
|
|
"learning_rate": 1.984143567294594e-09,
|
|
"logits/chosen": -2.089970350265503,
|
|
"logits/rejected": -2.082474946975708,
|
|
"logps/chosen": -0.2673383355140686,
|
|
"logps/rejected": -0.2954670786857605,
|
|
"loss": 4439.4287109375,
|
|
"loss/core": 4439.421875,
|
|
"loss/preference_sft": 0.2673383355140686,
|
|
"loss/reference_anchor": 0.32883888483047485,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.314150333404541,
|
|
"rewards/margins": 0.4091102182865143,
|
|
"rewards/rejected": 1.9050400257110596,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19237197935581207,
|
|
"drift/rejected_abs": 0.10806970298290253,
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 544.0,
|
|
"learning_rate": 1.577110336711096e-09,
|
|
"logits/chosen": -2.185058116912842,
|
|
"logits/rejected": -2.2582504749298096,
|
|
"logps/chosen": -0.29970070719718933,
|
|
"logps/rejected": -0.30701369047164917,
|
|
"loss": 4433.2958984375,
|
|
"loss/core": 4433.291015625,
|
|
"loss/preference_sft": 0.29970070719718933,
|
|
"loss/reference_anchor": 0.2221534699201584,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.921847939491272,
|
|
"rewards/margins": 0.8417887687683105,
|
|
"rewards/rejected": 1.0800589323043823,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26305049657821655,
|
|
"drift/rejected_abs": 0.12108287960290909,
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 736.0,
|
|
"learning_rate": 1.2164914073037048e-09,
|
|
"logits/chosen": -2.147150754928589,
|
|
"logits/rejected": -2.2282862663269043,
|
|
"logps/chosen": -0.29672080278396606,
|
|
"logps/rejected": -0.28275054693222046,
|
|
"loss": 4426.12451171875,
|
|
"loss/core": 4426.11181640625,
|
|
"loss/preference_sft": 0.29672080278396606,
|
|
"loss/reference_anchor": 0.6080988049507141,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.629103183746338,
|
|
"rewards/margins": 1.4199514389038086,
|
|
"rewards/rejected": 1.2091517448425293,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4451891779899597,
|
|
"drift/rejected_abs": 0.21213407814502716,
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 2024.0,
|
|
"learning_rate": 9.024223931035357e-10,
|
|
"logits/chosen": -1.9941089153289795,
|
|
"logits/rejected": -1.9594297409057617,
|
|
"logps/chosen": -0.2970310151576996,
|
|
"logps/rejected": -0.2791312336921692,
|
|
"loss": 4414.5986328125,
|
|
"loss/core": 4414.580078125,
|
|
"loss/preference_sft": 0.2970310151576996,
|
|
"loss/reference_anchor": 0.8715806007385254,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.448572158813477,
|
|
"rewards/margins": 2.3311219215393066,
|
|
"rewards/rejected": 2.1174495220184326,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19540363550186157,
|
|
"drift/rejected_abs": 0.13800457119941711,
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 440.0,
|
|
"learning_rate": 6.350214026223516e-10,
|
|
"logits/chosen": -2.0932259559631348,
|
|
"logits/rejected": -2.0954394340515137,
|
|
"logps/chosen": -0.30296754837036133,
|
|
"logps/rejected": -0.3049096465110779,
|
|
"loss": 4436.95068359375,
|
|
"loss/core": 4436.94580078125,
|
|
"loss/preference_sft": 0.30296754837036133,
|
|
"loss/reference_anchor": 0.24102571606636047,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.953322172164917,
|
|
"rewards/margins": 0.5735519528388977,
|
|
"rewards/rejected": 1.379770278930664,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37028616666793823,
|
|
"drift/rejected_abs": 0.15220406651496887,
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 360.0,
|
|
"learning_rate": 4.143889944367429e-10,
|
|
"logits/chosen": -2.00821590423584,
|
|
"logits/rejected": -1.9785652160644531,
|
|
"logps/chosen": -0.28579357266426086,
|
|
"logps/rejected": -0.30298155546188354,
|
|
"loss": 4416.18603515625,
|
|
"loss/core": 4416.17138671875,
|
|
"loss/preference_sft": 0.28579357266426086,
|
|
"loss/reference_anchor": 0.7266721725463867,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.702655076980591,
|
|
"rewards/margins": 2.1888763904571533,
|
|
"rewards/rejected": 1.5137783288955688,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15521343052387238,
|
|
"drift/rejected_abs": 0.07931456714868546,
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 2080.0,
|
|
"learning_rate": 2.406081393722531e-10,
|
|
"logits/chosen": -2.103982448577881,
|
|
"logits/rejected": -2.132598876953125,
|
|
"logps/chosen": -0.29826053977012634,
|
|
"logps/rejected": -0.3063696026802063,
|
|
"loss": 4434.3544921875,
|
|
"loss/core": 4434.3525390625,
|
|
"loss/preference_sft": 0.29826053977012634,
|
|
"loss/reference_anchor": 0.05555032566189766,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5512726306915283,
|
|
"rewards/margins": 0.7596304416656494,
|
|
"rewards/rejected": 0.7916421890258789,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18553657829761505,
|
|
"drift/rejected_abs": 0.1623484194278717,
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 1152.0,
|
|
"learning_rate": 1.1374418930135133e-10,
|
|
"logits/chosen": -2.1390254497528076,
|
|
"logits/rejected": -2.191925525665283,
|
|
"logps/chosen": -0.26982808113098145,
|
|
"logps/rejected": -0.28849366307258606,
|
|
"loss": 4441.68017578125,
|
|
"loss/core": 4441.67431640625,
|
|
"loss/preference_sft": 0.26982808113098145,
|
|
"loss/reference_anchor": 0.2661302983760834,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8522002696990967,
|
|
"rewards/margins": 0.22871632874011993,
|
|
"rewards/rejected": 1.6234838962554932,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21298833191394806,
|
|
"drift/rejected_abs": 0.09839890152215958,
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 5824.0,
|
|
"learning_rate": 3.3844852567285756e-11,
|
|
"logits/chosen": -2.102201223373413,
|
|
"logits/rejected": -2.060621738433838,
|
|
"logps/chosen": -0.29058605432510376,
|
|
"logps/rejected": -0.2914868891239166,
|
|
"loss": 4429.2578125,
|
|
"loss/core": 4429.2548828125,
|
|
"loss/preference_sft": 0.29058605432510376,
|
|
"loss/reference_anchor": 0.13424192368984222,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.129883289337158,
|
|
"rewards/margins": 1.1480109691619873,
|
|
"rewards/rejected": 0.9818723797798157,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29029321670532227,
|
|
"drift/rejected_abs": 0.14265631139278412,
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 7520.0,
|
|
"learning_rate": 9.401760429905703e-13,
|
|
"logits/chosen": -2.05942964553833,
|
|
"logits/rejected": -2.0243592262268066,
|
|
"logps/chosen": -0.2692040205001831,
|
|
"logps/rejected": -0.27301639318466187,
|
|
"loss": 4425.076171875,
|
|
"loss/core": 4425.06787109375,
|
|
"loss/preference_sft": 0.2692040205001831,
|
|
"loss/reference_anchor": 0.38637253642082214,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.9029033184051514,
|
|
"rewards/margins": 1.476595163345337,
|
|
"rewards/rejected": 1.4263079166412354,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 4429.236100260417,
|
|
"train_runtime": 7053.367,
|
|
"train_samples_per_second": 2.042,
|
|
"train_steps_per_second": 0.128
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|