2924 lines
102 KiB
JSON
2924 lines
102 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 0.142578125,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -1.9159685373306274,
|
|
"logits/rejected": -1.9266211986541748,
|
|
"logps/chosen": -0.3117416799068451,
|
|
"logps/rejected": -0.2903628647327423,
|
|
"loss": 0.689449667930603,
|
|
"loss/core": 0.689449667930603,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2060601711273193,
|
|
"rewards/margins": 1.513831377029419,
|
|
"rewards/rejected": 0.6922284960746765,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 0.42578125,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -1.8667621612548828,
|
|
"logits/rejected": -1.8464374542236328,
|
|
"logps/chosen": -0.2869036793708801,
|
|
"logps/rejected": -0.3021157681941986,
|
|
"loss": 0.6924150586128235,
|
|
"loss/core": 0.6924150586128235,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.997037649154663,
|
|
"rewards/margins": 0.35158514976501465,
|
|
"rewards/rejected": 2.6454522609710693,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 0.310546875,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -1.91054368019104,
|
|
"logits/rejected": -1.87662672996521,
|
|
"logps/chosen": -0.2590453028678894,
|
|
"logps/rejected": -0.27234840393066406,
|
|
"loss": 0.6913942098617554,
|
|
"loss/core": 0.6913942098617554,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.7586113214492798,
|
|
"rewards/margins": 0.7126884460449219,
|
|
"rewards/rejected": 1.045923113822937,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 0.1298828125,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -1.680210828781128,
|
|
"logits/rejected": -1.7510398626327515,
|
|
"logps/chosen": -0.27236178517341614,
|
|
"logps/rejected": -0.27088457345962524,
|
|
"loss": 0.687203049659729,
|
|
"loss/core": 0.687203049659729,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.647186279296875,
|
|
"rewards/margins": 2.474025011062622,
|
|
"rewards/rejected": 1.173161268234253,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 0.1787109375,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -1.7065041065216064,
|
|
"logits/rejected": -1.7718740701675415,
|
|
"logps/chosen": -0.27959316968917847,
|
|
"logps/rejected": -0.27071651816368103,
|
|
"loss": 0.6902543902397156,
|
|
"loss/core": 0.6902543902397156,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.143491744995117,
|
|
"rewards/margins": 1.1622021198272705,
|
|
"rewards/rejected": 0.9812896847724915,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 0.12890625,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -1.9748786687850952,
|
|
"logits/rejected": -1.9123010635375977,
|
|
"logps/chosen": -0.2816716730594635,
|
|
"logps/rejected": -0.2625161409378052,
|
|
"loss": 0.688904345035553,
|
|
"loss/core": 0.688904345035553,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.795006275177002,
|
|
"rewards/margins": 1.7435691356658936,
|
|
"rewards/rejected": 1.0514370203018188,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 0.0830078125,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -1.93292236328125,
|
|
"logits/rejected": -1.9241135120391846,
|
|
"logps/chosen": -0.29125192761421204,
|
|
"logps/rejected": -0.2757991552352905,
|
|
"loss": 0.6889916658401489,
|
|
"loss/core": 0.6889916658401489,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.532140016555786,
|
|
"rewards/margins": 1.7434587478637695,
|
|
"rewards/rejected": 0.7886813879013062,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 0.265625,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -1.8440186977386475,
|
|
"logits/rejected": -1.928789734840393,
|
|
"logps/chosen": -0.266449898481369,
|
|
"logps/rejected": -0.2915545105934143,
|
|
"loss": 0.6877603530883789,
|
|
"loss/core": 0.6877603530883789,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.920603036880493,
|
|
"rewards/margins": 2.235374927520752,
|
|
"rewards/rejected": 0.6852277517318726,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 0.162109375,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -1.8754253387451172,
|
|
"logits/rejected": -1.8980233669281006,
|
|
"logps/chosen": -0.26845940947532654,
|
|
"logps/rejected": -0.2764447033405304,
|
|
"loss": 0.6887972950935364,
|
|
"loss/core": 0.6887972950935364,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.0673203468322754,
|
|
"rewards/margins": 1.7863553762435913,
|
|
"rewards/rejected": 1.280964970588684,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -1.7663570642471313,
|
|
"logits/rejected": -1.810603380203247,
|
|
"logps/chosen": -0.2882184386253357,
|
|
"logps/rejected": -0.26946529746055603,
|
|
"loss": 0.6920673847198486,
|
|
"loss/core": 0.6920673847198486,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6537147760391235,
|
|
"rewards/margins": 0.43855220079421997,
|
|
"rewards/rejected": 1.2151625156402588,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 0.59375,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -1.8167644739151,
|
|
"logits/rejected": -1.847123146057129,
|
|
"logps/chosen": -0.28940513730049133,
|
|
"logps/rejected": -0.28783121705055237,
|
|
"loss": 0.6892070174217224,
|
|
"loss/core": 0.6892070174217224,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3337934017181396,
|
|
"rewards/margins": 1.6257107257843018,
|
|
"rewards/rejected": 0.7080828547477722,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -1.7945712804794312,
|
|
"logits/rejected": -1.765467882156372,
|
|
"logps/chosen": -0.2750927805900574,
|
|
"logps/rejected": -0.277031809091568,
|
|
"loss": 0.6882904171943665,
|
|
"loss/core": 0.6882904171943665,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.7785580158233643,
|
|
"rewards/margins": 2.080836772918701,
|
|
"rewards/rejected": 1.6977211236953735,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 0.7890625,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -1.7736009359359741,
|
|
"logits/rejected": -1.8232080936431885,
|
|
"logps/chosen": -0.2970460057258606,
|
|
"logps/rejected": -0.2842680811882019,
|
|
"loss": 0.6880653500556946,
|
|
"loss/core": 0.6880653500556946,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.2521681785583496,
|
|
"rewards/margins": 2.1032259464263916,
|
|
"rewards/rejected": 1.148942232131958,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 0.224609375,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -1.8815971612930298,
|
|
"logits/rejected": -1.8222999572753906,
|
|
"logps/chosen": -0.27899742126464844,
|
|
"logps/rejected": -0.27459582686424255,
|
|
"loss": 0.6906721591949463,
|
|
"loss/core": 0.6906721591949463,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.500136613845825,
|
|
"rewards/margins": 1.0100083351135254,
|
|
"rewards/rejected": 1.4901282787322998,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 0.109375,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -1.6210107803344727,
|
|
"logits/rejected": -1.6766055822372437,
|
|
"logps/chosen": -0.30045056343078613,
|
|
"logps/rejected": -0.3084052801132202,
|
|
"loss": 0.6901770830154419,
|
|
"loss/core": 0.6901770830154419,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.262094020843506,
|
|
"rewards/margins": 1.2137267589569092,
|
|
"rewards/rejected": 1.0483672618865967,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 0.07373046875,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -1.8754079341888428,
|
|
"logits/rejected": -1.9132652282714844,
|
|
"logps/chosen": -0.29582279920578003,
|
|
"logps/rejected": -0.28720319271087646,
|
|
"loss": 0.6921669244766235,
|
|
"loss/core": 0.6921669244766235,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.3508808612823486,
|
|
"rewards/margins": 0.3969760835170746,
|
|
"rewards/rejected": 0.9539046287536621,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 1.8515625,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -2.0209007263183594,
|
|
"logits/rejected": -2.061509132385254,
|
|
"logps/chosen": -0.27408546209335327,
|
|
"logps/rejected": -0.2787121832370758,
|
|
"loss": 0.6931945085525513,
|
|
"loss/core": 0.6931945085525513,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.682008147239685,
|
|
"rewards/margins": 0.032013483345508575,
|
|
"rewards/rejected": 1.6499946117401123,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 0.287109375,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -1.8627064228057861,
|
|
"logits/rejected": -1.919704794883728,
|
|
"logps/chosen": -0.26979389786720276,
|
|
"logps/rejected": -0.3131831884384155,
|
|
"loss": 0.6907284259796143,
|
|
"loss/core": 0.6907284259796143,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.5674381256103516,
|
|
"rewards/margins": 1.0534178018569946,
|
|
"rewards/rejected": 1.5140199661254883,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 0.4375,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -1.7471721172332764,
|
|
"logits/rejected": -1.6695648431777954,
|
|
"logps/chosen": -0.29016250371932983,
|
|
"logps/rejected": -0.2957083284854889,
|
|
"loss": 0.6901000142097473,
|
|
"loss/core": 0.6901000142097473,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2820587158203125,
|
|
"rewards/margins": 1.2528122663497925,
|
|
"rewards/rejected": 1.029246211051941,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 0.1552734375,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -1.9470646381378174,
|
|
"logits/rejected": -1.9981377124786377,
|
|
"logps/chosen": -0.2694973051548004,
|
|
"logps/rejected": -0.281091570854187,
|
|
"loss": 0.6881608963012695,
|
|
"loss/core": 0.6881608963012695,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.707597017288208,
|
|
"rewards/margins": 2.0738112926483154,
|
|
"rewards/rejected": 1.633785605430603,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 0.265625,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -1.66718327999115,
|
|
"logits/rejected": -1.654313087463379,
|
|
"logps/chosen": -0.3189978301525116,
|
|
"logps/rejected": -0.30316898226737976,
|
|
"loss": 0.6870630979537964,
|
|
"loss/core": 0.6870630979537964,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.3393523693084717,
|
|
"rewards/margins": 2.53385329246521,
|
|
"rewards/rejected": 0.805498480796814,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 0.109375,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -1.7715818881988525,
|
|
"logits/rejected": -1.7875267267227173,
|
|
"logps/chosen": -0.2973600924015045,
|
|
"logps/rejected": -0.3144645392894745,
|
|
"loss": 0.6913233399391174,
|
|
"loss/core": 0.6913233399391174,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6638648509979248,
|
|
"rewards/margins": 0.7346082925796509,
|
|
"rewards/rejected": 0.9292564392089844,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 0.2216796875,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -1.8163177967071533,
|
|
"logits/rejected": -1.8914730548858643,
|
|
"logps/chosen": -0.30195850133895874,
|
|
"logps/rejected": -0.2931588590145111,
|
|
"loss": 0.6890204548835754,
|
|
"loss/core": 0.6890204548835754,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8905234336853027,
|
|
"rewards/margins": 1.713559865951538,
|
|
"rewards/rejected": 1.1769633293151855,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -1.784044861793518,
|
|
"logits/rejected": -1.784518837928772,
|
|
"logps/chosen": -0.30433687567710876,
|
|
"logps/rejected": -0.30718523263931274,
|
|
"loss": 0.6902703642845154,
|
|
"loss/core": 0.6902703642845154,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3598580360412598,
|
|
"rewards/margins": 1.1782281398773193,
|
|
"rewards/rejected": 1.18163001537323,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 0.1533203125,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -1.6721748113632202,
|
|
"logits/rejected": -1.5910634994506836,
|
|
"logps/chosen": -0.29361215233802795,
|
|
"logps/rejected": -0.3014775812625885,
|
|
"loss": 0.6942087411880493,
|
|
"loss/core": 0.6942087411880493,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8780733346939087,
|
|
"rewards/margins": -0.34356021881103516,
|
|
"rewards/rejected": 2.2216334342956543,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -2.0185725688934326,
|
|
"logits/rejected": -1.9341987371444702,
|
|
"logps/chosen": -0.27574944496154785,
|
|
"logps/rejected": -0.2965439558029175,
|
|
"loss": 0.6927915811538696,
|
|
"loss/core": 0.6927915811538696,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2239298820495605,
|
|
"rewards/margins": 0.20492222905158997,
|
|
"rewards/rejected": 2.019007682800293,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -1.713491439819336,
|
|
"logits/rejected": -1.7976768016815186,
|
|
"logps/chosen": -0.345987468957901,
|
|
"logps/rejected": -0.3146022856235504,
|
|
"loss": 0.6902076005935669,
|
|
"loss/core": 0.6902076005935669,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6916372776031494,
|
|
"rewards/margins": 1.2287349700927734,
|
|
"rewards/rejected": 1.4629024267196655,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -1.644727349281311,
|
|
"logits/rejected": -1.6644747257232666,
|
|
"logps/chosen": -0.33096668124198914,
|
|
"logps/rejected": -0.34381940960884094,
|
|
"loss": 0.6915746927261353,
|
|
"loss/core": 0.6915746927261353,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.1107561588287354,
|
|
"rewards/margins": 0.7399330735206604,
|
|
"rewards/rejected": 2.3708226680755615,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 0.19140625,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.6278400421142578,
|
|
"logits/rejected": -1.64529287815094,
|
|
"logps/chosen": -0.3012077808380127,
|
|
"logps/rejected": -0.28633007407188416,
|
|
"loss": 0.690815269947052,
|
|
"loss/core": 0.690815269947052,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7844654321670532,
|
|
"rewards/margins": 0.944085419178009,
|
|
"rewards/rejected": 0.840380072593689,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -1.5810550451278687,
|
|
"logits/rejected": -1.5913165807724,
|
|
"logps/chosen": -0.28546205163002014,
|
|
"logps/rejected": -0.27281785011291504,
|
|
"loss": 0.6898037195205688,
|
|
"loss/core": 0.6898037195205688,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6366374492645264,
|
|
"rewards/margins": 1.3896093368530273,
|
|
"rewards/rejected": 1.247028112411499,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 0.1923828125,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -1.5124685764312744,
|
|
"logits/rejected": -1.6061471700668335,
|
|
"logps/chosen": -0.2498733252286911,
|
|
"logps/rejected": -0.26588135957717896,
|
|
"loss": 0.6902016997337341,
|
|
"loss/core": 0.6902016997337341,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6172432899475098,
|
|
"rewards/margins": 1.1884584426879883,
|
|
"rewards/rejected": 1.4287846088409424,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 0.388671875,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -1.886030912399292,
|
|
"logits/rejected": -1.8260345458984375,
|
|
"logps/chosen": -0.2788711190223694,
|
|
"logps/rejected": -0.2940537631511688,
|
|
"loss": 0.6895097494125366,
|
|
"loss/core": 0.6895097494125366,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.654872417449951,
|
|
"rewards/margins": 1.5049922466278076,
|
|
"rewards/rejected": 1.149880290031433,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 0.15234375,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -1.7723588943481445,
|
|
"logits/rejected": -1.740827202796936,
|
|
"logps/chosen": -0.307345986366272,
|
|
"logps/rejected": -0.30495208501815796,
|
|
"loss": 0.6911855340003967,
|
|
"loss/core": 0.6911855340003967,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7815402746200562,
|
|
"rewards/margins": 0.7892364859580994,
|
|
"rewards/rejected": 0.9923038482666016,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 0.08642578125,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -1.828007459640503,
|
|
"logits/rejected": -1.7083508968353271,
|
|
"logps/chosen": -0.3094002306461334,
|
|
"logps/rejected": -0.32762759923934937,
|
|
"loss": 0.6900734901428223,
|
|
"loss/core": 0.6900734901428223,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.83378005027771,
|
|
"rewards/margins": 1.247715950012207,
|
|
"rewards/rejected": 0.586064338684082,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -1.920536994934082,
|
|
"logits/rejected": -1.9467909336090088,
|
|
"logps/chosen": -0.2732824683189392,
|
|
"logps/rejected": -0.2819834053516388,
|
|
"loss": 0.6904429197311401,
|
|
"loss/core": 0.6904429197311401,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.7944914102554321,
|
|
"rewards/margins": 1.0950840711593628,
|
|
"rewards/rejected": 0.699407160282135,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 0.189453125,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -1.8890149593353271,
|
|
"logits/rejected": -1.8630263805389404,
|
|
"logps/chosen": -0.2616122364997864,
|
|
"logps/rejected": -0.2847253978252411,
|
|
"loss": 0.6903570294380188,
|
|
"loss/core": 0.6903570294380188,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4658520221710205,
|
|
"rewards/margins": 1.127223014831543,
|
|
"rewards/rejected": 1.3386290073394775,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 1.53125,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -1.936354398727417,
|
|
"logits/rejected": -1.9784005880355835,
|
|
"logps/chosen": -0.30487221479415894,
|
|
"logps/rejected": -0.30455270409584045,
|
|
"loss": 0.6924332976341248,
|
|
"loss/core": 0.6924332976341248,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8872802257537842,
|
|
"rewards/margins": 0.34668928384780884,
|
|
"rewards/rejected": 1.5405908823013306,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 1.34375,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -1.8787858486175537,
|
|
"logits/rejected": -1.877399206161499,
|
|
"logps/chosen": -0.28796452283859253,
|
|
"logps/rejected": -0.2709272801876068,
|
|
"loss": 0.6896687746047974,
|
|
"loss/core": 0.6896687746047974,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.630368709564209,
|
|
"rewards/margins": 1.4462611675262451,
|
|
"rewards/rejected": 1.1841075420379639,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 0.140625,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -1.689291000366211,
|
|
"logits/rejected": -1.709299087524414,
|
|
"logps/chosen": -0.2852824032306671,
|
|
"logps/rejected": -0.29218631982803345,
|
|
"loss": 0.692372739315033,
|
|
"loss/core": 0.692372739315033,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6775140762329102,
|
|
"rewards/margins": 0.32127708196640015,
|
|
"rewards/rejected": 1.3562370538711548,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 0.25390625,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -1.7467161417007446,
|
|
"logits/rejected": -1.8196022510528564,
|
|
"logps/chosen": -0.27380579710006714,
|
|
"logps/rejected": -0.27410024404525757,
|
|
"loss": 0.6902837157249451,
|
|
"loss/core": 0.6902837157249451,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.508232593536377,
|
|
"rewards/margins": 1.1675052642822266,
|
|
"rewards/rejected": 1.3407273292541504,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 0.1318359375,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -1.8901665210723877,
|
|
"logits/rejected": -1.9521496295928955,
|
|
"logps/chosen": -0.28199759125709534,
|
|
"logps/rejected": -0.29019278287887573,
|
|
"loss": 0.6913297772407532,
|
|
"loss/core": 0.6913297772407532,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.3713281154632568,
|
|
"rewards/margins": 0.7334555983543396,
|
|
"rewards/rejected": 0.6378724575042725,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 0.2255859375,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -1.899572730064392,
|
|
"logits/rejected": -1.8477237224578857,
|
|
"logps/chosen": -0.28993353247642517,
|
|
"logps/rejected": -0.28719550371170044,
|
|
"loss": 0.6906791925430298,
|
|
"loss/core": 0.6906791925430298,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.508605480194092,
|
|
"rewards/margins": 0.9953238368034363,
|
|
"rewards/rejected": 1.5132817029953003,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 0.1396484375,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -1.7165281772613525,
|
|
"logits/rejected": -1.7286529541015625,
|
|
"logps/chosen": -0.2964112162590027,
|
|
"logps/rejected": -0.2740923762321472,
|
|
"loss": 0.6906309723854065,
|
|
"loss/core": 0.6906309723854065,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0557491779327393,
|
|
"rewards/margins": 1.0147982835769653,
|
|
"rewards/rejected": 1.0409510135650635,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 0.123046875,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -1.7657253742218018,
|
|
"logits/rejected": -1.7964322566986084,
|
|
"logps/chosen": -0.2821979224681854,
|
|
"logps/rejected": -0.2735520005226135,
|
|
"loss": 0.6908767819404602,
|
|
"loss/core": 0.6908767819404602,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.770012617111206,
|
|
"rewards/margins": 0.9165030717849731,
|
|
"rewards/rejected": 0.8535095453262329,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 0.265625,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -1.840344786643982,
|
|
"logits/rejected": -1.8655697107315063,
|
|
"logps/chosen": -0.3119342029094696,
|
|
"logps/rejected": -0.30928319692611694,
|
|
"loss": 0.6905477643013,
|
|
"loss/core": 0.6905477643013,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.063857078552246,
|
|
"rewards/margins": 1.085402488708496,
|
|
"rewards/rejected": 1.97845458984375,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 0.287109375,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -1.8058847188949585,
|
|
"logits/rejected": -1.7770591974258423,
|
|
"logps/chosen": -0.2736867368221283,
|
|
"logps/rejected": -0.2652318775653839,
|
|
"loss": 0.6898475289344788,
|
|
"loss/core": 0.6898475289344788,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0367789268493652,
|
|
"rewards/margins": 1.344928503036499,
|
|
"rewards/rejected": 1.6918503046035767,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 0.1015625,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -1.8290420770645142,
|
|
"logits/rejected": -1.9100230932235718,
|
|
"logps/chosen": -0.2853394150733948,
|
|
"logps/rejected": -0.2796414792537689,
|
|
"loss": 0.6906234622001648,
|
|
"loss/core": 0.6906234622001648,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7059528827667236,
|
|
"rewards/margins": 1.020578145980835,
|
|
"rewards/rejected": 0.6853744387626648,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 0.1953125,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -1.8122670650482178,
|
|
"logits/rejected": -1.8503730297088623,
|
|
"logps/chosen": -0.2719040811061859,
|
|
"logps/rejected": -0.28884705901145935,
|
|
"loss": 0.6911881566047668,
|
|
"loss/core": 0.6911881566047668,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.7802484035491943,
|
|
"rewards/margins": 0.8196385502815247,
|
|
"rewards/rejected": 1.960609793663025,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 1.609375,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -1.7871983051300049,
|
|
"logits/rejected": -1.8605146408081055,
|
|
"logps/chosen": -0.3058835566043854,
|
|
"logps/rejected": -0.29272013902664185,
|
|
"loss": 0.6895998120307922,
|
|
"loss/core": 0.6895998120307922,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.141623020172119,
|
|
"rewards/margins": 1.4836769104003906,
|
|
"rewards/rejected": 0.6579458117485046,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 0.1728515625,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -1.5382251739501953,
|
|
"logits/rejected": -1.5597591400146484,
|
|
"logps/chosen": -0.3067648708820343,
|
|
"logps/rejected": -0.30108946561813354,
|
|
"loss": 0.6889725923538208,
|
|
"loss/core": 0.6889725923538208,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.3333942890167236,
|
|
"rewards/margins": 1.7330058813095093,
|
|
"rewards/rejected": 1.6003879308700562,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 0.1123046875,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -1.8064721822738647,
|
|
"logits/rejected": -1.7805678844451904,
|
|
"logps/chosen": -0.28534120321273804,
|
|
"logps/rejected": -0.29203271865844727,
|
|
"loss": 0.6893185973167419,
|
|
"loss/core": 0.6893185973167419,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7705674171447754,
|
|
"rewards/margins": 1.617296576499939,
|
|
"rewards/rejected": 1.1532708406448364,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -1.767867088317871,
|
|
"logits/rejected": -1.7515289783477783,
|
|
"logps/chosen": -0.30535268783569336,
|
|
"logps/rejected": -0.3085159361362457,
|
|
"loss": 0.6914692521095276,
|
|
"loss/core": 0.6914692521095276,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9446821212768555,
|
|
"rewards/margins": 0.6940180063247681,
|
|
"rewards/rejected": 1.2506643533706665,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 0.1357421875,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -1.7518078088760376,
|
|
"logits/rejected": -1.70755136013031,
|
|
"logps/chosen": -0.27699360251426697,
|
|
"logps/rejected": -0.28743523359298706,
|
|
"loss": 0.6904750466346741,
|
|
"loss/core": 0.6904750466346741,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.121932029724121,
|
|
"rewards/margins": 1.081810474395752,
|
|
"rewards/rejected": 1.0401216745376587,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 0.08837890625,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -1.7966949939727783,
|
|
"logits/rejected": -1.7754967212677002,
|
|
"logps/chosen": -0.3063514232635498,
|
|
"logps/rejected": -0.3063802719116211,
|
|
"loss": 0.6925251483917236,
|
|
"loss/core": 0.6925251483917236,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6563818454742432,
|
|
"rewards/margins": 0.26670390367507935,
|
|
"rewards/rejected": 1.389677882194519,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 0.1552734375,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -1.8795068264007568,
|
|
"logits/rejected": -1.963724136352539,
|
|
"logps/chosen": -0.29647111892700195,
|
|
"logps/rejected": -0.2953276038169861,
|
|
"loss": 0.6920241117477417,
|
|
"loss/core": 0.6920241117477417,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.034947156906128,
|
|
"rewards/margins": 0.4514775276184082,
|
|
"rewards/rejected": 0.5834697484970093,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -1.7779220342636108,
|
|
"logits/rejected": -1.7593568563461304,
|
|
"logps/chosen": -0.30200910568237305,
|
|
"logps/rejected": -0.30040618777275085,
|
|
"loss": 0.6907645463943481,
|
|
"loss/core": 0.6907645463943481,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8066829442977905,
|
|
"rewards/margins": 0.9670276641845703,
|
|
"rewards/rejected": 0.8396552801132202,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 0.185546875,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -1.9163156747817993,
|
|
"logits/rejected": -1.8812675476074219,
|
|
"logps/chosen": -0.2867981791496277,
|
|
"logps/rejected": -0.2877008616924286,
|
|
"loss": 0.6907505989074707,
|
|
"loss/core": 0.6907505989074707,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9703712463378906,
|
|
"rewards/margins": 1.0003026723861694,
|
|
"rewards/rejected": 1.9700685739517212,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 1.5625,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -1.8704643249511719,
|
|
"logits/rejected": -1.9002145528793335,
|
|
"logps/chosen": -0.2817132771015167,
|
|
"logps/rejected": -0.27127501368522644,
|
|
"loss": 0.6853945851325989,
|
|
"loss/core": 0.6853945851325989,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.546133041381836,
|
|
"rewards/margins": 3.254213809967041,
|
|
"rewards/rejected": 1.2919186353683472,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 0.26953125,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -1.634866714477539,
|
|
"logits/rejected": -1.7080461978912354,
|
|
"logps/chosen": -0.31140029430389404,
|
|
"logps/rejected": -0.3021450638771057,
|
|
"loss": 0.6891725659370422,
|
|
"loss/core": 0.6891725659370422,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7522711753845215,
|
|
"rewards/margins": 1.6253259181976318,
|
|
"rewards/rejected": 1.1269454956054688,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 0.12353515625,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.887028455734253,
|
|
"logits/rejected": -1.9402997493743896,
|
|
"logps/chosen": -0.2800259292125702,
|
|
"logps/rejected": -0.3047648072242737,
|
|
"loss": 0.6893907785415649,
|
|
"loss/core": 0.6893907785415649,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.504063844680786,
|
|
"rewards/margins": 1.5716952085494995,
|
|
"rewards/rejected": 1.9323689937591553,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 0.8984375,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -1.5624942779541016,
|
|
"logits/rejected": -1.5863012075424194,
|
|
"logps/chosen": -0.30193594098091125,
|
|
"logps/rejected": -0.3065473437309265,
|
|
"loss": 0.690039336681366,
|
|
"loss/core": 0.690039336681366,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.7968106269836426,
|
|
"rewards/margins": 1.314178228378296,
|
|
"rewards/rejected": 2.4826323986053467,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -1.7193517684936523,
|
|
"logits/rejected": -1.706871747970581,
|
|
"logps/chosen": -0.277077317237854,
|
|
"logps/rejected": -0.27818840742111206,
|
|
"loss": 0.6909591555595398,
|
|
"loss/core": 0.6909591555595398,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3378546237945557,
|
|
"rewards/margins": 0.8950866460800171,
|
|
"rewards/rejected": 1.4427679777145386,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 0.24609375,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -1.7371066808700562,
|
|
"logits/rejected": -1.7233482599258423,
|
|
"logps/chosen": -0.3089512884616852,
|
|
"logps/rejected": -0.3026643395423889,
|
|
"loss": 0.6932816505432129,
|
|
"loss/core": 0.6932816505432129,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.1888813972473145,
|
|
"rewards/margins": -0.031425733119249344,
|
|
"rewards/rejected": 1.2203071117401123,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 1.984375,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -1.4896411895751953,
|
|
"logits/rejected": -1.460703730583191,
|
|
"logps/chosen": -0.3152364492416382,
|
|
"logps/rejected": -0.31800299882888794,
|
|
"loss": 0.6897588968276978,
|
|
"loss/core": 0.6897588968276978,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.180734634399414,
|
|
"rewards/margins": 1.4465312957763672,
|
|
"rewards/rejected": 1.7342031002044678,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 0.09619140625,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -1.7158210277557373,
|
|
"logits/rejected": -1.7079627513885498,
|
|
"logps/chosen": -0.28858762979507446,
|
|
"logps/rejected": -0.2933047413825989,
|
|
"loss": 0.6915832757949829,
|
|
"loss/core": 0.6915832757949829,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.429247260093689,
|
|
"rewards/margins": 0.630083441734314,
|
|
"rewards/rejected": 0.799163818359375,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 2.671875,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -1.802638053894043,
|
|
"logits/rejected": -1.7583692073822021,
|
|
"logps/chosen": -0.31713518500328064,
|
|
"logps/rejected": -0.3377764821052551,
|
|
"loss": 0.6928405165672302,
|
|
"loss/core": 0.6928405165672302,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.3360774517059326,
|
|
"rewards/margins": 0.1572122424840927,
|
|
"rewards/rejected": 1.1788651943206787,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 0.1533203125,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -1.9015684127807617,
|
|
"logits/rejected": -1.9405295848846436,
|
|
"logps/chosen": -0.3054332733154297,
|
|
"logps/rejected": -0.3069126307964325,
|
|
"loss": 0.6918673515319824,
|
|
"loss/core": 0.6918673515319824,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.0781052112579346,
|
|
"rewards/margins": 0.5133062601089478,
|
|
"rewards/rejected": 0.5647989511489868,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 0.1416015625,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -1.830971360206604,
|
|
"logits/rejected": -1.8372182846069336,
|
|
"logps/chosen": -0.28015151619911194,
|
|
"logps/rejected": -0.2828627824783325,
|
|
"loss": 0.6899048686027527,
|
|
"loss/core": 0.6899048686027527,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2420430183410645,
|
|
"rewards/margins": 1.3439006805419922,
|
|
"rewards/rejected": 0.8981423377990723,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 0.61328125,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -1.8057060241699219,
|
|
"logits/rejected": -1.802642822265625,
|
|
"logps/chosen": -0.26754963397979736,
|
|
"logps/rejected": -0.273385614156723,
|
|
"loss": 0.69206303358078,
|
|
"loss/core": 0.69206303358078,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1810123920440674,
|
|
"rewards/margins": 0.46981877088546753,
|
|
"rewards/rejected": 1.7111934423446655,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 0.1796875,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -1.942135214805603,
|
|
"logits/rejected": -1.9843320846557617,
|
|
"logps/chosen": -0.2976142466068268,
|
|
"logps/rejected": -0.30821579694747925,
|
|
"loss": 0.6909466981887817,
|
|
"loss/core": 0.6909466981887817,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.338165283203125,
|
|
"rewards/margins": 0.9373272657394409,
|
|
"rewards/rejected": 1.4008381366729736,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 0.302734375,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -1.8573143482208252,
|
|
"logits/rejected": -1.8273032903671265,
|
|
"logps/chosen": -0.28388920426368713,
|
|
"logps/rejected": -0.2705972492694855,
|
|
"loss": 0.6890053749084473,
|
|
"loss/core": 0.6890053749084473,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.5264713764190674,
|
|
"rewards/margins": 1.7132002115249634,
|
|
"rewards/rejected": 1.8132712841033936,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 0.921875,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -1.7900562286376953,
|
|
"logits/rejected": -1.820005178451538,
|
|
"logps/chosen": -0.2741461396217346,
|
|
"logps/rejected": -0.27509093284606934,
|
|
"loss": 0.6885601282119751,
|
|
"loss/core": 0.6885601282119751,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.2590813636779785,
|
|
"rewards/margins": 1.9007847309112549,
|
|
"rewards/rejected": 1.3582968711853027,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 0.4140625,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -1.9214216470718384,
|
|
"logits/rejected": -1.9572467803955078,
|
|
"logps/chosen": -0.30636709928512573,
|
|
"logps/rejected": -0.30526402592658997,
|
|
"loss": 0.6896119713783264,
|
|
"loss/core": 0.6896119713783264,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.470032215118408,
|
|
"rewards/margins": 1.4553961753845215,
|
|
"rewards/rejected": 1.0146360397338867,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 0.1845703125,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.049393653869629,
|
|
"logits/rejected": -1.9477535486221313,
|
|
"logps/chosen": -0.2906716465950012,
|
|
"logps/rejected": -0.3007729649543762,
|
|
"loss": 0.6913913488388062,
|
|
"loss/core": 0.6913913488388062,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4209463596343994,
|
|
"rewards/margins": 0.750097393989563,
|
|
"rewards/rejected": 1.6708488464355469,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 0.1591796875,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -1.738469123840332,
|
|
"logits/rejected": -1.6882975101470947,
|
|
"logps/chosen": -0.2867928743362427,
|
|
"logps/rejected": -0.28549641370773315,
|
|
"loss": 0.6875568628311157,
|
|
"loss/core": 0.6875568628311157,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.7997069358825684,
|
|
"rewards/margins": 2.3964896202087402,
|
|
"rewards/rejected": 1.4032175540924072,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 0.74609375,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -1.6411170959472656,
|
|
"logits/rejected": -1.74252450466156,
|
|
"logps/chosen": -0.28902941942214966,
|
|
"logps/rejected": -0.30169445276260376,
|
|
"loss": 0.6886734962463379,
|
|
"loss/core": 0.6886734962463379,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.515756607055664,
|
|
"rewards/margins": 1.8329532146453857,
|
|
"rewards/rejected": 1.6828029155731201,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 1.734375,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -1.9373714923858643,
|
|
"logits/rejected": -1.9126183986663818,
|
|
"logps/chosen": -0.30250269174575806,
|
|
"logps/rejected": -0.3157595694065094,
|
|
"loss": 0.6906894445419312,
|
|
"loss/core": 0.6906894445419312,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.1111555099487305,
|
|
"rewards/margins": 1.0718204975128174,
|
|
"rewards/rejected": 2.039334535598755,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 0.1328125,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -1.8262889385223389,
|
|
"logits/rejected": -1.826817274093628,
|
|
"logps/chosen": -0.29866036772727966,
|
|
"logps/rejected": -0.29913249611854553,
|
|
"loss": 0.6902146935462952,
|
|
"loss/core": 0.6902146935462952,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0936100482940674,
|
|
"rewards/margins": 1.1971681118011475,
|
|
"rewards/rejected": 0.8964420557022095,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 0.1572265625,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -1.9917919635772705,
|
|
"logits/rejected": -2.0477356910705566,
|
|
"logps/chosen": -0.2745470404624939,
|
|
"logps/rejected": -0.26887795329093933,
|
|
"loss": 0.6905907392501831,
|
|
"loss/core": 0.6905907392501831,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0287718772888184,
|
|
"rewards/margins": 1.0297727584838867,
|
|
"rewards/rejected": 0.998999297618866,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 0.318359375,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -1.9308719635009766,
|
|
"logits/rejected": -1.8756221532821655,
|
|
"logps/chosen": -0.2740345895290375,
|
|
"logps/rejected": -0.2740941047668457,
|
|
"loss": 0.6921342015266418,
|
|
"loss/core": 0.6921342015266418,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0607590675354004,
|
|
"rewards/margins": 0.43019723892211914,
|
|
"rewards/rejected": 1.6305620670318604,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 1.25,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -1.9905240535736084,
|
|
"logits/rejected": -2.03885817527771,
|
|
"logps/chosen": -0.265903115272522,
|
|
"logps/rejected": -0.26458966732025146,
|
|
"loss": 0.6899409294128418,
|
|
"loss/core": 0.6899409294128418,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.5544772148132324,
|
|
"rewards/margins": 1.3087366819381714,
|
|
"rewards/rejected": 1.2457404136657715,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 0.244140625,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -1.6438277959823608,
|
|
"logits/rejected": -1.7215983867645264,
|
|
"logps/chosen": -0.30950814485549927,
|
|
"logps/rejected": -0.30294355750083923,
|
|
"loss": 0.6917771697044373,
|
|
"loss/core": 0.6917771697044373,
|
|
"rewards/accuracies": 0.7749999761581421,
|
|
"rewards/chosen": 1.2238495349884033,
|
|
"rewards/margins": 0.5521842241287231,
|
|
"rewards/rejected": 0.6716650724411011,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 0.12890625,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -1.944154143333435,
|
|
"logits/rejected": -1.9149882793426514,
|
|
"logps/chosen": -0.2952657639980316,
|
|
"logps/rejected": -0.28746429085731506,
|
|
"loss": 0.6903901696205139,
|
|
"loss/core": 0.6903901696205139,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.458098888397217,
|
|
"rewards/margins": 1.1424970626831055,
|
|
"rewards/rejected": 1.3156019449234009,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 1.4375,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -1.852236032485962,
|
|
"logits/rejected": -1.9032227993011475,
|
|
"logps/chosen": -0.32367467880249023,
|
|
"logps/rejected": -0.3032284379005432,
|
|
"loss": 0.6876433491706848,
|
|
"loss/core": 0.6876433491706848,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.3354249000549316,
|
|
"rewards/margins": 2.3274753093719482,
|
|
"rewards/rejected": 1.007949709892273,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 0.30859375,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -1.7561454772949219,
|
|
"logits/rejected": -1.8125495910644531,
|
|
"logps/chosen": -0.3123387396335602,
|
|
"logps/rejected": -0.31348609924316406,
|
|
"loss": 0.6884433627128601,
|
|
"loss/core": 0.6884433627128601,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.085703134536743,
|
|
"rewards/margins": 1.986158013343811,
|
|
"rewards/rejected": 1.0995451211929321,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 0.98828125,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -1.8188142776489258,
|
|
"logits/rejected": -1.914798378944397,
|
|
"logps/chosen": -0.2659410238265991,
|
|
"logps/rejected": -0.27096885442733765,
|
|
"loss": 0.691714346408844,
|
|
"loss/core": 0.691714346408844,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.4955499172210693,
|
|
"rewards/margins": 0.5815773606300354,
|
|
"rewards/rejected": 0.9139726758003235,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 2.078125,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -1.769033670425415,
|
|
"logits/rejected": -1.7553457021713257,
|
|
"logps/chosen": -0.2833646237850189,
|
|
"logps/rejected": -0.26877862215042114,
|
|
"loss": 0.6868944764137268,
|
|
"loss/core": 0.6868944764137268,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.937399387359619,
|
|
"rewards/margins": 2.5866665840148926,
|
|
"rewards/rejected": 1.3507325649261475,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -1.8661785125732422,
|
|
"logits/rejected": -1.8362553119659424,
|
|
"logps/chosen": -0.29633772373199463,
|
|
"logps/rejected": -0.2867165207862854,
|
|
"loss": 0.6908766031265259,
|
|
"loss/core": 0.6908766031265259,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.151101589202881,
|
|
"rewards/margins": 0.9135802388191223,
|
|
"rewards/rejected": 1.2375215291976929,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 0.63671875,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -1.6893161535263062,
|
|
"logits/rejected": -1.7709661722183228,
|
|
"logps/chosen": -0.2839089035987854,
|
|
"logps/rejected": -0.27923282980918884,
|
|
"loss": 0.6885705590248108,
|
|
"loss/core": 0.6885705590248108,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0020592212677,
|
|
"rewards/margins": 1.8747539520263672,
|
|
"rewards/rejected": 1.127305507659912,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 3.0625,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -1.8707926273345947,
|
|
"logits/rejected": -1.9706052541732788,
|
|
"logps/chosen": -0.26380446553230286,
|
|
"logps/rejected": -0.2769017517566681,
|
|
"loss": 0.6897034645080566,
|
|
"loss/core": 0.6897034645080566,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.89481782913208,
|
|
"rewards/margins": 1.4606375694274902,
|
|
"rewards/rejected": 1.4341800212860107,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 0.10302734375,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -1.648950219154358,
|
|
"logits/rejected": -1.645956039428711,
|
|
"logps/chosen": -0.30575722455978394,
|
|
"logps/rejected": -0.30025702714920044,
|
|
"loss": 0.6909690499305725,
|
|
"loss/core": 0.6909690499305725,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.719103455543518,
|
|
"rewards/margins": 0.8803337216377258,
|
|
"rewards/rejected": 0.8387696146965027,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 0.2001953125,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -1.7238233089447021,
|
|
"logits/rejected": -1.8354371786117554,
|
|
"logps/chosen": -0.2844577133655548,
|
|
"logps/rejected": -0.2817632555961609,
|
|
"loss": 0.6913126111030579,
|
|
"loss/core": 0.6913126111030579,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.263275623321533,
|
|
"rewards/margins": 0.7599185109138489,
|
|
"rewards/rejected": 1.5033574104309082,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 0.1796875,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -1.869998574256897,
|
|
"logits/rejected": -1.823256492614746,
|
|
"logps/chosen": -0.27499106526374817,
|
|
"logps/rejected": -0.28210312128067017,
|
|
"loss": 0.6916349530220032,
|
|
"loss/core": 0.6916349530220032,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.8145194053649902,
|
|
"rewards/margins": 0.7166808843612671,
|
|
"rewards/rejected": 2.0978386402130127,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 0.443359375,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -1.927408218383789,
|
|
"logits/rejected": -1.9314380884170532,
|
|
"logps/chosen": -0.26992565393447876,
|
|
"logps/rejected": -0.28807181119918823,
|
|
"loss": 0.6917387247085571,
|
|
"loss/core": 0.6917387247085571,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2258682250976562,
|
|
"rewards/margins": 0.6031666994094849,
|
|
"rewards/rejected": 1.6227014064788818,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 0.2578125,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.002894878387451,
|
|
"logits/rejected": -2.003776788711548,
|
|
"logps/chosen": -0.27657777070999146,
|
|
"logps/rejected": -0.2728041708469391,
|
|
"loss": 0.6912546753883362,
|
|
"loss/core": 0.6912546753883362,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.364955425262451,
|
|
"rewards/margins": 0.7624953985214233,
|
|
"rewards/rejected": 1.6024599075317383,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -1.601522445678711,
|
|
"logits/rejected": -1.6724109649658203,
|
|
"logps/chosen": -0.279160737991333,
|
|
"logps/rejected": -0.2730087637901306,
|
|
"loss": 0.683729350566864,
|
|
"loss/core": 0.683729350566864,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 5.24552059173584,
|
|
"rewards/margins": 4.030712127685547,
|
|
"rewards/rejected": 1.2148076295852661,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -1.8503739833831787,
|
|
"logits/rejected": -1.8288978338241577,
|
|
"logps/chosen": -0.28329378366470337,
|
|
"logps/rejected": -0.30308961868286133,
|
|
"loss": 0.6905882954597473,
|
|
"loss/core": 0.6905882954597473,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0931591987609863,
|
|
"rewards/margins": 1.066819429397583,
|
|
"rewards/rejected": 1.0263397693634033,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 1.15625,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -1.9194366931915283,
|
|
"logits/rejected": -1.8568189144134521,
|
|
"logps/chosen": -0.28827863931655884,
|
|
"logps/rejected": -0.3087393343448639,
|
|
"loss": 0.6923280954360962,
|
|
"loss/core": 0.6923280954360962,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.665645956993103,
|
|
"rewards/margins": 0.3457864820957184,
|
|
"rewards/rejected": 1.3198593854904175,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 2.453125,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -1.7917439937591553,
|
|
"logits/rejected": -1.7728865146636963,
|
|
"logps/chosen": -0.316561222076416,
|
|
"logps/rejected": -0.30622977018356323,
|
|
"loss": 0.6886661648750305,
|
|
"loss/core": 0.6886661648750305,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.6381404399871826,
|
|
"rewards/margins": 1.929628610610962,
|
|
"rewards/rejected": 1.7085119485855103,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 1.890625,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -1.771681785583496,
|
|
"logits/rejected": -1.731675386428833,
|
|
"logps/chosen": -0.28252094984054565,
|
|
"logps/rejected": -0.26652899384498596,
|
|
"loss": 0.6878234148025513,
|
|
"loss/core": 0.6878234148025513,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.460325241088867,
|
|
"rewards/margins": 2.2089626789093018,
|
|
"rewards/rejected": 1.2513624429702759,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 0.2109375,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -1.5761404037475586,
|
|
"logits/rejected": -1.5784422159194946,
|
|
"logps/chosen": -0.3069811463356018,
|
|
"logps/rejected": -0.3080241084098816,
|
|
"loss": 0.6885954737663269,
|
|
"loss/core": 0.6885954737663269,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2021026611328125,
|
|
"rewards/margins": 1.8541738986968994,
|
|
"rewards/rejected": 1.347928762435913,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 0.74609375,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -1.889041543006897,
|
|
"logits/rejected": -1.9575011730194092,
|
|
"logps/chosen": -0.26323431730270386,
|
|
"logps/rejected": -0.2795872688293457,
|
|
"loss": 0.689049482345581,
|
|
"loss/core": 0.689049482345581,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.0836408138275146,
|
|
"rewards/margins": 1.6842677593231201,
|
|
"rewards/rejected": 1.399373173713684,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 0.169921875,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -1.6385066509246826,
|
|
"logits/rejected": -1.6468448638916016,
|
|
"logps/chosen": -0.2980702519416809,
|
|
"logps/rejected": -0.3428047299385071,
|
|
"loss": 0.6906419992446899,
|
|
"loss/core": 0.6906419992446899,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.289318799972534,
|
|
"rewards/margins": 1.0524483919143677,
|
|
"rewards/rejected": 1.236870527267456,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 0.08837890625,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -1.5919002294540405,
|
|
"logits/rejected": -1.7469463348388672,
|
|
"logps/chosen": -0.3116455674171448,
|
|
"logps/rejected": -0.29385876655578613,
|
|
"loss": 0.6895068883895874,
|
|
"loss/core": 0.6895068883895874,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.45418643951416,
|
|
"rewards/margins": 1.5170066356658936,
|
|
"rewards/rejected": 0.937179446220398,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 0.061279296875,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -1.790310263633728,
|
|
"logits/rejected": -1.8553049564361572,
|
|
"logps/chosen": -0.27847370505332947,
|
|
"logps/rejected": -0.2731352150440216,
|
|
"loss": 0.688349723815918,
|
|
"loss/core": 0.688349723815918,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.402092456817627,
|
|
"rewards/margins": 1.995959997177124,
|
|
"rewards/rejected": 1.406132459640503,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 0.126953125,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -1.7230796813964844,
|
|
"logits/rejected": -1.798443078994751,
|
|
"logps/chosen": -0.29598766565322876,
|
|
"logps/rejected": -0.2965576946735382,
|
|
"loss": 0.6906968951225281,
|
|
"loss/core": 0.6906968951225281,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8174304962158203,
|
|
"rewards/margins": 0.9899489283561707,
|
|
"rewards/rejected": 0.8274815678596497,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 0.1904296875,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -1.7792017459869385,
|
|
"logits/rejected": -1.7719032764434814,
|
|
"logps/chosen": -0.28997790813446045,
|
|
"logps/rejected": -0.28154560923576355,
|
|
"loss": 0.6893754005432129,
|
|
"loss/core": 0.6893754005432129,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.301804304122925,
|
|
"rewards/margins": 1.5316702127456665,
|
|
"rewards/rejected": 1.7701339721679688,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 0.5234375,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -1.7738841772079468,
|
|
"logits/rejected": -1.7625110149383545,
|
|
"logps/chosen": -0.29042699933052063,
|
|
"logps/rejected": -0.28381985425949097,
|
|
"loss": 0.6905869245529175,
|
|
"loss/core": 0.6905869245529175,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9816631078720093,
|
|
"rewards/margins": 1.0423338413238525,
|
|
"rewards/rejected": 0.9393291473388672,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -1.829310417175293,
|
|
"logits/rejected": -1.8881504535675049,
|
|
"logps/chosen": -0.2865942716598511,
|
|
"logps/rejected": -0.2712253928184509,
|
|
"loss": 0.686186671257019,
|
|
"loss/core": 0.686186671257019,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.7003753185272217,
|
|
"rewards/margins": 2.869210720062256,
|
|
"rewards/rejected": 0.831164538860321,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 0.2294921875,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -1.6887779235839844,
|
|
"logits/rejected": -1.7008845806121826,
|
|
"logps/chosen": -0.28278762102127075,
|
|
"logps/rejected": -0.2788473069667816,
|
|
"loss": 0.6903247237205505,
|
|
"loss/core": 0.6903247237205505,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.2481186389923096,
|
|
"rewards/margins": 1.135740041732788,
|
|
"rewards/rejected": 1.112378716468811,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 0.2080078125,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -1.5623390674591064,
|
|
"logits/rejected": -1.6467643976211548,
|
|
"logps/chosen": -0.3176479935646057,
|
|
"logps/rejected": -0.29388684034347534,
|
|
"loss": 0.6890052556991577,
|
|
"loss/core": 0.6890052556991577,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.222991943359375,
|
|
"rewards/margins": 1.7007945775985718,
|
|
"rewards/rejected": 1.5221973657608032,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 0.2099609375,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -1.622319221496582,
|
|
"logits/rejected": -1.63738214969635,
|
|
"logps/chosen": -0.3008866310119629,
|
|
"logps/rejected": -0.2952221930027008,
|
|
"loss": 0.6901183724403381,
|
|
"loss/core": 0.6901183724403381,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5821938514709473,
|
|
"rewards/margins": 1.2760289907455444,
|
|
"rewards/rejected": 1.3061645030975342,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -1.7124159336090088,
|
|
"logits/rejected": -1.7910178899765015,
|
|
"logps/chosen": -0.27922117710113525,
|
|
"logps/rejected": -0.2886210083961487,
|
|
"loss": 0.6854382753372192,
|
|
"loss/core": 0.6854382753372192,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 5.158889293670654,
|
|
"rewards/margins": 3.2606253623962402,
|
|
"rewards/rejected": 1.8982641696929932,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 0.7890625,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -1.8085368871688843,
|
|
"logits/rejected": -1.7837501764297485,
|
|
"logps/chosen": -0.31605297327041626,
|
|
"logps/rejected": -0.30117732286453247,
|
|
"loss": 0.692598283290863,
|
|
"loss/core": 0.692598283290863,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.354279041290283,
|
|
"rewards/margins": 0.28271275758743286,
|
|
"rewards/rejected": 2.071566343307495,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 0.361328125,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.0771026611328125,
|
|
"logits/rejected": -2.058582305908203,
|
|
"logps/chosen": -0.28092527389526367,
|
|
"logps/rejected": -0.2910299301147461,
|
|
"loss": 0.6897355914115906,
|
|
"loss/core": 0.6897355914115906,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.395536422729492,
|
|
"rewards/margins": 1.4038153886795044,
|
|
"rewards/rejected": 0.9917212724685669,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 0.169921875,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -1.6925674676895142,
|
|
"logits/rejected": -1.7791401147842407,
|
|
"logps/chosen": -0.3049921989440918,
|
|
"logps/rejected": -0.3050501346588135,
|
|
"loss": 0.6895653605461121,
|
|
"loss/core": 0.6895653605461121,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.476440191268921,
|
|
"rewards/margins": 1.4556043148040771,
|
|
"rewards/rejected": 1.0208359956741333,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -1.778446912765503,
|
|
"logits/rejected": -1.8237922191619873,
|
|
"logps/chosen": -0.28048455715179443,
|
|
"logps/rejected": -0.28305408358573914,
|
|
"loss": 0.6914597153663635,
|
|
"loss/core": 0.6914597153663635,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6184022426605225,
|
|
"rewards/margins": 0.748033344745636,
|
|
"rewards/rejected": 1.8703689575195312,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 0.75390625,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -1.7187820672988892,
|
|
"logits/rejected": -1.6614373922348022,
|
|
"logps/chosen": -0.3019927144050598,
|
|
"logps/rejected": -0.29929202795028687,
|
|
"loss": 0.6907371282577515,
|
|
"loss/core": 0.6907371282577515,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.7743518352508545,
|
|
"rewards/margins": 1.0062803030014038,
|
|
"rewards/rejected": 1.7680715322494507,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 0.1376953125,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -1.7278330326080322,
|
|
"logits/rejected": -1.6942561864852905,
|
|
"logps/chosen": -0.27367183566093445,
|
|
"logps/rejected": -0.28410977125167847,
|
|
"loss": 0.6944975852966309,
|
|
"loss/core": 0.6944975852966309,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.822975516319275,
|
|
"rewards/margins": -0.4711933732032776,
|
|
"rewards/rejected": 2.294168710708618,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 0.244140625,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -1.8173329830169678,
|
|
"logits/rejected": -1.748335599899292,
|
|
"logps/chosen": -0.29630497097969055,
|
|
"logps/rejected": -0.30823907256126404,
|
|
"loss": 0.6916288733482361,
|
|
"loss/core": 0.6916288733482361,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.783888816833496,
|
|
"rewards/margins": 0.6391955614089966,
|
|
"rewards/rejected": 1.14469313621521,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 0.310546875,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -1.9312461614608765,
|
|
"logits/rejected": -1.9855235815048218,
|
|
"logps/chosen": -0.2915689945220947,
|
|
"logps/rejected": -0.3033396601676941,
|
|
"loss": 0.6898959279060364,
|
|
"loss/core": 0.6898959279060364,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.9401935338974,
|
|
"rewards/margins": 1.346240758895874,
|
|
"rewards/rejected": 0.5939527153968811,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 0.1435546875,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -1.854900598526001,
|
|
"logits/rejected": -1.9152843952178955,
|
|
"logps/chosen": -0.30195021629333496,
|
|
"logps/rejected": -0.3078872561454773,
|
|
"loss": 0.6913703680038452,
|
|
"loss/core": 0.6913703680038452,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3067500591278076,
|
|
"rewards/margins": 0.7192279696464539,
|
|
"rewards/rejected": 1.5875221490859985,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 0.07373046875,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -1.8532918691635132,
|
|
"logits/rejected": -1.834735631942749,
|
|
"logps/chosen": -0.3193635642528534,
|
|
"logps/rejected": -0.3099674880504608,
|
|
"loss": 0.6915178894996643,
|
|
"loss/core": 0.6915178894996643,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.416029930114746,
|
|
"rewards/margins": 0.6589576005935669,
|
|
"rewards/rejected": 0.7570723295211792,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 0.1806640625,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -1.9541394710540771,
|
|
"logits/rejected": -1.946727991104126,
|
|
"logps/chosen": -0.26166877150535583,
|
|
"logps/rejected": -0.2776499092578888,
|
|
"loss": 0.6918145418167114,
|
|
"loss/core": 0.6918145418167114,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.89385187625885,
|
|
"rewards/margins": 0.5465883612632751,
|
|
"rewards/rejected": 1.3472635746002197,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 0.322265625,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.755165696144104,
|
|
"logits/rejected": -1.7259944677352905,
|
|
"logps/chosen": -0.2936655580997467,
|
|
"logps/rejected": -0.298048198223114,
|
|
"loss": 0.6916288137435913,
|
|
"loss/core": 0.6916288137435913,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5220264196395874,
|
|
"rewards/margins": 0.609094500541687,
|
|
"rewards/rejected": 0.9129319190979004,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -1.9866920709609985,
|
|
"logits/rejected": -1.915894865989685,
|
|
"logps/chosen": -0.2786349654197693,
|
|
"logps/rejected": -0.2918104827404022,
|
|
"loss": 0.6923294067382812,
|
|
"loss/core": 0.6923294067382812,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.2574752569198608,
|
|
"rewards/margins": 0.3320327401161194,
|
|
"rewards/rejected": 0.9254426956176758,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 0.060302734375,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -1.8373003005981445,
|
|
"logits/rejected": -1.7937291860580444,
|
|
"logps/chosen": -0.3085290789604187,
|
|
"logps/rejected": -0.29286837577819824,
|
|
"loss": 0.6876273155212402,
|
|
"loss/core": 0.6876273155212402,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1967198848724365,
|
|
"rewards/margins": 2.337005615234375,
|
|
"rewards/rejected": 0.8597143292427063,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 0.7265625,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -1.621816873550415,
|
|
"logits/rejected": -1.822488784790039,
|
|
"logps/chosen": -0.2964431643486023,
|
|
"logps/rejected": -0.2829035520553589,
|
|
"loss": 0.6889544129371643,
|
|
"loss/core": 0.6889544129371643,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.3975021839141846,
|
|
"rewards/margins": 1.7253767251968384,
|
|
"rewards/rejected": 0.6721251606941223,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 0.28515625,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -1.8435766696929932,
|
|
"logits/rejected": -1.9550691843032837,
|
|
"logps/chosen": -0.28372493386268616,
|
|
"logps/rejected": -0.2617836594581604,
|
|
"loss": 0.6860982179641724,
|
|
"loss/core": 0.6860982179641724,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.548431873321533,
|
|
"rewards/margins": 2.9798524379730225,
|
|
"rewards/rejected": 1.5685795545578003,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 0.1845703125,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -1.938122034072876,
|
|
"logits/rejected": -1.9469550848007202,
|
|
"logps/chosen": -0.29863983392715454,
|
|
"logps/rejected": -0.28583261370658875,
|
|
"loss": 0.6872848868370056,
|
|
"loss/core": 0.6872848868370056,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.3786215782165527,
|
|
"rewards/margins": 2.4727911949157715,
|
|
"rewards/rejected": 0.9058302044868469,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 0.314453125,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -1.600602149963379,
|
|
"logits/rejected": -1.5954480171203613,
|
|
"logps/chosen": -0.27909520268440247,
|
|
"logps/rejected": -0.30399781465530396,
|
|
"loss": 0.6903051733970642,
|
|
"loss/core": 0.6903051733970642,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.860823154449463,
|
|
"rewards/margins": 1.1584161520004272,
|
|
"rewards/rejected": 1.702406883239746,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 0.2001953125,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -2.0156798362731934,
|
|
"logits/rejected": -2.0649049282073975,
|
|
"logps/chosen": -0.2738130986690521,
|
|
"logps/rejected": -0.2789636552333832,
|
|
"loss": 0.6907733678817749,
|
|
"loss/core": 0.6907733678817749,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0494155883789062,
|
|
"rewards/margins": 0.9572056531906128,
|
|
"rewards/rejected": 1.092210054397583,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 0.2890625,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -1.7956726551055908,
|
|
"logits/rejected": -1.876203179359436,
|
|
"logps/chosen": -0.2982097268104553,
|
|
"logps/rejected": -0.30547016859054565,
|
|
"loss": 0.6916817426681519,
|
|
"loss/core": 0.6916817426681519,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.7903995513916016,
|
|
"rewards/margins": 0.6054316759109497,
|
|
"rewards/rejected": 1.1849677562713623,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 0.1650390625,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -1.754360556602478,
|
|
"logits/rejected": -1.8462966680526733,
|
|
"logps/chosen": -0.28413721919059753,
|
|
"logps/rejected": -0.29304325580596924,
|
|
"loss": 0.6911409497261047,
|
|
"loss/core": 0.6911409497261047,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.00300931930542,
|
|
"rewards/margins": 0.8140490651130676,
|
|
"rewards/rejected": 1.188960313796997,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 0.1806640625,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -1.7482950687408447,
|
|
"logits/rejected": -1.8495619297027588,
|
|
"logps/chosen": -0.29564136266708374,
|
|
"logps/rejected": -0.2883380055427551,
|
|
"loss": 0.6910102367401123,
|
|
"loss/core": 0.6910102367401123,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9195715188980103,
|
|
"rewards/margins": 0.8625022768974304,
|
|
"rewards/rejected": 1.0570690631866455,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 2.09375,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -1.7818498611450195,
|
|
"logits/rejected": -1.819561243057251,
|
|
"logps/chosen": -0.2841910719871521,
|
|
"logps/rejected": -0.2850389778614044,
|
|
"loss": 0.693510890007019,
|
|
"loss/core": 0.693510890007019,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.2231769561767578,
|
|
"rewards/margins": -0.12083403766155243,
|
|
"rewards/rejected": 1.3440110683441162,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -1.8370325565338135,
|
|
"logits/rejected": -1.8255866765975952,
|
|
"logps/chosen": -0.29680460691452026,
|
|
"logps/rejected": -0.2980230748653412,
|
|
"loss": 0.6934405565261841,
|
|
"loss/core": 0.6934405565261841,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.4498194456100464,
|
|
"rewards/margins": -0.0671045333147049,
|
|
"rewards/rejected": 1.5169239044189453,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 0.302734375,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -1.761301040649414,
|
|
"logits/rejected": -1.7649517059326172,
|
|
"logps/chosen": -0.26283127069473267,
|
|
"logps/rejected": -0.2820115387439728,
|
|
"loss": 0.6897014379501343,
|
|
"loss/core": 0.6897014379501343,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0530502796173096,
|
|
"rewards/margins": 1.409324049949646,
|
|
"rewards/rejected": 1.6437263488769531,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 0.15625,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -1.8872359991073608,
|
|
"logits/rejected": -1.8546419143676758,
|
|
"logps/chosen": -0.29460957646369934,
|
|
"logps/rejected": -0.30844348669052124,
|
|
"loss": 0.6914256811141968,
|
|
"loss/core": 0.6914256811141968,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4490922689437866,
|
|
"rewards/margins": 0.6981182098388672,
|
|
"rewards/rejected": 0.750974178314209,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -1.8494236469268799,
|
|
"logits/rejected": -1.9358432292938232,
|
|
"logps/chosen": -0.28461378812789917,
|
|
"logps/rejected": -0.26845136284828186,
|
|
"loss": 0.6891158819198608,
|
|
"loss/core": 0.6891158819198608,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.0717930793762207,
|
|
"rewards/margins": 1.6619094610214233,
|
|
"rewards/rejected": 1.409883737564087,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 1.1328125,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -1.827734351158142,
|
|
"logits/rejected": -1.946860909461975,
|
|
"logps/chosen": -0.27680468559265137,
|
|
"logps/rejected": -0.275119811296463,
|
|
"loss": 0.6926502585411072,
|
|
"loss/core": 0.6926502585411072,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.4091708660125732,
|
|
"rewards/margins": 0.20720620453357697,
|
|
"rewards/rejected": 1.2019646167755127,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 0.1279296875,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -1.6843173503875732,
|
|
"logits/rejected": -1.7385330200195312,
|
|
"logps/chosen": -0.28421929478645325,
|
|
"logps/rejected": -0.2876400053501129,
|
|
"loss": 0.69017493724823,
|
|
"loss/core": 0.69017493724823,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.562863826751709,
|
|
"rewards/margins": 1.2080352306365967,
|
|
"rewards/rejected": 1.3548285961151123,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 0.0947265625,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -1.7102231979370117,
|
|
"logits/rejected": -1.7559185028076172,
|
|
"logps/chosen": -0.2854920029640198,
|
|
"logps/rejected": -0.29401493072509766,
|
|
"loss": 0.6895767450332642,
|
|
"loss/core": 0.6895767450332642,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8428616523742676,
|
|
"rewards/margins": 1.45379638671875,
|
|
"rewards/rejected": 1.389065146446228,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 0.11328125,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -1.742907166481018,
|
|
"logits/rejected": -1.7834516763687134,
|
|
"logps/chosen": -0.2981284260749817,
|
|
"logps/rejected": -0.29392343759536743,
|
|
"loss": 0.6902607679367065,
|
|
"loss/core": 0.6902607679367065,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.236698627471924,
|
|
"rewards/margins": 1.1834802627563477,
|
|
"rewards/rejected": 1.053218126296997,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 0.1552734375,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -1.7728607654571533,
|
|
"logits/rejected": -1.7845735549926758,
|
|
"logps/chosen": -0.3191523551940918,
|
|
"logps/rejected": -0.3145609498023987,
|
|
"loss": 0.6922028660774231,
|
|
"loss/core": 0.6922028660774231,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.1376005411148071,
|
|
"rewards/margins": 0.3802829086780548,
|
|
"rewards/rejected": 0.7573176622390747,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 0.298828125,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -1.832451581954956,
|
|
"logits/rejected": -1.820267915725708,
|
|
"logps/chosen": -0.2969440519809723,
|
|
"logps/rejected": -0.30008524656295776,
|
|
"loss": 0.6890357732772827,
|
|
"loss/core": 0.6890357732772827,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.5468456745147705,
|
|
"rewards/margins": 1.685861349105835,
|
|
"rewards/rejected": 0.8609840273857117,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 0.236328125,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -1.8389718532562256,
|
|
"logits/rejected": -1.8534542322158813,
|
|
"logps/chosen": -0.2692399024963379,
|
|
"logps/rejected": -0.2879648208618164,
|
|
"loss": 0.6913638710975647,
|
|
"loss/core": 0.6913638710975647,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.036916971206665,
|
|
"rewards/margins": 0.7281818389892578,
|
|
"rewards/rejected": 1.3087352514266968,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 0.1708984375,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -1.733703851699829,
|
|
"logits/rejected": -1.7701585292816162,
|
|
"logps/chosen": -0.26635271310806274,
|
|
"logps/rejected": -0.2835581600666046,
|
|
"loss": 0.6906028985977173,
|
|
"loss/core": 0.6906028985977173,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2045063972473145,
|
|
"rewards/margins": 1.0283342599868774,
|
|
"rewards/rejected": 1.1761722564697266,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 0.396484375,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -1.5449193716049194,
|
|
"logits/rejected": -1.5420639514923096,
|
|
"logps/chosen": -0.28244468569755554,
|
|
"logps/rejected": -0.26813778281211853,
|
|
"loss": 0.6907768249511719,
|
|
"loss/core": 0.6907768249511719,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1589250564575195,
|
|
"rewards/margins": 1.085134744644165,
|
|
"rewards/rejected": 2.0737905502319336,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 0.1708984375,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -1.8882849216461182,
|
|
"logits/rejected": -1.9425636529922485,
|
|
"logps/chosen": -0.25149840116500854,
|
|
"logps/rejected": -0.2820515036582947,
|
|
"loss": 0.6905457973480225,
|
|
"loss/core": 0.6905457973480225,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5088672637939453,
|
|
"rewards/margins": 1.0693387985229492,
|
|
"rewards/rejected": 1.4395287036895752,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 0.0771484375,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -1.9825334548950195,
|
|
"logits/rejected": -2.009000778198242,
|
|
"logps/chosen": -0.3606415390968323,
|
|
"logps/rejected": -0.28844404220581055,
|
|
"loss": 0.6889899969100952,
|
|
"loss/core": 0.6889899969100952,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.009084701538086,
|
|
"rewards/margins": 1.8813291788101196,
|
|
"rewards/rejected": 2.1277554035186768,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 0.130859375,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -1.849302887916565,
|
|
"logits/rejected": -1.8440090417861938,
|
|
"logps/chosen": -0.29616793990135193,
|
|
"logps/rejected": -0.2985708713531494,
|
|
"loss": 0.6910385489463806,
|
|
"loss/core": 0.6910385489463806,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5953813791275024,
|
|
"rewards/margins": 0.8663269281387329,
|
|
"rewards/rejected": 0.72905433177948,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 0.1796875,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -2.037290096282959,
|
|
"logits/rejected": -2.077535390853882,
|
|
"logps/chosen": -0.27280181646347046,
|
|
"logps/rejected": -0.2824481725692749,
|
|
"loss": 0.6917025446891785,
|
|
"loss/core": 0.6917025446891785,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.4865820407867432,
|
|
"rewards/margins": 0.5794221758842468,
|
|
"rewards/rejected": 0.907159686088562,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 0.453125,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -1.942304253578186,
|
|
"logits/rejected": -1.9689109325408936,
|
|
"logps/chosen": -0.29773375391960144,
|
|
"logps/rejected": -0.2885454595088959,
|
|
"loss": 0.6907824277877808,
|
|
"loss/core": 0.6907824277877808,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.037071943283081,
|
|
"rewards/margins": 0.9551854133605957,
|
|
"rewards/rejected": 1.0818865299224854,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 0.4921875,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -1.9432833194732666,
|
|
"logits/rejected": -1.9501100778579712,
|
|
"logps/chosen": -0.26834672689437866,
|
|
"logps/rejected": -0.269378662109375,
|
|
"loss": 0.6902218461036682,
|
|
"loss/core": 0.6902218461036682,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.0740108489990234,
|
|
"rewards/margins": 1.1815000772476196,
|
|
"rewards/rejected": 0.8925108909606934,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 0.349609375,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -1.6919254064559937,
|
|
"logits/rejected": -1.7820037603378296,
|
|
"logps/chosen": -0.28077149391174316,
|
|
"logps/rejected": -0.279075562953949,
|
|
"loss": 0.6903265118598938,
|
|
"loss/core": 0.6903265118598938,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.868180513381958,
|
|
"rewards/margins": 1.1484382152557373,
|
|
"rewards/rejected": 1.7197420597076416,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 0.38671875,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.6962074041366577,
|
|
"logits/rejected": -1.6986114978790283,
|
|
"logps/chosen": -0.28555965423583984,
|
|
"logps/rejected": -0.2935461401939392,
|
|
"loss": 0.6910402178764343,
|
|
"loss/core": 0.6910402178764343,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4226348400115967,
|
|
"rewards/margins": 0.897823691368103,
|
|
"rewards/rejected": 1.5248112678527832,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 0.14453125,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -1.844538927078247,
|
|
"logits/rejected": -1.9082237482070923,
|
|
"logps/chosen": -0.307687908411026,
|
|
"logps/rejected": -0.3395816683769226,
|
|
"loss": 0.6907016634941101,
|
|
"loss/core": 0.6907016634941101,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7747430801391602,
|
|
"rewards/margins": 0.9889305233955383,
|
|
"rewards/rejected": 0.7858127951622009,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 1.015625,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -1.975682020187378,
|
|
"logits/rejected": -1.9615514278411865,
|
|
"logps/chosen": -0.2598734200000763,
|
|
"logps/rejected": -0.27107658982276917,
|
|
"loss": 0.6921221613883972,
|
|
"loss/core": 0.6921221613883972,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.4536190032958984,
|
|
"rewards/margins": 0.45077699422836304,
|
|
"rewards/rejected": 2.0028419494628906,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 0.2158203125,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -1.632433533668518,
|
|
"logits/rejected": -1.7214616537094116,
|
|
"logps/chosen": -0.308523565530777,
|
|
"logps/rejected": -0.28264203667640686,
|
|
"loss": 0.6870818138122559,
|
|
"loss/core": 0.6870818138122559,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.5316734313964844,
|
|
"rewards/margins": 2.579163074493408,
|
|
"rewards/rejected": 0.9525102376937866,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -1.7808490991592407,
|
|
"logits/rejected": -1.8207063674926758,
|
|
"logps/chosen": -0.2825844883918762,
|
|
"logps/rejected": -0.27922552824020386,
|
|
"loss": 0.6903389692306519,
|
|
"loss/core": 0.6903389692306519,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.0911669731140137,
|
|
"rewards/margins": 1.1479694843292236,
|
|
"rewards/rejected": 0.9431974291801453,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -2.021035671234131,
|
|
"logits/rejected": -2.0233137607574463,
|
|
"logps/chosen": -0.2777281403541565,
|
|
"logps/rejected": -0.2643401622772217,
|
|
"loss": 0.6903270483016968,
|
|
"loss/core": 0.6903270483016968,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9988548755645752,
|
|
"rewards/margins": 1.1564974784851074,
|
|
"rewards/rejected": 0.8423575162887573,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 1.8671875,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -1.9014087915420532,
|
|
"logits/rejected": -1.8085308074951172,
|
|
"logps/chosen": -0.2891666293144226,
|
|
"logps/rejected": -0.30600792169570923,
|
|
"loss": 0.6911376118659973,
|
|
"loss/core": 0.6911376118659973,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9695777893066406,
|
|
"rewards/margins": 0.8112414479255676,
|
|
"rewards/rejected": 1.1583364009857178,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 0.458984375,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -1.7752872705459595,
|
|
"logits/rejected": -1.7699896097183228,
|
|
"logps/chosen": -0.28837233781814575,
|
|
"logps/rejected": -0.2897118926048279,
|
|
"loss": 0.6898549199104309,
|
|
"loss/core": 0.6898549199104309,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.560025691986084,
|
|
"rewards/margins": 1.3317620754241943,
|
|
"rewards/rejected": 1.2282639741897583,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 0.1025390625,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -1.6815214157104492,
|
|
"logits/rejected": -1.7447792291641235,
|
|
"logps/chosen": -0.30126506090164185,
|
|
"logps/rejected": -0.3436921536922455,
|
|
"loss": 0.6893914341926575,
|
|
"loss/core": 0.6893914341926575,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2814574241638184,
|
|
"rewards/margins": 1.542607069015503,
|
|
"rewards/rejected": 0.738849937915802,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 0.2451171875,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -1.8093611001968384,
|
|
"logits/rejected": -1.790492057800293,
|
|
"logps/chosen": -0.2983977794647217,
|
|
"logps/rejected": -0.28687578439712524,
|
|
"loss": 0.6904627680778503,
|
|
"loss/core": 0.6904627680778503,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8845103979110718,
|
|
"rewards/margins": 1.0901850461959839,
|
|
"rewards/rejected": 0.7943253517150879,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 0.193359375,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -1.8425931930541992,
|
|
"logits/rejected": -1.853624939918518,
|
|
"logps/chosen": -0.2532172203063965,
|
|
"logps/rejected": -0.2740766704082489,
|
|
"loss": 0.6899970173835754,
|
|
"loss/core": 0.6899970173835754,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.117034673690796,
|
|
"rewards/margins": 1.288293480873108,
|
|
"rewards/rejected": 0.8287409543991089,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 1.3671875,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -1.739816427230835,
|
|
"logits/rejected": -1.735145926475525,
|
|
"logps/chosen": -0.2897264361381531,
|
|
"logps/rejected": -0.2904459238052368,
|
|
"loss": 0.6874421834945679,
|
|
"loss/core": 0.6874421834945679,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.566680431365967,
|
|
"rewards/margins": 2.4187304973602295,
|
|
"rewards/rejected": 1.147949457168579,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 0.4609375,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -1.5910301208496094,
|
|
"logits/rejected": -1.6553596258163452,
|
|
"logps/chosen": -0.3020200729370117,
|
|
"logps/rejected": -0.3062165379524231,
|
|
"loss": 0.6924393773078918,
|
|
"loss/core": 0.6924393773078918,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.691601276397705,
|
|
"rewards/margins": 0.3428772985935211,
|
|
"rewards/rejected": 2.348723888397217,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 0.2890625,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -1.8377548456192017,
|
|
"logits/rejected": -1.8844318389892578,
|
|
"logps/chosen": -0.2867121696472168,
|
|
"logps/rejected": -0.29250529408454895,
|
|
"loss": 0.6900558471679688,
|
|
"loss/core": 0.6900558471679688,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.395780086517334,
|
|
"rewards/margins": 1.2476097345352173,
|
|
"rewards/rejected": 1.1481703519821167,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 0.302734375,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -1.7527811527252197,
|
|
"logits/rejected": -1.7593837976455688,
|
|
"logps/chosen": -0.2845557630062103,
|
|
"logps/rejected": -0.28174346685409546,
|
|
"loss": 0.690881609916687,
|
|
"loss/core": 0.690881609916687,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.1602466106414795,
|
|
"rewards/margins": 0.9179478883743286,
|
|
"rewards/rejected": 1.2422987222671509,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 0.177734375,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -1.7969452142715454,
|
|
"logits/rejected": -1.8759748935699463,
|
|
"logps/chosen": -0.2859663963317871,
|
|
"logps/rejected": -0.33153361082077026,
|
|
"loss": 0.6913708448410034,
|
|
"loss/core": 0.6913708448410034,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.9288288354873657,
|
|
"rewards/margins": 0.7228490114212036,
|
|
"rewards/rejected": 1.205979585647583,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 0.91796875,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -1.6951520442962646,
|
|
"logits/rejected": -1.7336347103118896,
|
|
"logps/chosen": -0.25996583700180054,
|
|
"logps/rejected": -0.29030436277389526,
|
|
"loss": 0.6907094120979309,
|
|
"loss/core": 0.6907094120979309,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2786126136779785,
|
|
"rewards/margins": 0.9912854433059692,
|
|
"rewards/rejected": 1.2873269319534302,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 1.3828125,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -1.884939432144165,
|
|
"logits/rejected": -1.8319370746612549,
|
|
"logps/chosen": -0.30168288946151733,
|
|
"logps/rejected": -0.3016626834869385,
|
|
"loss": 0.690568745136261,
|
|
"loss/core": 0.690568745136261,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4173693656921387,
|
|
"rewards/margins": 1.0916273593902588,
|
|
"rewards/rejected": 1.3257420063018799,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -1.948432207107544,
|
|
"logits/rejected": -1.7609401941299438,
|
|
"logps/chosen": -0.27557021379470825,
|
|
"logps/rejected": -0.29795706272125244,
|
|
"loss": 0.6937984228134155,
|
|
"loss/core": 0.6937984228134155,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.6998504400253296,
|
|
"rewards/margins": -0.23256230354309082,
|
|
"rewards/rejected": 1.9324127435684204,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 0.2255859375,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -1.8939878940582275,
|
|
"logits/rejected": -1.914552092552185,
|
|
"logps/chosen": -0.307658851146698,
|
|
"logps/rejected": -0.3075157403945923,
|
|
"loss": 0.6896311044692993,
|
|
"loss/core": 0.6896311044692993,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8853094577789307,
|
|
"rewards/margins": 1.4759790897369385,
|
|
"rewards/rejected": 1.409330129623413,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 0.166015625,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -1.8575178384780884,
|
|
"logits/rejected": -1.897189736366272,
|
|
"logps/chosen": -0.30906686186790466,
|
|
"logps/rejected": -0.29951316118240356,
|
|
"loss": 0.6915870904922485,
|
|
"loss/core": 0.6915870904922485,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.1715130805969238,
|
|
"rewards/margins": 0.6273240447044373,
|
|
"rewards/rejected": 0.5441890954971313,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 0.76171875,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -1.9069395065307617,
|
|
"logits/rejected": -1.957416296005249,
|
|
"logps/chosen": -0.3348519206047058,
|
|
"logps/rejected": -0.3277924954891205,
|
|
"loss": 0.6899693012237549,
|
|
"loss/core": 0.6899693012237549,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.047415256500244,
|
|
"rewards/margins": 1.303015947341919,
|
|
"rewards/rejected": 0.7443991899490356,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 0.23828125,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -1.7169866561889648,
|
|
"logits/rejected": -1.6992985010147095,
|
|
"logps/chosen": -0.2847689688205719,
|
|
"logps/rejected": -0.27514931559562683,
|
|
"loss": 0.6903687715530396,
|
|
"loss/core": 0.6903687715530396,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.195614814758301,
|
|
"rewards/margins": 1.1351045370101929,
|
|
"rewards/rejected": 1.060510516166687,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -1.8456823825836182,
|
|
"logits/rejected": -1.849439024925232,
|
|
"logps/chosen": -0.31287848949432373,
|
|
"logps/rejected": -0.31777575612068176,
|
|
"loss": 0.6863356828689575,
|
|
"loss/core": 0.6863356828689575,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.444868087768555,
|
|
"rewards/margins": 2.881870985031128,
|
|
"rewards/rejected": 1.5629966259002686,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.6902858657307095,
|
|
"train_runtime": 8330.5288,
|
|
"train_samples_per_second": 1.729,
|
|
"train_steps_per_second": 0.108
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|