4904 lines
186 KiB
JSON
4904 lines
186 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.2860548271752086,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 900,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2889330983161926,
|
||
|
|
"drift/rejected_abs": 0.1752604842185974,
|
||
|
|
"epoch": 0.0015891934843067143,
|
||
|
|
"grad_norm": 1.015625,
|
||
|
|
"learning_rate": 1.111111111111111e-09,
|
||
|
|
"logits/chosen": -2.2279419898986816,
|
||
|
|
"logits/rejected": -2.1537013053894043,
|
||
|
|
"logps/chosen": -0.2503497004508972,
|
||
|
|
"logps/rejected": -0.25458794832229614,
|
||
|
|
"loss": 0.2848242223262787,
|
||
|
|
"loss/core": 0.2454325258731842,
|
||
|
|
"loss/preference_sft": 0.24757125973701477,
|
||
|
|
"loss/reference_anchor": 0.36916011571884155,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.8893306255340576,
|
||
|
|
"rewards/margins": 1.1367456912994385,
|
||
|
|
"rewards/rejected": 1.7525850534439087,
|
||
|
|
"ronpo/logit": 0.1136745810508728,
|
||
|
|
"ronpo/residual": 0.10866384208202362,
|
||
|
|
"ronpo/residual_abs": 0.2132132351398468,
|
||
|
|
"ronpo/target": 0.005010743625462055,
|
||
|
|
"ronpo/target_abs": 0.04510241001844406,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.251609742641449,
|
||
|
|
"drift/rejected_abs": 0.17321258783340454,
|
||
|
|
"epoch": 0.0031783869686134287,
|
||
|
|
"grad_norm": 5.75,
|
||
|
|
"learning_rate": 2.5e-09,
|
||
|
|
"logits/chosen": -1.9813182353973389,
|
||
|
|
"logits/rejected": -1.9225966930389404,
|
||
|
|
"logps/chosen": -0.2639704942703247,
|
||
|
|
"logps/rejected": -0.27192094922065735,
|
||
|
|
"loss": 0.19277027249336243,
|
||
|
|
"loss/core": 0.15751448273658752,
|
||
|
|
"loss/preference_sft": 0.2704908847808838,
|
||
|
|
"loss/reference_anchor": 0.32550862431526184,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.5153849124908447,
|
||
|
|
"rewards/margins": 0.7842203974723816,
|
||
|
|
"rewards/rejected": 1.7311643362045288,
|
||
|
|
"ronpo/logit": 0.07842205464839935,
|
||
|
|
"ronpo/residual": 0.07000575214624405,
|
||
|
|
"ronpo/residual_abs": 0.17156489193439484,
|
||
|
|
"ronpo/target": 0.008416297845542431,
|
||
|
|
"ronpo/target_abs": 0.04953983426094055,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14724932610988617,
|
||
|
|
"drift/rejected_abs": 0.07930952310562134,
|
||
|
|
"epoch": 0.004767580452920143,
|
||
|
|
"grad_norm": 7.96875,
|
||
|
|
"learning_rate": 3.8888888888888884e-09,
|
||
|
|
"logits/chosen": -2.2914788722991943,
|
||
|
|
"logits/rejected": -2.24648118019104,
|
||
|
|
"logps/chosen": -0.27627187967300415,
|
||
|
|
"logps/rejected": -0.2831260859966278,
|
||
|
|
"loss": 0.05049755424261093,
|
||
|
|
"loss/core": 0.041736677289009094,
|
||
|
|
"loss/preference_sft": 0.2777397632598877,
|
||
|
|
"loss/reference_anchor": 0.05983476713299751,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.4721348285675049,
|
||
|
|
"rewards/margins": 0.6800366640090942,
|
||
|
|
"rewards/rejected": 0.7920981645584106,
|
||
|
|
"ronpo/logit": 0.06800366193056107,
|
||
|
|
"ronpo/residual": 0.07898763567209244,
|
||
|
|
"ronpo/residual_abs": 0.12248041480779648,
|
||
|
|
"ronpo/target": -0.010983972810208797,
|
||
|
|
"ronpo/target_abs": 0.053494714200496674,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14157260954380035,
|
||
|
|
"drift/rejected_abs": 0.0700564831495285,
|
||
|
|
"epoch": 0.006356773937226857,
|
||
|
|
"grad_norm": 1.5859375,
|
||
|
|
"learning_rate": 5.277777777777778e-09,
|
||
|
|
"logits/chosen": -2.291292905807495,
|
||
|
|
"logits/rejected": -2.2172799110412598,
|
||
|
|
"logps/chosen": -0.275204598903656,
|
||
|
|
"logps/rejected": -0.2790767550468445,
|
||
|
|
"loss": 0.03427541255950928,
|
||
|
|
"loss/core": 0.02626476064324379,
|
||
|
|
"loss/preference_sft": 0.27796709537506104,
|
||
|
|
"loss/reference_anchor": 0.05230981111526489,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 1.4151142835617065,
|
||
|
|
"rewards/margins": 0.718280553817749,
|
||
|
|
"rewards/rejected": 0.6968337297439575,
|
||
|
|
"ronpo/logit": 0.07182805240154266,
|
||
|
|
"ronpo/residual": 0.07484658807516098,
|
||
|
|
"ronpo/residual_abs": 0.10549838840961456,
|
||
|
|
"ronpo/target": -0.003018544986844063,
|
||
|
|
"ronpo/target_abs": 0.050488390028476715,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23460185527801514,
|
||
|
|
"drift/rejected_abs": 0.07755617797374725,
|
||
|
|
"epoch": 0.007945967421533572,
|
||
|
|
"grad_norm": 1.375,
|
||
|
|
"learning_rate": 6.666666666666666e-09,
|
||
|
|
"logits/chosen": -2.354492664337158,
|
||
|
|
"logits/rejected": -2.394556999206543,
|
||
|
|
"logps/chosen": -0.23961284756660461,
|
||
|
|
"logps/rejected": -0.2479882687330246,
|
||
|
|
"loss": 0.3154865503311157,
|
||
|
|
"loss/core": 0.2928386330604553,
|
||
|
|
"loss/preference_sft": 0.24814143776893616,
|
||
|
|
"loss/reference_anchor": 0.2016647309064865,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.3450961112976074,
|
||
|
|
"rewards/margins": 1.571350336074829,
|
||
|
|
"rewards/rejected": 0.7737457156181335,
|
||
|
|
"ronpo/logit": 0.1571349948644638,
|
||
|
|
"ronpo/residual": 0.16688285768032074,
|
||
|
|
"ronpo/residual_abs": 0.1950080841779709,
|
||
|
|
"ronpo/target": -0.009747850708663464,
|
||
|
|
"ronpo/target_abs": 0.05017418786883354,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2047700136899948,
|
||
|
|
"drift/rejected_abs": 0.08747217059135437,
|
||
|
|
"epoch": 0.009535160905840286,
|
||
|
|
"grad_norm": 346.0,
|
||
|
|
"learning_rate": 8.055555555555556e-09,
|
||
|
|
"logits/chosen": -2.229592800140381,
|
||
|
|
"logits/rejected": -2.125317096710205,
|
||
|
|
"logps/chosen": -0.2870704233646393,
|
||
|
|
"logps/rejected": -0.2829613983631134,
|
||
|
|
"loss": 0.17466267943382263,
|
||
|
|
"loss/core": 0.15042664110660553,
|
||
|
|
"loss/preference_sft": 0.2907489836215973,
|
||
|
|
"loss/reference_anchor": 0.21328547596931458,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.047576904296875,
|
||
|
|
"rewards/margins": 1.1739780902862549,
|
||
|
|
"rewards/rejected": 0.8735988736152649,
|
||
|
|
"ronpo/logit": 0.11739782243967056,
|
||
|
|
"ronpo/residual": 0.1012236699461937,
|
||
|
|
"ronpo/residual_abs": 0.14672869443893433,
|
||
|
|
"ronpo/target": 0.016174152493476868,
|
||
|
|
"ronpo/target_abs": 0.05148453637957573,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20920784771442413,
|
||
|
|
"drift/rejected_abs": 0.13015516102313995,
|
||
|
|
"epoch": 0.011124354390147001,
|
||
|
|
"grad_norm": 4.375,
|
||
|
|
"learning_rate": 9.444444444444444e-09,
|
||
|
|
"logits/chosen": -2.1240811347961426,
|
||
|
|
"logits/rejected": -2.0613348484039307,
|
||
|
|
"logps/chosen": -0.27063313126564026,
|
||
|
|
"logps/rejected": -0.28193506598472595,
|
||
|
|
"loss": 0.058792613446712494,
|
||
|
|
"loss/core": 0.03939628228545189,
|
||
|
|
"loss/preference_sft": 0.2742695212364197,
|
||
|
|
"loss/reference_anchor": 0.1665363758802414,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.0917978286743164,
|
||
|
|
"rewards/margins": 0.7923383116722107,
|
||
|
|
"rewards/rejected": 1.299459457397461,
|
||
|
|
"ronpo/logit": 0.07923384010791779,
|
||
|
|
"ronpo/residual": 0.06704284250736237,
|
||
|
|
"ronpo/residual_abs": 0.1055685505270958,
|
||
|
|
"ronpo/target": 0.012190988287329674,
|
||
|
|
"ronpo/target_abs": 0.04947342351078987,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.13401487469673157,
|
||
|
|
"drift/rejected_abs": 0.10863766819238663,
|
||
|
|
"epoch": 0.012713547874453715,
|
||
|
|
"grad_norm": 452.0,
|
||
|
|
"learning_rate": 1.0833333333333333e-08,
|
||
|
|
"logits/chosen": -2.08085298538208,
|
||
|
|
"logits/rejected": -2.0408239364624023,
|
||
|
|
"logps/chosen": -0.318928062915802,
|
||
|
|
"logps/rejected": -0.32496583461761475,
|
||
|
|
"loss": 0.24572625756263733,
|
||
|
|
"loss/core": 0.2277563512325287,
|
||
|
|
"loss/preference_sft": 0.31747809052467346,
|
||
|
|
"loss/reference_anchor": 0.1479511708021164,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.3394415378570557,
|
||
|
|
"rewards/margins": 0.2542641758918762,
|
||
|
|
"rewards/rejected": 1.0851774215698242,
|
||
|
|
"ronpo/logit": 0.02542642131447792,
|
||
|
|
"ronpo/residual": 0.011654033325612545,
|
||
|
|
"ronpo/residual_abs": 0.14836682379245758,
|
||
|
|
"ronpo/target": 0.01377238892018795,
|
||
|
|
"ronpo/target_abs": 0.05200153589248657,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1999109983444214,
|
||
|
|
"drift/rejected_abs": 0.10404626280069351,
|
||
|
|
"epoch": 0.014302741358760428,
|
||
|
|
"grad_norm": 3.84375,
|
||
|
|
"learning_rate": 1.2222222222222222e-08,
|
||
|
|
"logits/chosen": -2.1482348442077637,
|
||
|
|
"logits/rejected": -2.0771121978759766,
|
||
|
|
"logps/chosen": -0.3045925498008728,
|
||
|
|
"logps/rejected": -0.29603320360183716,
|
||
|
|
"loss": 0.0930178314447403,
|
||
|
|
"loss/core": 0.07558077573776245,
|
||
|
|
"loss/preference_sft": 0.30196070671081543,
|
||
|
|
"loss/reference_anchor": 0.14417441189289093,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.9981660842895508,
|
||
|
|
"rewards/margins": 0.9596048593521118,
|
||
|
|
"rewards/rejected": 1.0385613441467285,
|
||
|
|
"ronpo/logit": 0.09596048295497894,
|
||
|
|
"ronpo/residual": 0.08603992313146591,
|
||
|
|
"ronpo/residual_abs": 0.13150832056999207,
|
||
|
|
"ronpo/target": 0.00992056354880333,
|
||
|
|
"ronpo/target_abs": 0.05012957379221916,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21057943999767303,
|
||
|
|
"drift/rejected_abs": 0.1738637387752533,
|
||
|
|
"epoch": 0.015891934843067144,
|
||
|
|
"grad_norm": 14.6875,
|
||
|
|
"learning_rate": 1.361111111111111e-08,
|
||
|
|
"logits/chosen": -2.2325432300567627,
|
||
|
|
"logits/rejected": -2.280646324157715,
|
||
|
|
"logps/chosen": -0.38519901037216187,
|
||
|
|
"logps/rejected": -0.3755597174167633,
|
||
|
|
"loss": 0.520218551158905,
|
||
|
|
"loss/core": 0.46836939454078674,
|
||
|
|
"loss/preference_sft": 0.3819790482521057,
|
||
|
|
"loss/reference_anchor": 0.4802929759025574,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.1057944297790527,
|
||
|
|
"rewards/margins": 1.4202427864074707,
|
||
|
|
"rewards/rejected": 0.6855517625808716,
|
||
|
|
"ronpo/logit": 0.14202429354190826,
|
||
|
|
"ronpo/residual": 0.14623448252677917,
|
||
|
|
"ronpo/residual_abs": 0.21223580837249756,
|
||
|
|
"ronpo/target": -0.004210175015032291,
|
||
|
|
"ronpo/target_abs": 0.04790406674146652,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24303218722343445,
|
||
|
|
"drift/rejected_abs": 0.11335645616054535,
|
||
|
|
"epoch": 0.01748112832737386,
|
||
|
|
"grad_norm": 1.3359375,
|
||
|
|
"learning_rate": 1.5e-08,
|
||
|
|
"logits/chosen": -2.134533643722534,
|
||
|
|
"logits/rejected": -2.106968402862549,
|
||
|
|
"logps/chosen": -0.3017352223396301,
|
||
|
|
"logps/rejected": -0.2939698100090027,
|
||
|
|
"loss": 0.29720133543014526,
|
||
|
|
"loss/core": 0.27017584443092346,
|
||
|
|
"loss/preference_sft": 0.29091566801071167,
|
||
|
|
"loss/reference_anchor": 0.2411632239818573,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.428701877593994,
|
||
|
|
"rewards/margins": 1.2995102405548096,
|
||
|
|
"rewards/rejected": 1.1291919946670532,
|
||
|
|
"ronpo/logit": 0.12995100021362305,
|
||
|
|
"ronpo/residual": 0.1421266496181488,
|
||
|
|
"ronpo/residual_abs": 0.17218372225761414,
|
||
|
|
"ronpo/target": -0.012175637297332287,
|
||
|
|
"ronpo/target_abs": 0.05182241275906563,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.286194771528244,
|
||
|
|
"drift/rejected_abs": 0.07608653604984283,
|
||
|
|
"epoch": 0.01907032181168057,
|
||
|
|
"grad_norm": 8.0,
|
||
|
|
"learning_rate": 1.6388888888888888e-08,
|
||
|
|
"logits/chosen": -2.018968105316162,
|
||
|
|
"logits/rejected": -2.0368776321411133,
|
||
|
|
"logps/chosen": -0.25825974345207214,
|
||
|
|
"logps/rejected": -0.26028722524642944,
|
||
|
|
"loss": 0.7146841287612915,
|
||
|
|
"loss/core": 0.6727110147476196,
|
||
|
|
"loss/preference_sft": 0.26158055663108826,
|
||
|
|
"loss/reference_anchor": 0.39357417821884155,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.861429214477539,
|
||
|
|
"rewards/margins": 2.1107606887817383,
|
||
|
|
"rewards/rejected": 0.7506682872772217,
|
||
|
|
"ronpo/logit": 0.2110760658979416,
|
||
|
|
"ronpo/residual": 0.21510767936706543,
|
||
|
|
"ronpo/residual_abs": 0.2415662556886673,
|
||
|
|
"ronpo/target": -0.0040316274389624596,
|
||
|
|
"ronpo/target_abs": 0.04714924842119217,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23433828353881836,
|
||
|
|
"drift/rejected_abs": 0.084614098072052,
|
||
|
|
"epoch": 0.020659515295987287,
|
||
|
|
"grad_norm": 173.0,
|
||
|
|
"learning_rate": 1.7777777777777777e-08,
|
||
|
|
"logits/chosen": -2.3905935287475586,
|
||
|
|
"logits/rejected": -2.4606516361236572,
|
||
|
|
"logps/chosen": -0.25088340044021606,
|
||
|
|
"logps/rejected": -0.2440684586763382,
|
||
|
|
"loss": 0.27581101655960083,
|
||
|
|
"loss/core": 0.2562420666217804,
|
||
|
|
"loss/preference_sft": 0.24446015059947968,
|
||
|
|
"loss/reference_anchor": 0.17124369740486145,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.3433830738067627,
|
||
|
|
"rewards/margins": 1.4972563982009888,
|
||
|
|
"rewards/rejected": 0.8461267352104187,
|
||
|
|
"ronpo/logit": 0.14972564578056335,
|
||
|
|
"ronpo/residual": 0.13597126305103302,
|
||
|
|
"ronpo/residual_abs": 0.17988578975200653,
|
||
|
|
"ronpo/target": 0.013754380866885185,
|
||
|
|
"ronpo/target_abs": 0.051927365362644196,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.218926340341568,
|
||
|
|
"drift/rejected_abs": 0.11191727966070175,
|
||
|
|
"epoch": 0.022248708780294002,
|
||
|
|
"grad_norm": 2.734375,
|
||
|
|
"learning_rate": 1.9166666666666666e-08,
|
||
|
|
"logits/chosen": -2.131840944290161,
|
||
|
|
"logits/rejected": -2.131176233291626,
|
||
|
|
"logps/chosen": -0.2633002996444702,
|
||
|
|
"logps/rejected": -0.26801466941833496,
|
||
|
|
"loss": 0.07617291808128357,
|
||
|
|
"loss/core": 0.062096256762742996,
|
||
|
|
"loss/preference_sft": 0.2654517590999603,
|
||
|
|
"loss/reference_anchor": 0.11422140896320343,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.188795804977417,
|
||
|
|
"rewards/margins": 1.0696232318878174,
|
||
|
|
"rewards/rejected": 1.1191728115081787,
|
||
|
|
"ronpo/logit": 0.10696232318878174,
|
||
|
|
"ronpo/residual": 0.10355587303638458,
|
||
|
|
"ronpo/residual_abs": 0.12823593616485596,
|
||
|
|
"ronpo/target": 0.003406452713534236,
|
||
|
|
"ronpo/target_abs": 0.05231611803174019,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2827184796333313,
|
||
|
|
"drift/rejected_abs": 0.147368922829628,
|
||
|
|
"epoch": 0.023837902264600714,
|
||
|
|
"grad_norm": 4.875,
|
||
|
|
"learning_rate": 2.0555555555555552e-08,
|
||
|
|
"logits/chosen": -2.1087307929992676,
|
||
|
|
"logits/rejected": -2.042454242706299,
|
||
|
|
"logps/chosen": -0.2781427502632141,
|
||
|
|
"logps/rejected": -0.27138060331344604,
|
||
|
|
"loss": 0.6821389198303223,
|
||
|
|
"loss/core": 0.637114942073822,
|
||
|
|
"loss/preference_sft": 0.270221084356308,
|
||
|
|
"loss/reference_anchor": 0.4232177734375,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.826986789703369,
|
||
|
|
"rewards/margins": 1.3705655336380005,
|
||
|
|
"rewards/rejected": 1.4564214944839478,
|
||
|
|
"ronpo/logit": 0.13705654442310333,
|
||
|
|
"ronpo/residual": 0.13803334534168243,
|
||
|
|
"ronpo/residual_abs": 0.2660602927207947,
|
||
|
|
"ronpo/target": -0.0009768044110387564,
|
||
|
|
"ronpo/target_abs": 0.050860919058322906,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.32659146189689636,
|
||
|
|
"drift/rejected_abs": 0.13844743371009827,
|
||
|
|
"epoch": 0.02542709574890743,
|
||
|
|
"grad_norm": 38.5,
|
||
|
|
"learning_rate": 2.1944444444444445e-08,
|
||
|
|
"logits/chosen": -2.052938222885132,
|
||
|
|
"logits/rejected": -2.021271228790283,
|
||
|
|
"logps/chosen": -0.270699679851532,
|
||
|
|
"logps/rejected": -0.2735486626625061,
|
||
|
|
"loss": 0.4138568341732025,
|
||
|
|
"loss/core": 0.37115219235420227,
|
||
|
|
"loss/preference_sft": 0.2724604308605194,
|
||
|
|
"loss/reference_anchor": 0.39979997277259827,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.2659149169921875,
|
||
|
|
"rewards/margins": 1.8850905895233154,
|
||
|
|
"rewards/rejected": 1.3808239698410034,
|
||
|
|
"ronpo/logit": 0.18850906193256378,
|
||
|
|
"ronpo/residual": 0.17836426198482513,
|
||
|
|
"ronpo/residual_abs": 0.21546652913093567,
|
||
|
|
"ronpo/target": 0.010144800879061222,
|
||
|
|
"ronpo/target_abs": 0.04853076487779617,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2298695147037506,
|
||
|
|
"drift/rejected_abs": 0.07848798483610153,
|
||
|
|
"epoch": 0.027016289233214145,
|
||
|
|
"grad_norm": 1024.0,
|
||
|
|
"learning_rate": 2.3333333333333334e-08,
|
||
|
|
"logits/chosen": -2.3688607215881348,
|
||
|
|
"logits/rejected": -2.3101439476013184,
|
||
|
|
"logps/chosen": -0.2924603819847107,
|
||
|
|
"logps/rejected": -0.26657044887542725,
|
||
|
|
"loss": 0.5983717441558838,
|
||
|
|
"loss/core": 0.5626398324966431,
|
||
|
|
"loss/preference_sft": 0.27175357937812805,
|
||
|
|
"loss/reference_anchor": 0.3301441967487335,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.297607898712158,
|
||
|
|
"rewards/margins": 1.5177080631256104,
|
||
|
|
"rewards/rejected": 0.7798997163772583,
|
||
|
|
"ronpo/logit": 0.15177080035209656,
|
||
|
|
"ronpo/residual": 0.14059045910835266,
|
||
|
|
"ronpo/residual_abs": 0.18269233405590057,
|
||
|
|
"ronpo/target": 0.011180354282259941,
|
||
|
|
"ronpo/target_abs": 0.053492844104766846,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19050000607967377,
|
||
|
|
"drift/rejected_abs": 0.15556380152702332,
|
||
|
|
"epoch": 0.028605482717520857,
|
||
|
|
"grad_norm": 1.3984375,
|
||
|
|
"learning_rate": 2.4722222222222223e-08,
|
||
|
|
"logits/chosen": -2.261676788330078,
|
||
|
|
"logits/rejected": -2.2711334228515625,
|
||
|
|
"logps/chosen": -0.2823393940925598,
|
||
|
|
"logps/rejected": -0.2829485237598419,
|
||
|
|
"loss": 0.24817046523094177,
|
||
|
|
"loss/core": 0.2262168675661087,
|
||
|
|
"loss/preference_sft": 0.2772825360298157,
|
||
|
|
"loss/reference_anchor": 0.1918078511953354,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.9050003290176392,
|
||
|
|
"rewards/margins": 0.3525739014148712,
|
||
|
|
"rewards/rejected": 1.5524263381958008,
|
||
|
|
"ronpo/logit": 0.03525739163160324,
|
||
|
|
"ronpo/residual": 0.03311719745397568,
|
||
|
|
"ronpo/residual_abs": 0.1650288850069046,
|
||
|
|
"ronpo/target": 0.0021401962731033564,
|
||
|
|
"ronpo/target_abs": 0.05777706578373909,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2254537045955658,
|
||
|
|
"drift/rejected_abs": 0.1015145406126976,
|
||
|
|
"epoch": 0.030194676201827572,
|
||
|
|
"grad_norm": 3.328125,
|
||
|
|
"learning_rate": 2.6111111111111112e-08,
|
||
|
|
"logits/chosen": -2.3401074409484863,
|
||
|
|
"logits/rejected": -2.2592806816101074,
|
||
|
|
"logps/chosen": -0.2731631100177765,
|
||
|
|
"logps/rejected": -0.2693111002445221,
|
||
|
|
"loss": 0.3633732199668884,
|
||
|
|
"loss/core": 0.33972230553627014,
|
||
|
|
"loss/preference_sft": 0.2640019357204437,
|
||
|
|
"loss/reference_anchor": 0.21010908484458923,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.2541868686676025,
|
||
|
|
"rewards/margins": 1.2461910247802734,
|
||
|
|
"rewards/rejected": 1.007995843887329,
|
||
|
|
"ronpo/logit": 0.12461910396814346,
|
||
|
|
"ronpo/residual": 0.12321188300848007,
|
||
|
|
"ronpo/residual_abs": 0.19888217747211456,
|
||
|
|
"ronpo/target": 0.0014072132762521505,
|
||
|
|
"ronpo/target_abs": 0.055929411202669144,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2529149651527405,
|
||
|
|
"drift/rejected_abs": 0.10006153583526611,
|
||
|
|
"epoch": 0.03178386968613429,
|
||
|
|
"grad_norm": 9.75,
|
||
|
|
"learning_rate": 2.75e-08,
|
||
|
|
"logits/chosen": -1.9696271419525146,
|
||
|
|
"logits/rejected": -2.041968822479248,
|
||
|
|
"logps/chosen": -0.2982648015022278,
|
||
|
|
"logps/rejected": -0.2892451584339142,
|
||
|
|
"loss": 0.2479362040758133,
|
||
|
|
"loss/core": 0.22514848411083221,
|
||
|
|
"loss/preference_sft": 0.28658321499824524,
|
||
|
|
"loss/reference_anchor": 0.1992187649011612,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.52838134765625,
|
||
|
|
"rewards/margins": 1.5295547246932983,
|
||
|
|
"rewards/rejected": 0.9988266229629517,
|
||
|
|
"ronpo/logit": 0.15295547246932983,
|
||
|
|
"ronpo/residual": 0.1562143713235855,
|
||
|
|
"ronpo/residual_abs": 0.18405328691005707,
|
||
|
|
"ronpo/target": -0.0032588993199169636,
|
||
|
|
"ronpo/target_abs": 0.04928561672568321,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3901594877243042,
|
||
|
|
"drift/rejected_abs": 0.12727132439613342,
|
||
|
|
"epoch": 0.033373063170441,
|
||
|
|
"grad_norm": 96.0,
|
||
|
|
"learning_rate": 2.8888888888888884e-08,
|
||
|
|
"logits/chosen": -1.9293874502182007,
|
||
|
|
"logits/rejected": -1.936822533607483,
|
||
|
|
"logps/chosen": -0.2803095877170563,
|
||
|
|
"logps/rejected": -0.283048152923584,
|
||
|
|
"loss": 0.7342885732650757,
|
||
|
|
"loss/core": 0.6701232194900513,
|
||
|
|
"loss/preference_sft": 0.27669522166252136,
|
||
|
|
"loss/reference_anchor": 0.6139832735061646,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.9012649059295654,
|
||
|
|
"rewards/margins": 2.641878604888916,
|
||
|
|
"rewards/rejected": 1.259386420249939,
|
||
|
|
"ronpo/logit": 0.26418787240982056,
|
||
|
|
"ronpo/residual": 0.2662436068058014,
|
||
|
|
"ronpo/residual_abs": 0.29203924536705017,
|
||
|
|
"ronpo/target": -0.0020557777024805546,
|
||
|
|
"ronpo/target_abs": 0.053755033761262894,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2434467077255249,
|
||
|
|
"drift/rejected_abs": 0.1235206350684166,
|
||
|
|
"epoch": 0.03496225665474772,
|
||
|
|
"grad_norm": 8.1875,
|
||
|
|
"learning_rate": 3.027777777777777e-08,
|
||
|
|
"logits/chosen": -2.2449328899383545,
|
||
|
|
"logits/rejected": -2.2041220664978027,
|
||
|
|
"logps/chosen": -0.2943647503852844,
|
||
|
|
"logps/rejected": -0.2864592969417572,
|
||
|
|
"loss": 0.12976226210594177,
|
||
|
|
"loss/core": 0.10662249475717545,
|
||
|
|
"loss/preference_sft": 0.2895354628562927,
|
||
|
|
"loss/reference_anchor": 0.20244407653808594,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.431950807571411,
|
||
|
|
"rewards/margins": 1.2087242603302002,
|
||
|
|
"rewards/rejected": 1.223226547241211,
|
||
|
|
"ronpo/logit": 0.12087241560220718,
|
||
|
|
"ronpo/residual": 0.12740285694599152,
|
||
|
|
"ronpo/residual_abs": 0.15507419407367706,
|
||
|
|
"ronpo/target": -0.006530435290187597,
|
||
|
|
"ronpo/target_abs": 0.048901356756687164,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27528631687164307,
|
||
|
|
"drift/rejected_abs": 0.13778968155384064,
|
||
|
|
"epoch": 0.03655145013905443,
|
||
|
|
"grad_norm": 1.6640625,
|
||
|
|
"learning_rate": 3.1666666666666666e-08,
|
||
|
|
"logits/chosen": -1.8844751119613647,
|
||
|
|
"logits/rejected": -1.7757047414779663,
|
||
|
|
"logps/chosen": -0.2790004312992096,
|
||
|
|
"logps/rejected": -0.28186291456222534,
|
||
|
|
"loss": 0.16678151488304138,
|
||
|
|
"loss/core": 0.1427289992570877,
|
||
|
|
"loss/preference_sft": 0.2831977903842926,
|
||
|
|
"loss/reference_anchor": 0.2122054547071457,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.7513701915740967,
|
||
|
|
"rewards/margins": 1.3792320489883423,
|
||
|
|
"rewards/rejected": 1.3721381425857544,
|
||
|
|
"ronpo/logit": 0.1379232108592987,
|
||
|
|
"ronpo/residual": 0.1344049870967865,
|
||
|
|
"ronpo/residual_abs": 0.1711626797914505,
|
||
|
|
"ronpo/target": 0.0035182249266654253,
|
||
|
|
"ronpo/target_abs": 0.05670144036412239,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.251396119594574,
|
||
|
|
"drift/rejected_abs": 0.11713407188653946,
|
||
|
|
"epoch": 0.03814064362336114,
|
||
|
|
"grad_norm": 1.4765625,
|
||
|
|
"learning_rate": 3.305555555555555e-08,
|
||
|
|
"logits/chosen": -2.0189380645751953,
|
||
|
|
"logits/rejected": -2.066065549850464,
|
||
|
|
"logps/chosen": -0.3038211464881897,
|
||
|
|
"logps/rejected": -0.30019038915634155,
|
||
|
|
"loss": 0.3087940514087677,
|
||
|
|
"loss/core": 0.27471792697906494,
|
||
|
|
"loss/preference_sft": 0.3073151111602783,
|
||
|
|
"loss/reference_anchor": 0.31002968549728394,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.5131027698516846,
|
||
|
|
"rewards/margins": 1.34487783908844,
|
||
|
|
"rewards/rejected": 1.168224573135376,
|
||
|
|
"ronpo/logit": 0.1344877928495407,
|
||
|
|
"ronpo/residual": 0.13394805788993835,
|
||
|
|
"ronpo/residual_abs": 0.19293567538261414,
|
||
|
|
"ronpo/target": 0.0005397285567596555,
|
||
|
|
"ronpo/target_abs": 0.04854871332645416,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.12723875045776367,
|
||
|
|
"drift/rejected_abs": 0.07009278982877731,
|
||
|
|
"epoch": 0.03972983710766786,
|
||
|
|
"grad_norm": 0.9375,
|
||
|
|
"learning_rate": 3.4444444444444444e-08,
|
||
|
|
"logits/chosen": -2.273139715194702,
|
||
|
|
"logits/rejected": -2.194185972213745,
|
||
|
|
"logps/chosen": -0.2742195725440979,
|
||
|
|
"logps/rejected": -0.2820054292678833,
|
||
|
|
"loss": 0.04828890785574913,
|
||
|
|
"loss/core": 0.03853292018175125,
|
||
|
|
"loss/preference_sft": 0.2750937342643738,
|
||
|
|
"loss/reference_anchor": 0.07005052268505096,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.2721223831176758,
|
||
|
|
"rewards/margins": 0.5717689990997314,
|
||
|
|
"rewards/rejected": 0.7003533840179443,
|
||
|
|
"ronpo/logit": 0.057176899164915085,
|
||
|
|
"ronpo/residual": 0.06078102067112923,
|
||
|
|
"ronpo/residual_abs": 0.09998279809951782,
|
||
|
|
"ronpo/target": -0.0036041145212948322,
|
||
|
|
"ronpo/target_abs": 0.05498850345611572,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36905351281166077,
|
||
|
|
"drift/rejected_abs": 0.15415672957897186,
|
||
|
|
"epoch": 0.04131903059197457,
|
||
|
|
"grad_norm": 15.1875,
|
||
|
|
"learning_rate": 3.583333333333333e-08,
|
||
|
|
"logits/chosen": -2.0486526489257812,
|
||
|
|
"logits/rejected": -1.976944923400879,
|
||
|
|
"logps/chosen": -0.28115013241767883,
|
||
|
|
"logps/rejected": -0.29946258664131165,
|
||
|
|
"loss": 0.5215417146682739,
|
||
|
|
"loss/core": 0.4482294023036957,
|
||
|
|
"loss/preference_sft": 0.27702564001083374,
|
||
|
|
"loss/reference_anchor": 0.7054204940795898,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.690535068511963,
|
||
|
|
"rewards/margins": 2.1494040489196777,
|
||
|
|
"rewards/rejected": 1.541130781173706,
|
||
|
|
"ronpo/logit": 0.21494042873382568,
|
||
|
|
"ronpo/residual": 0.21962769329547882,
|
||
|
|
"ronpo/residual_abs": 0.2594267427921295,
|
||
|
|
"ronpo/target": -0.004687254782766104,
|
||
|
|
"ronpo/target_abs": 0.05417006462812424,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2299853265285492,
|
||
|
|
"drift/rejected_abs": 0.1746150255203247,
|
||
|
|
"epoch": 0.04290822407628129,
|
||
|
|
"grad_norm": 104.5,
|
||
|
|
"learning_rate": 3.722222222222222e-08,
|
||
|
|
"logits/chosen": -2.1584811210632324,
|
||
|
|
"logits/rejected": -2.1437926292419434,
|
||
|
|
"logps/chosen": -0.25327831506729126,
|
||
|
|
"logps/rejected": -0.27471664547920227,
|
||
|
|
"loss": 0.4051688313484192,
|
||
|
|
"loss/core": 0.37376266717910767,
|
||
|
|
"loss/preference_sft": 0.2582192122936249,
|
||
|
|
"loss/reference_anchor": 0.2882392406463623,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.298335552215576,
|
||
|
|
"rewards/margins": 0.5546636581420898,
|
||
|
|
"rewards/rejected": 1.7436714172363281,
|
||
|
|
"ronpo/logit": 0.05546638369560242,
|
||
|
|
"ronpo/residual": 0.06129222363233566,
|
||
|
|
"ronpo/residual_abs": 0.2028910219669342,
|
||
|
|
"ronpo/target": -0.005825856234878302,
|
||
|
|
"ronpo/target_abs": 0.05126802995800972,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21368154883384705,
|
||
|
|
"drift/rejected_abs": 0.1560516655445099,
|
||
|
|
"epoch": 0.044497417560588004,
|
||
|
|
"grad_norm": 42.5,
|
||
|
|
"learning_rate": 3.861111111111111e-08,
|
||
|
|
"logits/chosen": -2.065540075302124,
|
||
|
|
"logits/rejected": -2.036073684692383,
|
||
|
|
"logps/chosen": -0.28536534309387207,
|
||
|
|
"logps/rejected": -0.2856103181838989,
|
||
|
|
"loss": 0.2617502212524414,
|
||
|
|
"loss/core": 0.23660394549369812,
|
||
|
|
"loss/preference_sft": 0.28660038113594055,
|
||
|
|
"loss/reference_anchor": 0.22280272841453552,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.1362662315368652,
|
||
|
|
"rewards/margins": 0.5771859884262085,
|
||
|
|
"rewards/rejected": 1.5590801239013672,
|
||
|
|
"ronpo/logit": 0.05771859735250473,
|
||
|
|
"ronpo/residual": 0.0572664849460125,
|
||
|
|
"ronpo/residual_abs": 0.18445725739002228,
|
||
|
|
"ronpo/target": 0.0004521068185567856,
|
||
|
|
"ronpo/target_abs": 0.05590490251779556,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3109515607357025,
|
||
|
|
"drift/rejected_abs": 0.1370420604944229,
|
||
|
|
"epoch": 0.04608661104489471,
|
||
|
|
"grad_norm": 242.0,
|
||
|
|
"learning_rate": 4e-08,
|
||
|
|
"logits/chosen": -2.100191116333008,
|
||
|
|
"logits/rejected": -2.1371798515319824,
|
||
|
|
"logps/chosen": -0.2746989130973816,
|
||
|
|
"logps/rejected": -0.2657720148563385,
|
||
|
|
"loss": 0.2308829128742218,
|
||
|
|
"loss/core": 0.19054999947547913,
|
||
|
|
"loss/preference_sft": 0.2726476788520813,
|
||
|
|
"loss/reference_anchor": 0.3760644197463989,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.108795642852783,
|
||
|
|
"rewards/margins": 1.7410615682601929,
|
||
|
|
"rewards/rejected": 1.3677339553833008,
|
||
|
|
"ronpo/logit": 0.174106165766716,
|
||
|
|
"ronpo/residual": 0.17656587064266205,
|
||
|
|
"ronpo/residual_abs": 0.20304152369499207,
|
||
|
|
"ronpo/target": -0.002459695329889655,
|
||
|
|
"ronpo/target_abs": 0.04878217726945877,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4026366174221039,
|
||
|
|
"drift/rejected_abs": 0.11420154571533203,
|
||
|
|
"epoch": 0.04767580452920143,
|
||
|
|
"grad_norm": 3.046875,
|
||
|
|
"learning_rate": 4.138888888888889e-08,
|
||
|
|
"logits/chosen": -2.0290703773498535,
|
||
|
|
"logits/rejected": -2.1320388317108154,
|
||
|
|
"logps/chosen": -0.2950589656829834,
|
||
|
|
"logps/rejected": -0.27424511313438416,
|
||
|
|
"loss": 1.086280107498169,
|
||
|
|
"loss/core": 1.0141551494598389,
|
||
|
|
"loss/preference_sft": 0.2878630757331848,
|
||
|
|
"loss/reference_anchor": 0.692463755607605,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 4.026366233825684,
|
||
|
|
"rewards/margins": 2.884765625,
|
||
|
|
"rewards/rejected": 1.1416008472442627,
|
||
|
|
"ronpo/logit": 0.28847652673721313,
|
||
|
|
"ronpo/residual": 0.2900107502937317,
|
||
|
|
"ronpo/residual_abs": 0.32643136382102966,
|
||
|
|
"ronpo/target": -0.0015342000406235456,
|
||
|
|
"ronpo/target_abs": 0.05635756999254227,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31187811493873596,
|
||
|
|
"drift/rejected_abs": 0.10443463176488876,
|
||
|
|
"epoch": 0.04926499801350814,
|
||
|
|
"grad_norm": 3.46875,
|
||
|
|
"learning_rate": 4.277777777777777e-08,
|
||
|
|
"logits/chosen": -2.167552947998047,
|
||
|
|
"logits/rejected": -2.173797607421875,
|
||
|
|
"logps/chosen": -0.2655718922615051,
|
||
|
|
"logps/rejected": -0.26215478777885437,
|
||
|
|
"loss": 0.7132057547569275,
|
||
|
|
"loss/core": 0.6582164764404297,
|
||
|
|
"loss/preference_sft": 0.2656114995479584,
|
||
|
|
"loss/reference_anchor": 0.523330807685852,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.1172947883605957,
|
||
|
|
"rewards/margins": 2.077317714691162,
|
||
|
|
"rewards/rejected": 1.0399768352508545,
|
||
|
|
"ronpo/logit": 0.20773179829120636,
|
||
|
|
"ronpo/residual": 0.21012091636657715,
|
||
|
|
"ronpo/residual_abs": 0.2313835173845291,
|
||
|
|
"ronpo/target": -0.002389120403677225,
|
||
|
|
"ronpo/target_abs": 0.04883997514843941,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2020689994096756,
|
||
|
|
"drift/rejected_abs": 0.06760167330503464,
|
||
|
|
"epoch": 0.05085419149781486,
|
||
|
|
"grad_norm": 900.0,
|
||
|
|
"learning_rate": 4.4166666666666665e-08,
|
||
|
|
"logits/chosen": -2.207207202911377,
|
||
|
|
"logits/rejected": -2.223909378051758,
|
||
|
|
"logps/chosen": -0.3178792893886566,
|
||
|
|
"logps/rejected": -0.2741866707801819,
|
||
|
|
"loss": 0.596080482006073,
|
||
|
|
"loss/core": 0.5607853531837463,
|
||
|
|
"loss/preference_sft": 0.2734844386577606,
|
||
|
|
"loss/reference_anchor": 0.32560402154922485,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0203135013580322,
|
||
|
|
"rewards/margins": 1.346987009048462,
|
||
|
|
"rewards/rejected": 0.6733266115188599,
|
||
|
|
"ronpo/logit": 0.13469868898391724,
|
||
|
|
"ronpo/residual": 0.1416359394788742,
|
||
|
|
"ronpo/residual_abs": 0.1742027997970581,
|
||
|
|
"ronpo/target": -0.006937255151569843,
|
||
|
|
"ronpo/target_abs": 0.0489855594933033,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.13068225979804993,
|
||
|
|
"drift/rejected_abs": 0.05099822208285332,
|
||
|
|
"epoch": 0.052443384982121574,
|
||
|
|
"grad_norm": 2.859375,
|
||
|
|
"learning_rate": 4.555555555555555e-08,
|
||
|
|
"logits/chosen": -2.1282591819763184,
|
||
|
|
"logits/rejected": -2.0065226554870605,
|
||
|
|
"logps/chosen": -0.2849346101284027,
|
||
|
|
"logps/rejected": -0.29358747601509094,
|
||
|
|
"loss": 0.10835715383291245,
|
||
|
|
"loss/core": 0.09872569143772125,
|
||
|
|
"loss/preference_sft": 0.28516682982444763,
|
||
|
|
"loss/reference_anchor": 0.06779788434505463,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.3054677248001099,
|
||
|
|
"rewards/margins": 0.7985641360282898,
|
||
|
|
"rewards/rejected": 0.5069035887718201,
|
||
|
|
"ronpo/logit": 0.07985641807317734,
|
||
|
|
"ronpo/residual": 0.08017878234386444,
|
||
|
|
"ronpo/residual_abs": 0.11188485473394394,
|
||
|
|
"ronpo/target": -0.0003223681706003845,
|
||
|
|
"ronpo/target_abs": 0.04679277911782265,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3152261972427368,
|
||
|
|
"drift/rejected_abs": 0.16839492321014404,
|
||
|
|
"epoch": 0.05403257846642829,
|
||
|
|
"grad_norm": 1.734375,
|
||
|
|
"learning_rate": 4.6944444444444443e-08,
|
||
|
|
"logits/chosen": -2.0259292125701904,
|
||
|
|
"logits/rejected": -2.019106149673462,
|
||
|
|
"logps/chosen": -0.2906624674797058,
|
||
|
|
"logps/rejected": -0.2745984196662903,
|
||
|
|
"loss": 0.4489654004573822,
|
||
|
|
"loss/core": 0.4040234088897705,
|
||
|
|
"loss/preference_sft": 0.2755510210990906,
|
||
|
|
"loss/reference_anchor": 0.4218643307685852,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.1518421173095703,
|
||
|
|
"rewards/margins": 1.471212387084961,
|
||
|
|
"rewards/rejected": 1.680630087852478,
|
||
|
|
"ronpo/logit": 0.14712122082710266,
|
||
|
|
"ronpo/residual": 0.1320314109325409,
|
||
|
|
"ronpo/residual_abs": 0.2297447919845581,
|
||
|
|
"ronpo/target": 0.015089799650013447,
|
||
|
|
"ronpo/target_abs": 0.05001477152109146,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1986486166715622,
|
||
|
|
"drift/rejected_abs": 0.11206654459238052,
|
||
|
|
"epoch": 0.055621771950735005,
|
||
|
|
"grad_norm": 31.5,
|
||
|
|
"learning_rate": 4.833333333333333e-08,
|
||
|
|
"logits/chosen": -2.2629666328430176,
|
||
|
|
"logits/rejected": -2.200096845626831,
|
||
|
|
"logps/chosen": -0.29371052980422974,
|
||
|
|
"logps/rejected": -0.3042106628417969,
|
||
|
|
"loss": 0.06993673741817474,
|
||
|
|
"loss/core": 0.05088762566447258,
|
||
|
|
"loss/preference_sft": 0.2960934638977051,
|
||
|
|
"loss/reference_anchor": 0.16088177263736725,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.9864861965179443,
|
||
|
|
"rewards/margins": 0.8661035299301147,
|
||
|
|
"rewards/rejected": 1.1203826665878296,
|
||
|
|
"ronpo/logit": 0.0866103544831276,
|
||
|
|
"ronpo/residual": 0.08271974325180054,
|
||
|
|
"ronpo/residual_abs": 0.11550427973270416,
|
||
|
|
"ronpo/target": 0.0038906182162463665,
|
||
|
|
"ronpo/target_abs": 0.049135539680719376,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.47210416197776794,
|
||
|
|
"drift/rejected_abs": 0.2260768860578537,
|
||
|
|
"epoch": 0.057210965435041714,
|
||
|
|
"grad_norm": 47.0,
|
||
|
|
"learning_rate": 4.972222222222222e-08,
|
||
|
|
"logits/chosen": -2.071915864944458,
|
||
|
|
"logits/rejected": -2.048391103744507,
|
||
|
|
"logps/chosen": -0.2908160388469696,
|
||
|
|
"logps/rejected": -0.28190651535987854,
|
||
|
|
"loss": 0.9919503927230835,
|
||
|
|
"loss/core": 0.8761954307556152,
|
||
|
|
"loss/preference_sft": 0.28307628631591797,
|
||
|
|
"loss/reference_anchor": 1.129240870475769,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.721041679382324,
|
||
|
|
"rewards/margins": 2.460272789001465,
|
||
|
|
"rewards/rejected": 2.2607688903808594,
|
||
|
|
"ronpo/logit": 0.24602727591991425,
|
||
|
|
"ronpo/residual": 0.250581294298172,
|
||
|
|
"ronpo/residual_abs": 0.29279235005378723,
|
||
|
|
"ronpo/target": -0.004554013721644878,
|
||
|
|
"ronpo/target_abs": 0.043643683195114136,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1224537044763565,
|
||
|
|
"drift/rejected_abs": 0.06466648727655411,
|
||
|
|
"epoch": 0.05880015891934843,
|
||
|
|
"grad_norm": 5.75,
|
||
|
|
"learning_rate": 4.999619237890978e-08,
|
||
|
|
"logits/chosen": -2.226485013961792,
|
||
|
|
"logits/rejected": -2.3286421298980713,
|
||
|
|
"logps/chosen": -0.29493987560272217,
|
||
|
|
"logps/rejected": -0.29172560572624207,
|
||
|
|
"loss": 0.03143535554409027,
|
||
|
|
"loss/core": 0.024793710559606552,
|
||
|
|
"loss/preference_sft": 0.2961380183696747,
|
||
|
|
"loss/reference_anchor": 0.03680265694856644,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.2238212823867798,
|
||
|
|
"rewards/margins": 0.5794644355773926,
|
||
|
|
"rewards/rejected": 0.6443568468093872,
|
||
|
|
"ronpo/logit": 0.05794644355773926,
|
||
|
|
"ronpo/residual": 0.07471181452274323,
|
||
|
|
"ronpo/residual_abs": 0.10397426784038544,
|
||
|
|
"ronpo/target": -0.01676536537706852,
|
||
|
|
"ronpo/target_abs": 0.05582839995622635,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.12504041194915771,
|
||
|
|
"drift/rejected_abs": 0.055094581097364426,
|
||
|
|
"epoch": 0.060389352403655144,
|
||
|
|
"grad_norm": 0.8984375,
|
||
|
|
"learning_rate": 4.9980725906018076e-08,
|
||
|
|
"logits/chosen": -2.314112663269043,
|
||
|
|
"logits/rejected": -2.4101805686950684,
|
||
|
|
"logps/chosen": -0.3051087260246277,
|
||
|
|
"logps/rejected": -0.3032669425010681,
|
||
|
|
"loss": 0.05326960235834122,
|
||
|
|
"loss/core": 0.04397609829902649,
|
||
|
|
"loss/preference_sft": 0.3047112822532654,
|
||
|
|
"loss/reference_anchor": 0.0624639168381691,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.2492070198059082,
|
||
|
|
"rewards/margins": 0.6990978717803955,
|
||
|
|
"rewards/rejected": 0.5501092076301575,
|
||
|
|
"ronpo/logit": 0.06990978866815567,
|
||
|
|
"ronpo/residual": 0.07269351929426193,
|
||
|
|
"ronpo/residual_abs": 0.11123079061508179,
|
||
|
|
"ronpo/target": -0.00278373877517879,
|
||
|
|
"ronpo/target_abs": 0.05910308286547661,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21298687160015106,
|
||
|
|
"drift/rejected_abs": 0.1180461049079895,
|
||
|
|
"epoch": 0.06197854588796186,
|
||
|
|
"grad_norm": 6.0625,
|
||
|
|
"learning_rate": 4.995336996054667e-08,
|
||
|
|
"logits/chosen": -2.0543062686920166,
|
||
|
|
"logits/rejected": -2.029996156692505,
|
||
|
|
"logps/chosen": -0.2776458263397217,
|
||
|
|
"logps/rejected": -0.2725464403629303,
|
||
|
|
"loss": 0.06012827157974243,
|
||
|
|
"loss/core": 0.04569826275110245,
|
||
|
|
"loss/preference_sft": 0.2735288739204407,
|
||
|
|
"loss/reference_anchor": 0.11694715172052383,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.128607988357544,
|
||
|
|
"rewards/margins": 0.9508472681045532,
|
||
|
|
"rewards/rejected": 1.1777608394622803,
|
||
|
|
"ronpo/logit": 0.09508472681045532,
|
||
|
|
"ronpo/residual": 0.09097838401794434,
|
||
|
|
"ronpo/residual_abs": 0.11423306167125702,
|
||
|
|
"ronpo/target": 0.004106341395527124,
|
||
|
|
"ronpo/target_abs": 0.04503884166479111,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2828276753425598,
|
||
|
|
"drift/rejected_abs": 0.08978013694286346,
|
||
|
|
"epoch": 0.06356773937226858,
|
||
|
|
"grad_norm": 9.625,
|
||
|
|
"learning_rate": 4.991413756244404e-08,
|
||
|
|
"logits/chosen": -2.0380451679229736,
|
||
|
|
"logits/rejected": -2.1581947803497314,
|
||
|
|
"logps/chosen": -0.2897427976131439,
|
||
|
|
"logps/rejected": -0.2790738642215729,
|
||
|
|
"loss": 0.5045924186706543,
|
||
|
|
"loss/core": 0.4699951708316803,
|
||
|
|
"loss/preference_sft": 0.28447893261909485,
|
||
|
|
"loss/reference_anchor": 0.31752467155456543,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 2.827852964401245,
|
||
|
|
"rewards/margins": 1.9351660013198853,
|
||
|
|
"rewards/rejected": 0.8926870226860046,
|
||
|
|
"ronpo/logit": 0.19351661205291748,
|
||
|
|
"ronpo/residual": 0.18979412317276,
|
||
|
|
"ronpo/residual_abs": 0.22088727355003357,
|
||
|
|
"ronpo/target": 0.003722475143149495,
|
||
|
|
"ronpo/target_abs": 0.04679802060127258,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2945198118686676,
|
||
|
|
"drift/rejected_abs": 0.10771737992763519,
|
||
|
|
"epoch": 0.06515693285657528,
|
||
|
|
"grad_norm": 98.0,
|
||
|
|
"learning_rate": 4.986304738420683e-08,
|
||
|
|
"logits/chosen": -2.0024924278259277,
|
||
|
|
"logits/rejected": -2.0828309059143066,
|
||
|
|
"logps/chosen": -0.28146928548812866,
|
||
|
|
"logps/rejected": -0.2874804735183716,
|
||
|
|
"loss": 0.7829475998878479,
|
||
|
|
"loss/core": 0.7258787155151367,
|
||
|
|
"loss/preference_sft": 0.28588175773620605,
|
||
|
|
"loss/reference_anchor": 0.5421004295349121,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.942234992980957,
|
||
|
|
"rewards/margins": 1.8656885623931885,
|
||
|
|
"rewards/rejected": 1.076546549797058,
|
||
|
|
"ronpo/logit": 0.18656882643699646,
|
||
|
|
"ronpo/residual": 0.19427460432052612,
|
||
|
|
"ronpo/residual_abs": 0.22354765236377716,
|
||
|
|
"ronpo/target": -0.007705748081207275,
|
||
|
|
"ronpo/target_abs": 0.053630173206329346,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21879103779792786,
|
||
|
|
"drift/rejected_abs": 0.057897716760635376,
|
||
|
|
"epoch": 0.066746126340882,
|
||
|
|
"grad_norm": 4.0625,
|
||
|
|
"learning_rate": 4.980012374199287e-08,
|
||
|
|
"logits/chosen": -2.2520713806152344,
|
||
|
|
"logits/rejected": -2.1708178520202637,
|
||
|
|
"logps/chosen": -0.308249294757843,
|
||
|
|
"logps/rejected": -0.315049946308136,
|
||
|
|
"loss": 1.0403220653533936,
|
||
|
|
"loss/core": 0.9826607704162598,
|
||
|
|
"loss/preference_sft": 0.30766767263412476,
|
||
|
|
"loss/reference_anchor": 0.5458493828773499,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.1878044605255127,
|
||
|
|
"rewards/margins": 1.6101818084716797,
|
||
|
|
"rewards/rejected": 0.5776225924491882,
|
||
|
|
"ronpo/logit": 0.16101817786693573,
|
||
|
|
"ronpo/residual": 0.17357835173606873,
|
||
|
|
"ronpo/residual_abs": 0.19822609424591064,
|
||
|
|
"ronpo/target": -0.012560190632939339,
|
||
|
|
"ronpo/target_abs": 0.052357811480760574,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3271009922027588,
|
||
|
|
"drift/rejected_abs": 0.16482210159301758,
|
||
|
|
"epoch": 0.06833531982518871,
|
||
|
|
"grad_norm": 1.59375,
|
||
|
|
"learning_rate": 4.9725396584047915e-08,
|
||
|
|
"logits/chosen": -2.1691105365753174,
|
||
|
|
"logits/rejected": -2.1548848152160645,
|
||
|
|
"logps/chosen": -0.2653549015522003,
|
||
|
|
"logps/rejected": -0.2719860374927521,
|
||
|
|
"loss": 0.32540175318717957,
|
||
|
|
"loss/core": 0.26584428548812866,
|
||
|
|
"loss/preference_sft": 0.270608127117157,
|
||
|
|
"loss/reference_anchor": 0.5685140490531921,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 3.271010160446167,
|
||
|
|
"rewards/margins": 1.6233627796173096,
|
||
|
|
"rewards/rejected": 1.6476472616195679,
|
||
|
|
"ronpo/logit": 0.16233626008033752,
|
||
|
|
"ronpo/residual": 0.1596986949443817,
|
||
|
|
"ronpo/residual_abs": 0.18813616037368774,
|
||
|
|
"ronpo/target": 0.0026375583838671446,
|
||
|
|
"ronpo/target_abs": 0.04490605369210243,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.11970488727092743,
|
||
|
|
"drift/rejected_abs": 0.08118265867233276,
|
||
|
|
"epoch": 0.06992451330949544,
|
||
|
|
"grad_norm": 0.36328125,
|
||
|
|
"learning_rate": 4.963890147645195e-08,
|
||
|
|
"logits/chosen": -2.2142767906188965,
|
||
|
|
"logits/rejected": -2.081634759902954,
|
||
|
|
"logps/chosen": -0.3046541213989258,
|
||
|
|
"logps/rejected": -0.2997341454029083,
|
||
|
|
"loss": 0.05313153192400932,
|
||
|
|
"loss/core": 0.045478448271751404,
|
||
|
|
"loss/preference_sft": 0.3017224669456482,
|
||
|
|
"loss/reference_anchor": 0.04635859280824661,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.1934808492660522,
|
||
|
|
"rewards/margins": 0.384704053401947,
|
||
|
|
"rewards/rejected": 0.8087766766548157,
|
||
|
|
"ronpo/logit": 0.03847040981054306,
|
||
|
|
"ronpo/residual": 0.03697671741247177,
|
||
|
|
"ronpo/residual_abs": 0.10223197937011719,
|
||
|
|
"ronpo/target": 0.0014936840161681175,
|
||
|
|
"ronpo/target_abs": 0.04688495397567749,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2311873733997345,
|
||
|
|
"drift/rejected_abs": 0.10326783359050751,
|
||
|
|
"epoch": 0.07151370679380215,
|
||
|
|
"grad_norm": 5.09375,
|
||
|
|
"learning_rate": 4.95406795861916e-08,
|
||
|
|
"logits/chosen": -2.177318572998047,
|
||
|
|
"logits/rejected": -2.1946113109588623,
|
||
|
|
"logps/chosen": -0.2876010835170746,
|
||
|
|
"logps/rejected": -0.2852502465248108,
|
||
|
|
"loss": 0.20730121433734894,
|
||
|
|
"loss/core": 0.18707375228405,
|
||
|
|
"loss/preference_sft": 0.2840942442417145,
|
||
|
|
"loss/reference_anchor": 0.17386510968208313,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.3118736743927,
|
||
|
|
"rewards/margins": 1.279848575592041,
|
||
|
|
"rewards/rejected": 1.0320250988006592,
|
||
|
|
"ronpo/logit": 0.12798485159873962,
|
||
|
|
"ronpo/residual": 0.12988588213920593,
|
||
|
|
"ronpo/residual_abs": 0.16303423047065735,
|
||
|
|
"ronpo/target": -0.0019010124960914254,
|
||
|
|
"ronpo/target_abs": 0.056721486151218414,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2658592760562897,
|
||
|
|
"drift/rejected_abs": 0.09674189984798431,
|
||
|
|
"epoch": 0.07310290027810885,
|
||
|
|
"grad_norm": 2.359375,
|
||
|
|
"learning_rate": 4.943077766156697e-08,
|
||
|
|
"logits/chosen": -2.2422900199890137,
|
||
|
|
"logits/rejected": -2.2595908641815186,
|
||
|
|
"logps/chosen": -0.2615046203136444,
|
||
|
|
"logps/rejected": -0.26030460000038147,
|
||
|
|
"loss": 0.4114556908607483,
|
||
|
|
"loss/core": 0.38013729453086853,
|
||
|
|
"loss/preference_sft": 0.2594269812107086,
|
||
|
|
"loss/reference_anchor": 0.28724151849746704,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.6584160327911377,
|
||
|
|
"rewards/margins": 1.691518783569336,
|
||
|
|
"rewards/rejected": 0.9668973684310913,
|
||
|
|
"ronpo/logit": 0.16915185749530792,
|
||
|
|
"ronpo/residual": 0.17316488921642303,
|
||
|
|
"ronpo/residual_abs": 0.19615182280540466,
|
||
|
|
"ronpo/target": -0.004013021010905504,
|
||
|
|
"ronpo/target_abs": 0.05004221946001053,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4598860740661621,
|
||
|
|
"drift/rejected_abs": 0.2359577715396881,
|
||
|
|
"epoch": 0.07469209376241558,
|
||
|
|
"grad_norm": 64.5,
|
||
|
|
"learning_rate": 4.930924800994191e-08,
|
||
|
|
"logits/chosen": -2.114593982696533,
|
||
|
|
"logits/rejected": -2.116748809814453,
|
||
|
|
"logps/chosen": -0.2821466028690338,
|
||
|
|
"logps/rejected": -0.2878660261631012,
|
||
|
|
"loss": 1.2597297430038452,
|
||
|
|
"loss/core": 1.1379563808441162,
|
||
|
|
"loss/preference_sft": 0.27529674768447876,
|
||
|
|
"loss/reference_anchor": 1.1902048587799072,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.597031593322754,
|
||
|
|
"rewards/margins": 2.3334996700286865,
|
||
|
|
"rewards/rejected": 2.2635321617126465,
|
||
|
|
"ronpo/logit": 0.2333499640226364,
|
||
|
|
"ronpo/residual": 0.23511190712451935,
|
||
|
|
"ronpo/residual_abs": 0.37244728207588196,
|
||
|
|
"ronpo/target": -0.0017619434511289,
|
||
|
|
"ronpo/target_abs": 0.04279688373208046,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29015716910362244,
|
||
|
|
"drift/rejected_abs": 0.16504958271980286,
|
||
|
|
"epoch": 0.07628128724672228,
|
||
|
|
"grad_norm": 149.0,
|
||
|
|
"learning_rate": 4.9176148472848576e-08,
|
||
|
|
"logits/chosen": -2.139037609100342,
|
||
|
|
"logits/rejected": -2.214994192123413,
|
||
|
|
"logps/chosen": -0.26754242181777954,
|
||
|
|
"logps/rejected": -0.287071168422699,
|
||
|
|
"loss": 0.24881801009178162,
|
||
|
|
"loss/core": 0.19462670385837555,
|
||
|
|
"loss/preference_sft": 0.28026410937309265,
|
||
|
|
"loss/reference_anchor": 0.5138863325119019,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.9009900093078613,
|
||
|
|
"rewards/margins": 1.2524168491363525,
|
||
|
|
"rewards/rejected": 1.6485732793807983,
|
||
|
|
"ronpo/logit": 0.12524166703224182,
|
||
|
|
"ronpo/residual": 0.1251114010810852,
|
||
|
|
"ronpo/residual_abs": 0.15745453536510468,
|
||
|
|
"ronpo/target": 0.00013028904504608363,
|
||
|
|
"ronpo/target_abs": 0.05093258619308472,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1876174658536911,
|
||
|
|
"drift/rejected_abs": 0.07340504974126816,
|
||
|
|
"epoch": 0.07787048073102901,
|
||
|
|
"grad_norm": 1.609375,
|
||
|
|
"learning_rate": 4.903154239845797e-08,
|
||
|
|
"logits/chosen": -2.1785225868225098,
|
||
|
|
"logits/rejected": -2.1970629692077637,
|
||
|
|
"logps/chosen": -0.29958945512771606,
|
||
|
|
"logps/rejected": -0.3053562045097351,
|
||
|
|
"loss": 0.28825491666793823,
|
||
|
|
"loss/core": 0.26845070719718933,
|
||
|
|
"loss/preference_sft": 0.31076282262802124,
|
||
|
|
"loss/reference_anchor": 0.16696596145629883,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8747392892837524,
|
||
|
|
"rewards/margins": 1.171970009803772,
|
||
|
|
"rewards/rejected": 0.7027692198753357,
|
||
|
|
"ronpo/logit": 0.11719699203968048,
|
||
|
|
"ronpo/residual": 0.11347774416208267,
|
||
|
|
"ronpo/residual_abs": 0.15537722408771515,
|
||
|
|
"ronpo/target": 0.0037192567251622677,
|
||
|
|
"ronpo/target_abs": 0.05189714580774307,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36311304569244385,
|
||
|
|
"drift/rejected_abs": 0.08675487339496613,
|
||
|
|
"epoch": 0.07945967421533572,
|
||
|
|
"grad_norm": 7.0625,
|
||
|
|
"learning_rate": 4.887549861142967e-08,
|
||
|
|
"logits/chosen": -2.1118674278259277,
|
||
|
|
"logits/rejected": -2.088712215423584,
|
||
|
|
"logps/chosen": -0.3086296021938324,
|
||
|
|
"logps/rejected": -0.3087017834186554,
|
||
|
|
"loss": 1.2505347728729248,
|
||
|
|
"loss/core": 1.1789346933364868,
|
||
|
|
"loss/preference_sft": 0.30625787377357483,
|
||
|
|
"loss/reference_anchor": 0.6853760480880737,
|
||
|
|
"rewards/accuracies": 0.7875000238418579,
|
||
|
|
"rewards/chosen": 3.6298019886016846,
|
||
|
|
"rewards/margins": 2.7633728981018066,
|
||
|
|
"rewards/rejected": 0.8664296269416809,
|
||
|
|
"ronpo/logit": 0.27633726596832275,
|
||
|
|
"ronpo/residual": 0.2702479362487793,
|
||
|
|
"ronpo/residual_abs": 0.30587834119796753,
|
||
|
|
"ronpo/target": 0.006089328322559595,
|
||
|
|
"ronpo/target_abs": 0.04417353868484497,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.44501981139183044,
|
||
|
|
"drift/rejected_abs": 0.19686473906040192,
|
||
|
|
"epoch": 0.08104886769964244,
|
||
|
|
"grad_norm": 14.0,
|
||
|
|
"learning_rate": 4.870809138015498e-08,
|
||
|
|
"logits/chosen": -2.0998764038085938,
|
||
|
|
"logits/rejected": -2.103111505508423,
|
||
|
|
"logps/chosen": -0.26382118463516235,
|
||
|
|
"logps/rejected": -0.27121174335479736,
|
||
|
|
"loss": 0.7604058384895325,
|
||
|
|
"loss/core": 0.6632653474807739,
|
||
|
|
"loss/preference_sft": 0.2692394554615021,
|
||
|
|
"loss/reference_anchor": 0.9444811940193176,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 4.449104309082031,
|
||
|
|
"rewards/margins": 2.5365662574768066,
|
||
|
|
"rewards/rejected": 1.9125378131866455,
|
||
|
|
"ronpo/logit": 0.25365665555000305,
|
||
|
|
"ronpo/residual": 0.25510507822036743,
|
||
|
|
"ronpo/residual_abs": 0.28715819120407104,
|
||
|
|
"ronpo/target": -0.0014484294224530458,
|
||
|
|
"ronpo/target_abs": 0.056101828813552856,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.32862967252731323,
|
||
|
|
"drift/rejected_abs": 0.08690224587917328,
|
||
|
|
"epoch": 0.08263806118394915,
|
||
|
|
"grad_norm": 976.0,
|
||
|
|
"learning_rate": 4.852940038140926e-08,
|
||
|
|
"logits/chosen": -2.1928184032440186,
|
||
|
|
"logits/rejected": -2.1674201488494873,
|
||
|
|
"logps/chosen": -0.2829035818576813,
|
||
|
|
"logps/rejected": -0.2785223722457886,
|
||
|
|
"loss": 0.8419889211654663,
|
||
|
|
"loss/core": 0.789025068283081,
|
||
|
|
"loss/preference_sft": 0.27425065636634827,
|
||
|
|
"loss/reference_anchor": 0.5022132992744446,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.285032272338867,
|
||
|
|
"rewards/margins": 2.417757034301758,
|
||
|
|
"rewards/rejected": 0.867275059223175,
|
||
|
|
"ronpo/logit": 0.24177570641040802,
|
||
|
|
"ronpo/residual": 0.23542478680610657,
|
||
|
|
"ronpo/residual_abs": 0.2721962332725525,
|
||
|
|
"ronpo/target": 0.006350924260914326,
|
||
|
|
"ronpo/target_abs": 0.055059123784303665,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18202148377895355,
|
||
|
|
"drift/rejected_abs": 0.08143140375614166,
|
||
|
|
"epoch": 0.08422725466825585,
|
||
|
|
"grad_norm": 16.25,
|
||
|
|
"learning_rate": 4.833951066243004e-08,
|
||
|
|
"logits/chosen": -2.1530301570892334,
|
||
|
|
"logits/rejected": -2.1527392864227295,
|
||
|
|
"logps/chosen": -0.28495079278945923,
|
||
|
|
"logps/rejected": -0.29102346301078796,
|
||
|
|
"loss": 0.08294971287250519,
|
||
|
|
"loss/core": 0.07107045501470566,
|
||
|
|
"loss/preference_sft": 0.2882830500602722,
|
||
|
|
"loss/reference_anchor": 0.08996426314115524,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.8192421197891235,
|
||
|
|
"rewards/margins": 1.0141630172729492,
|
||
|
|
"rewards/rejected": 0.8050788640975952,
|
||
|
|
"ronpo/logit": 0.10141630470752716,
|
||
|
|
"ronpo/residual": 0.10920605808496475,
|
||
|
|
"ronpo/residual_abs": 0.13682009279727936,
|
||
|
|
"ronpo/target": -0.0077897594310343266,
|
||
|
|
"ronpo/target_abs": 0.05120920389890671,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2922496199607849,
|
||
|
|
"drift/rejected_abs": 0.1604980230331421,
|
||
|
|
"epoch": 0.08581644815256258,
|
||
|
|
"grad_norm": 5.625,
|
||
|
|
"learning_rate": 4.813851260043916e-08,
|
||
|
|
"logits/chosen": -2.22133469581604,
|
||
|
|
"logits/rejected": -2.1806182861328125,
|
||
|
|
"logps/chosen": -0.2800171375274658,
|
||
|
|
"logps/rejected": -0.2896736264228821,
|
||
|
|
"loss": 0.3876208961009979,
|
||
|
|
"loss/core": 0.31458601355552673,
|
||
|
|
"loss/preference_sft": 0.2803085148334503,
|
||
|
|
"loss/reference_anchor": 0.7023177146911621,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.9203429222106934,
|
||
|
|
"rewards/margins": 1.317029595375061,
|
||
|
|
"rewards/rejected": 1.6033132076263428,
|
||
|
|
"ronpo/logit": 0.13170292973518372,
|
||
|
|
"ronpo/residual": 0.12294062227010727,
|
||
|
|
"ronpo/residual_abs": 0.16244474053382874,
|
||
|
|
"ronpo/target": 0.008762339130043983,
|
||
|
|
"ronpo/target_abs": 0.058000266551971436,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17858698964118958,
|
||
|
|
"drift/rejected_abs": 0.10436340421438217,
|
||
|
|
"epoch": 0.08740564163686929,
|
||
|
|
"grad_norm": 9.125,
|
||
|
|
"learning_rate": 4.79265018596281e-08,
|
||
|
|
"logits/chosen": -2.271712064743042,
|
||
|
|
"logits/rejected": -2.3456873893737793,
|
||
|
|
"logps/chosen": -0.279397577047348,
|
||
|
|
"logps/rejected": -0.26725754141807556,
|
||
|
|
"loss": 0.04921260103583336,
|
||
|
|
"loss/core": 0.03643937036395073,
|
||
|
|
"loss/preference_sft": 0.2758616507053375,
|
||
|
|
"loss/reference_anchor": 0.10014615207910538,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.785718560218811,
|
||
|
|
"rewards/margins": 0.7422192096710205,
|
||
|
|
"rewards/rejected": 1.043499231338501,
|
||
|
|
"ronpo/logit": 0.07422193139791489,
|
||
|
|
"ronpo/residual": 0.06601117551326752,
|
||
|
|
"ronpo/residual_abs": 0.10768099874258041,
|
||
|
|
"ronpo/target": 0.008210750296711922,
|
||
|
|
"ronpo/target_abs": 0.05532875657081604,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.169549822807312,
|
||
|
|
"drift/rejected_abs": 0.11006991565227509,
|
||
|
|
"epoch": 0.08899483512117601,
|
||
|
|
"grad_norm": 55.75,
|
||
|
|
"learning_rate": 4.770357934562703e-08,
|
||
|
|
"logits/chosen": -2.1815786361694336,
|
||
|
|
"logits/rejected": -2.182666063308716,
|
||
|
|
"logps/chosen": -0.28446492552757263,
|
||
|
|
"logps/rejected": -0.291333943605423,
|
||
|
|
"loss": 0.11231124401092529,
|
||
|
|
"loss/core": 0.09980011731386185,
|
||
|
|
"loss/preference_sft": 0.2837428152561188,
|
||
|
|
"loss/reference_anchor": 0.09673698246479034,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.6950247287750244,
|
||
|
|
"rewards/margins": 0.5955981612205505,
|
||
|
|
"rewards/rejected": 1.099426507949829,
|
||
|
|
"ronpo/logit": 0.05955982953310013,
|
||
|
|
"ronpo/residual": 0.06428845226764679,
|
||
|
|
"ronpo/residual_abs": 0.13956977427005768,
|
||
|
|
"ronpo/target": -0.004728636238723993,
|
||
|
|
"ronpo/target_abs": 0.04768453910946846,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.43819236755371094,
|
||
|
|
"drift/rejected_abs": 0.13878020644187927,
|
||
|
|
"epoch": 0.09058402860548272,
|
||
|
|
"grad_norm": 18.625,
|
||
|
|
"learning_rate": 4.746985115747917e-08,
|
||
|
|
"logits/chosen": -2.2233951091766357,
|
||
|
|
"logits/rejected": -2.1950364112854004,
|
||
|
|
"logps/chosen": -0.280063658952713,
|
||
|
|
"logps/rejected": -0.2928066849708557,
|
||
|
|
"loss": 1.5658351182937622,
|
||
|
|
"loss/core": 1.4394580125808716,
|
||
|
|
"loss/preference_sft": 0.2897217869758606,
|
||
|
|
"loss/reference_anchor": 1.2347972393035889,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 4.379513740539551,
|
||
|
|
"rewards/margins": 2.9923789501190186,
|
||
|
|
"rewards/rejected": 1.3871352672576904,
|
||
|
|
"ronpo/logit": 0.2992379069328308,
|
||
|
|
"ronpo/residual": 0.29981642961502075,
|
||
|
|
"ronpo/residual_abs": 0.33501166105270386,
|
||
|
|
"ronpo/target": -0.00057854043552652,
|
||
|
|
"ronpo/target_abs": 0.04776735231280327,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2989239990711212,
|
||
|
|
"drift/rejected_abs": 0.13974739611148834,
|
||
|
|
"epoch": 0.09217322208978942,
|
||
|
|
"grad_norm": 136.0,
|
||
|
|
"learning_rate": 4.722542853714341e-08,
|
||
|
|
"logits/chosen": -2.012983560562134,
|
||
|
|
"logits/rejected": -2.0874452590942383,
|
||
|
|
"logps/chosen": -0.27262991666793823,
|
||
|
|
"logps/rejected": -0.28246259689331055,
|
||
|
|
"loss": 0.27830126881599426,
|
||
|
|
"loss/core": 0.2490129917860031,
|
||
|
|
"loss/preference_sft": 0.27377456426620483,
|
||
|
|
"loss/reference_anchor": 0.26550552248954773,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.9884445667266846,
|
||
|
|
"rewards/margins": 1.591313123703003,
|
||
|
|
"rewards/rejected": 1.3971314430236816,
|
||
|
|
"ronpo/logit": 0.15913130342960358,
|
||
|
|
"ronpo/residual": 0.1600702404975891,
|
||
|
|
"ronpo/residual_abs": 0.21637621521949768,
|
||
|
|
"ronpo/target": -0.0009389538317918777,
|
||
|
|
"ronpo/target_abs": 0.048052363097667694,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19468945264816284,
|
||
|
|
"drift/rejected_abs": 0.07088744640350342,
|
||
|
|
"epoch": 0.09376241557409615,
|
||
|
|
"grad_norm": 192.0,
|
||
|
|
"learning_rate": 4.697042781654913e-08,
|
||
|
|
"logits/chosen": -2.1905837059020996,
|
||
|
|
"logits/rejected": -2.1836931705474854,
|
||
|
|
"logps/chosen": -0.30156785249710083,
|
||
|
|
"logps/rejected": -0.3063206672668457,
|
||
|
|
"loss": 0.2443789690732956,
|
||
|
|
"loss/core": 0.22306425869464874,
|
||
|
|
"loss/preference_sft": 0.29774826765060425,
|
||
|
|
"loss/reference_anchor": 0.18337255716323853,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.9455058574676514,
|
||
|
|
"rewards/margins": 1.2405574321746826,
|
||
|
|
"rewards/rejected": 0.7049483060836792,
|
||
|
|
"ronpo/logit": 0.12405575811862946,
|
||
|
|
"ronpo/residual": 0.11795137822628021,
|
||
|
|
"ronpo/residual_abs": 0.15991681814193726,
|
||
|
|
"ronpo/target": 0.006104385945945978,
|
||
|
|
"ronpo/target_abs": 0.05217462033033371,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.13699117302894592,
|
||
|
|
"drift/rejected_abs": 0.08163593709468842,
|
||
|
|
"epoch": 0.09535160905840286,
|
||
|
|
"grad_norm": 6.125,
|
||
|
|
"learning_rate": 4.6704970362228555e-08,
|
||
|
|
"logits/chosen": -2.405043840408325,
|
||
|
|
"logits/rejected": -2.4092018604278564,
|
||
|
|
"logps/chosen": -0.29529136419296265,
|
||
|
|
"logps/rejected": -0.2865511476993561,
|
||
|
|
"loss": 0.032488640397787094,
|
||
|
|
"loss/core": 0.024509349837899208,
|
||
|
|
"loss/preference_sft": 0.29015666246414185,
|
||
|
|
"loss/reference_anchor": 0.05077719688415527,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.3689161539077759,
|
||
|
|
"rewards/margins": 0.5563350915908813,
|
||
|
|
"rewards/rejected": 0.8125810623168945,
|
||
|
|
"ronpo/logit": 0.055633507668972015,
|
||
|
|
"ronpo/residual": 0.055543310940265656,
|
||
|
|
"ronpo/residual_abs": 0.09124297648668289,
|
||
|
|
"ronpo/target": 9.020241122925654e-05,
|
||
|
|
"ronpo/target_abs": 0.051511455327272415,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22262239456176758,
|
||
|
|
"drift/rejected_abs": 0.10259328037500381,
|
||
|
|
"epoch": 0.09694080254270958,
|
||
|
|
"grad_norm": 1.5546875,
|
||
|
|
"learning_rate": 4.6429182517552804e-08,
|
||
|
|
"logits/chosen": -2.077086925506592,
|
||
|
|
"logits/rejected": -2.105011463165283,
|
||
|
|
"logps/chosen": -0.3044688105583191,
|
||
|
|
"logps/rejected": -0.292930543422699,
|
||
|
|
"loss": 0.2044544517993927,
|
||
|
|
"loss/core": 0.1758301556110382,
|
||
|
|
"loss/preference_sft": 0.30388787388801575,
|
||
|
|
"loss/reference_anchor": 0.25585389137268066,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.2251529693603516,
|
||
|
|
"rewards/margins": 1.2022806406021118,
|
||
|
|
"rewards/rejected": 1.0228722095489502,
|
||
|
|
"ronpo/logit": 0.12022805213928223,
|
||
|
|
"ronpo/residual": 0.11487364768981934,
|
||
|
|
"ronpo/residual_abs": 0.14453622698783875,
|
||
|
|
"ronpo/target": 0.005354395601898432,
|
||
|
|
"ronpo/target_abs": 0.05369141697883606,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23092417418956757,
|
||
|
|
"drift/rejected_abs": 0.1479761302471161,
|
||
|
|
"epoch": 0.09852999602701629,
|
||
|
|
"grad_norm": 201.0,
|
||
|
|
"learning_rate": 4.614319554259933e-08,
|
||
|
|
"logits/chosen": -2.096682071685791,
|
||
|
|
"logits/rejected": -1.9904273748397827,
|
||
|
|
"logps/chosen": -0.3039776682853699,
|
||
|
|
"logps/rejected": -0.3026920557022095,
|
||
|
|
"loss": 0.10154803842306137,
|
||
|
|
"loss/core": 0.06678663194179535,
|
||
|
|
"loss/preference_sft": 0.2993779182434082,
|
||
|
|
"loss/reference_anchor": 0.3176761865615845,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.3076138496398926,
|
||
|
|
"rewards/margins": 0.8309847712516785,
|
||
|
|
"rewards/rejected": 1.4766290187835693,
|
||
|
|
"ronpo/logit": 0.08309847861528397,
|
||
|
|
"ronpo/residual": 0.0783224031329155,
|
||
|
|
"ronpo/residual_abs": 0.13604936003684998,
|
||
|
|
"ronpo/target": 0.004776070825755596,
|
||
|
|
"ronpo/target_abs": 0.05459970235824585,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21857643127441406,
|
||
|
|
"drift/rejected_abs": 0.17166323959827423,
|
||
|
|
"epoch": 0.10011918951132301,
|
||
|
|
"grad_norm": 1.859375,
|
||
|
|
"learning_rate": 4.58471455516792e-08,
|
||
|
|
"logits/chosen": -2.208028793334961,
|
||
|
|
"logits/rejected": -2.2415013313293457,
|
||
|
|
"logps/chosen": -0.28857478499412537,
|
||
|
|
"logps/rejected": -0.2832663655281067,
|
||
|
|
"loss": 0.16358520090579987,
|
||
|
|
"loss/core": 0.08099617063999176,
|
||
|
|
"loss/preference_sft": 0.2877857983112335,
|
||
|
|
"loss/reference_anchor": 0.7971116304397583,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.18546724319458,
|
||
|
|
"rewards/margins": 0.4695800840854645,
|
||
|
|
"rewards/rejected": 1.7158870697021484,
|
||
|
|
"ronpo/logit": 0.04695800319314003,
|
||
|
|
"ronpo/residual": 0.04399949312210083,
|
||
|
|
"ronpo/residual_abs": 0.11905704438686371,
|
||
|
|
"ronpo/target": 0.0029585096053779125,
|
||
|
|
"ronpo/target_abs": 0.05115630105137825,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.30618441104888916,
|
||
|
|
"drift/rejected_abs": 0.15159662067890167,
|
||
|
|
"epoch": 0.10170838299562972,
|
||
|
|
"grad_norm": 15.6875,
|
||
|
|
"learning_rate": 4.5541173448554095e-08,
|
||
|
|
"logits/chosen": -2.0477356910705566,
|
||
|
|
"logits/rejected": -2.046473979949951,
|
||
|
|
"logps/chosen": -0.26754650473594666,
|
||
|
|
"logps/rejected": -0.26750463247299194,
|
||
|
|
"loss": 0.45313024520874023,
|
||
|
|
"loss/core": 0.3808836340904236,
|
||
|
|
"loss/preference_sft": 0.26480069756507874,
|
||
|
|
"loss/reference_anchor": 0.6959866285324097,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.0603110790252686,
|
||
|
|
"rewards/margins": 1.54562246799469,
|
||
|
|
"rewards/rejected": 1.5146887302398682,
|
||
|
|
"ronpo/logit": 0.15456224977970123,
|
||
|
|
"ronpo/residual": 0.1472359597682953,
|
||
|
|
"ronpo/residual_abs": 0.19273725152015686,
|
||
|
|
"ronpo/target": 0.007326282560825348,
|
||
|
|
"ronpo/target_abs": 0.047880660742521286,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27261799573898315,
|
||
|
|
"drift/rejected_abs": 0.06441514194011688,
|
||
|
|
"epoch": 0.10329757647993643,
|
||
|
|
"grad_norm": 24.25,
|
||
|
|
"learning_rate": 4.522542485937369e-08,
|
||
|
|
"logits/chosen": -2.213456153869629,
|
||
|
|
"logits/rejected": -2.237545967102051,
|
||
|
|
"logps/chosen": -0.3028256595134735,
|
||
|
|
"logps/rejected": -0.29002341628074646,
|
||
|
|
"loss": 1.2407621145248413,
|
||
|
|
"loss/core": 1.174018144607544,
|
||
|
|
"loss/preference_sft": 0.2956632375717163,
|
||
|
|
"loss/reference_anchor": 0.6378735899925232,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.7259883880615234,
|
||
|
|
"rewards/margins": 2.0907344818115234,
|
||
|
|
"rewards/rejected": 0.6352539658546448,
|
||
|
|
"ronpo/logit": 0.20907345414161682,
|
||
|
|
"ronpo/residual": 0.21372437477111816,
|
||
|
|
"ronpo/residual_abs": 0.24546225368976593,
|
||
|
|
"ronpo/target": -0.004650958813726902,
|
||
|
|
"ronpo/target_abs": 0.04953046515583992,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2498444765806198,
|
||
|
|
"drift/rejected_abs": 0.127003476023674,
|
||
|
|
"epoch": 0.10488676996424315,
|
||
|
|
"grad_norm": 11.0625,
|
||
|
|
"learning_rate": 4.490005006336554e-08,
|
||
|
|
"logits/chosen": -2.322620391845703,
|
||
|
|
"logits/rejected": -2.3220651149749756,
|
||
|
|
"logps/chosen": -0.26767295598983765,
|
||
|
|
"logps/rejected": -0.26008275151252747,
|
||
|
|
"loss": 0.614111065864563,
|
||
|
|
"loss/core": 0.5768492221832275,
|
||
|
|
"loss/preference_sft": 0.2576170861721039,
|
||
|
|
"loss/reference_anchor": 0.34685665369033813,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.4984447956085205,
|
||
|
|
"rewards/margins": 1.2349770069122314,
|
||
|
|
"rewards/rejected": 1.263467788696289,
|
||
|
|
"ronpo/logit": 0.12349770218133926,
|
||
|
|
"ronpo/residual": 0.13091148436069489,
|
||
|
|
"ronpo/residual_abs": 0.23883113265037537,
|
||
|
|
"ronpo/target": -0.007413757033646107,
|
||
|
|
"ronpo/target_abs": 0.054652709513902664,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4506363868713379,
|
||
|
|
"drift/rejected_abs": 0.2148597538471222,
|
||
|
|
"epoch": 0.10647596344854986,
|
||
|
|
"grad_norm": 13.6875,
|
||
|
|
"learning_rate": 4.456520392131034e-08,
|
||
|
|
"logits/chosen": -1.9939444065093994,
|
||
|
|
"logits/rejected": -1.9420192241668701,
|
||
|
|
"logps/chosen": -0.2896828055381775,
|
||
|
|
"logps/rejected": -0.2902643084526062,
|
||
|
|
"loss": 0.8482235670089722,
|
||
|
|
"loss/core": 0.6990407705307007,
|
||
|
|
"loss/preference_sft": 0.29258525371551514,
|
||
|
|
"loss/reference_anchor": 1.4625699520111084,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 4.504998207092285,
|
||
|
|
"rewards/margins": 2.361280918121338,
|
||
|
|
"rewards/rejected": 2.1437175273895264,
|
||
|
|
"ronpo/logit": 0.2361280918121338,
|
||
|
|
"ronpo/residual": 0.23645956814289093,
|
||
|
|
"ronpo/residual_abs": 0.2861987054347992,
|
||
|
|
"ronpo/target": -0.00033148974762298167,
|
||
|
|
"ronpo/target_abs": 0.04662436991930008,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4584285616874695,
|
||
|
|
"drift/rejected_abs": 0.16387048363685608,
|
||
|
|
"epoch": 0.10806515693285658,
|
||
|
|
"grad_norm": 85.5,
|
||
|
|
"learning_rate": 4.422104580183649e-08,
|
||
|
|
"logits/chosen": -1.8824104070663452,
|
||
|
|
"logits/rejected": -1.9405462741851807,
|
||
|
|
"logps/chosen": -0.28784456849098206,
|
||
|
|
"logps/rejected": -0.27431824803352356,
|
||
|
|
"loss": 1.0368645191192627,
|
||
|
|
"loss/core": 0.9412893056869507,
|
||
|
|
"loss/preference_sft": 0.2768869400024414,
|
||
|
|
"loss/reference_anchor": 0.9280635714530945,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.582031726837158,
|
||
|
|
"rewards/margins": 2.948826313018799,
|
||
|
|
"rewards/rejected": 1.633205771446228,
|
||
|
|
"ronpo/logit": 0.294882595539093,
|
||
|
|
"ronpo/residual": 0.2988366186618805,
|
||
|
|
"ronpo/residual_abs": 0.3257350027561188,
|
||
|
|
"ronpo/target": -0.003954009618610144,
|
||
|
|
"ronpo/target_abs": 0.047440093010663986,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24579569697380066,
|
||
|
|
"drift/rejected_abs": 0.12019170820713043,
|
||
|
|
"epoch": 0.10965435041716329,
|
||
|
|
"grad_norm": 64.5,
|
||
|
|
"learning_rate": 4.38677395055693e-08,
|
||
|
|
"logits/chosen": -2.1810965538024902,
|
||
|
|
"logits/rejected": -2.252776622772217,
|
||
|
|
"logps/chosen": -0.27312830090522766,
|
||
|
|
"logps/rejected": -0.27882593870162964,
|
||
|
|
"loss": 0.1356753408908844,
|
||
|
|
"loss/core": 0.10093406587839127,
|
||
|
|
"loss/preference_sft": 0.2720806300640106,
|
||
|
|
"loss/reference_anchor": 0.3202047348022461,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.456185817718506,
|
||
|
|
"rewards/margins": 1.2865262031555176,
|
||
|
|
"rewards/rejected": 1.1696596145629883,
|
||
|
|
"ronpo/logit": 0.1286526322364807,
|
||
|
|
"ronpo/residual": 0.1406368762254715,
|
||
|
|
"ronpo/residual_abs": 0.15837253630161285,
|
||
|
|
"ronpo/target": -0.011984249576926231,
|
||
|
|
"ronpo/target_abs": 0.04884602874517441,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4141915440559387,
|
||
|
|
"drift/rejected_abs": 0.16395843029022217,
|
||
|
|
"epoch": 0.11124354390147001,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 4.35054531871708e-08,
|
||
|
|
"logits/chosen": -2.269747257232666,
|
||
|
|
"logits/rejected": -2.2979190349578857,
|
||
|
|
"logps/chosen": -0.23346249759197235,
|
||
|
|
"logps/rejected": -0.2476530522108078,
|
||
|
|
"loss": 1.0521429777145386,
|
||
|
|
"loss/core": 0.9406593441963196,
|
||
|
|
"loss/preference_sft": 0.24042610824108124,
|
||
|
|
"loss/reference_anchor": 1.090794324874878,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.141915321350098,
|
||
|
|
"rewards/margins": 2.502330780029297,
|
||
|
|
"rewards/rejected": 1.6395843029022217,
|
||
|
|
"ronpo/logit": 0.2502330541610718,
|
||
|
|
"ronpo/residual": 0.25256186723709106,
|
||
|
|
"ronpo/residual_abs": 0.2842928171157837,
|
||
|
|
"ronpo/target": -0.002328807720914483,
|
||
|
|
"ronpo/target_abs": 0.04533926397562027,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2807723879814148,
|
||
|
|
"drift/rejected_abs": 0.10302431881427765,
|
||
|
|
"epoch": 0.11283273738577672,
|
||
|
|
"grad_norm": 15.125,
|
||
|
|
"learning_rate": 4.313435927530719e-08,
|
||
|
|
"logits/chosen": -2.1268205642700195,
|
||
|
|
"logits/rejected": -2.025103807449341,
|
||
|
|
"logps/chosen": -0.2836920917034149,
|
||
|
|
"logps/rejected": -0.2762809693813324,
|
||
|
|
"loss": 0.4303739070892334,
|
||
|
|
"loss/core": 0.3964649438858032,
|
||
|
|
"loss/preference_sft": 0.2680888772010803,
|
||
|
|
"loss/reference_anchor": 0.31228065490722656,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8077235221862793,
|
||
|
|
"rewards/margins": 1.7782020568847656,
|
||
|
|
"rewards/rejected": 1.0295217037200928,
|
||
|
|
"ronpo/logit": 0.17782019078731537,
|
||
|
|
"ronpo/residual": 0.17538082599639893,
|
||
|
|
"ronpo/residual_abs": 0.21557548642158508,
|
||
|
|
"ronpo/target": 0.0024393785279244184,
|
||
|
|
"ronpo/target_abs": 0.04851159825921059,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2171783745288849,
|
||
|
|
"drift/rejected_abs": 0.08919801563024521,
|
||
|
|
"epoch": 0.11442193087008343,
|
||
|
|
"grad_norm": 11.9375,
|
||
|
|
"learning_rate": 4.2754634390582136e-08,
|
||
|
|
"logits/chosen": -2.2076804637908936,
|
||
|
|
"logits/rejected": -2.2731592655181885,
|
||
|
|
"logps/chosen": -0.26778849959373474,
|
||
|
|
"logps/rejected": -0.2578774094581604,
|
||
|
|
"loss": 0.09732436388731003,
|
||
|
|
"loss/core": 0.08528517186641693,
|
||
|
|
"loss/preference_sft": 0.2644139230251312,
|
||
|
|
"loss/reference_anchor": 0.0939505323767662,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.1707053184509277,
|
||
|
|
"rewards/margins": 1.27962327003479,
|
||
|
|
"rewards/rejected": 0.8910821676254272,
|
||
|
|
"ronpo/logit": 0.12796232104301453,
|
||
|
|
"ronpo/residual": 0.1286647766828537,
|
||
|
|
"ronpo/residual_abs": 0.1663980484008789,
|
||
|
|
"ronpo/target": -0.0007024522055871785,
|
||
|
|
"ronpo/target_abs": 0.04947920888662338,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2075362503528595,
|
||
|
|
"drift/rejected_abs": 0.13140513002872467,
|
||
|
|
"epoch": 0.11601112435439015,
|
||
|
|
"grad_norm": 2.171875,
|
||
|
|
"learning_rate": 4.236645926147493e-08,
|
||
|
|
"logits/chosen": -2.0933029651641846,
|
||
|
|
"logits/rejected": -2.131420373916626,
|
||
|
|
"logps/chosen": -0.28616708517074585,
|
||
|
|
"logps/rejected": -0.3243045210838318,
|
||
|
|
"loss": 0.07761023193597794,
|
||
|
|
"loss/core": 0.036653272807598114,
|
||
|
|
"loss/preference_sft": 0.321381539106369,
|
||
|
|
"loss/reference_anchor": 0.3774314522743225,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.0736491680145264,
|
||
|
|
"rewards/margins": 0.7846614718437195,
|
||
|
|
"rewards/rejected": 1.2889881134033203,
|
||
|
|
"ronpo/logit": 0.07846614718437195,
|
||
|
|
"ronpo/residual": 0.08032883703708649,
|
||
|
|
"ronpo/residual_abs": 0.11741244792938232,
|
||
|
|
"ronpo/target": -0.0018626858945935965,
|
||
|
|
"ronpo/target_abs": 0.05729759484529495,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31861263513565063,
|
||
|
|
"drift/rejected_abs": 0.2158060520887375,
|
||
|
|
"epoch": 0.11760031783869686,
|
||
|
|
"grad_norm": 36.25,
|
||
|
|
"learning_rate": 4.197001863832354e-08,
|
||
|
|
"logits/chosen": -2.1318399906158447,
|
||
|
|
"logits/rejected": -2.1656715869903564,
|
||
|
|
"logps/chosen": -0.2784048318862915,
|
||
|
|
"logps/rejected": -0.28124070167541504,
|
||
|
|
"loss": 0.22356906533241272,
|
||
|
|
"loss/core": 0.10556735098361969,
|
||
|
|
"loss/preference_sft": 0.28225523233413696,
|
||
|
|
"loss/reference_anchor": 1.1517915725708008,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.1856679916381836,
|
||
|
|
"rewards/margins": 1.0293043851852417,
|
||
|
|
"rewards/rejected": 2.1563634872436523,
|
||
|
|
"ronpo/logit": 0.10293042659759521,
|
||
|
|
"ronpo/residual": 0.09626283496618271,
|
||
|
|
"ronpo/residual_abs": 0.14562055468559265,
|
||
|
|
"ronpo/target": 0.006667587906122208,
|
||
|
|
"ronpo/target_abs": 0.05160321667790413,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21357055008411407,
|
||
|
|
"drift/rejected_abs": 0.0986676812171936,
|
||
|
|
"epoch": 0.11918951132300358,
|
||
|
|
"grad_norm": 18.875,
|
||
|
|
"learning_rate": 4.156550120539344e-08,
|
||
|
|
"logits/chosen": -2.2781190872192383,
|
||
|
|
"logits/rejected": -2.3018202781677246,
|
||
|
|
"logps/chosen": -0.2936388850212097,
|
||
|
|
"logps/rejected": -0.2881937623023987,
|
||
|
|
"loss": 0.13922074437141418,
|
||
|
|
"loss/core": 0.12170878797769547,
|
||
|
|
"loss/preference_sft": 0.28684449195861816,
|
||
|
|
"loss/reference_anchor": 0.14643503725528717,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.1355700492858887,
|
||
|
|
"rewards/margins": 1.1490267515182495,
|
||
|
|
"rewards/rejected": 0.9865430593490601,
|
||
|
|
"ronpo/logit": 0.11490269005298615,
|
||
|
|
"ronpo/residual": 0.10873659700155258,
|
||
|
|
"ronpo/residual_abs": 0.14544065296649933,
|
||
|
|
"ronpo/target": 0.006166078150272369,
|
||
|
|
"ronpo/target_abs": 0.05552104860544205,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2752930521965027,
|
||
|
|
"drift/rejected_abs": 0.10053251683712006,
|
||
|
|
"epoch": 0.12077870480731029,
|
||
|
|
"grad_norm": 1.046875,
|
||
|
|
"learning_rate": 4.1153099491074094e-08,
|
||
|
|
"logits/chosen": -2.1597490310668945,
|
||
|
|
"logits/rejected": -2.1769750118255615,
|
||
|
|
"logps/chosen": -0.28902846574783325,
|
||
|
|
"logps/rejected": -0.282781720161438,
|
||
|
|
"loss": 0.529904842376709,
|
||
|
|
"loss/core": 0.4918580651283264,
|
||
|
|
"loss/preference_sft": 0.28759074211120605,
|
||
|
|
"loss/reference_anchor": 0.3517104983329773,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.750469207763672,
|
||
|
|
"rewards/margins": 1.7569026947021484,
|
||
|
|
"rewards/rejected": 0.993566632270813,
|
||
|
|
"ronpo/logit": 0.17569026350975037,
|
||
|
|
"ronpo/residual": 0.17634570598602295,
|
||
|
|
"ronpo/residual_abs": 0.20597271621227264,
|
||
|
|
"ronpo/target": -0.000655452604405582,
|
||
|
|
"ronpo/target_abs": 0.04808509349822998,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25277113914489746,
|
||
|
|
"drift/rejected_abs": 0.1480860561132431,
|
||
|
|
"epoch": 0.12236789829161701,
|
||
|
|
"grad_norm": 61.25,
|
||
|
|
"learning_rate": 4.073300977624594e-08,
|
||
|
|
"logits/chosen": -2.277191162109375,
|
||
|
|
"logits/rejected": -2.2477023601531982,
|
||
|
|
"logps/chosen": -0.27077725529670715,
|
||
|
|
"logps/rejected": -0.26460614800453186,
|
||
|
|
"loss": 0.1768111288547516,
|
||
|
|
"loss/core": 0.151821106672287,
|
||
|
|
"loss/preference_sft": 0.2671385705471039,
|
||
|
|
"loss/reference_anchor": 0.22318613529205322,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.526538133621216,
|
||
|
|
"rewards/margins": 1.048628330230713,
|
||
|
|
"rewards/rejected": 1.477910041809082,
|
||
|
|
"ronpo/logit": 0.10486283153295517,
|
||
|
|
"ronpo/residual": 0.09956089407205582,
|
||
|
|
"ronpo/residual_abs": 0.1376153975725174,
|
||
|
|
"ronpo/target": 0.005301934666931629,
|
||
|
|
"ronpo/target_abs": 0.05118630453944206,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21099695563316345,
|
||
|
|
"drift/rejected_abs": 0.12248991429805756,
|
||
|
|
"epoch": 0.12395709177592372,
|
||
|
|
"grad_norm": 3.078125,
|
||
|
|
"learning_rate": 4.030543200086123e-08,
|
||
|
|
"logits/chosen": -2.0786709785461426,
|
||
|
|
"logits/rejected": -2.1199498176574707,
|
||
|
|
"logps/chosen": -0.3045576214790344,
|
||
|
|
"logps/rejected": -0.29476088285446167,
|
||
|
|
"loss": 0.0992443710565567,
|
||
|
|
"loss/core": 0.07167808711528778,
|
||
|
|
"loss/preference_sft": 0.29941990971565247,
|
||
|
|
"loss/reference_anchor": 0.24572083353996277,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.1079843044281006,
|
||
|
|
"rewards/margins": 0.8853281140327454,
|
||
|
|
"rewards/rejected": 1.222656011581421,
|
||
|
|
"ronpo/logit": 0.08853280544281006,
|
||
|
|
"ronpo/residual": 0.0776071846485138,
|
||
|
|
"ronpo/residual_abs": 0.12177129089832306,
|
||
|
|
"ronpo/target": 0.01092563010752201,
|
||
|
|
"ronpo/target_abs": 0.04902078956365585,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21322067081928253,
|
||
|
|
"drift/rejected_abs": 0.06863544136285782,
|
||
|
|
"epoch": 0.12554628526023043,
|
||
|
|
"grad_norm": 3.21875,
|
||
|
|
"learning_rate": 3.987056966878353e-08,
|
||
|
|
"logits/chosen": -2.158668041229248,
|
||
|
|
"logits/rejected": -2.221909999847412,
|
||
|
|
"logps/chosen": -0.31034913659095764,
|
||
|
|
"logps/rejected": -0.29723021388053894,
|
||
|
|
"loss": 0.3569168448448181,
|
||
|
|
"loss/core": 0.33192580938339233,
|
||
|
|
"loss/preference_sft": 0.28455764055252075,
|
||
|
|
"loss/reference_anchor": 0.22145459055900574,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.131547689437866,
|
||
|
|
"rewards/margins": 1.4457036256790161,
|
||
|
|
"rewards/rejected": 0.6858440637588501,
|
||
|
|
"ronpo/logit": 0.14457036554813385,
|
||
|
|
"ronpo/residual": 0.1422502100467682,
|
||
|
|
"ronpo/residual_abs": 0.1726013422012329,
|
||
|
|
"ronpo/target": 0.002320161322131753,
|
||
|
|
"ronpo/target_abs": 0.0495462641119957,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1918472945690155,
|
||
|
|
"drift/rejected_abs": 0.08430652320384979,
|
||
|
|
"epoch": 0.12713547874453715,
|
||
|
|
"grad_norm": 10.5625,
|
||
|
|
"learning_rate": 3.9428629750930845e-08,
|
||
|
|
"logits/chosen": -2.2298977375030518,
|
||
|
|
"logits/rejected": -2.319603443145752,
|
||
|
|
"logps/chosen": -0.2567993700504303,
|
||
|
|
"logps/rejected": -0.26063793897628784,
|
||
|
|
"loss": 0.07930155098438263,
|
||
|
|
"loss/core": 0.06460737437009811,
|
||
|
|
"loss/preference_sft": 0.254304975271225,
|
||
|
|
"loss/reference_anchor": 0.12151118367910385,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 1.9175297021865845,
|
||
|
|
"rewards/margins": 1.078814148902893,
|
||
|
|
"rewards/rejected": 0.8387155532836914,
|
||
|
|
"ronpo/logit": 0.10788138955831528,
|
||
|
|
"ronpo/residual": 0.11212491989135742,
|
||
|
|
"ronpo/residual_abs": 0.1347557008266449,
|
||
|
|
"ronpo/target": -0.004243509843945503,
|
||
|
|
"ronpo/target_abs": 0.048654936254024506,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17124497890472412,
|
||
|
|
"drift/rejected_abs": 0.1462007462978363,
|
||
|
|
"epoch": 0.12872467222884387,
|
||
|
|
"grad_norm": 2.515625,
|
||
|
|
"learning_rate": 3.8979822586768666e-08,
|
||
|
|
"logits/chosen": -2.290794610977173,
|
||
|
|
"logits/rejected": -2.2985095977783203,
|
||
|
|
"logps/chosen": -0.2945011258125305,
|
||
|
|
"logps/rejected": -0.2886204123497009,
|
||
|
|
"loss": 0.30920687317848206,
|
||
|
|
"loss/core": 0.2832431197166443,
|
||
|
|
"loss/preference_sft": 0.2873281240463257,
|
||
|
|
"loss/reference_anchor": 0.23090443015098572,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7103097438812256,
|
||
|
|
"rewards/margins": 0.24959950149059296,
|
||
|
|
"rewards/rejected": 1.4607102870941162,
|
||
|
|
"ronpo/logit": 0.024959946051239967,
|
||
|
|
"ronpo/residual": 0.02641598880290985,
|
||
|
|
"ronpo/residual_abs": 0.16461867094039917,
|
||
|
|
"ronpo/target": -0.0014560356503352523,
|
||
|
|
"ronpo/target_abs": 0.055648498237133026,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1872716248035431,
|
||
|
|
"drift/rejected_abs": 0.057905036956071854,
|
||
|
|
"epoch": 0.13031386571315057,
|
||
|
|
"grad_norm": 0.67578125,
|
||
|
|
"learning_rate": 3.852436178419985e-08,
|
||
|
|
"logits/chosen": -2.2279515266418457,
|
||
|
|
"logits/rejected": -2.141326904296875,
|
||
|
|
"logps/chosen": -0.28925570845603943,
|
||
|
|
"logps/rejected": -0.28760525584220886,
|
||
|
|
"loss": 0.3526923656463623,
|
||
|
|
"loss/core": 0.33196255564689636,
|
||
|
|
"loss/preference_sft": 0.28182536363601685,
|
||
|
|
"loss/reference_anchor": 0.17911583185195923,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.870459794998169,
|
||
|
|
"rewards/margins": 1.3097550868988037,
|
||
|
|
"rewards/rejected": 0.5607047080993652,
|
||
|
|
"ronpo/logit": 0.13097552955150604,
|
||
|
|
"ronpo/residual": 0.13209015130996704,
|
||
|
|
"ronpo/residual_abs": 0.16732732951641083,
|
||
|
|
"ronpo/target": -0.001114630838856101,
|
||
|
|
"ronpo/target_abs": 0.05251222103834152,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.12823328375816345,
|
||
|
|
"drift/rejected_abs": 0.05924879387021065,
|
||
|
|
"epoch": 0.1319030591974573,
|
||
|
|
"grad_norm": 18.375,
|
||
|
|
"learning_rate": 3.806246411789872e-08,
|
||
|
|
"logits/chosen": -2.2676451206207275,
|
||
|
|
"logits/rejected": -2.2973361015319824,
|
||
|
|
"logps/chosen": -0.2848006784915924,
|
||
|
|
"logps/rejected": -0.27459055185317993,
|
||
|
|
"loss": 0.040306154638528824,
|
||
|
|
"loss/core": 0.033028461039066315,
|
||
|
|
"loss/preference_sft": 0.2755371332168579,
|
||
|
|
"loss/reference_anchor": 0.04522323980927467,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.2822843790054321,
|
||
|
|
"rewards/margins": 0.6960135698318481,
|
||
|
|
"rewards/rejected": 0.5862708687782288,
|
||
|
|
"ronpo/logit": 0.06960136443376541,
|
||
|
|
"ronpo/residual": 0.0626053735613823,
|
||
|
|
"ronpo/residual_abs": 0.09992660582065582,
|
||
|
|
"ronpo/target": 0.006995986215770245,
|
||
|
|
"ronpo/target_abs": 0.05004571005702019,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.387541264295578,
|
||
|
|
"drift/rejected_abs": 0.1291671097278595,
|
||
|
|
"epoch": 0.133492252681764,
|
||
|
|
"grad_norm": 3.625,
|
||
|
|
"learning_rate": 3.7594349426138154e-08,
|
||
|
|
"logits/chosen": -2.113781452178955,
|
||
|
|
"logits/rejected": -2.233093738555908,
|
||
|
|
"logps/chosen": -0.2736510634422302,
|
||
|
|
"logps/rejected": -0.27200594544410706,
|
||
|
|
"loss": 0.9080482721328735,
|
||
|
|
"loss/core": 0.8385866284370422,
|
||
|
|
"loss/preference_sft": 0.2741411626338959,
|
||
|
|
"loss/reference_anchor": 0.667202353477478,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.875412702560425,
|
||
|
|
"rewards/margins": 2.708035945892334,
|
||
|
|
"rewards/rejected": 1.1673767566680908,
|
||
|
|
"ronpo/logit": 0.27080363035202026,
|
||
|
|
"ronpo/residual": 0.2666502892971039,
|
||
|
|
"ronpo/residual_abs": 0.3021991550922394,
|
||
|
|
"ronpo/target": 0.004153319634497166,
|
||
|
|
"ronpo/target_abs": 0.04889250919222832,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.42560648918151855,
|
||
|
|
"drift/rejected_abs": 0.13950805366039276,
|
||
|
|
"epoch": 0.1350814461660707,
|
||
|
|
"grad_norm": 304.0,
|
||
|
|
"learning_rate": 3.712024050615843e-08,
|
||
|
|
"logits/chosen": -2.164858341217041,
|
||
|
|
"logits/rejected": -2.1596226692199707,
|
||
|
|
"logps/chosen": -0.2971101403236389,
|
||
|
|
"logps/rejected": -0.29576635360717773,
|
||
|
|
"loss": 1.2745723724365234,
|
||
|
|
"loss/core": 1.1691445112228394,
|
||
|
|
"loss/preference_sft": 0.28978782892227173,
|
||
|
|
"loss/reference_anchor": 1.0252996683120728,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 4.2549614906311035,
|
||
|
|
"rewards/margins": 2.861567735671997,
|
||
|
|
"rewards/rejected": 1.393393874168396,
|
||
|
|
"ronpo/logit": 0.2861567735671997,
|
||
|
|
"ronpo/residual": 0.288315087556839,
|
||
|
|
"ronpo/residual_abs": 0.3225054144859314,
|
||
|
|
"ronpo/target": -0.0021583049092441797,
|
||
|
|
"ronpo/target_abs": 0.05234162136912346,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1872471123933792,
|
||
|
|
"drift/rejected_abs": 0.09143074601888657,
|
||
|
|
"epoch": 0.13667063965037743,
|
||
|
|
"grad_norm": 11.625,
|
||
|
|
"learning_rate": 3.6640363008127786e-08,
|
||
|
|
"logits/chosen": -2.432072162628174,
|
||
|
|
"logits/rejected": -2.4184775352478027,
|
||
|
|
"logps/chosen": -0.2796880304813385,
|
||
|
|
"logps/rejected": -0.27485981583595276,
|
||
|
|
"loss": 0.1038626879453659,
|
||
|
|
"loss/core": 0.08703657239675522,
|
||
|
|
"loss/preference_sft": 0.2725329101085663,
|
||
|
|
"loss/reference_anchor": 0.1410079151391983,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.8717552423477173,
|
||
|
|
"rewards/margins": 0.9586809277534485,
|
||
|
|
"rewards/rejected": 0.9130741357803345,
|
||
|
|
"ronpo/logit": 0.09586808830499649,
|
||
|
|
"ronpo/residual": 0.08066330850124359,
|
||
|
|
"ronpo/residual_abs": 0.12503182888031006,
|
||
|
|
"ronpo/target": 0.01520479004830122,
|
||
|
|
"ronpo/target_abs": 0.049617547541856766,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17280510067939758,
|
||
|
|
"drift/rejected_abs": 0.09314249455928802,
|
||
|
|
"epoch": 0.13825983313468415,
|
||
|
|
"grad_norm": 12.1875,
|
||
|
|
"learning_rate": 3.615494532774522e-08,
|
||
|
|
"logits/chosen": -2.1827242374420166,
|
||
|
|
"logits/rejected": -2.1762850284576416,
|
||
|
|
"logps/chosen": -0.280172199010849,
|
||
|
|
"logps/rejected": -0.28061169385910034,
|
||
|
|
"loss": 0.043207619339227676,
|
||
|
|
"loss/core": 0.03257911652326584,
|
||
|
|
"loss/preference_sft": 0.27778077125549316,
|
||
|
|
"loss/reference_anchor": 0.07850693166255951,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.7262004613876343,
|
||
|
|
"rewards/margins": 0.7954661846160889,
|
||
|
|
"rewards/rejected": 0.9307343363761902,
|
||
|
|
"ronpo/logit": 0.07954661548137665,
|
||
|
|
"ronpo/residual": 0.06194382905960083,
|
||
|
|
"ronpo/residual_abs": 0.10705231130123138,
|
||
|
|
"ronpo/target": 0.01760278083384037,
|
||
|
|
"ronpo/target_abs": 0.04770347476005554,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3187355399131775,
|
||
|
|
"drift/rejected_abs": 0.13290651142597198,
|
||
|
|
"epoch": 0.13984902661899087,
|
||
|
|
"grad_norm": 4.125,
|
||
|
|
"learning_rate": 3.566421849753646e-08,
|
||
|
|
"logits/chosen": -2.2726149559020996,
|
||
|
|
"logits/rejected": -2.297689437866211,
|
||
|
|
"logps/chosen": -0.2947022020816803,
|
||
|
|
"logps/rejected": -0.2913546562194824,
|
||
|
|
"loss": 0.7720416784286499,
|
||
|
|
"loss/core": 0.6805315017700195,
|
||
|
|
"loss/preference_sft": 0.300406277179718,
|
||
|
|
"loss/reference_anchor": 0.8850604295730591,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.1873040199279785,
|
||
|
|
"rewards/margins": 1.865846872329712,
|
||
|
|
"rewards/rejected": 1.3214576244354248,
|
||
|
|
"ronpo/logit": 0.18658466637134552,
|
||
|
|
"ronpo/residual": 0.18800434470176697,
|
||
|
|
"ronpo/residual_abs": 0.21804949641227722,
|
||
|
|
"ronpo/target": -0.0014196861302480102,
|
||
|
|
"ronpo/target_abs": 0.051555730402469635,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.190468430519104,
|
||
|
|
"drift/rejected_abs": 0.1678762435913086,
|
||
|
|
"epoch": 0.14143822010329757,
|
||
|
|
"grad_norm": 300.0,
|
||
|
|
"learning_rate": 3.516841607689501e-08,
|
||
|
|
"logits/chosen": -2.2291038036346436,
|
||
|
|
"logits/rejected": -2.1525654792785645,
|
||
|
|
"logps/chosen": -0.2803241014480591,
|
||
|
|
"logps/rejected": -0.27273237705230713,
|
||
|
|
"loss": 0.2623751163482666,
|
||
|
|
"loss/core": 0.23482990264892578,
|
||
|
|
"loss/preference_sft": 0.27626365423202515,
|
||
|
|
"loss/reference_anchor": 0.24782554805278778,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 1.9043534994125366,
|
||
|
|
"rewards/margins": 0.22563567757606506,
|
||
|
|
"rewards/rejected": 1.678717851638794,
|
||
|
|
"ronpo/logit": 0.022563569247722626,
|
||
|
|
"ronpo/residual": 0.02049192786216736,
|
||
|
|
"ronpo/residual_abs": 0.18030428886413574,
|
||
|
|
"ronpo/target": 0.002071630908176303,
|
||
|
|
"ronpo/target_abs": 0.052827440202236176,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3158915638923645,
|
||
|
|
"drift/rejected_abs": 0.10503937304019928,
|
||
|
|
"epoch": 0.1430274135876043,
|
||
|
|
"grad_norm": 30.25,
|
||
|
|
"learning_rate": 3.4667774040920516e-08,
|
||
|
|
"logits/chosen": -2.262582778930664,
|
||
|
|
"logits/rejected": -2.3268892765045166,
|
||
|
|
"logps/chosen": -0.27218812704086304,
|
||
|
|
"logps/rejected": -0.2808297276496887,
|
||
|
|
"loss": 0.7278597950935364,
|
||
|
|
"loss/core": 0.6705979704856873,
|
||
|
|
"loss/preference_sft": 0.26000097393989563,
|
||
|
|
"loss/reference_anchor": 0.5466181039810181,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.158046007156372,
|
||
|
|
"rewards/margins": 2.112226963043213,
|
||
|
|
"rewards/rejected": 1.0458192825317383,
|
||
|
|
"ronpo/logit": 0.21122269332408905,
|
||
|
|
"ronpo/residual": 0.22126352787017822,
|
||
|
|
"ronpo/residual_abs": 0.25051742792129517,
|
||
|
|
"ronpo/target": -0.010040839202702045,
|
||
|
|
"ronpo/target_abs": 0.054663896560668945,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20814290642738342,
|
||
|
|
"drift/rejected_abs": 0.09922685474157333,
|
||
|
|
"epoch": 0.144616607071911,
|
||
|
|
"grad_norm": 11.1875,
|
||
|
|
"learning_rate": 3.416253066810743e-08,
|
||
|
|
"logits/chosen": -2.3161652088165283,
|
||
|
|
"logits/rejected": -2.3485805988311768,
|
||
|
|
"logps/chosen": -0.28402179479599,
|
||
|
|
"logps/rejected": -0.2825237512588501,
|
||
|
|
"loss": 0.11367638409137726,
|
||
|
|
"loss/core": 0.09682723134756088,
|
||
|
|
"loss/preference_sft": 0.28498321771621704,
|
||
|
|
"loss/reference_anchor": 0.13999313116073608,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.0814290046691895,
|
||
|
|
"rewards/margins": 1.0895073413848877,
|
||
|
|
"rewards/rejected": 0.9919217228889465,
|
||
|
|
"ronpo/logit": 0.10895073413848877,
|
||
|
|
"ronpo/residual": 0.11198439449071884,
|
||
|
|
"ronpo/residual_abs": 0.1464233696460724,
|
||
|
|
"ronpo/target": -0.00303367106243968,
|
||
|
|
"ronpo/target_abs": 0.04594602435827255,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3775826096534729,
|
||
|
|
"drift/rejected_abs": 0.16409118473529816,
|
||
|
|
"epoch": 0.1462058005562177,
|
||
|
|
"grad_norm": 119.0,
|
||
|
|
"learning_rate": 3.365292642693732e-08,
|
||
|
|
"logits/chosen": -1.8064380884170532,
|
||
|
|
"logits/rejected": -1.66707444190979,
|
||
|
|
"logps/chosen": -0.28099215030670166,
|
||
|
|
"logps/rejected": -0.3002210259437561,
|
||
|
|
"loss": 0.5818950533866882,
|
||
|
|
"loss/core": 0.5199607610702515,
|
||
|
|
"loss/preference_sft": 0.2856101095676422,
|
||
|
|
"loss/reference_anchor": 0.5907815098762512,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.770456314086914,
|
||
|
|
"rewards/margins": 2.13781476020813,
|
||
|
|
"rewards/rejected": 1.6326417922973633,
|
||
|
|
"ronpo/logit": 0.2137814462184906,
|
||
|
|
"ronpo/residual": 0.21489810943603516,
|
||
|
|
"ronpo/residual_abs": 0.2495894730091095,
|
||
|
|
"ronpo/target": -0.0011166559997946024,
|
||
|
|
"ronpo/target_abs": 0.05269808694720268,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3443106412887573,
|
||
|
|
"drift/rejected_abs": 0.23541593551635742,
|
||
|
|
"epoch": 0.14779499404052443,
|
||
|
|
"grad_norm": 366.0,
|
||
|
|
"learning_rate": 3.313920386142892e-08,
|
||
|
|
"logits/chosen": -2.094979763031006,
|
||
|
|
"logits/rejected": -2.115394115447998,
|
||
|
|
"logps/chosen": -0.26985684037208557,
|
||
|
|
"logps/rejected": -0.2721291780471802,
|
||
|
|
"loss": 0.859258770942688,
|
||
|
|
"loss/core": 0.7965814471244812,
|
||
|
|
"loss/preference_sft": 0.26244303584098816,
|
||
|
|
"loss/reference_anchor": 0.6005281209945679,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.441904067993164,
|
||
|
|
"rewards/margins": 1.089691400527954,
|
||
|
|
"rewards/rejected": 2.35221266746521,
|
||
|
|
"ronpo/logit": 0.10896915197372437,
|
||
|
|
"ronpo/residual": 0.10788600146770477,
|
||
|
|
"ronpo/residual_abs": 0.293454110622406,
|
||
|
|
"ronpo/target": 0.0010831309482455254,
|
||
|
|
"ronpo/target_abs": 0.052142005413770676,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3005021810531616,
|
||
|
|
"drift/rejected_abs": 0.11847560107707977,
|
||
|
|
"epoch": 0.14938418752483115,
|
||
|
|
"grad_norm": 2.921875,
|
||
|
|
"learning_rate": 3.262160747570027e-08,
|
||
|
|
"logits/chosen": -2.0722744464874268,
|
||
|
|
"logits/rejected": -2.0413122177124023,
|
||
|
|
"logps/chosen": -0.253449410200119,
|
||
|
|
"logps/rejected": -0.2583273947238922,
|
||
|
|
"loss": 0.804643988609314,
|
||
|
|
"loss/core": 0.7490277886390686,
|
||
|
|
"loss/preference_sft": 0.2524202764034271,
|
||
|
|
"loss/reference_anchor": 0.5309203267097473,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.0032966136932373,
|
||
|
|
"rewards/margins": 1.820664644241333,
|
||
|
|
"rewards/rejected": 1.1826322078704834,
|
||
|
|
"ronpo/logit": 0.18206645548343658,
|
||
|
|
"ronpo/residual": 0.18194696307182312,
|
||
|
|
"ronpo/residual_abs": 0.2314409464597702,
|
||
|
|
"ronpo/target": 0.00011948794417548925,
|
||
|
|
"ronpo/target_abs": 0.04965994879603386,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25074881315231323,
|
||
|
|
"drift/rejected_abs": 0.0969192162156105,
|
||
|
|
"epoch": 0.15097338100913787,
|
||
|
|
"grad_norm": 7.875,
|
||
|
|
"learning_rate": 3.210038361759807e-08,
|
||
|
|
"logits/chosen": -2.196094512939453,
|
||
|
|
"logits/rejected": -2.1309614181518555,
|
||
|
|
"logps/chosen": -0.3051135540008545,
|
||
|
|
"logps/rejected": -0.2989034950733185,
|
||
|
|
"loss": 0.2300875186920166,
|
||
|
|
"loss/core": 0.21017341315746307,
|
||
|
|
"loss/preference_sft": 0.30538254976272583,
|
||
|
|
"loss/reference_anchor": 0.16860318183898926,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.5062077045440674,
|
||
|
|
"rewards/margins": 1.53762686252594,
|
||
|
|
"rewards/rejected": 0.9685808420181274,
|
||
|
|
"ronpo/logit": 0.15376266837120056,
|
||
|
|
"ronpo/residual": 0.15053051710128784,
|
||
|
|
"ronpo/residual_abs": 0.19873568415641785,
|
||
|
|
"ronpo/target": 0.0032321817707270384,
|
||
|
|
"ronpo/target_abs": 0.05262786149978638,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22192128002643585,
|
||
|
|
"drift/rejected_abs": 0.109128437936306,
|
||
|
|
"epoch": 0.15256257449344457,
|
||
|
|
"grad_norm": 1.515625,
|
||
|
|
"learning_rate": 3.157578036144937e-08,
|
||
|
|
"logits/chosen": -2.1180901527404785,
|
||
|
|
"logits/rejected": -2.140153169631958,
|
||
|
|
"logps/chosen": -0.27319201827049255,
|
||
|
|
"logps/rejected": -0.2684481739997864,
|
||
|
|
"loss": 0.08363901823759079,
|
||
|
|
"loss/core": 0.06819086521863937,
|
||
|
|
"loss/preference_sft": 0.2708436846733093,
|
||
|
|
"loss/reference_anchor": 0.12739713490009308,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.2160263061523438,
|
||
|
|
"rewards/margins": 1.133367896080017,
|
||
|
|
"rewards/rejected": 1.0826584100723267,
|
||
|
|
"ronpo/logit": 0.11333680152893066,
|
||
|
|
"ronpo/residual": 0.1177079826593399,
|
||
|
|
"ronpo/residual_abs": 0.14758066833019257,
|
||
|
|
"ronpo/target": -0.004371200688183308,
|
||
|
|
"ronpo/target_abs": 0.050663966685533524,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29827818274497986,
|
||
|
|
"drift/rejected_abs": 0.16067150235176086,
|
||
|
|
"epoch": 0.1541517679777513,
|
||
|
|
"grad_norm": 6.59375,
|
||
|
|
"learning_rate": 3.104804738999169e-08,
|
||
|
|
"logits/chosen": -1.9035288095474243,
|
||
|
|
"logits/rejected": -2.026747226715088,
|
||
|
|
"logps/chosen": -0.28207504749298096,
|
||
|
|
"logps/rejected": -0.27979451417922974,
|
||
|
|
"loss": 0.14665299654006958,
|
||
|
|
"loss/core": 0.11166276037693024,
|
||
|
|
"loss/preference_sft": 0.2819157540798187,
|
||
|
|
"loss/reference_anchor": 0.32171088457107544,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.982020139694214,
|
||
|
|
"rewards/margins": 1.3762387037277222,
|
||
|
|
"rewards/rejected": 1.6057815551757812,
|
||
|
|
"ronpo/logit": 0.1376238763332367,
|
||
|
|
"ronpo/residual": 0.13066348433494568,
|
||
|
|
"ronpo/residual_abs": 0.17018786072731018,
|
||
|
|
"ronpo/target": 0.006960384547710419,
|
||
|
|
"ronpo/target_abs": 0.05105891078710556,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.368802547454834,
|
||
|
|
"drift/rejected_abs": 0.15182280540466309,
|
||
|
|
"epoch": 0.15574096146205801,
|
||
|
|
"grad_norm": 288.0,
|
||
|
|
"learning_rate": 3.0517435875537536e-08,
|
||
|
|
"logits/chosen": -2.1045501232147217,
|
||
|
|
"logits/rejected": -2.08597993850708,
|
||
|
|
"logps/chosen": -0.2816130220890045,
|
||
|
|
"logps/rejected": -0.2822210490703583,
|
||
|
|
"loss": 0.7077787518501282,
|
||
|
|
"loss/core": 0.6370621919631958,
|
||
|
|
"loss/preference_sft": 0.28190574049949646,
|
||
|
|
"loss/reference_anchor": 0.6789735555648804,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.685183048248291,
|
||
|
|
"rewards/margins": 2.1684653759002686,
|
||
|
|
"rewards/rejected": 1.5167176723480225,
|
||
|
|
"ronpo/logit": 0.2168465107679367,
|
||
|
|
"ronpo/residual": 0.20779669284820557,
|
||
|
|
"ronpo/residual_abs": 0.25034013390541077,
|
||
|
|
"ronpo/target": 0.009049828164279461,
|
||
|
|
"ronpo/target_abs": 0.05838625505566597,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21987831592559814,
|
||
|
|
"drift/rejected_abs": 0.18404293060302734,
|
||
|
|
"epoch": 0.1573301549463647,
|
||
|
|
"grad_norm": 496.0,
|
||
|
|
"learning_rate": 2.998419836042993e-08,
|
||
|
|
"logits/chosen": -2.125863552093506,
|
||
|
|
"logits/rejected": -2.0966343879699707,
|
||
|
|
"logps/chosen": -0.28703004121780396,
|
||
|
|
"logps/rejected": -0.2746201157569885,
|
||
|
|
"loss": 1.2086600065231323,
|
||
|
|
"loss/core": 1.1395339965820312,
|
||
|
|
"loss/preference_sft": 0.2791707217693329,
|
||
|
|
"loss/reference_anchor": 0.6633409261703491,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.197369337081909,
|
||
|
|
"rewards/margins": 0.36156225204467773,
|
||
|
|
"rewards/rejected": 1.8358070850372314,
|
||
|
|
"ronpo/logit": 0.03615620732307434,
|
||
|
|
"ronpo/residual": 0.03486531972885132,
|
||
|
|
"ronpo/residual_abs": 0.2834121584892273,
|
||
|
|
"ronpo/target": 0.0012908976059406996,
|
||
|
|
"ronpo/target_abs": 0.05236129090189934,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24376916885375977,
|
||
|
|
"drift/rejected_abs": 0.1475829780101776,
|
||
|
|
"epoch": 0.15891934843067143,
|
||
|
|
"grad_norm": 4.6875,
|
||
|
|
"learning_rate": 2.9448588636846045e-08,
|
||
|
|
"logits/chosen": -2.2326369285583496,
|
||
|
|
"logits/rejected": -2.090822696685791,
|
||
|
|
"logps/chosen": -0.28119930624961853,
|
||
|
|
"logps/rejected": -0.2759275734424591,
|
||
|
|
"loss": 0.24097900092601776,
|
||
|
|
"loss/core": 0.20966272056102753,
|
||
|
|
"loss/preference_sft": 0.2712128758430481,
|
||
|
|
"loss/reference_anchor": 0.28604134917259216,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.4376919269561768,
|
||
|
|
"rewards/margins": 0.9774544835090637,
|
||
|
|
"rewards/rejected": 1.4602375030517578,
|
||
|
|
"ronpo/logit": 0.09774543344974518,
|
||
|
|
"ronpo/residual": 0.09335900098085403,
|
||
|
|
"ronpo/residual_abs": 0.19132252037525177,
|
||
|
|
"ronpo/target": 0.004386442247778177,
|
||
|
|
"ronpo/target_abs": 0.05662950128316879,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16036273539066315,
|
||
|
|
"drift/rejected_abs": 0.106539785861969,
|
||
|
|
"epoch": 0.16050854191497815,
|
||
|
|
"grad_norm": 4.46875,
|
||
|
|
"learning_rate": 2.8910861626005773e-08,
|
||
|
|
"logits/chosen": -2.283858299255371,
|
||
|
|
"logits/rejected": -2.20324444770813,
|
||
|
|
"logps/chosen": -0.2959023714065552,
|
||
|
|
"logps/rejected": -0.28080320358276367,
|
||
|
|
"loss": 0.032736729830503464,
|
||
|
|
"loss/core": 0.022858766838908195,
|
||
|
|
"loss/preference_sft": 0.28779202699661255,
|
||
|
|
"loss/reference_anchor": 0.07000039517879486,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.6006122827529907,
|
||
|
|
"rewards/margins": 0.5359739661216736,
|
||
|
|
"rewards/rejected": 1.064638376235962,
|
||
|
|
"ronpo/logit": 0.053597401827573776,
|
||
|
|
"ronpo/residual": 0.04705274850130081,
|
||
|
|
"ronpo/residual_abs": 0.09807590395212173,
|
||
|
|
"ronpo/target": 0.006544650532305241,
|
||
|
|
"ronpo/target_abs": 0.05331002548336983,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31569427251815796,
|
||
|
|
"drift/rejected_abs": 0.16049371659755707,
|
||
|
|
"epoch": 0.16209773539928488,
|
||
|
|
"grad_norm": 2.625,
|
||
|
|
"learning_rate": 2.8371273256843075e-08,
|
||
|
|
"logits/chosen": -1.9404888153076172,
|
||
|
|
"logits/rejected": -1.963454008102417,
|
||
|
|
"logps/chosen": -0.2612414062023163,
|
||
|
|
"logps/rejected": -0.2635475993156433,
|
||
|
|
"loss": 0.46916890144348145,
|
||
|
|
"loss/core": 0.4046415686607361,
|
||
|
|
"loss/preference_sft": 0.2662178874015808,
|
||
|
|
"loss/reference_anchor": 0.6186517477035522,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.1566901206970215,
|
||
|
|
"rewards/margins": 1.5550490617752075,
|
||
|
|
"rewards/rejected": 1.601641058921814,
|
||
|
|
"ronpo/logit": 0.15550491213798523,
|
||
|
|
"ronpo/residual": 0.15648245811462402,
|
||
|
|
"ronpo/residual_abs": 0.18898537755012512,
|
||
|
|
"ronpo/target": -0.0009775443468242884,
|
||
|
|
"ronpo/target_abs": 0.04822375625371933,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2568056285381317,
|
||
|
|
"drift/rejected_abs": 0.15386393666267395,
|
||
|
|
"epoch": 0.16368692888359157,
|
||
|
|
"grad_norm": 5.0,
|
||
|
|
"learning_rate": 2.7830080344197672e-08,
|
||
|
|
"logits/chosen": -2.1424307823181152,
|
||
|
|
"logits/rejected": -2.210108995437622,
|
||
|
|
"logps/chosen": -0.290412038564682,
|
||
|
|
"logps/rejected": -0.2754800617694855,
|
||
|
|
"loss": 0.3476158082485199,
|
||
|
|
"loss/core": 0.317764550447464,
|
||
|
|
"loss/preference_sft": 0.2852453589439392,
|
||
|
|
"loss/reference_anchor": 0.26998820900917053,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.568056344985962,
|
||
|
|
"rewards/margins": 1.0325605869293213,
|
||
|
|
"rewards/rejected": 1.5354955196380615,
|
||
|
|
"ronpo/logit": 0.10325608402490616,
|
||
|
|
"ronpo/residual": 0.1111401692032814,
|
||
|
|
"ronpo/residual_abs": 0.21046268939971924,
|
||
|
|
"ronpo/target": -0.007884078659117222,
|
||
|
|
"ronpo/target_abs": 0.05043824389576912,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21701356768608093,
|
||
|
|
"drift/rejected_abs": 0.10474522411823273,
|
||
|
|
"epoch": 0.1652761223678983,
|
||
|
|
"grad_norm": 3.859375,
|
||
|
|
"learning_rate": 2.7287540466585063e-08,
|
||
|
|
"logits/chosen": -2.1483113765716553,
|
||
|
|
"logits/rejected": -2.1584086418151855,
|
||
|
|
"logps/chosen": -0.29263603687286377,
|
||
|
|
"logps/rejected": -0.30166834592819214,
|
||
|
|
"loss": 0.28205159306526184,
|
||
|
|
"loss/core": 0.22824864089488983,
|
||
|
|
"loss/preference_sft": 0.2935117483139038,
|
||
|
|
"loss/reference_anchor": 0.508678138256073,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.169997215270996,
|
||
|
|
"rewards/margins": 1.130325436592102,
|
||
|
|
"rewards/rejected": 1.0396720170974731,
|
||
|
|
"ronpo/logit": 0.11303253471851349,
|
||
|
|
"ronpo/residual": 0.10396580398082733,
|
||
|
|
"ronpo/residual_abs": 0.13924115896224976,
|
||
|
|
"ronpo/target": 0.009066728875041008,
|
||
|
|
"ronpo/target_abs": 0.052173011004924774,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22040951251983643,
|
||
|
|
"drift/rejected_abs": 0.0650288462638855,
|
||
|
|
"epoch": 0.16686531585220502,
|
||
|
|
"grad_norm": 7.59375,
|
||
|
|
"learning_rate": 2.6743911843603128e-08,
|
||
|
|
"logits/chosen": -1.9065526723861694,
|
||
|
|
"logits/rejected": -2.050851583480835,
|
||
|
|
"logps/chosen": -0.31132742762565613,
|
||
|
|
"logps/rejected": -0.3201202154159546,
|
||
|
|
"loss": 0.2915444076061249,
|
||
|
|
"loss/core": 0.2698300778865814,
|
||
|
|
"loss/preference_sft": 0.3184563219547272,
|
||
|
|
"loss/reference_anchor": 0.1852976679801941,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.2039241790771484,
|
||
|
|
"rewards/margins": 1.5547595024108887,
|
||
|
|
"rewards/rejected": 0.6491645574569702,
|
||
|
|
"ronpo/logit": 0.15547595918178558,
|
||
|
|
"ronpo/residual": 0.1664518564939499,
|
||
|
|
"ronpo/residual_abs": 0.18785083293914795,
|
||
|
|
"ronpo/target": -0.010975899174809456,
|
||
|
|
"ronpo/target_abs": 0.049125999212265015,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1532839685678482,
|
||
|
|
"drift/rejected_abs": 0.06518113613128662,
|
||
|
|
"epoch": 0.1684545093365117,
|
||
|
|
"grad_norm": 1.0390625,
|
||
|
|
"learning_rate": 2.6199453213033594e-08,
|
||
|
|
"logits/chosen": -2.3217177391052246,
|
||
|
|
"logits/rejected": -2.3166098594665527,
|
||
|
|
"logps/chosen": -0.2668350636959076,
|
||
|
|
"logps/rejected": -0.26081985235214233,
|
||
|
|
"loss": 0.05902089923620224,
|
||
|
|
"loss/core": 0.05048280954360962,
|
||
|
|
"loss/preference_sft": 0.2698516249656677,
|
||
|
|
"loss/reference_anchor": 0.05839569494128227,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 1.5320897102355957,
|
||
|
|
"rewards/margins": 0.8829070329666138,
|
||
|
|
"rewards/rejected": 0.6491826772689819,
|
||
|
|
"ronpo/logit": 0.08829069137573242,
|
||
|
|
"ronpo/residual": 0.08748365938663483,
|
||
|
|
"ronpo/residual_abs": 0.11516772210597992,
|
||
|
|
"ronpo/target": 0.0008070375770330429,
|
||
|
|
"ronpo/target_abs": 0.05328748747706413,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3506208658218384,
|
||
|
|
"drift/rejected_abs": 0.1504441797733307,
|
||
|
|
"epoch": 0.17004370282081843,
|
||
|
|
"grad_norm": 3.34375,
|
||
|
|
"learning_rate": 2.5654423707696833e-08,
|
||
|
|
"logits/chosen": -2.242438793182373,
|
||
|
|
"logits/rejected": -2.2580552101135254,
|
||
|
|
"logps/chosen": -0.3120534420013428,
|
||
|
|
"logps/rejected": -0.3226138949394226,
|
||
|
|
"loss": 1.6565309762954712,
|
||
|
|
"loss/core": 1.5435130596160889,
|
||
|
|
"loss/preference_sft": 0.32026439905166626,
|
||
|
|
"loss/reference_anchor": 1.0981547832489014,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.5062084197998047,
|
||
|
|
"rewards/margins": 2.2400765419006348,
|
||
|
|
"rewards/rejected": 1.2661319971084595,
|
||
|
|
"ronpo/logit": 0.22400769591331482,
|
||
|
|
"ronpo/residual": 0.24428728222846985,
|
||
|
|
"ronpo/residual_abs": 0.32966747879981995,
|
||
|
|
"ronpo/target": -0.02027963474392891,
|
||
|
|
"ronpo/target_abs": 0.054383207112550735,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2452022284269333,
|
||
|
|
"drift/rejected_abs": 0.10782649368047714,
|
||
|
|
"epoch": 0.17163289630512515,
|
||
|
|
"grad_norm": 13.0,
|
||
|
|
"learning_rate": 2.5109082732118664e-08,
|
||
|
|
"logits/chosen": -2.201077938079834,
|
||
|
|
"logits/rejected": -2.2382140159606934,
|
||
|
|
"logps/chosen": -0.2791464030742645,
|
||
|
|
"logps/rejected": -0.2900802791118622,
|
||
|
|
"loss": 0.23707738518714905,
|
||
|
|
"loss/core": 0.197381854057312,
|
||
|
|
"loss/preference_sft": 0.290230393409729,
|
||
|
|
"loss/reference_anchor": 0.36793252825737,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.4503917694091797,
|
||
|
|
"rewards/margins": 1.3725776672363281,
|
||
|
|
"rewards/rejected": 1.0778138637542725,
|
||
|
|
"ronpo/logit": 0.13725778460502625,
|
||
|
|
"ronpo/residual": 0.14476758241653442,
|
||
|
|
"ronpo/residual_abs": 0.17167086899280548,
|
||
|
|
"ronpo/target": -0.007509827613830566,
|
||
|
|
"ronpo/target_abs": 0.04967178776860237,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.398145467042923,
|
||
|
|
"drift/rejected_abs": 0.24695797264575958,
|
||
|
|
"epoch": 0.17322208978943188,
|
||
|
|
"grad_norm": 5.3125,
|
||
|
|
"learning_rate": 2.456368983906791e-08,
|
||
|
|
"logits/chosen": -1.9901946783065796,
|
||
|
|
"logits/rejected": -2.0143864154815674,
|
||
|
|
"logps/chosen": -0.261131227016449,
|
||
|
|
"logps/rejected": -0.27835986018180847,
|
||
|
|
"loss": 0.4815913736820221,
|
||
|
|
"loss/core": 0.3962658643722534,
|
||
|
|
"loss/preference_sft": 0.27543601393699646,
|
||
|
|
"loss/reference_anchor": 0.8257115483283997,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.9795594215393066,
|
||
|
|
"rewards/margins": 1.5161901712417603,
|
||
|
|
"rewards/rejected": 2.463369369506836,
|
||
|
|
"ronpo/logit": 0.15161903202533722,
|
||
|
|
"ronpo/residual": 0.14838778972625732,
|
||
|
|
"ronpo/residual_abs": 0.2382703274488449,
|
||
|
|
"ronpo/target": 0.003231218084692955,
|
||
|
|
"ronpo/target_abs": 0.047506727278232574,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.275449275970459,
|
||
|
|
"drift/rejected_abs": 0.13889315724372864,
|
||
|
|
"epoch": 0.17481128327373857,
|
||
|
|
"grad_norm": 1.390625,
|
||
|
|
"learning_rate": 2.401850460602329e-08,
|
||
|
|
"logits/chosen": -2.1420063972473145,
|
||
|
|
"logits/rejected": -2.098811149597168,
|
||
|
|
"logps/chosen": -0.25522199273109436,
|
||
|
|
"logps/rejected": -0.2641642689704895,
|
||
|
|
"loss": 0.3213452696800232,
|
||
|
|
"loss/core": 0.28511881828308105,
|
||
|
|
"loss/preference_sft": 0.26377683877944946,
|
||
|
|
"loss/reference_anchor": 0.33588656783103943,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.7539238929748535,
|
||
|
|
"rewards/margins": 1.3655030727386475,
|
||
|
|
"rewards/rejected": 1.388420820236206,
|
||
|
|
"ronpo/logit": 0.13655030727386475,
|
||
|
|
"ronpo/residual": 0.1357690542936325,
|
||
|
|
"ronpo/residual_abs": 0.1875356137752533,
|
||
|
|
"ronpo/target": 0.0007812596159055829,
|
||
|
|
"ronpo/target_abs": 0.05187007039785385,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.12571373581886292,
|
||
|
|
"drift/rejected_abs": 0.055512648075819016,
|
||
|
|
"epoch": 0.1764004767580453,
|
||
|
|
"grad_norm": 0.447265625,
|
||
|
|
"learning_rate": 2.3473786511628573e-08,
|
||
|
|
"logits/chosen": -2.2189087867736816,
|
||
|
|
"logits/rejected": -2.261380434036255,
|
||
|
|
"logps/chosen": -0.31099390983581543,
|
||
|
|
"logps/rejected": -0.3084818720817566,
|
||
|
|
"loss": 0.0710396021604538,
|
||
|
|
"loss/core": 0.06051533669233322,
|
||
|
|
"loss/preference_sft": 0.30863305926322937,
|
||
|
|
"loss/reference_anchor": 0.0743793398141861,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.2571372985839844,
|
||
|
|
"rewards/margins": 0.7021911144256592,
|
||
|
|
"rewards/rejected": 0.5549461245536804,
|
||
|
|
"ronpo/logit": 0.07021911442279816,
|
||
|
|
"ronpo/residual": 0.07059456408023834,
|
||
|
|
"ronpo/residual_abs": 0.11249272525310516,
|
||
|
|
"ronpo/target": -0.0003754418285097927,
|
||
|
|
"ronpo/target_abs": 0.05336303636431694,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2654932141304016,
|
||
|
|
"drift/rejected_abs": 0.12235883623361588,
|
||
|
|
"epoch": 0.17798967024235202,
|
||
|
|
"grad_norm": 16.875,
|
||
|
|
"learning_rate": 2.2929794812194894e-08,
|
||
|
|
"logits/chosen": -2.0882561206817627,
|
||
|
|
"logits/rejected": -2.0949742794036865,
|
||
|
|
"logps/chosen": -0.2720016837120056,
|
||
|
|
"logps/rejected": -0.2790417969226837,
|
||
|
|
"loss": 0.26181310415267944,
|
||
|
|
"loss/core": 0.23417861759662628,
|
||
|
|
"loss/preference_sft": 0.2686235010623932,
|
||
|
|
"loss/reference_anchor": 0.24948279559612274,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.6532604694366455,
|
||
|
|
"rewards/margins": 1.4311665296554565,
|
||
|
|
"rewards/rejected": 1.222094178199768,
|
||
|
|
"ronpo/logit": 0.14311663806438446,
|
||
|
|
"ronpo/residual": 0.13579586148262024,
|
||
|
|
"ronpo/residual_abs": 0.1613430231809616,
|
||
|
|
"ronpo/target": 0.007320776581764221,
|
||
|
|
"ronpo/target_abs": 0.045001320540905,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25932857394218445,
|
||
|
|
"drift/rejected_abs": 0.11433260142803192,
|
||
|
|
"epoch": 0.1795788637266587,
|
||
|
|
"grad_norm": 29.625,
|
||
|
|
"learning_rate": 2.2386788418308666e-08,
|
||
|
|
"logits/chosen": -2.0426180362701416,
|
||
|
|
"logits/rejected": -2.067936658859253,
|
||
|
|
"logps/chosen": -0.3414672017097473,
|
||
|
|
"logps/rejected": -0.3361096978187561,
|
||
|
|
"loss": 0.38202768564224243,
|
||
|
|
"loss/core": 0.33821603655815125,
|
||
|
|
"loss/preference_sft": 0.3408833146095276,
|
||
|
|
"loss/reference_anchor": 0.40402793884277344,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.5908656120300293,
|
||
|
|
"rewards/margins": 1.4496145248413086,
|
||
|
|
"rewards/rejected": 1.1412513256072998,
|
||
|
|
"ronpo/logit": 0.14496144652366638,
|
||
|
|
"ronpo/residual": 0.14660702645778656,
|
||
|
|
"ronpo/residual_abs": 0.17200031876564026,
|
||
|
|
"ronpo/target": -0.001645585522055626,
|
||
|
|
"ronpo/target_abs": 0.04828788712620735,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4130350947380066,
|
||
|
|
"drift/rejected_abs": 0.10012461245059967,
|
||
|
|
"epoch": 0.18116805721096543,
|
||
|
|
"grad_norm": 18.875,
|
||
|
|
"learning_rate": 2.184502577160426e-08,
|
||
|
|
"logits/chosen": -2.149862289428711,
|
||
|
|
"logits/rejected": -2.119326114654541,
|
||
|
|
"logps/chosen": -0.27440696954727173,
|
||
|
|
"logps/rejected": -0.26447421312332153,
|
||
|
|
"loss": 1.456210970878601,
|
||
|
|
"loss/core": 1.372689962387085,
|
||
|
|
"loss/preference_sft": 0.25947293639183044,
|
||
|
|
"loss/reference_anchor": 0.8092624545097351,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 4.13002872467041,
|
||
|
|
"rewards/margins": 3.131789445877075,
|
||
|
|
"rewards/rejected": 0.9982389211654663,
|
||
|
|
"ronpo/logit": 0.3131789565086365,
|
||
|
|
"ronpo/residual": 0.3171713352203369,
|
||
|
|
"ronpo/residual_abs": 0.35084468126296997,
|
||
|
|
"ronpo/target": -0.00399237172678113,
|
||
|
|
"ronpo/target_abs": 0.05224527791142464,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19252817332744598,
|
||
|
|
"drift/rejected_abs": 0.0961676687002182,
|
||
|
|
"epoch": 0.18275725069527216,
|
||
|
|
"grad_norm": 3.796875,
|
||
|
|
"learning_rate": 2.130476472175973e-08,
|
||
|
|
"logits/chosen": -2.009521722793579,
|
||
|
|
"logits/rejected": -2.0971667766571045,
|
||
|
|
"logps/chosen": -0.282725065946579,
|
||
|
|
"logps/rejected": -0.27961233258247375,
|
||
|
|
"loss": 0.06870145350694656,
|
||
|
|
"loss/core": 0.05739961937069893,
|
||
|
|
"loss/preference_sft": 0.2835802137851715,
|
||
|
|
"loss/reference_anchor": 0.08466026186943054,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 1.9235862493515015,
|
||
|
|
"rewards/margins": 0.9644415974617004,
|
||
|
|
"rewards/rejected": 0.9591447114944458,
|
||
|
|
"ronpo/logit": 0.09644416719675064,
|
||
|
|
"ronpo/residual": 0.1159183606505394,
|
||
|
|
"ronpo/residual_abs": 0.14433664083480835,
|
||
|
|
"ronpo/target": -0.01947418972849846,
|
||
|
|
"ronpo/target_abs": 0.0581415593624115,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3638065457344055,
|
||
|
|
"drift/rejected_abs": 0.16699595749378204,
|
||
|
|
"epoch": 0.18434644417957885,
|
||
|
|
"grad_norm": 3.46875,
|
||
|
|
"learning_rate": 2.0766262403774385e-08,
|
||
|
|
"logits/chosen": -2.002498149871826,
|
||
|
|
"logits/rejected": -2.032501697540283,
|
||
|
|
"logps/chosen": -0.28927743434906006,
|
||
|
|
"logps/rejected": -0.26778870820999146,
|
||
|
|
"loss": 0.6307193040847778,
|
||
|
|
"loss/core": 0.5643881559371948,
|
||
|
|
"loss/preference_sft": 0.2721135914325714,
|
||
|
|
"loss/reference_anchor": 0.6360998153686523,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.637876033782959,
|
||
|
|
"rewards/margins": 1.9686304330825806,
|
||
|
|
"rewards/rejected": 1.669245719909668,
|
||
|
|
"ronpo/logit": 0.19686304032802582,
|
||
|
|
"ronpo/residual": 0.1981230527162552,
|
||
|
|
"ronpo/residual_abs": 0.24869951605796814,
|
||
|
|
"ronpo/target": -0.0012599985348060727,
|
||
|
|
"ronpo/target_abs": 0.04539477080106735,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23143327236175537,
|
||
|
|
"drift/rejected_abs": 0.09587498754262924,
|
||
|
|
"epoch": 0.18593563766388557,
|
||
|
|
"grad_norm": 1.796875,
|
||
|
|
"learning_rate": 2.0229775115586378e-08,
|
||
|
|
"logits/chosen": -1.9558576345443726,
|
||
|
|
"logits/rejected": -1.9806501865386963,
|
||
|
|
"logps/chosen": -0.28681617975234985,
|
||
|
|
"logps/rejected": -0.28583577275276184,
|
||
|
|
"loss": 0.16384239494800568,
|
||
|
|
"loss/core": 0.14417728781700134,
|
||
|
|
"loss/preference_sft": 0.2843136489391327,
|
||
|
|
"loss/reference_anchor": 0.1682194620370865,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 2.3142428398132324,
|
||
|
|
"rewards/margins": 1.3574857711791992,
|
||
|
|
"rewards/rejected": 0.9567572474479675,
|
||
|
|
"ronpo/logit": 0.13574858009815216,
|
||
|
|
"ronpo/residual": 0.13210666179656982,
|
||
|
|
"ronpo/residual_abs": 0.16300050914287567,
|
||
|
|
"ronpo/target": 0.00364193064160645,
|
||
|
|
"ronpo/target_abs": 0.05011861398816109,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25644367933273315,
|
||
|
|
"drift/rejected_abs": 0.123075470328331,
|
||
|
|
"epoch": 0.1875248311481923,
|
||
|
|
"grad_norm": 1.671875,
|
||
|
|
"learning_rate": 1.9695558196088843e-08,
|
||
|
|
"logits/chosen": -2.1181397438049316,
|
||
|
|
"logits/rejected": -2.086034059524536,
|
||
|
|
"logps/chosen": -0.31038814783096313,
|
||
|
|
"logps/rejected": -0.31416597962379456,
|
||
|
|
"loss": 0.5733916163444519,
|
||
|
|
"loss/core": 0.526887059211731,
|
||
|
|
"loss/preference_sft": 0.30243176221847534,
|
||
|
|
"loss/reference_anchor": 0.4348035454750061,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.560229778289795,
|
||
|
|
"rewards/margins": 1.3305037021636963,
|
||
|
|
"rewards/rejected": 1.2297260761260986,
|
||
|
|
"ronpo/logit": 0.1330503672361374,
|
||
|
|
"ronpo/residual": 0.1380425989627838,
|
||
|
|
"ronpo/residual_abs": 0.1819685846567154,
|
||
|
|
"ronpo/target": -0.0049922168254852295,
|
||
|
|
"ronpo/target_abs": 0.04599359631538391,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.232018381357193,
|
||
|
|
"drift/rejected_abs": 0.11203984171152115,
|
||
|
|
"epoch": 0.18911402463249902,
|
||
|
|
"grad_norm": 5.28125,
|
||
|
|
"learning_rate": 1.9163865903602372e-08,
|
||
|
|
"logits/chosen": -1.9771127700805664,
|
||
|
|
"logits/rejected": -1.9363014698028564,
|
||
|
|
"logps/chosen": -0.28953754901885986,
|
||
|
|
"logps/rejected": -0.2934724986553192,
|
||
|
|
"loss": 0.2176361083984375,
|
||
|
|
"loss/core": 0.19568440318107605,
|
||
|
|
"loss/preference_sft": 0.29219508171081543,
|
||
|
|
"loss/reference_anchor": 0.19029754400253296,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.3178868293762207,
|
||
|
|
"rewards/margins": 1.200582504272461,
|
||
|
|
"rewards/rejected": 1.1173043251037598,
|
||
|
|
"ronpo/logit": 0.12005823850631714,
|
||
|
|
"ronpo/residual": 0.12189905345439911,
|
||
|
|
"ronpo/residual_abs": 0.17195966839790344,
|
||
|
|
"ronpo/target": -0.0018408190226182342,
|
||
|
|
"ronpo/target_abs": 0.048326022922992706,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20352081954479218,
|
||
|
|
"drift/rejected_abs": 0.14214810729026794,
|
||
|
|
"epoch": 0.1907032181168057,
|
||
|
|
"grad_norm": 154.0,
|
||
|
|
"learning_rate": 1.8634951294861806e-08,
|
||
|
|
"logits/chosen": -2.1343958377838135,
|
||
|
|
"logits/rejected": -2.1073379516601562,
|
||
|
|
"logps/chosen": -0.2995579242706299,
|
||
|
|
"logps/rejected": -0.3202636241912842,
|
||
|
|
"loss": 0.4734867513179779,
|
||
|
|
"loss/core": 0.4431106150150299,
|
||
|
|
"loss/preference_sft": 0.3036237061023712,
|
||
|
|
"loss/reference_anchor": 0.27340036630630493,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.0335845947265625,
|
||
|
|
"rewards/margins": 0.6139718890190125,
|
||
|
|
"rewards/rejected": 1.4196126461029053,
|
||
|
|
"ronpo/logit": 0.06139718368649483,
|
||
|
|
"ronpo/residual": 0.06256367266178131,
|
||
|
|
"ronpo/residual_abs": 0.2166094332933426,
|
||
|
|
"ronpo/target": -0.0011664947960525751,
|
||
|
|
"ronpo/target_abs": 0.049785248935222626,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.34506934881210327,
|
||
|
|
"drift/rejected_abs": 0.1576377898454666,
|
||
|
|
"epoch": 0.19229241160111243,
|
||
|
|
"grad_norm": 1.796875,
|
||
|
|
"learning_rate": 1.810906610457502e-08,
|
||
|
|
"logits/chosen": -2.029658794403076,
|
||
|
|
"logits/rejected": -1.9551337957382202,
|
||
|
|
"logps/chosen": -0.28458595275878906,
|
||
|
|
"logps/rejected": -0.2794283330440521,
|
||
|
|
"loss": 0.507605254650116,
|
||
|
|
"loss/core": 0.44724521040916443,
|
||
|
|
"loss/preference_sft": 0.2836672067642212,
|
||
|
|
"loss/reference_anchor": 0.5752339363098145,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.4416699409484863,
|
||
|
|
"rewards/margins": 1.8752292394638062,
|
||
|
|
"rewards/rejected": 1.5664407014846802,
|
||
|
|
"ronpo/logit": 0.18752291798591614,
|
||
|
|
"ronpo/residual": 0.1868840903043747,
|
||
|
|
"ronpo/residual_abs": 0.22322769463062286,
|
||
|
|
"ronpo/target": 0.0006388464826159179,
|
||
|
|
"ronpo/target_abs": 0.05182533711194992,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3366909623146057,
|
||
|
|
"drift/rejected_abs": 0.1535976231098175,
|
||
|
|
"epoch": 0.19388160508541916,
|
||
|
|
"grad_norm": 92.0,
|
||
|
|
"learning_rate": 1.7586460625610727e-08,
|
||
|
|
"logits/chosen": -2.0207197666168213,
|
||
|
|
"logits/rejected": -1.9711004495620728,
|
||
|
|
"logps/chosen": -0.30812549591064453,
|
||
|
|
"logps/rejected": -0.29248955845832825,
|
||
|
|
"loss": 0.3734953701496124,
|
||
|
|
"loss/core": 0.3218352198600769,
|
||
|
|
"loss/preference_sft": 0.30512887239456177,
|
||
|
|
"loss/reference_anchor": 0.48608869314193726,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.365083694458008,
|
||
|
|
"rewards/margins": 1.8310083150863647,
|
||
|
|
"rewards/rejected": 1.534075140953064,
|
||
|
|
"ronpo/logit": 0.1831008344888687,
|
||
|
|
"ronpo/residual": 0.18432757258415222,
|
||
|
|
"ronpo/residual_abs": 0.2148648202419281,
|
||
|
|
"ronpo/target": -0.00122673565056175,
|
||
|
|
"ronpo/target_abs": 0.05218973010778427,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25414878129959106,
|
||
|
|
"drift/rejected_abs": 0.1142294853925705,
|
||
|
|
"epoch": 0.19547079856972585,
|
||
|
|
"grad_norm": 129.0,
|
||
|
|
"learning_rate": 1.70673835898727e-08,
|
||
|
|
"logits/chosen": -2.2016429901123047,
|
||
|
|
"logits/rejected": -2.16279673576355,
|
||
|
|
"logps/chosen": -0.2886617183685303,
|
||
|
|
"logps/rejected": -0.28729248046875,
|
||
|
|
"loss": 0.32489103078842163,
|
||
|
|
"loss/core": 0.2884674370288849,
|
||
|
|
"loss/preference_sft": 0.2909470200538635,
|
||
|
|
"loss/reference_anchor": 0.3351413607597351,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.5414881706237793,
|
||
|
|
"rewards/margins": 1.4145499467849731,
|
||
|
|
"rewards/rejected": 1.1269382238388062,
|
||
|
|
"ronpo/logit": 0.14145499467849731,
|
||
|
|
"ronpo/residual": 0.140377014875412,
|
||
|
|
"ronpo/residual_abs": 0.17693236470222473,
|
||
|
|
"ronpo/target": 0.0010779699077829719,
|
||
|
|
"ronpo/target_abs": 0.05794703960418701,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22227022051811218,
|
||
|
|
"drift/rejected_abs": 0.10932586342096329,
|
||
|
|
"epoch": 0.19705999205403257,
|
||
|
|
"grad_norm": 13.6875,
|
||
|
|
"learning_rate": 1.6552082049916824e-08,
|
||
|
|
"logits/chosen": -2.0862820148468018,
|
||
|
|
"logits/rejected": -2.021430253982544,
|
||
|
|
"logps/chosen": -0.291514128446579,
|
||
|
|
"logps/rejected": -0.28724825382232666,
|
||
|
|
"loss": 0.11000826209783554,
|
||
|
|
"loss/core": 0.09267652034759521,
|
||
|
|
"loss/preference_sft": 0.2868395745754242,
|
||
|
|
"loss/reference_anchor": 0.1446334570646286,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.2227022647857666,
|
||
|
|
"rewards/margins": 1.1303623914718628,
|
||
|
|
"rewards/rejected": 1.0923398733139038,
|
||
|
|
"ronpo/logit": 0.11303625255823135,
|
||
|
|
"ronpo/residual": 0.1041765958070755,
|
||
|
|
"ronpo/residual_abs": 0.14824050664901733,
|
||
|
|
"ronpo/target": 0.00885964184999466,
|
||
|
|
"ronpo/target_abs": 0.05069901794195175,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24614998698234558,
|
||
|
|
"drift/rejected_abs": 0.0928288996219635,
|
||
|
|
"epoch": 0.1986491855383393,
|
||
|
|
"grad_norm": 43.75,
|
||
|
|
"learning_rate": 1.604080126136749e-08,
|
||
|
|
"logits/chosen": -2.038576126098633,
|
||
|
|
"logits/rejected": -2.042659044265747,
|
||
|
|
"logps/chosen": -0.2963336408138275,
|
||
|
|
"logps/rejected": -0.29823175072669983,
|
||
|
|
"loss": 0.34140458703041077,
|
||
|
|
"loss/core": 0.3157057762145996,
|
||
|
|
"loss/preference_sft": 0.2944321036338806,
|
||
|
|
"loss/reference_anchor": 0.22754497826099396,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.4598302841186523,
|
||
|
|
"rewards/margins": 1.5353715419769287,
|
||
|
|
"rewards/rejected": 0.9244586825370789,
|
||
|
|
"ronpo/logit": 0.15353716909885406,
|
||
|
|
"ronpo/residual": 0.15676531195640564,
|
||
|
|
"ronpo/residual_abs": 0.19076670706272125,
|
||
|
|
"ronpo/target": -0.0032281391322612762,
|
||
|
|
"ronpo/target_abs": 0.052575696259737015,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23410654067993164,
|
||
|
|
"drift/rejected_abs": 0.08934091031551361,
|
||
|
|
"epoch": 0.20023837902264602,
|
||
|
|
"grad_norm": 6.875,
|
||
|
|
"learning_rate": 1.5533784566189177e-08,
|
||
|
|
"logits/chosen": -2.328397035598755,
|
||
|
|
"logits/rejected": -2.4190592765808105,
|
||
|
|
"logps/chosen": -0.2671896815299988,
|
||
|
|
"logps/rejected": -0.2637612819671631,
|
||
|
|
"loss": 0.32761818170547485,
|
||
|
|
"loss/core": 0.29400014877319336,
|
||
|
|
"loss/preference_sft": 0.26751917600631714,
|
||
|
|
"loss/reference_anchor": 0.309428334236145,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 2.3410582542419434,
|
||
|
|
"rewards/margins": 1.4506725072860718,
|
||
|
|
"rewards/rejected": 0.8903859853744507,
|
||
|
|
"ronpo/logit": 0.1450672447681427,
|
||
|
|
"ronpo/residual": 0.14449718594551086,
|
||
|
|
"ronpo/residual_abs": 0.17810878157615662,
|
||
|
|
"ronpo/target": 0.0005700558540411294,
|
||
|
|
"ronpo/target_abs": 0.04920119792222977,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2907083034515381,
|
||
|
|
"drift/rejected_abs": 0.1089189276099205,
|
||
|
|
"epoch": 0.2018275725069527,
|
||
|
|
"grad_norm": 15.75,
|
||
|
|
"learning_rate": 1.5031273276868844e-08,
|
||
|
|
"logits/chosen": -2.112438440322876,
|
||
|
|
"logits/rejected": -2.1512842178344727,
|
||
|
|
"logps/chosen": -0.26441654562950134,
|
||
|
|
"logps/rejected": -0.2715560495853424,
|
||
|
|
"loss": 0.3996767997741699,
|
||
|
|
"loss/core": 0.3656352162361145,
|
||
|
|
"loss/preference_sft": 0.265216588973999,
|
||
|
|
"loss/reference_anchor": 0.31389421224594116,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.905461549758911,
|
||
|
|
"rewards/margins": 1.824873685836792,
|
||
|
|
"rewards/rejected": 1.0805878639221191,
|
||
|
|
"ronpo/logit": 0.1824873685836792,
|
||
|
|
"ronpo/residual": 0.18542273342609406,
|
||
|
|
"ronpo/residual_abs": 0.21292610466480255,
|
||
|
|
"ronpo/target": -0.002935352735221386,
|
||
|
|
"ronpo/target_abs": 0.04666103422641754,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.324721097946167,
|
||
|
|
"drift/rejected_abs": 0.09727884829044342,
|
||
|
|
"epoch": 0.20341676599125944,
|
||
|
|
"grad_norm": 1.25,
|
||
|
|
"learning_rate": 1.4533506561564305e-08,
|
||
|
|
"logits/chosen": -2.0161423683166504,
|
||
|
|
"logits/rejected": -2.0071678161621094,
|
||
|
|
"logps/chosen": -0.31316065788269043,
|
||
|
|
"logps/rejected": -0.3185476064682007,
|
||
|
|
"loss": 0.8060309290885925,
|
||
|
|
"loss/core": 0.7467989921569824,
|
||
|
|
"loss/preference_sft": 0.30903545022010803,
|
||
|
|
"loss/reference_anchor": 0.5614163875579834,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.2465310096740723,
|
||
|
|
"rewards/margins": 2.313912868499756,
|
||
|
|
"rewards/rejected": 0.9326180219650269,
|
||
|
|
"ronpo/logit": 0.23139123618602753,
|
||
|
|
"ronpo/residual": 0.22939686477184296,
|
||
|
|
"ronpo/residual_abs": 0.26494738459587097,
|
||
|
|
"ronpo/target": 0.0019944149535149336,
|
||
|
|
"ronpo/target_abs": 0.05449249595403671,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2655789256095886,
|
||
|
|
"drift/rejected_abs": 0.11475946009159088,
|
||
|
|
"epoch": 0.20500595947556616,
|
||
|
|
"grad_norm": 9.1875,
|
||
|
|
"learning_rate": 1.4040721330273062e-08,
|
||
|
|
"logits/chosen": -2.1235086917877197,
|
||
|
|
"logits/rejected": -2.1292612552642822,
|
||
|
|
"logps/chosen": -0.3084896206855774,
|
||
|
|
"logps/rejected": -0.2943161129951477,
|
||
|
|
"loss": 0.36062735319137573,
|
||
|
|
"loss/core": 0.3258729875087738,
|
||
|
|
"loss/preference_sft": 0.29873496294021606,
|
||
|
|
"loss/reference_anchor": 0.3176696300506592,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.6545615196228027,
|
||
|
|
"rewards/margins": 1.5085389614105225,
|
||
|
|
"rewards/rejected": 1.1460225582122803,
|
||
|
|
"ronpo/logit": 0.15085388720035553,
|
||
|
|
"ronpo/residual": 0.14222462475299835,
|
||
|
|
"ronpo/residual_abs": 0.18793636560440063,
|
||
|
|
"ronpo/target": 0.008629274554550648,
|
||
|
|
"ronpo/target_abs": 0.05275329202413559,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15096431970596313,
|
||
|
|
"drift/rejected_abs": 0.09060250222682953,
|
||
|
|
"epoch": 0.20659515295987285,
|
||
|
|
"grad_norm": 1.53125,
|
||
|
|
"learning_rate": 1.3553152122076079e-08,
|
||
|
|
"logits/chosen": -2.2055015563964844,
|
||
|
|
"logits/rejected": -2.133873462677002,
|
||
|
|
"logps/chosen": -0.30167868733406067,
|
||
|
|
"logps/rejected": -0.29324105381965637,
|
||
|
|
"loss": 0.056435346603393555,
|
||
|
|
"loss/core": 0.046567730605602264,
|
||
|
|
"loss/preference_sft": 0.30248433351516724,
|
||
|
|
"loss/reference_anchor": 0.06842776387929916,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.5087251663208008,
|
||
|
|
"rewards/margins": 0.652277410030365,
|
||
|
|
"rewards/rejected": 0.856447696685791,
|
||
|
|
"ronpo/logit": 0.06522773951292038,
|
||
|
|
"ronpo/residual": 0.06795114278793335,
|
||
|
|
"ronpo/residual_abs": 0.11372838169336319,
|
||
|
|
"ronpo/target": -0.0027233988512307405,
|
||
|
|
"ronpo/target_abs": 0.05443141609430313,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20415695011615753,
|
||
|
|
"drift/rejected_abs": 0.06474515795707703,
|
||
|
|
"epoch": 0.20818434644417957,
|
||
|
|
"grad_norm": 169.0,
|
||
|
|
"learning_rate": 1.3071030993509787e-08,
|
||
|
|
"logits/chosen": -2.2794108390808105,
|
||
|
|
"logits/rejected": -2.285342216491699,
|
||
|
|
"logps/chosen": -0.2840138077735901,
|
||
|
|
"logps/rejected": -0.29859381914138794,
|
||
|
|
"loss": 0.38816702365875244,
|
||
|
|
"loss/core": 0.3634727895259857,
|
||
|
|
"loss/preference_sft": 0.2865355908870697,
|
||
|
|
"loss/reference_anchor": 0.21828866004943848,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.04097843170166,
|
||
|
|
"rewards/margins": 1.3973244428634644,
|
||
|
|
"rewards/rejected": 0.6436540484428406,
|
||
|
|
"ronpo/logit": 0.13973243534564972,
|
||
|
|
"ronpo/residual": 0.15421327948570251,
|
||
|
|
"ronpo/residual_abs": 0.18282660841941833,
|
||
|
|
"ronpo/target": -0.014480856247246265,
|
||
|
|
"ronpo/target_abs": 0.05253823474049568,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1760435551404953,
|
||
|
|
"drift/rejected_abs": 0.07354947924613953,
|
||
|
|
"epoch": 0.2097735399284863,
|
||
|
|
"grad_norm": 6.5,
|
||
|
|
"learning_rate": 1.2594587408119802e-08,
|
||
|
|
"logits/chosen": -2.2539172172546387,
|
||
|
|
"logits/rejected": -2.2968709468841553,
|
||
|
|
"logps/chosen": -0.2949877381324768,
|
||
|
|
"logps/rejected": -0.2898675203323364,
|
||
|
|
"loss": 0.14153003692626953,
|
||
|
|
"loss/core": 0.1278771013021469,
|
||
|
|
"loss/preference_sft": 0.2885059118270874,
|
||
|
|
"loss/reference_anchor": 0.10767867416143417,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 1.758472204208374,
|
||
|
|
"rewards/margins": 1.0266904830932617,
|
||
|
|
"rewards/rejected": 0.7317818999290466,
|
||
|
|
"ronpo/logit": 0.10266906023025513,
|
||
|
|
"ronpo/residual": 0.09563291817903519,
|
||
|
|
"ronpo/residual_abs": 0.12419185787439346,
|
||
|
|
"ronpo/target": 0.007036135531961918,
|
||
|
|
"ronpo/target_abs": 0.051446299999952316,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23514601588249207,
|
||
|
|
"drift/rejected_abs": 0.13599976897239685,
|
||
|
|
"epoch": 0.21136273341279302,
|
||
|
|
"grad_norm": 0.4765625,
|
||
|
|
"learning_rate": 1.2124048127248643e-08,
|
||
|
|
"logits/chosen": -2.0788931846618652,
|
||
|
|
"logits/rejected": -2.023658514022827,
|
||
|
|
"logps/chosen": -0.2821238040924072,
|
||
|
|
"logps/rejected": -0.2862771153450012,
|
||
|
|
"loss": 0.2474207878112793,
|
||
|
|
"loss/core": 0.2166004627943039,
|
||
|
|
"loss/preference_sft": 0.28421351313591003,
|
||
|
|
"loss/reference_anchor": 0.2797810733318329,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.348618268966675,
|
||
|
|
"rewards/margins": 0.9887655377388,
|
||
|
|
"rewards/rejected": 1.35985267162323,
|
||
|
|
"ronpo/logit": 0.09887654334306717,
|
||
|
|
"ronpo/residual": 0.09633339941501617,
|
||
|
|
"ronpo/residual_abs": 0.14032281935214996,
|
||
|
|
"ronpo/target": 0.002543145790696144,
|
||
|
|
"ronpo/target_abs": 0.044390588998794556,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21025514602661133,
|
||
|
|
"drift/rejected_abs": 0.07857976108789444,
|
||
|
|
"epoch": 0.2129519268970997,
|
||
|
|
"grad_norm": 5.65625,
|
||
|
|
"learning_rate": 1.1659637102109713e-08,
|
||
|
|
"logits/chosen": -2.3492414951324463,
|
||
|
|
"logits/rejected": -2.356902599334717,
|
||
|
|
"logps/chosen": -0.2673395574092865,
|
||
|
|
"logps/rejected": -0.2719830572605133,
|
||
|
|
"loss": 0.20967233180999756,
|
||
|
|
"loss/core": 0.19076594710350037,
|
||
|
|
"loss/preference_sft": 0.2634381651878357,
|
||
|
|
"loss/reference_anchor": 0.1627199351787567,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.1009294986724854,
|
||
|
|
"rewards/margins": 1.3163961172103882,
|
||
|
|
"rewards/rejected": 0.7845333814620972,
|
||
|
|
"ronpo/logit": 0.13163962960243225,
|
||
|
|
"ronpo/residual": 0.11512235552072525,
|
||
|
|
"ronpo/residual_abs": 0.15650084614753723,
|
||
|
|
"ronpo/target": 0.016517281532287598,
|
||
|
|
"ronpo/target_abs": 0.05511252209544182,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.40095004439353943,
|
||
|
|
"drift/rejected_abs": 0.08112253993749619,
|
||
|
|
"epoch": 0.21454112038140644,
|
||
|
|
"grad_norm": 2.3125,
|
||
|
|
"learning_rate": 1.1201575367198545e-08,
|
||
|
|
"logits/chosen": -1.9876753091812134,
|
||
|
|
"logits/rejected": -2.1013565063476562,
|
||
|
|
"logps/chosen": -0.27906253933906555,
|
||
|
|
"logps/rejected": -0.26979774236679077,
|
||
|
|
"loss": 1.4776394367218018,
|
||
|
|
"loss/core": 1.3953857421875,
|
||
|
|
"loss/preference_sft": 0.26779764890670776,
|
||
|
|
"loss/reference_anchor": 0.7957568764686584,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 4.009209156036377,
|
||
|
|
"rewards/margins": 3.201277256011963,
|
||
|
|
"rewards/rejected": 0.8079322576522827,
|
||
|
|
"ronpo/logit": 0.3201276957988739,
|
||
|
|
"ronpo/residual": 0.3296274244785309,
|
||
|
|
"ronpo/residual_abs": 0.34753426909446716,
|
||
|
|
"ronpo/target": -0.009499681182205677,
|
||
|
|
"ronpo/target_abs": 0.053924452513456345,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19604100286960602,
|
||
|
|
"drift/rejected_abs": 0.1168225035071373,
|
||
|
|
"epoch": 0.21613031386571316,
|
||
|
|
"grad_norm": 2.265625,
|
||
|
|
"learning_rate": 1.0750080935092422e-08,
|
||
|
|
"logits/chosen": -1.9559427499771118,
|
||
|
|
"logits/rejected": -1.9504743814468384,
|
||
|
|
"logps/chosen": -0.28676366806030273,
|
||
|
|
"logps/rejected": -0.2847217917442322,
|
||
|
|
"loss": 0.07230311632156372,
|
||
|
|
"loss/core": 0.05109725520014763,
|
||
|
|
"loss/preference_sft": 0.28723257780075073,
|
||
|
|
"loss/reference_anchor": 0.18333539366722107,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.9591118097305298,
|
||
|
|
"rewards/margins": 0.791954755783081,
|
||
|
|
"rewards/rejected": 1.1671569347381592,
|
||
|
|
"ronpo/logit": 0.07919548451900482,
|
||
|
|
"ronpo/residual": 0.0781867653131485,
|
||
|
|
"ronpo/residual_abs": 0.1194804310798645,
|
||
|
|
"ronpo/target": 0.0010087231639772654,
|
||
|
|
"ronpo/target_abs": 0.05514944717288017,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18207547068595886,
|
||
|
|
"drift/rejected_abs": 0.09372608363628387,
|
||
|
|
"epoch": 0.21771950735001985,
|
||
|
|
"grad_norm": 3.109375,
|
||
|
|
"learning_rate": 1.0305368692688174e-08,
|
||
|
|
"logits/chosen": -1.9711679220199585,
|
||
|
|
"logits/rejected": -2.0610291957855225,
|
||
|
|
"logps/chosen": -0.29287078976631165,
|
||
|
|
"logps/rejected": -0.30058807134628296,
|
||
|
|
"loss": 0.05506179481744766,
|
||
|
|
"loss/core": 0.04439196363091469,
|
||
|
|
"loss/preference_sft": 0.2933097183704376,
|
||
|
|
"loss/reference_anchor": 0.07736729830503464,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.8186616897583008,
|
||
|
|
"rewards/margins": 0.8839622735977173,
|
||
|
|
"rewards/rejected": 0.9346992373466492,
|
||
|
|
"ronpo/logit": 0.08839622884988785,
|
||
|
|
"ronpo/residual": 0.08107195794582367,
|
||
|
|
"ronpo/residual_abs": 0.11638529598712921,
|
||
|
|
"ronpo/target": 0.007324265781790018,
|
||
|
|
"ronpo/target_abs": 0.04978422075510025,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4522300362586975,
|
||
|
|
"drift/rejected_abs": 0.1868419349193573,
|
||
|
|
"epoch": 0.21930870083432658,
|
||
|
|
"grad_norm": 440.0,
|
||
|
|
"learning_rate": 9.867650298927644e-09,
|
||
|
|
"logits/chosen": -1.9316879510879517,
|
||
|
|
"logits/rejected": -1.9761654138565063,
|
||
|
|
"logps/chosen": -0.27664968371391296,
|
||
|
|
"logps/rejected": -0.27031582593917847,
|
||
|
|
"loss": 0.9360002279281616,
|
||
|
|
"loss/core": 0.8222877383232117,
|
||
|
|
"loss/preference_sft": 0.2849580645561218,
|
||
|
|
"loss/reference_anchor": 1.108628511428833,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 4.52152156829834,
|
||
|
|
"rewards/margins": 2.65468692779541,
|
||
|
|
"rewards/rejected": 1.866834044456482,
|
||
|
|
"ronpo/logit": 0.2654687762260437,
|
||
|
|
"ronpo/residual": 0.2594831585884094,
|
||
|
|
"ronpo/residual_abs": 0.2916156053543091,
|
||
|
|
"ronpo/target": 0.0059855724684894085,
|
||
|
|
"ronpo/target_abs": 0.048761725425720215,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18377462029457092,
|
||
|
|
"drift/rejected_abs": 0.10978679358959198,
|
||
|
|
"epoch": 0.2208978943186333,
|
||
|
|
"grad_norm": 12.5,
|
||
|
|
"learning_rate": 9.437134084059515e-09,
|
||
|
|
"logits/chosen": -2.163043260574341,
|
||
|
|
"logits/rejected": -2.150045156478882,
|
||
|
|
"logps/chosen": -0.2978960871696472,
|
||
|
|
"logps/rejected": -0.30089619755744934,
|
||
|
|
"loss": 0.11494803428649902,
|
||
|
|
"loss/core": 0.09975998103618622,
|
||
|
|
"loss/preference_sft": 0.29669278860092163,
|
||
|
|
"loss/reference_anchor": 0.12221113592386246,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.8371896743774414,
|
||
|
|
"rewards/margins": 0.7406196594238281,
|
||
|
|
"rewards/rejected": 1.0965700149536133,
|
||
|
|
"ronpo/logit": 0.07406197488307953,
|
||
|
|
"ronpo/residual": 0.07061334699392319,
|
||
|
|
"ronpo/residual_abs": 0.1484992951154709,
|
||
|
|
"ronpo/target": 0.003448624163866043,
|
||
|
|
"ronpo/target_abs": 0.05069219321012497,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 695
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22288422286510468,
|
||
|
|
"drift/rejected_abs": 0.13863565027713776,
|
||
|
|
"epoch": 0.22248708780294002,
|
||
|
|
"grad_norm": 5.9375,
|
||
|
|
"learning_rate": 9.014024950485382e-09,
|
||
|
|
"logits/chosen": -2.21722674369812,
|
||
|
|
"logits/rejected": -2.178441047668457,
|
||
|
|
"logps/chosen": -0.32178056240081787,
|
||
|
|
"logps/rejected": -0.31281206011772156,
|
||
|
|
"loss": 0.460977703332901,
|
||
|
|
"loss/core": 0.4184110760688782,
|
||
|
|
"loss/preference_sft": 0.31066298484802246,
|
||
|
|
"loss/reference_anchor": 0.3945997357368469,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.2285068035125732,
|
||
|
|
"rewards/margins": 0.8443525433540344,
|
||
|
|
"rewards/rejected": 1.3841540813446045,
|
||
|
|
"ronpo/logit": 0.08443526178598404,
|
||
|
|
"ronpo/residual": 0.08723791688680649,
|
||
|
|
"ronpo/residual_abs": 0.1965109258890152,
|
||
|
|
"ronpo/target": -0.002802655566483736,
|
||
|
|
"ronpo/target_abs": 0.05208230018615723,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18381109833717346,
|
||
|
|
"drift/rejected_abs": 0.08396384865045547,
|
||
|
|
"epoch": 0.22407628128724671,
|
||
|
|
"grad_norm": 2.359375,
|
||
|
|
"learning_rate": 8.59852427523732e-09,
|
||
|
|
"logits/chosen": -2.262627601623535,
|
||
|
|
"logits/rejected": -2.367927312850952,
|
||
|
|
"logps/chosen": -0.2676858901977539,
|
||
|
|
"logps/rejected": -0.27092617750167847,
|
||
|
|
"loss": 0.08023377507925034,
|
||
|
|
"loss/core": 0.06743164360523224,
|
||
|
|
"loss/preference_sft": 0.2751052975654602,
|
||
|
|
"loss/reference_anchor": 0.10051079839468002,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.8374383449554443,
|
||
|
|
"rewards/margins": 1.0038096904754639,
|
||
|
|
"rewards/rejected": 0.8336284756660461,
|
||
|
|
"ronpo/logit": 0.10038097947835922,
|
||
|
|
"ronpo/residual": 0.10365533828735352,
|
||
|
|
"ronpo/residual_abs": 0.1367378532886505,
|
||
|
|
"ronpo/target": -0.0032743613701313734,
|
||
|
|
"ronpo/target_abs": 0.05378307029604912,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28812941908836365,
|
||
|
|
"drift/rejected_abs": 0.10422752052545547,
|
||
|
|
"epoch": 0.22566547477155344,
|
||
|
|
"grad_norm": 103.5,
|
||
|
|
"learning_rate": 8.190829814133293e-09,
|
||
|
|
"logits/chosen": -2.1671175956726074,
|
||
|
|
"logits/rejected": -2.1793408393859863,
|
||
|
|
"logps/chosen": -0.30027687549591064,
|
||
|
|
"logps/rejected": -0.29821285605430603,
|
||
|
|
"loss": 0.7324048280715942,
|
||
|
|
"loss/core": 0.6899200677871704,
|
||
|
|
"loss/preference_sft": 0.29592448472976685,
|
||
|
|
"loss/reference_anchor": 0.3952544927597046,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.881293773651123,
|
||
|
|
"rewards/margins": 1.9280929565429688,
|
||
|
|
"rewards/rejected": 0.9532011151313782,
|
||
|
|
"ronpo/logit": 0.19280926883220673,
|
||
|
|
"ronpo/residual": 0.19424277544021606,
|
||
|
|
"ronpo/residual_abs": 0.24397198855876923,
|
||
|
|
"ronpo/target": -0.0014334864681586623,
|
||
|
|
"ronpo/target_abs": 0.055146049708127975,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1660590022802353,
|
||
|
|
"drift/rejected_abs": 0.09195362776517868,
|
||
|
|
"epoch": 0.22725466825586016,
|
||
|
|
"grad_norm": 1.328125,
|
||
|
|
"learning_rate": 7.791135607656146e-09,
|
||
|
|
"logits/chosen": -2.0482685565948486,
|
||
|
|
"logits/rejected": -1.9684371948242188,
|
||
|
|
"logps/chosen": -0.3031107187271118,
|
||
|
|
"logps/rejected": -0.3003300726413727,
|
||
|
|
"loss": 0.07312048971652985,
|
||
|
|
"loss/core": 0.06187158077955246,
|
||
|
|
"loss/preference_sft": 0.3050219416618347,
|
||
|
|
"loss/reference_anchor": 0.08198684453964233,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.6589847803115845,
|
||
|
|
"rewards/margins": 0.7399789690971375,
|
||
|
|
"rewards/rejected": 0.9190058708190918,
|
||
|
|
"ronpo/logit": 0.07399789243936539,
|
||
|
|
"ronpo/residual": 0.06657494604587555,
|
||
|
|
"ronpo/residual_abs": 0.1202554851770401,
|
||
|
|
"ronpo/target": 0.007422941271215677,
|
||
|
|
"ronpo/target_abs": 0.04994440823793411,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3468184471130371,
|
||
|
|
"drift/rejected_abs": 0.14277954399585724,
|
||
|
|
"epoch": 0.22884386174016685,
|
||
|
|
"grad_norm": 452.0,
|
||
|
|
"learning_rate": 7.399631888600796e-09,
|
||
|
|
"logits/chosen": -1.9520069360733032,
|
||
|
|
"logits/rejected": -2.0314078330993652,
|
||
|
|
"logps/chosen": -0.29010164737701416,
|
||
|
|
"logps/rejected": -0.28294092416763306,
|
||
|
|
"loss": 0.6354092359542847,
|
||
|
|
"loss/core": 0.5749737024307251,
|
||
|
|
"loss/preference_sft": 0.2856560945510864,
|
||
|
|
"loss/reference_anchor": 0.5757890939712524,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.46818470954895,
|
||
|
|
"rewards/margins": 2.045009136199951,
|
||
|
|
"rewards/rejected": 1.4231754541397095,
|
||
|
|
"ronpo/logit": 0.20450091361999512,
|
||
|
|
"ronpo/residual": 0.214862659573555,
|
||
|
|
"ronpo/residual_abs": 0.2341841459274292,
|
||
|
|
"ronpo/target": -0.010361749678850174,
|
||
|
|
"ronpo/target_abs": 0.049705568701028824,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2965972125530243,
|
||
|
|
"drift/rejected_abs": 0.06823800504207611,
|
||
|
|
"epoch": 0.23043305522447358,
|
||
|
|
"grad_norm": 4.65625,
|
||
|
|
"learning_rate": 7.016504991533726e-09,
|
||
|
|
"logits/chosen": -2.0876588821411133,
|
||
|
|
"logits/rejected": -2.0283491611480713,
|
||
|
|
"logps/chosen": -0.28961247205734253,
|
||
|
|
"logps/rejected": -0.29187989234924316,
|
||
|
|
"loss": 1.162423849105835,
|
||
|
|
"loss/core": 1.1012930870056152,
|
||
|
|
"loss/preference_sft": 0.29563257098197937,
|
||
|
|
"loss/reference_anchor": 0.5817441344261169,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.964808464050293,
|
||
|
|
"rewards/margins": 2.2854580879211426,
|
||
|
|
"rewards/rejected": 0.6793509721755981,
|
||
|
|
"ronpo/logit": 0.22854574024677277,
|
||
|
|
"ronpo/residual": 0.23741579055786133,
|
||
|
|
"ronpo/residual_abs": 0.267715722322464,
|
||
|
|
"ronpo/target": -0.008870027028024197,
|
||
|
|
"ronpo/target_abs": 0.0548769049346447,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3146856725215912,
|
||
|
|
"drift/rejected_abs": 0.093979112803936,
|
||
|
|
"epoch": 0.2320222487087803,
|
||
|
|
"grad_norm": 2032.0,
|
||
|
|
"learning_rate": 6.641937264107866e-09,
|
||
|
|
"logits/chosen": -2.183267593383789,
|
||
|
|
"logits/rejected": -2.218003511428833,
|
||
|
|
"logps/chosen": -0.29960209131240845,
|
||
|
|
"logps/rejected": -0.29752200841903687,
|
||
|
|
"loss": 0.9546008110046387,
|
||
|
|
"loss/core": 0.8912256360054016,
|
||
|
|
"loss/preference_sft": 0.29938754439353943,
|
||
|
|
"loss/reference_anchor": 0.6038151383399963,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.1452383995056152,
|
||
|
|
"rewards/margins": 2.208786725997925,
|
||
|
|
"rewards/rejected": 0.9364517331123352,
|
||
|
|
"ronpo/logit": 0.22087864577770233,
|
||
|
|
"ronpo/residual": 0.21167048811912537,
|
||
|
|
"ronpo/residual_abs": 0.24570854008197784,
|
||
|
|
"ronpo/target": 0.009208188392221928,
|
||
|
|
"ronpo/target_abs": 0.050097811967134476,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4118995666503906,
|
||
|
|
"drift/rejected_abs": 0.15235108137130737,
|
||
|
|
"epoch": 0.23361144219308702,
|
||
|
|
"grad_norm": 11.6875,
|
||
|
|
"learning_rate": 6.276106980274945e-09,
|
||
|
|
"logits/chosen": -1.8743293285369873,
|
||
|
|
"logits/rejected": -1.925328016281128,
|
||
|
|
"logps/chosen": -0.31055787205696106,
|
||
|
|
"logps/rejected": -0.2830975651741028,
|
||
|
|
"loss": 1.4325848817825317,
|
||
|
|
"loss/core": 1.343813180923462,
|
||
|
|
"loss/preference_sft": 0.2843840718269348,
|
||
|
|
"loss/reference_anchor": 0.8592769503593445,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 4.117542266845703,
|
||
|
|
"rewards/margins": 2.6022226810455322,
|
||
|
|
"rewards/rejected": 1.5153193473815918,
|
||
|
|
"ronpo/logit": 0.2602222263813019,
|
||
|
|
"ronpo/residual": 0.2596417963504791,
|
||
|
|
"ronpo/residual_abs": 0.3390171229839325,
|
||
|
|
"ronpo/target": 0.0005804725224152207,
|
||
|
|
"ronpo/target_abs": 0.052908122539520264,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 735
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.40725356340408325,
|
||
|
|
"drift/rejected_abs": 0.17515242099761963,
|
||
|
|
"epoch": 0.23520063567739372,
|
||
|
|
"grad_norm": 9.1875,
|
||
|
|
"learning_rate": 5.919188255436777e-09,
|
||
|
|
"logits/chosen": -2.062140941619873,
|
||
|
|
"logits/rejected": -2.053471088409424,
|
||
|
|
"logps/chosen": -0.2751580774784088,
|
||
|
|
"logps/rejected": -0.2721211016178131,
|
||
|
|
"loss": 0.9029391407966614,
|
||
|
|
"loss/core": 0.7974163889884949,
|
||
|
|
"loss/preference_sft": 0.274557888507843,
|
||
|
|
"loss/reference_anchor": 1.0277706384658813,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 4.072251796722412,
|
||
|
|
"rewards/margins": 2.321990966796875,
|
||
|
|
"rewards/rejected": 1.7502609491348267,
|
||
|
|
"ronpo/logit": 0.23219911754131317,
|
||
|
|
"ronpo/residual": 0.22244659066200256,
|
||
|
|
"ronpo/residual_abs": 0.27284926176071167,
|
||
|
|
"ronpo/target": 0.009752524085342884,
|
||
|
|
"ronpo/target_abs": 0.04651670157909393,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2788258194923401,
|
||
|
|
"drift/rejected_abs": 0.13999952375888824,
|
||
|
|
"epoch": 0.23678982916170044,
|
||
|
|
"grad_norm": 1.1640625,
|
||
|
|
"learning_rate": 5.571350963575727e-09,
|
||
|
|
"logits/chosen": -1.965963363647461,
|
||
|
|
"logits/rejected": -1.9410635232925415,
|
||
|
|
"logps/chosen": -0.3108576834201813,
|
||
|
|
"logps/rejected": -0.29554474353790283,
|
||
|
|
"loss": 0.42476820945739746,
|
||
|
|
"loss/core": 0.38765430450439453,
|
||
|
|
"loss/preference_sft": 0.3039025664329529,
|
||
|
|
"loss/reference_anchor": 0.34074899554252625,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.785062313079834,
|
||
|
|
"rewards/margins": 1.4028537273406982,
|
||
|
|
"rewards/rejected": 1.382208228111267,
|
||
|
|
"ronpo/logit": 0.14028538763523102,
|
||
|
|
"ronpo/residual": 0.1309804618358612,
|
||
|
|
"ronpo/residual_abs": 0.1938273310661316,
|
||
|
|
"ronpo/target": 0.009304934181272984,
|
||
|
|
"ronpo/target_abs": 0.05095517635345459,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 745
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.49405422806739807,
|
||
|
|
"drift/rejected_abs": 0.08988231420516968,
|
||
|
|
"epoch": 0.23837902264600716,
|
||
|
|
"grad_norm": 28.875,
|
||
|
|
"learning_rate": 5.2327606564039225e-09,
|
||
|
|
"logits/chosen": -1.8724329471588135,
|
||
|
|
"logits/rejected": -2.024134397506714,
|
||
|
|
"logps/chosen": -0.3255074918270111,
|
||
|
|
"logps/rejected": -0.3025345206260681,
|
||
|
|
"loss": 1.7781035900115967,
|
||
|
|
"loss/core": 1.674705147743225,
|
||
|
|
"loss/preference_sft": 0.2975428104400635,
|
||
|
|
"loss/reference_anchor": 1.0042304992675781,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.939142227172852,
|
||
|
|
"rewards/margins": 4.043478012084961,
|
||
|
|
"rewards/rejected": 0.8956640958786011,
|
||
|
|
"ronpo/logit": 0.40434780716896057,
|
||
|
|
"ronpo/residual": 0.41805681586265564,
|
||
|
|
"ronpo/residual_abs": 0.43893271684646606,
|
||
|
|
"ronpo/target": -0.013708988204598427,
|
||
|
|
"ronpo/target_abs": 0.04939410090446472,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.550828754901886,
|
||
|
|
"drift/rejected_abs": 0.25195178389549255,
|
||
|
|
"epoch": 0.23996821613031386,
|
||
|
|
"grad_norm": 183.0,
|
||
|
|
"learning_rate": 4.903578484569568e-09,
|
||
|
|
"logits/chosen": -1.7783501148223877,
|
||
|
|
"logits/rejected": -1.706907033920288,
|
||
|
|
"logps/chosen": -0.2732313275337219,
|
||
|
|
"logps/rejected": -0.2803798317909241,
|
||
|
|
"loss": 1.803733468055725,
|
||
|
|
"loss/core": 1.6817384958267212,
|
||
|
|
"loss/preference_sft": 0.25985172390937805,
|
||
|
|
"loss/reference_anchor": 1.1939656734466553,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 5.507561683654785,
|
||
|
|
"rewards/margins": 3.0016417503356934,
|
||
|
|
"rewards/rejected": 2.505919933319092,
|
||
|
|
"ronpo/logit": 0.3001641631126404,
|
||
|
|
"ronpo/residual": 0.2940429151058197,
|
||
|
|
"ronpo/residual_abs": 0.4681456685066223,
|
||
|
|
"ronpo/target": 0.006121277809143066,
|
||
|
|
"ronpo/target_abs": 0.05067913979291916,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 755
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25603729486465454,
|
||
|
|
"drift/rejected_abs": 0.10197993367910385,
|
||
|
|
"epoch": 0.24155740961462058,
|
||
|
|
"grad_norm": 1.84375,
|
||
|
|
"learning_rate": 4.583961120958027e-09,
|
||
|
|
"logits/chosen": -2.2031126022338867,
|
||
|
|
"logits/rejected": -2.253415107727051,
|
||
|
|
"logps/chosen": -0.26150673627853394,
|
||
|
|
"logps/rejected": -0.26170969009399414,
|
||
|
|
"loss": 0.41260021924972534,
|
||
|
|
"loss/core": 0.379812628030777,
|
||
|
|
"loss/preference_sft": 0.2550763189792633,
|
||
|
|
"loss/reference_anchor": 0.3023683428764343,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.5603137016296387,
|
||
|
|
"rewards/margins": 1.5461097955703735,
|
||
|
|
"rewards/rejected": 1.0142039060592651,
|
||
|
|
"ronpo/logit": 0.15461096167564392,
|
||
|
|
"ronpo/residual": 0.1472678929567337,
|
||
|
|
"ronpo/residual_abs": 0.19386346638202667,
|
||
|
|
"ronpo/target": 0.007343077566474676,
|
||
|
|
"ronpo/target_abs": 0.05549933761358261,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16723796725273132,
|
||
|
|
"drift/rejected_abs": 0.09124104678630829,
|
||
|
|
"epoch": 0.2431466030989273,
|
||
|
|
"grad_norm": 1.125,
|
||
|
|
"learning_rate": 4.2740606861239594e-09,
|
||
|
|
"logits/chosen": -2.399120330810547,
|
||
|
|
"logits/rejected": -2.3372297286987305,
|
||
|
|
"logps/chosen": -0.2863905429840088,
|
||
|
|
"logps/rejected": -0.29001763463020325,
|
||
|
|
"loss": 0.04658041149377823,
|
||
|
|
"loss/core": 0.033091410994529724,
|
||
|
|
"loss/preference_sft": 0.28594884276390076,
|
||
|
|
"loss/reference_anchor": 0.1062951311469078,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6717811822891235,
|
||
|
|
"rewards/margins": 0.7596646547317505,
|
||
|
|
"rewards/rejected": 0.9121164083480835,
|
||
|
|
"ronpo/logit": 0.07596646994352341,
|
||
|
|
"ronpo/residual": 0.0702570378780365,
|
||
|
|
"ronpo/residual_abs": 0.10007613897323608,
|
||
|
|
"ronpo/target": 0.005709432996809483,
|
||
|
|
"ronpo/target_abs": 0.043748289346694946,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 765
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.33148249983787537,
|
||
|
|
"drift/rejected_abs": 0.10127073526382446,
|
||
|
|
"epoch": 0.24473579658323402,
|
||
|
|
"grad_norm": 65.0,
|
||
|
|
"learning_rate": 3.974024675890189e-09,
|
||
|
|
"logits/chosen": -1.9635722637176514,
|
||
|
|
"logits/rejected": -2.0935862064361572,
|
||
|
|
"logps/chosen": -0.254546195268631,
|
||
|
|
"logps/rejected": -0.27278387546539307,
|
||
|
|
"loss": 0.6218898296356201,
|
||
|
|
"loss/core": 0.5817257761955261,
|
||
|
|
"loss/preference_sft": 0.25621524453163147,
|
||
|
|
"loss/reference_anchor": 0.3760190010070801,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.3138370513916016,
|
||
|
|
"rewards/margins": 2.3918099403381348,
|
||
|
|
"rewards/rejected": 0.9220269322395325,
|
||
|
|
"ronpo/logit": 0.23918099701404572,
|
||
|
|
"ronpo/residual": 0.2456188201904297,
|
||
|
|
"ronpo/residual_abs": 0.27006497979164124,
|
||
|
|
"ronpo/target": -0.006437826901674271,
|
||
|
|
"ronpo/target_abs": 0.05173778533935547,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.344692587852478,
|
||
|
|
"drift/rejected_abs": 0.1439075767993927,
|
||
|
|
"epoch": 0.24632499006754072,
|
||
|
|
"grad_norm": 0.94921875,
|
||
|
|
"learning_rate": 3.6839958911476955e-09,
|
||
|
|
"logits/chosen": -1.8287601470947266,
|
||
|
|
"logits/rejected": -1.816575288772583,
|
||
|
|
"logps/chosen": -0.2944333553314209,
|
||
|
|
"logps/rejected": -0.29648080468177795,
|
||
|
|
"loss": 0.5895982980728149,
|
||
|
|
"loss/core": 0.5448721051216125,
|
||
|
|
"loss/preference_sft": 0.29511556029319763,
|
||
|
|
"loss/reference_anchor": 0.41775059700012207,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.4456286430358887,
|
||
|
|
"rewards/margins": 2.052269458770752,
|
||
|
|
"rewards/rejected": 1.393358826637268,
|
||
|
|
"ronpo/logit": 0.20522697269916534,
|
||
|
|
"ronpo/residual": 0.20050707459449768,
|
||
|
|
"ronpo/residual_abs": 0.23854902386665344,
|
||
|
|
"ronpo/target": 0.004719903226941824,
|
||
|
|
"ronpo/target_abs": 0.0534793958067894,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 775
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.35038870573043823,
|
||
|
|
"drift/rejected_abs": 0.1556740552186966,
|
||
|
|
"epoch": 0.24791418355184744,
|
||
|
|
"grad_norm": 3.421875,
|
||
|
|
"learning_rate": 3.404112369890108e-09,
|
||
|
|
"logits/chosen": -2.1595654487609863,
|
||
|
|
"logits/rejected": -2.1828014850616455,
|
||
|
|
"logps/chosen": -0.2698805332183838,
|
||
|
|
"logps/rejected": -0.27884024381637573,
|
||
|
|
"loss": 0.37909770011901855,
|
||
|
|
"loss/core": 0.3323928713798523,
|
||
|
|
"loss/preference_sft": 0.2687809467315674,
|
||
|
|
"loss/reference_anchor": 0.4401705265045166,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.50298810005188,
|
||
|
|
"rewards/margins": 1.947210669517517,
|
||
|
|
"rewards/rejected": 1.5557773113250732,
|
||
|
|
"ronpo/logit": 0.1947210729122162,
|
||
|
|
"ronpo/residual": 0.19420704245567322,
|
||
|
|
"ronpo/residual_abs": 0.26663556694984436,
|
||
|
|
"ronpo/target": 0.0005140554858371615,
|
||
|
|
"ronpo/target_abs": 0.05390627309679985,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4771912097930908,
|
||
|
|
"drift/rejected_abs": 0.10854575783014297,
|
||
|
|
"epoch": 0.24950337703615416,
|
||
|
|
"grad_norm": 4.1875,
|
||
|
|
"learning_rate": 3.1345073215151064e-09,
|
||
|
|
"logits/chosen": -2.103487491607666,
|
||
|
|
"logits/rejected": -2.214913845062256,
|
||
|
|
"logps/chosen": -0.2716423571109772,
|
||
|
|
"logps/rejected": -0.2626985013484955,
|
||
|
|
"loss": 1.8278989791870117,
|
||
|
|
"loss/core": 1.7319368124008179,
|
||
|
|
"loss/preference_sft": 0.2729416489601135,
|
||
|
|
"loss/reference_anchor": 0.9323285818099976,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 4.765838146209717,
|
||
|
|
"rewards/margins": 3.703094482421875,
|
||
|
|
"rewards/rejected": 1.0627435445785522,
|
||
|
|
"ronpo/logit": 0.370309442281723,
|
||
|
|
"ronpo/residual": 0.3793594241142273,
|
||
|
|
"ronpo/residual_abs": 0.40880364179611206,
|
||
|
|
"ronpo/target": -0.009049968793988228,
|
||
|
|
"ronpo/target_abs": 0.05032235383987427,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3605960011482239,
|
||
|
|
"drift/rejected_abs": 0.12088502943515778,
|
||
|
|
"epoch": 0.25109257052046086,
|
||
|
|
"grad_norm": 196.0,
|
||
|
|
"learning_rate": 2.8753090634239558e-09,
|
||
|
|
"logits/chosen": -2.0059657096862793,
|
||
|
|
"logits/rejected": -2.025968074798584,
|
||
|
|
"logps/chosen": -0.3111969530582428,
|
||
|
|
"logps/rejected": -0.29000169038772583,
|
||
|
|
"loss": 0.5503872036933899,
|
||
|
|
"loss/core": 0.5009673237800598,
|
||
|
|
"loss/preference_sft": 0.30275315046310425,
|
||
|
|
"loss/reference_anchor": 0.463923841714859,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.6052303314208984,
|
||
|
|
"rewards/margins": 2.423473596572876,
|
||
|
|
"rewards/rejected": 1.1817567348480225,
|
||
|
|
"ronpo/logit": 0.2423473298549652,
|
||
|
|
"ronpo/residual": 0.24348480999469757,
|
||
|
|
"ronpo/residual_abs": 0.29033395648002625,
|
||
|
|
"ronpo/target": -0.001137460581958294,
|
||
|
|
"ronpo/target_abs": 0.05171912908554077,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2400600016117096,
|
||
|
|
"drift/rejected_abs": 0.11403791606426239,
|
||
|
|
"epoch": 0.2526817640047676,
|
||
|
|
"grad_norm": 1.8125,
|
||
|
|
"learning_rate": 2.626640959949375e-09,
|
||
|
|
"logits/chosen": -2.132143497467041,
|
||
|
|
"logits/rejected": -2.166182041168213,
|
||
|
|
"logps/chosen": -0.2598033547401428,
|
||
|
|
"logps/rejected": -0.2518232464790344,
|
||
|
|
"loss": 0.12491835653781891,
|
||
|
|
"loss/core": 0.10928516089916229,
|
||
|
|
"loss/preference_sft": 0.2489791363477707,
|
||
|
|
"loss/reference_anchor": 0.13143403828144073,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.399862766265869,
|
||
|
|
"rewards/margins": 1.2609279155731201,
|
||
|
|
"rewards/rejected": 1.138934850692749,
|
||
|
|
"ronpo/logit": 0.1260928064584732,
|
||
|
|
"ronpo/residual": 0.13136708736419678,
|
||
|
|
"ronpo/residual_abs": 0.1680411547422409,
|
||
|
|
"ronpo/target": -0.00527427950873971,
|
||
|
|
"ronpo/target_abs": 0.05185922235250473,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.283835768699646,
|
||
|
|
"drift/rejected_abs": 0.1590053141117096,
|
||
|
|
"epoch": 0.2542709574890743,
|
||
|
|
"grad_norm": 18.875,
|
||
|
|
"learning_rate": 2.388621363640797e-09,
|
||
|
|
"logits/chosen": -2.1680550575256348,
|
||
|
|
"logits/rejected": -2.2392494678497314,
|
||
|
|
"logps/chosen": -0.2695561349391937,
|
||
|
|
"logps/rejected": -0.2701888978481293,
|
||
|
|
"loss": 0.2641626000404358,
|
||
|
|
"loss/core": 0.18199072778224945,
|
||
|
|
"loss/preference_sft": 0.27051591873168945,
|
||
|
|
"loss/reference_anchor": 0.7946673631668091,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.837977886199951,
|
||
|
|
"rewards/margins": 1.2485079765319824,
|
||
|
|
"rewards/rejected": 1.5894701480865479,
|
||
|
|
"ronpo/logit": 0.124850794672966,
|
||
|
|
"ronpo/residual": 0.12345902621746063,
|
||
|
|
"ronpo/residual_abs": 0.157121941447258,
|
||
|
|
"ronpo/target": 0.0013917500618845224,
|
||
|
|
"ronpo/target_abs": 0.04755866900086403,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3202213644981384,
|
||
|
|
"drift/rejected_abs": 0.13916157186031342,
|
||
|
|
"epoch": 0.255860150973381,
|
||
|
|
"grad_norm": 204.0,
|
||
|
|
"learning_rate": 2.1613635589349754e-09,
|
||
|
|
"logits/chosen": -2.27490234375,
|
||
|
|
"logits/rejected": -2.2372684478759766,
|
||
|
|
"logps/chosen": -0.30233731865882874,
|
||
|
|
"logps/rejected": -0.3073555827140808,
|
||
|
|
"loss": 0.9722781181335449,
|
||
|
|
"loss/core": 0.9004875421524048,
|
||
|
|
"loss/preference_sft": 0.28705883026123047,
|
||
|
|
"loss/reference_anchor": 0.6892002820968628,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.202214002609253,
|
||
|
|
"rewards/margins": 1.8725093603134155,
|
||
|
|
"rewards/rejected": 1.3297045230865479,
|
||
|
|
"ronpo/logit": 0.18725092709064484,
|
||
|
|
"ronpo/residual": 0.18470832705497742,
|
||
|
|
"ronpo/residual_abs": 0.2942502200603485,
|
||
|
|
"ronpo/target": 0.0025425744242966175,
|
||
|
|
"ronpo/target_abs": 0.050916336476802826,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 805
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21148745715618134,
|
||
|
|
"drift/rejected_abs": 0.1105772852897644,
|
||
|
|
"epoch": 0.25744934445768775,
|
||
|
|
"grad_norm": 8.4375,
|
||
|
|
"learning_rate": 1.944975708238708e-09,
|
||
|
|
"logits/chosen": -2.0623979568481445,
|
||
|
|
"logits/rejected": -2.1137001514434814,
|
||
|
|
"logps/chosen": -0.2832852602005005,
|
||
|
|
"logps/rejected": -0.2870853841304779,
|
||
|
|
"loss": 0.05209481716156006,
|
||
|
|
"loss/core": 0.03912948817014694,
|
||
|
|
"loss/preference_sft": 0.2837418019771576,
|
||
|
|
"loss/reference_anchor": 0.10127910226583481,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.1126019954681396,
|
||
|
|
"rewards/margins": 1.0071337223052979,
|
||
|
|
"rewards/rejected": 1.1054683923721313,
|
||
|
|
"ronpo/logit": 0.10071338713169098,
|
||
|
|
"ronpo/residual": 0.09261061996221542,
|
||
|
|
"ronpo/residual_abs": 0.12470117956399918,
|
||
|
|
"ronpo/target": 0.008102752268314362,
|
||
|
|
"ronpo/target_abs": 0.048462167382240295,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25939127802848816,
|
||
|
|
"drift/rejected_abs": 0.16253964602947235,
|
||
|
|
"epoch": 0.25903853794199444,
|
||
|
|
"grad_norm": 12.875,
|
||
|
|
"learning_rate": 1.7395608004493884e-09,
|
||
|
|
"logits/chosen": -2.110250949859619,
|
||
|
|
"logits/rejected": -2.0329298973083496,
|
||
|
|
"logps/chosen": -0.2672348916530609,
|
||
|
|
"logps/rejected": -0.2788865268230438,
|
||
|
|
"loss": 0.23105821013450623,
|
||
|
|
"loss/core": 0.20882940292358398,
|
||
|
|
"loss/preference_sft": 0.263536661863327,
|
||
|
|
"loss/reference_anchor": 0.19593465328216553,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.5937724113464355,
|
||
|
|
"rewards/margins": 0.9874376058578491,
|
||
|
|
"rewards/rejected": 1.606334924697876,
|
||
|
|
"ronpo/logit": 0.09874377399682999,
|
||
|
|
"ronpo/residual": 0.0899277850985527,
|
||
|
|
"ronpo/residual_abs": 0.20764711499214172,
|
||
|
|
"ronpo/target": 0.008815980516374111,
|
||
|
|
"ronpo/target_abs": 0.05238635092973709,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 815
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19982877373695374,
|
||
|
|
"drift/rejected_abs": 0.1136329397559166,
|
||
|
|
"epoch": 0.26062773142630113,
|
||
|
|
"grad_norm": 3.046875,
|
||
|
|
"learning_rate": 1.5452166019378988e-09,
|
||
|
|
"logits/chosen": -2.021080493927002,
|
||
|
|
"logits/rejected": -1.9773486852645874,
|
||
|
|
"logps/chosen": -0.2758404612541199,
|
||
|
|
"logps/rejected": -0.2889498174190521,
|
||
|
|
"loss": 0.07101830095052719,
|
||
|
|
"loss/core": 0.05234837532043457,
|
||
|
|
"loss/preference_sft": 0.27237778902053833,
|
||
|
|
"loss/reference_anchor": 0.15946142375469208,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.9943138360977173,
|
||
|
|
"rewards/margins": 0.8593200445175171,
|
||
|
|
"rewards/rejected": 1.1349937915802002,
|
||
|
|
"ronpo/logit": 0.08593200892210007,
|
||
|
|
"ronpo/residual": 0.08751102536916733,
|
||
|
|
"ronpo/residual_abs": 0.1283148229122162,
|
||
|
|
"ronpo/target": -0.0015790194738656282,
|
||
|
|
"ronpo/target_abs": 0.04799572750926018,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2875332832336426,
|
||
|
|
"drift/rejected_abs": 0.1883879154920578,
|
||
|
|
"epoch": 0.2622169249106079,
|
||
|
|
"grad_norm": 88.0,
|
||
|
|
"learning_rate": 1.3620356100170789e-09,
|
||
|
|
"logits/chosen": -2.214282512664795,
|
||
|
|
"logits/rejected": -2.162520170211792,
|
||
|
|
"logps/chosen": -0.2991814613342285,
|
||
|
|
"logps/rejected": -0.30333730578422546,
|
||
|
|
"loss": 0.39491066336631775,
|
||
|
|
"loss/core": 0.30867427587509155,
|
||
|
|
"loss/preference_sft": 0.2967020869255066,
|
||
|
|
"loss/reference_anchor": 0.8326936960220337,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.87361478805542,
|
||
|
|
"rewards/margins": 1.0094401836395264,
|
||
|
|
"rewards/rejected": 1.864174485206604,
|
||
|
|
"ronpo/logit": 0.10094402730464935,
|
||
|
|
"ronpo/residual": 0.09369300305843353,
|
||
|
|
"ronpo/residual_abs": 0.20826761424541473,
|
||
|
|
"ronpo/target": 0.007251021917909384,
|
||
|
|
"ronpo/target_abs": 0.058990202844142914,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 825
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18447622656822205,
|
||
|
|
"drift/rejected_abs": 0.11325931549072266,
|
||
|
|
"epoch": 0.2638061183949146,
|
||
|
|
"grad_norm": 31.875,
|
||
|
|
"learning_rate": 1.190105008918041e-09,
|
||
|
|
"logits/chosen": -2.2358505725860596,
|
||
|
|
"logits/rejected": -2.23689603805542,
|
||
|
|
"logps/chosen": -0.2778850197792053,
|
||
|
|
"logps/rejected": -0.2730768620967865,
|
||
|
|
"loss": 0.13070687651634216,
|
||
|
|
"loss/core": 0.11438991874456406,
|
||
|
|
"loss/preference_sft": 0.2781921625137329,
|
||
|
|
"loss/reference_anchor": 0.13535039126873016,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.8447624444961548,
|
||
|
|
"rewards/margins": 0.7130740880966187,
|
||
|
|
"rewards/rejected": 1.1316883563995361,
|
||
|
|
"ronpo/logit": 0.07130740582942963,
|
||
|
|
"ronpo/residual": 0.06643342226743698,
|
||
|
|
"ronpo/residual_abs": 0.14979875087738037,
|
||
|
|
"ronpo/target": 0.004873988218605518,
|
||
|
|
"ronpo/target_abs": 0.05250249058008194,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.40701180696487427,
|
||
|
|
"drift/rejected_abs": 0.162872314453125,
|
||
|
|
"epoch": 0.2653953118792213,
|
||
|
|
"grad_norm": 33.25,
|
||
|
|
"learning_rate": 1.0295066282951737e-09,
|
||
|
|
"logits/chosen": -2.190506935119629,
|
||
|
|
"logits/rejected": -2.155287504196167,
|
||
|
|
"logps/chosen": -0.24197907745838165,
|
||
|
|
"logps/rejected": -0.24426302313804626,
|
||
|
|
"loss": 1.1218777894973755,
|
||
|
|
"loss/core": 1.0501762628555298,
|
||
|
|
"loss/preference_sft": 0.24557368457317352,
|
||
|
|
"loss/reference_anchor": 0.6924582719802856,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 4.069449424743652,
|
||
|
|
"rewards/margins": 2.4409031867980957,
|
||
|
|
"rewards/rejected": 1.6285464763641357,
|
||
|
|
"ronpo/logit": 0.24409031867980957,
|
||
|
|
"ronpo/residual": 0.24195992946624756,
|
||
|
|
"ronpo/residual_abs": 0.3348272442817688,
|
||
|
|
"ronpo/target": 0.0021303787361830473,
|
||
|
|
"ronpo/target_abs": 0.051504336297512054,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 835
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2866782248020172,
|
||
|
|
"drift/rejected_abs": 0.13607479631900787,
|
||
|
|
"epoch": 0.266984505363528,
|
||
|
|
"grad_norm": 1.0546875,
|
||
|
|
"learning_rate": 8.803169042796765e-10,
|
||
|
|
"logits/chosen": -2.36369252204895,
|
||
|
|
"logits/rejected": -2.4254086017608643,
|
||
|
|
"logps/chosen": -0.2668394148349762,
|
||
|
|
"logps/rejected": -0.2654018998146057,
|
||
|
|
"loss": 0.3814314007759094,
|
||
|
|
"loss/core": 0.3395002484321594,
|
||
|
|
"loss/preference_sft": 0.26231107115745544,
|
||
|
|
"loss/reference_anchor": 0.39308053255081177,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8666539192199707,
|
||
|
|
"rewards/margins": 1.5065144300460815,
|
||
|
|
"rewards/rejected": 1.3601396083831787,
|
||
|
|
"ronpo/logit": 0.15065142512321472,
|
||
|
|
"ronpo/residual": 0.15467844903469086,
|
||
|
|
"ronpo/residual_abs": 0.1781061887741089,
|
||
|
|
"ronpo/target": -0.004027015529572964,
|
||
|
|
"ronpo/target_abs": 0.05107799917459488,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29828858375549316,
|
||
|
|
"drift/rejected_abs": 0.1848953515291214,
|
||
|
|
"epoch": 0.2685736988478347,
|
||
|
|
"grad_norm": 5.15625,
|
||
|
|
"learning_rate": 7.426068431000881e-10,
|
||
|
|
"logits/chosen": -2.143845558166504,
|
||
|
|
"logits/rejected": -2.135876178741455,
|
||
|
|
"logps/chosen": -0.24773220717906952,
|
||
|
|
"logps/rejected": -0.2345115840435028,
|
||
|
|
"loss": 0.10221978276968002,
|
||
|
|
"loss/core": 0.0692344605922699,
|
||
|
|
"loss/preference_sft": 0.23999452590942383,
|
||
|
|
"loss/reference_anchor": 0.3058537244796753,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 2.9807627201080322,
|
||
|
|
"rewards/margins": 1.1323689222335815,
|
||
|
|
"rewards/rejected": 1.8483936786651611,
|
||
|
|
"ronpo/logit": 0.11323690414428711,
|
||
|
|
"ronpo/residual": 0.10838621854782104,
|
||
|
|
"ronpo/residual_abs": 0.13771787285804749,
|
||
|
|
"ronpo/target": 0.004850681871175766,
|
||
|
|
"ronpo/target_abs": 0.0496140718460083,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.41046634316444397,
|
||
|
|
"drift/rejected_abs": 0.1459590494632721,
|
||
|
|
"epoch": 0.2701628923321414,
|
||
|
|
"grad_norm": 80.5,
|
||
|
|
"learning_rate": 6.164419872871834e-10,
|
||
|
|
"logits/chosen": -1.9537031650543213,
|
||
|
|
"logits/rejected": -1.8872268199920654,
|
||
|
|
"logps/chosen": -0.2728812098503113,
|
||
|
|
"logps/rejected": -0.28422796726226807,
|
||
|
|
"loss": 0.9486082196235657,
|
||
|
|
"loss/core": 0.8727816343307495,
|
||
|
|
"loss/preference_sft": 0.26899510622024536,
|
||
|
|
"loss/reference_anchor": 0.7313664555549622,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 4.103553771972656,
|
||
|
|
"rewards/margins": 2.6454689502716064,
|
||
|
|
"rewards/rejected": 1.4580847024917603,
|
||
|
|
"ronpo/logit": 0.2645469009876251,
|
||
|
|
"ronpo/residual": 0.2555916905403137,
|
||
|
|
"ronpo/residual_abs": 0.3104216456413269,
|
||
|
|
"ronpo/target": 0.008955182507634163,
|
||
|
|
"ronpo/target_abs": 0.05471789091825485,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20613236725330353,
|
||
|
|
"drift/rejected_abs": 0.0725497379899025,
|
||
|
|
"epoch": 0.27175208581644816,
|
||
|
|
"grad_norm": 2.890625,
|
||
|
|
"learning_rate": 5.018823844792602e-10,
|
||
|
|
"logits/chosen": -2.3794169425964355,
|
||
|
|
"logits/rejected": -2.366502046585083,
|
||
|
|
"logps/chosen": -0.2781080901622772,
|
||
|
|
"logps/rejected": -0.2757300138473511,
|
||
|
|
"loss": 0.4294048845767975,
|
||
|
|
"loss/core": 0.4002324938774109,
|
||
|
|
"loss/preference_sft": 0.27767062187194824,
|
||
|
|
"loss/reference_anchor": 0.26395726203918457,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.061227798461914,
|
||
|
|
"rewards/margins": 1.3357303142547607,
|
||
|
|
"rewards/rejected": 0.7254973649978638,
|
||
|
|
"ronpo/logit": 0.1335730254650116,
|
||
|
|
"ronpo/residual": 0.1437883824110031,
|
||
|
|
"ronpo/residual_abs": 0.17174139618873596,
|
||
|
|
"ronpo/target": -0.010215363465249538,
|
||
|
|
"ronpo/target_abs": 0.05027645081281662,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1879555881023407,
|
||
|
|
"drift/rejected_abs": 0.10524116456508636,
|
||
|
|
"epoch": 0.27334127930075486,
|
||
|
|
"grad_norm": 46.5,
|
||
|
|
"learning_rate": 3.9898255884272814e-10,
|
||
|
|
"logits/chosen": -2.2028138637542725,
|
||
|
|
"logits/rejected": -2.1289145946502686,
|
||
|
|
"logps/chosen": -0.2808927893638611,
|
||
|
|
"logps/rejected": -0.2781389653682709,
|
||
|
|
"loss": 0.13080891966819763,
|
||
|
|
"loss/core": 0.11396379768848419,
|
||
|
|
"loss/preference_sft": 0.2718280255794525,
|
||
|
|
"loss/reference_anchor": 0.1412682831287384,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.8792381286621094,
|
||
|
|
"rewards/margins": 0.8328661918640137,
|
||
|
|
"rewards/rejected": 1.0463716983795166,
|
||
|
|
"ronpo/logit": 0.08328662812709808,
|
||
|
|
"ronpo/residual": 0.07004024088382721,
|
||
|
|
"ronpo/residual_abs": 0.1364951878786087,
|
||
|
|
"ronpo/target": 0.013246382586658001,
|
||
|
|
"ronpo/target_abs": 0.05126240849494934,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24820244312286377,
|
||
|
|
"drift/rejected_abs": 0.12920786440372467,
|
||
|
|
"epoch": 0.2749304727850616,
|
||
|
|
"grad_norm": 0.474609375,
|
||
|
|
"learning_rate": 3.077914851215585e-10,
|
||
|
|
"logits/chosen": -2.0905251502990723,
|
||
|
|
"logits/rejected": -2.0355658531188965,
|
||
|
|
"logps/chosen": -0.2801133990287781,
|
||
|
|
"logps/rejected": -0.2894144058227539,
|
||
|
|
"loss": 0.11979007720947266,
|
||
|
|
"loss/core": 0.09946328401565552,
|
||
|
|
"loss/preference_sft": 0.2898562550544739,
|
||
|
|
"loss/reference_anchor": 0.1742822378873825,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.481431484222412,
|
||
|
|
"rewards/margins": 1.1907166242599487,
|
||
|
|
"rewards/rejected": 1.2907148599624634,
|
||
|
|
"ronpo/logit": 0.11907164752483368,
|
||
|
|
"ronpo/residual": 0.11746648699045181,
|
||
|
|
"ronpo/residual_abs": 0.15590353310108185,
|
||
|
|
"ronpo/target": 0.0016051754355430603,
|
||
|
|
"ronpo/target_abs": 0.046399109065532684,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 865
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.46365779638290405,
|
||
|
|
"drift/rejected_abs": 0.21867212653160095,
|
||
|
|
"epoch": 0.2765196662693683,
|
||
|
|
"grad_norm": 536.0,
|
||
|
|
"learning_rate": 2.283525653279439e-10,
|
||
|
|
"logits/chosen": -2.0990447998046875,
|
||
|
|
"logits/rejected": -2.087538003921509,
|
||
|
|
"logps/chosen": -0.2689060866832733,
|
||
|
|
"logps/rejected": -0.27739399671554565,
|
||
|
|
"loss": 0.7828746438026428,
|
||
|
|
"loss/core": 0.6796854734420776,
|
||
|
|
"loss/preference_sft": 0.2810099124908447,
|
||
|
|
"loss/reference_anchor": 1.0037904977798462,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 4.635507583618164,
|
||
|
|
"rewards/margins": 2.44916033744812,
|
||
|
|
"rewards/rejected": 2.186347484588623,
|
||
|
|
"ronpo/logit": 0.24491600692272186,
|
||
|
|
"ronpo/residual": 0.24868957698345184,
|
||
|
|
"ronpo/residual_abs": 0.2878829836845398,
|
||
|
|
"ronpo/target": -0.003773549571633339,
|
||
|
|
"ronpo/target_abs": 0.04606860131025314,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3057054877281189,
|
||
|
|
"drift/rejected_abs": 0.153896301984787,
|
||
|
|
"epoch": 0.278108859753675,
|
||
|
|
"grad_norm": 6.3125,
|
||
|
|
"learning_rate": 1.6070360808531359e-10,
|
||
|
|
"logits/chosen": -1.9459526538848877,
|
||
|
|
"logits/rejected": -1.9143660068511963,
|
||
|
|
"logps/chosen": -0.2711811065673828,
|
||
|
|
"logps/rejected": -0.27196747064590454,
|
||
|
|
"loss": 0.2286219298839569,
|
||
|
|
"loss/core": 0.1964854896068573,
|
||
|
|
"loss/preference_sft": 0.2671479284763336,
|
||
|
|
"loss/reference_anchor": 0.2946498394012451,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.0563113689422607,
|
||
|
|
"rewards/margins": 1.5175580978393555,
|
||
|
|
"rewards/rejected": 1.5387529134750366,
|
||
|
|
"ronpo/logit": 0.15175580978393555,
|
||
|
|
"ronpo/residual": 0.14992663264274597,
|
||
|
|
"ronpo/residual_abs": 0.18774335086345673,
|
||
|
|
"ronpo/target": 0.0018291939049959183,
|
||
|
|
"ronpo/target_abs": 0.049851469695568085,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24419137835502625,
|
||
|
|
"drift/rejected_abs": 0.09081504493951797,
|
||
|
|
"epoch": 0.27969805323798175,
|
||
|
|
"grad_norm": 5.125,
|
||
|
|
"learning_rate": 1.0487681063345854e-10,
|
||
|
|
"logits/chosen": -2.1151018142700195,
|
||
|
|
"logits/rejected": -2.156067371368408,
|
||
|
|
"logps/chosen": -0.2783779501914978,
|
||
|
|
"logps/rejected": -0.2911151945590973,
|
||
|
|
"loss": 0.4191954731941223,
|
||
|
|
"loss/core": 0.391798198223114,
|
||
|
|
"loss/preference_sft": 0.2866339087486267,
|
||
|
|
"loss/reference_anchor": 0.2453095018863678,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.4419138431549072,
|
||
|
|
"rewards/margins": 1.5502523183822632,
|
||
|
|
"rewards/rejected": 0.8916614651679993,
|
||
|
|
"ronpo/logit": 0.15502521395683289,
|
||
|
|
"ronpo/residual": 0.1629984974861145,
|
||
|
|
"ronpo/residual_abs": 0.1879701018333435,
|
||
|
|
"ronpo/target": -0.00797327235341072,
|
||
|
|
"ronpo/target_abs": 0.055687934160232544,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2644875645637512,
|
||
|
|
"drift/rejected_abs": 0.10445030778646469,
|
||
|
|
"epoch": 0.28128724672228844,
|
||
|
|
"grad_norm": 9.0625,
|
||
|
|
"learning_rate": 6.089874350439506e-11,
|
||
|
|
"logits/chosen": -2.162407636642456,
|
||
|
|
"logits/rejected": -2.2234578132629395,
|
||
|
|
"logps/chosen": -0.2983078062534332,
|
||
|
|
"logps/rejected": -0.2950352132320404,
|
||
|
|
"loss": 0.42109808325767517,
|
||
|
|
"loss/core": 0.3663472533226013,
|
||
|
|
"loss/preference_sft": 0.2981397211551666,
|
||
|
|
"loss/reference_anchor": 0.5176942348480225,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.643937110900879,
|
||
|
|
"rewards/margins": 1.6017318964004517,
|
||
|
|
"rewards/rejected": 1.0422050952911377,
|
||
|
|
"ronpo/logit": 0.1601731926202774,
|
||
|
|
"ronpo/residual": 0.1630907952785492,
|
||
|
|
"ronpo/residual_abs": 0.18677613139152527,
|
||
|
|
"ronpo/target": -0.0029176094103604555,
|
||
|
|
"ronpo/target_abs": 0.04585547372698784,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29927030205726624,
|
||
|
|
"drift/rejected_abs": 0.12154443562030792,
|
||
|
|
"epoch": 0.28287644020659514,
|
||
|
|
"grad_norm": 3.453125,
|
||
|
|
"learning_rate": 2.8790337876233307e-11,
|
||
|
|
"logits/chosen": -2.306915760040283,
|
||
|
|
"logits/rejected": -2.2651777267456055,
|
||
|
|
"logps/chosen": -0.28179579973220825,
|
||
|
|
"logps/rejected": -0.28089454770088196,
|
||
|
|
"loss": 0.43596991896629333,
|
||
|
|
"loss/core": 0.38446953892707825,
|
||
|
|
"loss/preference_sft": 0.28348347544670105,
|
||
|
|
"loss/reference_anchor": 0.48665523529052734,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.9908390045166016,
|
||
|
|
"rewards/margins": 1.7791423797607422,
|
||
|
|
"rewards/rejected": 1.2116966247558594,
|
||
|
|
"ronpo/logit": 0.17791421711444855,
|
||
|
|
"ronpo/residual": 0.17613449692726135,
|
||
|
|
"ronpo/residual_abs": 0.2077271044254303,
|
||
|
|
"ronpo/target": 0.0017797171603888273,
|
||
|
|
"ronpo/target_abs": 0.053874753415584564,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3066047728061676,
|
||
|
|
"drift/rejected_abs": 0.12381645292043686,
|
||
|
|
"epoch": 0.2844656336909019,
|
||
|
|
"grad_norm": 10.3125,
|
||
|
|
"learning_rate": 8.566875611068502e-12,
|
||
|
|
"logits/chosen": -2.080899477005005,
|
||
|
|
"logits/rejected": -2.0696678161621094,
|
||
|
|
"logps/chosen": -0.28545522689819336,
|
||
|
|
"logps/rejected": -0.2945188283920288,
|
||
|
|
"loss": 0.39896756410598755,
|
||
|
|
"loss/core": 0.35809260606765747,
|
||
|
|
"loss/preference_sft": 0.28190794587135315,
|
||
|
|
"loss/reference_anchor": 0.38055914640426636,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.0644736289978027,
|
||
|
|
"rewards/margins": 1.9215772151947021,
|
||
|
|
"rewards/rejected": 1.142896294593811,
|
||
|
|
"ronpo/logit": 0.19215771555900574,
|
||
|
|
"ronpo/residual": 0.1853954941034317,
|
||
|
|
"ronpo/residual_abs": 0.21138229966163635,
|
||
|
|
"ronpo/target": 0.0067622363567352295,
|
||
|
|
"ronpo/target_abs": 0.051556866616010666,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 895
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1592852622270584,
|
||
|
|
"drift/rejected_abs": 0.084005206823349,
|
||
|
|
"epoch": 0.2860548271752086,
|
||
|
|
"grad_norm": 17.75,
|
||
|
|
"learning_rate": 2.379819816378248e-13,
|
||
|
|
"logits/chosen": -2.1036276817321777,
|
||
|
|
"logits/rejected": -2.0870792865753174,
|
||
|
|
"logps/chosen": -0.31869596242904663,
|
||
|
|
"logps/rejected": -0.31833693385124207,
|
||
|
|
"loss": 0.14820127189159393,
|
||
|
|
"loss/core": 0.13302530348300934,
|
||
|
|
"loss/preference_sft": 0.31609418988227844,
|
||
|
|
"loss/reference_anchor": 0.12015028297901154,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.5925084352493286,
|
||
|
|
"rewards/margins": 0.8815566897392273,
|
||
|
|
"rewards/rejected": 0.7109516859054565,
|
||
|
|
"ronpo/logit": 0.08815567195415497,
|
||
|
|
"ronpo/residual": 0.09216871112585068,
|
||
|
|
"ronpo/residual_abs": 0.1262686550617218,
|
||
|
|
"ronpo/target": -0.00401304429396987,
|
||
|
|
"ronpo/target_abs": 0.050638485699892044,
|
||
|
|
"ronpo/weight": 1.0,
|
||
|
|
"ronpo/weight_max": 1.0,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2860548271752086,
|
||
|
|
"step": 900,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.4462373371918996,
|
||
|
|
"train_runtime": 7238.9345,
|
||
|
|
"train_samples_per_second": 1.989,
|
||
|
|
"train_steps_per_second": 0.124
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 900,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 900,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|