{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.8594449418084154, "eval_steps": 500, "global_step": 900, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.004774694121157864, "grad_norm": 60.5, "ht_mnpo/logit": 0.2415936291217804, "ht_mnpo/residual": 0.2340935915708542, "ht_mnpo/residual_abs": 0.23548129200935364, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.222222222222222e-08, "logits/chosen": -2.011902332305908, "logits/rejected": -2.257323741912842, "logps/chosen": -0.25824588537216187, "logps/rejected": -0.2757476568222046, "loss": 0.5226427316665649, "loss/core": 0.5226427316665649, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 3.5988030433654785, "rewards/margins": 2.41593599319458, "rewards/rejected": 1.1828668117523193, "step": 5 }, { "epoch": 0.009549388242315727, "grad_norm": 2736.0, "ht_mnpo/logit": 0.26693910360336304, "ht_mnpo/residual": 0.2594391107559204, "ht_mnpo/residual_abs": 0.2632564902305603, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 5e-08, "logits/chosen": -2.0542407035827637, "logits/rejected": -2.270714282989502, "logps/chosen": -0.2949259281158447, "logps/rejected": -0.29275721311569214, "loss": 1.0850975513458252, "loss/core": 1.0850975513458252, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.9222195148468018, "rewards/margins": 2.669390916824341, "rewards/rejected": 1.2528284788131714, "step": 10 }, { "epoch": 0.01432408236347359, "grad_norm": 10.8125, "ht_mnpo/logit": 0.19589567184448242, "ht_mnpo/residual": 0.188395693898201, "ht_mnpo/residual_abs": 0.19078858196735382, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 7.777777777777778e-08, "logits/chosen": -2.067415475845337, "logits/rejected": -2.327845811843872, "logps/chosen": -0.314541757106781, "logps/rejected": -0.29293060302734375, "loss": 0.5424398183822632, "loss/core": 0.5424398183822632, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.4876604080200195, "rewards/margins": 1.9589569568634033, "rewards/rejected": 0.5287034511566162, "step": 15 }, { "epoch": 0.019098776484631454, "grad_norm": 36.0, "ht_mnpo/logit": 0.3147750794887543, "ht_mnpo/residual": 0.30727505683898926, "ht_mnpo/residual_abs": 0.30989599227905273, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.0555555555555555e-07, "logits/chosen": -1.927666425704956, "logits/rejected": -2.2836575508117676, "logps/chosen": -0.3032557964324951, "logps/rejected": -0.2924278676509857, "loss": 1.1851537227630615, "loss/core": 1.1851537227630615, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.1310954093933105, "rewards/margins": 3.1477506160736084, "rewards/rejected": 0.9833448529243469, "step": 20 }, { "epoch": 0.023873470605789315, "grad_norm": 73.5, "ht_mnpo/logit": 0.12462173402309418, "ht_mnpo/residual": 0.11712173372507095, "ht_mnpo/residual_abs": 0.12095322459936142, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.3333333333333334e-07, "logits/chosen": -2.0298550128936768, "logits/rejected": -2.2757391929626465, "logps/chosen": -0.31602349877357483, "logps/rejected": -0.3301704525947571, "loss": 0.18879806995391846, "loss/core": 0.18879806995391846, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 1.6169111728668213, "rewards/margins": 1.2462173700332642, "rewards/rejected": 0.3706938624382019, "step": 25 }, { "epoch": 0.02864816472694718, "grad_norm": 368.0, "ht_mnpo/logit": 0.12316179275512695, "ht_mnpo/residual": 0.11566178500652313, "ht_mnpo/residual_abs": 0.11892789602279663, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.611111111111111e-07, "logits/chosen": -2.042910575866699, "logits/rejected": -2.2183661460876465, "logps/chosen": -0.28492701053619385, "logps/rejected": -0.3071616291999817, "loss": 0.1271527111530304, "loss/core": 0.1271527111530304, "rewards/accuracies": 0.9375, "rewards/chosen": 2.4336202144622803, "rewards/margins": 1.23161780834198, "rewards/rejected": 1.2020022869110107, "step": 30 }, { "epoch": 0.033422858848105044, "grad_norm": 9.625, "ht_mnpo/logit": 0.3371656835079193, "ht_mnpo/residual": 0.3296656310558319, "ht_mnpo/residual_abs": 0.3311299681663513, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.8888888888888888e-07, "logits/chosen": -1.9528906345367432, "logits/rejected": -2.2516660690307617, "logps/chosen": -0.27027541399002075, "logps/rejected": -0.27153652906417847, "loss": 1.1185133457183838, "loss/core": 1.1185133457183838, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.334385395050049, "rewards/margins": 3.3716564178466797, "rewards/rejected": 0.9627290964126587, "step": 35 }, { "epoch": 0.03819755296926291, "grad_norm": 14.1875, "ht_mnpo/logit": 0.21347995102405548, "ht_mnpo/residual": 0.20597994327545166, "ht_mnpo/residual_abs": 0.20741498470306396, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.1666666666666667e-07, "logits/chosen": -2.00959849357605, "logits/rejected": -2.2939441204071045, "logps/chosen": -0.26474690437316895, "logps/rejected": -0.276932954788208, "loss": 0.41715115308761597, "loss/core": 0.41715115308761597, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 3.2156689167022705, "rewards/margins": 2.1347994804382324, "rewards/rejected": 1.080869436264038, "step": 40 }, { "epoch": 0.04297224709042077, "grad_norm": 2.203125, "ht_mnpo/logit": 0.2520568370819092, "ht_mnpo/residual": 0.24455685913562775, "ht_mnpo/residual_abs": 0.24652080237865448, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.4444444444444445e-07, "logits/chosen": -2.1033449172973633, "logits/rejected": -2.3409695625305176, "logps/chosen": -0.2933202385902405, "logps/rejected": -0.3012697398662567, "loss": 0.9052979350090027, "loss/core": 0.9052979350090027, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.102382183074951, "rewards/margins": 2.520568609237671, "rewards/rejected": 0.5818136930465698, "step": 45 }, { "epoch": 0.04774694121157863, "grad_norm": 103.5, "ht_mnpo/logit": 0.2532837688922882, "ht_mnpo/residual": 0.24578377604484558, "ht_mnpo/residual_abs": 0.2527261972427368, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.7222222222222216e-07, "logits/chosen": -1.9192997217178345, "logits/rejected": -2.134883403778076, "logps/chosen": -0.26225295662879944, "logps/rejected": -0.29307669401168823, "loss": 0.6835693717002869, "loss/core": 0.6835693717002869, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.6984927654266357, "rewards/margins": 2.5328376293182373, "rewards/rejected": 1.1656550168991089, "step": 50 }, { "epoch": 0.052521635332736495, "grad_norm": 2.328125, "ht_mnpo/logit": 0.203359916806221, "ht_mnpo/residual": 0.1958599090576172, "ht_mnpo/residual_abs": 0.1985529363155365, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3e-07, "logits/chosen": -1.9192931652069092, "logits/rejected": -2.2161107063293457, "logps/chosen": -0.2721022963523865, "logps/rejected": -0.2915636897087097, "loss": 0.437203973531723, "loss/core": 0.437203973531723, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.0200881958007812, "rewards/margins": 2.0335988998413086, "rewards/rejected": 0.9864892959594727, "step": 55 }, { "epoch": 0.05729632945389436, "grad_norm": 19.5, "ht_mnpo/logit": 0.2052539587020874, "ht_mnpo/residual": 0.19775396585464478, "ht_mnpo/residual_abs": 0.20126383006572723, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.2777777777777776e-07, "logits/chosen": -1.9262555837631226, "logits/rejected": -2.166170120239258, "logps/chosen": -0.29561707377433777, "logps/rejected": -0.31034865975379944, "loss": 1.1692918539047241, "loss/core": 1.1692918539047241, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.606314182281494, "rewards/margins": 2.052539348602295, "rewards/rejected": 0.5537749528884888, "step": 60 }, { "epoch": 0.062071023575052224, "grad_norm": 924.0, "ht_mnpo/logit": 0.2645673453807831, "ht_mnpo/residual": 0.25706732273101807, "ht_mnpo/residual_abs": 0.2587380111217499, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.5555555555555553e-07, "logits/chosen": -2.020003080368042, "logits/rejected": -2.3714873790740967, "logps/chosen": -0.2805558741092682, "logps/rejected": -0.27380621433258057, "loss": 1.0117337703704834, "loss/core": 1.0117337703704834, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 3.615847110748291, "rewards/margins": 2.6456735134124756, "rewards/rejected": 0.9701736569404602, "step": 65 }, { "epoch": 0.06684571769621009, "grad_norm": 1.40625, "ht_mnpo/logit": 0.14507603645324707, "ht_mnpo/residual": 0.13757602870464325, "ht_mnpo/residual_abs": 0.13970807194709778, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.8333333333333335e-07, "logits/chosen": -1.9647560119628906, "logits/rejected": -2.2312729358673096, "logps/chosen": -0.28448420763015747, "logps/rejected": -0.29094865918159485, "loss": 0.08910230547189713, "loss/core": 0.08910230547189713, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.4837958812713623, "rewards/margins": 1.4507603645324707, "rewards/rejected": 1.0330355167388916, "step": 70 }, { "epoch": 0.07162041181736795, "grad_norm": 99.5, "ht_mnpo/logit": 0.34166499972343445, "ht_mnpo/residual": 0.3341650068759918, "ht_mnpo/residual_abs": 0.3365333080291748, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.1111111111111107e-07, "logits/chosen": -1.8882007598876953, "logits/rejected": -2.146094560623169, "logps/chosen": -0.28841090202331543, "logps/rejected": -0.28859004378318787, "loss": 0.9650213122367859, "loss/core": 0.9650213122367859, "rewards/accuracies": 0.9375, "rewards/chosen": 4.434047698974609, "rewards/margins": 3.4166500568389893, "rewards/rejected": 1.0173976421356201, "step": 75 }, { "epoch": 0.07639510593852582, "grad_norm": 98.0, "ht_mnpo/logit": 0.4781981110572815, "ht_mnpo/residual": 0.47069811820983887, "ht_mnpo/residual_abs": 0.47250255942344666, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.3888888888888884e-07, "logits/chosen": -2.0093605518341064, "logits/rejected": -2.3259353637695312, "logps/chosen": -0.2699472904205322, "logps/rejected": -0.2692564129829407, "loss": 2.545933485031128, "loss/core": 2.545933485031128, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 5.524721622467041, "rewards/margins": 4.781980991363525, "rewards/rejected": 0.7427404522895813, "step": 80 }, { "epoch": 0.08116980005968367, "grad_norm": 28.375, "ht_mnpo/logit": 0.3949056565761566, "ht_mnpo/residual": 0.3874056935310364, "ht_mnpo/residual_abs": 0.38943880796432495, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.6666666666666666e-07, "logits/chosen": -1.8436777591705322, "logits/rejected": -2.107656478881836, "logps/chosen": -0.27149876952171326, "logps/rejected": -0.26327431201934814, "loss": 1.4980366230010986, "loss/core": 1.4980366230010986, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 5.046863555908203, "rewards/margins": 3.949056625366211, "rewards/rejected": 1.097806692123413, "step": 85 }, { "epoch": 0.08594449418084155, "grad_norm": 9.6875, "ht_mnpo/logit": 0.2532029151916504, "ht_mnpo/residual": 0.24570293724536896, "ht_mnpo/residual_abs": 0.24952733516693115, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.944444444444445e-07, "logits/chosen": -1.9614741802215576, "logits/rejected": -2.207104444503784, "logps/chosen": -0.308493047952652, "logps/rejected": -0.2764720320701599, "loss": 0.856387734413147, "loss/core": 0.856387734413147, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.3289856910705566, "rewards/margins": 2.532029151916504, "rewards/rejected": 0.7969565391540527, "step": 90 }, { "epoch": 0.0907191883019994, "grad_norm": 504.0, "ht_mnpo/logit": 0.2930489778518677, "ht_mnpo/residual": 0.28554895520210266, "ht_mnpo/residual_abs": 0.2872154712677002, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.999699149323369e-07, "logits/chosen": -2.090327262878418, "logits/rejected": -2.431143283843994, "logps/chosen": -0.2864113748073578, "logps/rejected": -0.28018051385879517, "loss": 1.093916654586792, "loss/core": 1.093916654586792, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.887937545776367, "rewards/margins": 2.9304897785186768, "rewards/rejected": 0.9574478268623352, "step": 95 }, { "epoch": 0.09549388242315726, "grad_norm": 43.75, "ht_mnpo/logit": 0.13229675590991974, "ht_mnpo/residual": 0.12479674816131592, "ht_mnpo/residual_abs": 0.15026070177555084, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.998477067547739e-07, "logits/chosen": -1.9240186214447021, "logits/rejected": -2.271806001663208, "logps/chosen": -0.2857854664325714, "logps/rejected": -0.3030196726322174, "loss": 0.26169103384017944, "loss/core": 0.26169103384017944, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.2315335273742676, "rewards/margins": 1.322967529296875, "rewards/rejected": 0.908565878868103, "step": 100 }, { "epoch": 0.10026857654431513, "grad_norm": 788.0, "ht_mnpo/logit": 0.2821234464645386, "ht_mnpo/residual": 0.27462345361709595, "ht_mnpo/residual_abs": 0.275934636592865, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.996315410727229e-07, "logits/chosen": -2.103553533554077, "logits/rejected": -2.360391855239868, "logps/chosen": -0.32235783338546753, "logps/rejected": -0.29607582092285156, "loss": 0.564530611038208, "loss/core": 0.564530611038208, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 4.4333977699279785, "rewards/margins": 2.8212342262268066, "rewards/rejected": 1.6121633052825928, "step": 105 }, { "epoch": 0.10504327066547299, "grad_norm": 122.0, "ht_mnpo/logit": 0.29163163900375366, "ht_mnpo/residual": 0.28413161635398865, "ht_mnpo/residual_abs": 0.28595611453056335, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.993214991772562e-07, "logits/chosen": -2.0258805751800537, "logits/rejected": -2.2438912391662598, "logps/chosen": -0.289029598236084, "logps/rejected": -0.2787890136241913, "loss": 1.244662880897522, "loss/core": 1.244662880897522, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.081713676452637, "rewards/margins": 2.916316032409668, "rewards/rejected": 1.1653969287872314, "step": 110 }, { "epoch": 0.10981796478663086, "grad_norm": 58.5, "ht_mnpo/logit": 0.2460227906703949, "ht_mnpo/residual": 0.23852276802062988, "ht_mnpo/residual_abs": 0.24091574549674988, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.989176976624511e-07, "logits/chosen": -1.842900276184082, "logits/rejected": -2.1219844818115234, "logps/chosen": -0.27585095167160034, "logps/rejected": -0.26493367552757263, "loss": 1.031520128250122, "loss/core": 1.031520128250122, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.6426234245300293, "rewards/margins": 2.4602277278900146, "rewards/rejected": 1.1823960542678833, "step": 115 }, { "epoch": 0.11459265890778872, "grad_norm": 516.0, "ht_mnpo/logit": 0.33325010538101196, "ht_mnpo/residual": 0.32575008273124695, "ht_mnpo/residual_abs": 0.3277893662452698, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.984202883815428e-07, "logits/chosen": -2.084425687789917, "logits/rejected": -2.2928617000579834, "logps/chosen": -0.2503313422203064, "logps/rejected": -0.27189603447914124, "loss": 0.949650764465332, "loss/core": 0.949650764465332, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.397518157958984, "rewards/margins": 3.332500457763672, "rewards/rejected": 1.0650172233581543, "step": 120 }, { "epoch": 0.11936735302894658, "grad_norm": 756.0, "ht_mnpo/logit": 0.4765322208404541, "ht_mnpo/residual": 0.4690321981906891, "ht_mnpo/residual_abs": 0.47166210412979126, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.978294583898195e-07, "logits/chosen": -1.8222243785858154, "logits/rejected": -2.102375030517578, "logps/chosen": -0.2784605026245117, "logps/rejected": -0.2793668210506439, "loss": 2.5388500690460205, "loss/core": 2.5388500690460205, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 5.699808120727539, "rewards/margins": 4.765322208404541, "rewards/rejected": 0.9344862103462219, "step": 125 }, { "epoch": 0.12414204715010445, "grad_norm": 18.875, "ht_mnpo/logit": 0.28754621744155884, "ht_mnpo/residual": 0.2800462245941162, "ht_mnpo/residual_abs": 0.28318971395492554, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.971454298742779e-07, "logits/chosen": -1.8026081323623657, "logits/rejected": -2.1300437450408936, "logps/chosen": -0.3127506673336029, "logps/rejected": -0.2983434796333313, "loss": 0.685983419418335, "loss/core": 0.685983419418335, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.466167449951172, "rewards/margins": 2.875462293624878, "rewards/rejected": 0.590705156326294, "step": 130 }, { "epoch": 0.12891674127126232, "grad_norm": 248.0, "ht_mnpo/logit": 0.18734104931354523, "ht_mnpo/residual": 0.1798410415649414, "ht_mnpo/residual_abs": 0.18434356153011322, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.963684600700678e-07, "logits/chosen": -2.0444672107696533, "logits/rejected": -2.295093059539795, "logps/chosen": -0.28641682863235474, "logps/rejected": -0.29985663294792175, "loss": 0.27129822969436646, "loss/core": 0.27129822969436646, "rewards/accuracies": 0.9375, "rewards/chosen": 2.71679949760437, "rewards/margins": 1.8734104633331299, "rewards/rejected": 0.8433891534805298, "step": 135 }, { "epoch": 0.13369143539242018, "grad_norm": 2.171875, "ht_mnpo/logit": 0.1653221845626831, "ht_mnpo/residual": 0.15782217681407928, "ht_mnpo/residual_abs": 0.16011430323123932, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.954988411637571e-07, "logits/chosen": -1.931020736694336, "logits/rejected": -2.2295100688934326, "logps/chosen": -0.28395944833755493, "logps/rejected": -0.27869370579719543, "loss": 0.12267724424600601, "loss/core": 0.12267724424600601, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.6366994380950928, "rewards/margins": 1.6532217264175415, "rewards/rejected": 0.9834780693054199, "step": 140 }, { "epoch": 0.13846612951357803, "grad_norm": 0.75, "ht_mnpo/logit": 0.2931879162788391, "ht_mnpo/residual": 0.28568798303604126, "ht_mnpo/residual_abs": 0.2914429306983948, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.945369001834514e-07, "logits/chosen": -1.8905494213104248, "logits/rejected": -2.122390031814575, "logps/chosen": -0.3317320942878723, "logps/rejected": -0.30477237701416016, "loss": 1.277919054031372, "loss/core": 1.277919054031372, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.9621856212615967, "rewards/margins": 2.9318792819976807, "rewards/rejected": 1.0303064584732056, "step": 145 }, { "epoch": 0.1432408236347359, "grad_norm": 1.546875, "ht_mnpo/logit": 0.2134993076324463, "ht_mnpo/residual": 0.20599932968616486, "ht_mnpo/residual_abs": 0.20773494243621826, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.93482998875813e-07, "logits/chosen": -2.0588443279266357, "logits/rejected": -2.308776378631592, "logps/chosen": -0.2723400592803955, "logps/rejected": -0.2848970293998718, "loss": 0.3919183015823364, "loss/core": 0.3919183015823364, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.260389804840088, "rewards/margins": 2.134993076324463, "rewards/rejected": 1.1253963708877563, "step": 150 }, { "epoch": 0.14801551775589375, "grad_norm": 165.0, "ht_mnpo/logit": 0.32777029275894165, "ht_mnpo/residual": 0.32027024030685425, "ht_mnpo/residual_abs": 0.3234081268310547, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.923375335700223e-07, "logits/chosen": -2.0180845260620117, "logits/rejected": -2.2344937324523926, "logps/chosen": -0.3052632808685303, "logps/rejected": -0.2794122099876404, "loss": 1.312514066696167, "loss/core": 1.312514066696167, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.1154704093933105, "rewards/margins": 3.277703046798706, "rewards/rejected": 0.8377677798271179, "step": 155 }, { "epoch": 0.15279021187705163, "grad_norm": 89.5, "ht_mnpo/logit": 0.15068449079990387, "ht_mnpo/residual": 0.14318448305130005, "ht_mnpo/residual_abs": 0.14576324820518494, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.911009350287347e-07, "logits/chosen": -2.039623975753784, "logits/rejected": -2.4383645057678223, "logps/chosen": -0.26783841848373413, "logps/rejected": -0.27481716871261597, "loss": 0.16402561962604523, "loss/core": 0.16402561962604523, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.1534738540649414, "rewards/margins": 1.5068447589874268, "rewards/rejected": 0.6466289758682251, "step": 160 }, { "epoch": 0.1575649059982095, "grad_norm": 3.75, "ht_mnpo/logit": 0.37046578526496887, "ht_mnpo/residual": 0.36296579241752625, "ht_mnpo/residual_abs": 0.36714157462120056, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.897736682860885e-07, "logits/chosen": -2.004047393798828, "logits/rejected": -2.2582778930664062, "logps/chosen": -0.2819061875343323, "logps/rejected": -0.2967601418495178, "loss": 1.949097990989685, "loss/core": 1.949097990989685, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 4.285637855529785, "rewards/margins": 3.704658031463623, "rewards/rejected": 0.5809799432754517, "step": 165 }, { "epoch": 0.16233960011936735, "grad_norm": 12.625, "ht_mnpo/logit": 0.2548084855079651, "ht_mnpo/residual": 0.24730846285820007, "ht_mnpo/residual_abs": 0.24948911368846893, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.883562324728241e-07, "logits/chosen": -2.0560150146484375, "logits/rejected": -2.2419166564941406, "logps/chosen": -0.3087208867073059, "logps/rejected": -0.32812321186065674, "loss": 0.8445270657539368, "loss/core": 0.8445270657539368, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.7604167461395264, "rewards/margins": 2.5480847358703613, "rewards/rejected": 1.212332010269165, "step": 170 }, { "epoch": 0.1671142942405252, "grad_norm": 25.25, "ht_mnpo/logit": 0.20657019317150116, "ht_mnpo/residual": 0.19907021522521973, "ht_mnpo/residual_abs": 0.20315143465995789, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.868491606285823e-07, "logits/chosen": -2.158867597579956, "logits/rejected": -2.4179515838623047, "logps/chosen": -0.29173943400382996, "logps/rejected": -0.3005591630935669, "loss": 1.0301963090896606, "loss/core": 1.0301963090896606, "rewards/accuracies": 0.875, "rewards/chosen": 2.7829740047454834, "rewards/margins": 2.065701961517334, "rewards/rejected": 0.7172719240188599, "step": 175 }, { "epoch": 0.1718889883616831, "grad_norm": 2.28125, "ht_mnpo/logit": 0.25792625546455383, "ht_mnpo/residual": 0.2504262328147888, "ht_mnpo/residual_abs": 0.2523646354675293, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.852530195014489e-07, "logits/chosen": -1.9154199361801147, "logits/rejected": -2.108656644821167, "logps/chosen": -0.29020223021507263, "logps/rejected": -0.28535544872283936, "loss": 0.9428588151931763, "loss/core": 0.9428588151931763, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.405637741088867, "rewards/margins": 2.5792624950408936, "rewards/rejected": 0.8263753056526184, "step": 180 }, { "epoch": 0.17666368248284095, "grad_norm": 9.1875, "ht_mnpo/logit": 0.11725939810276031, "ht_mnpo/residual": 0.1097593903541565, "ht_mnpo/residual_abs": 0.1118379607796669, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.835684093348244e-07, "logits/chosen": -2.0307536125183105, "logits/rejected": -2.279958486557007, "logps/chosen": -0.2843839228153229, "logps/rejected": -0.2895074784755707, "loss": 0.05697331950068474, "loss/core": 0.05697331950068474, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 1.8723161220550537, "rewards/margins": 1.1725938320159912, "rewards/rejected": 0.6997222900390625, "step": 185 }, { "epoch": 0.1814383766039988, "grad_norm": 314.0, "ht_mnpo/logit": 0.2137770652770996, "ht_mnpo/residual": 0.2062770575284958, "ht_mnpo/residual_abs": 0.21894094347953796, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.817959636416969e-07, "logits/chosen": -1.7537132501602173, "logits/rejected": -1.9378242492675781, "logps/chosen": -0.29650992155075073, "logps/rejected": -0.2991756796836853, "loss": 0.4277607798576355, "loss/core": 0.4277607798576355, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.179126262664795, "rewards/margins": 2.137770652770996, "rewards/rejected": 2.041355609893799, "step": 190 }, { "epoch": 0.18621307072515667, "grad_norm": 616.0, "ht_mnpo/logit": 0.28763315081596375, "ht_mnpo/residual": 0.2801331579685211, "ht_mnpo/residual_abs": 0.2813577353954315, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.799363489664039e-07, "logits/chosen": -1.9013550281524658, "logits/rejected": -2.1618947982788086, "logps/chosen": -0.2702489197254181, "logps/rejected": -0.2578398883342743, "loss": 0.7751284837722778, "loss/core": 0.7751284837722778, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 4.433821678161621, "rewards/margins": 2.876331329345703, "rewards/rejected": 1.557490587234497, "step": 195 }, { "epoch": 0.19098776484631452, "grad_norm": 764.0, "ht_mnpo/logit": 0.36720559000968933, "ht_mnpo/residual": 0.3597055971622467, "ht_mnpo/residual_abs": 0.3611852824687958, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.779902646339721e-07, "logits/chosen": -1.9602934122085571, "logits/rejected": -2.284992218017578, "logps/chosen": -0.30019837617874146, "logps/rejected": -0.29457828402519226, "loss": 1.2417619228363037, "loss/core": 1.2417619228363037, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 4.696887969970703, "rewards/margins": 3.672055721282959, "rewards/rejected": 1.024832010269165, "step": 200 }, { "epoch": 0.1957624589674724, "grad_norm": 1.15625, "ht_mnpo/logit": 0.13184615969657898, "ht_mnpo/residual": 0.12434617429971695, "ht_mnpo/residual_abs": 0.12654730677604675, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.759584424871301e-07, "logits/chosen": -2.0989491939544678, "logits/rejected": -2.2950825691223145, "logps/chosen": -0.27947577834129333, "logps/rejected": -0.2976261079311371, "loss": 0.12396552413702011, "loss/core": 0.12396552413702011, "rewards/accuracies": 0.9375, "rewards/chosen": 2.6087183952331543, "rewards/margins": 1.3184616565704346, "rewards/rejected": 1.2902568578720093, "step": 205 }, { "epoch": 0.20053715308863027, "grad_norm": 1144.0, "ht_mnpo/logit": 0.2897939682006836, "ht_mnpo/residual": 0.2822939157485962, "ht_mnpo/residual_abs": 0.28489789366722107, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.738416466110917e-07, "logits/chosen": -1.7683238983154297, "logits/rejected": -2.0142149925231934, "logps/chosen": -0.30294543504714966, "logps/rejected": -0.29282546043395996, "loss": 0.8169538378715515, "loss/core": 0.8169538378715515, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.211258888244629, "rewards/margins": 2.8979392051696777, "rewards/rejected": 1.3133190870285034, "step": 210 }, { "epoch": 0.20531184720978812, "grad_norm": 2.859375, "ht_mnpo/logit": 0.19056037068367004, "ht_mnpo/residual": 0.18306037783622742, "ht_mnpo/residual_abs": 0.1847141683101654, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.716406730462153e-07, "logits/chosen": -2.104823589324951, "logits/rejected": -2.3929922580718994, "logps/chosen": -0.27915796637535095, "logps/rejected": -0.28238147497177124, "loss": 0.5005974769592285, "loss/core": 0.5005974769592285, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 2.5159075260162354, "rewards/margins": 1.9056037664413452, "rewards/rejected": 0.610303521156311, "step": 215 }, { "epoch": 0.21008654133094598, "grad_norm": 71.0, "ht_mnpo/logit": 0.2221338003873825, "ht_mnpo/residual": 0.21463382244110107, "ht_mnpo/residual_abs": 0.21771471202373505, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.693563494886454e-07, "logits/chosen": -1.793410301208496, "logits/rejected": -2.138627767562866, "logps/chosen": -0.2802343964576721, "logps/rejected": -0.29542267322540283, "loss": 0.4375545084476471, "loss/core": 0.4375545084476471, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.305910587310791, "rewards/margins": 2.2213380336761475, "rewards/rejected": 1.0845725536346436, "step": 220 }, { "epoch": 0.21486123545210384, "grad_norm": 23.5, "ht_mnpo/logit": 0.20983533561229706, "ht_mnpo/residual": 0.20233532786369324, "ht_mnpo/residual_abs": 0.20403678715229034, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.6698953497905016e-07, "logits/chosen": -2.118230104446411, "logits/rejected": -2.3553214073181152, "logps/chosen": -0.2552695572376251, "logps/rejected": -0.27933183312416077, "loss": 0.59180748462677, "loss/core": 0.59180748462677, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.013653039932251, "rewards/margins": 2.098353147506714, "rewards/rejected": 0.9152997732162476, "step": 225 }, { "epoch": 0.21963592957326172, "grad_norm": 4.125, "ht_mnpo/logit": 0.13968196511268616, "ht_mnpo/residual": 0.13218197226524353, "ht_mnpo/residual_abs": 0.13623417913913727, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.645411195795709e-07, "logits/chosen": -1.9094196557998657, "logits/rejected": -2.1887524127960205, "logps/chosen": -0.29535284638404846, "logps/rejected": -0.30087095499038696, "loss": 0.09830106049776077, "loss/core": 0.09830106049776077, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.2886228561401367, "rewards/margins": 1.3968197107315063, "rewards/rejected": 0.8918031454086304, "step": 230 }, { "epoch": 0.22441062369441958, "grad_norm": 10.5625, "ht_mnpo/logit": 0.2398790419101715, "ht_mnpo/residual": 0.2323790341615677, "ht_mnpo/residual_abs": 0.23418621718883514, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.6201202403910643e-07, "logits/chosen": -1.7939506769180298, "logits/rejected": -2.1177468299865723, "logps/chosen": -0.2927335500717163, "logps/rejected": -0.2929953336715698, "loss": 0.39657431840896606, "loss/core": 0.39657431840896606, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.5993685722351074, "rewards/margins": 2.3987903594970703, "rewards/rejected": 1.2005784511566162, "step": 235 }, { "epoch": 0.22918531781557744, "grad_norm": 0.52734375, "ht_mnpo/logit": 0.23736481368541718, "ht_mnpo/residual": 0.22986480593681335, "ht_mnpo/residual_abs": 0.23360922932624817, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.594031994470573e-07, "logits/chosen": -2.2269482612609863, "logits/rejected": -2.406406879425049, "logps/chosen": -0.30673980712890625, "logps/rejected": -0.2728860080242157, "loss": 0.7180436253547668, "loss/core": 0.7180436253547668, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.5076498985290527, "rewards/margins": 2.373648166656494, "rewards/rejected": 1.1340019702911377, "step": 240 }, { "epoch": 0.2339600119367353, "grad_norm": 14.9375, "ht_mnpo/logit": 0.11371958255767822, "ht_mnpo/residual": 0.1062195748090744, "ht_mnpo/residual_abs": 0.1083589419722557, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.567156268756593e-07, "logits/chosen": -2.077096700668335, "logits/rejected": -2.2862114906311035, "logps/chosen": -0.3035627007484436, "logps/rejected": -0.3022594153881073, "loss": 0.10577340424060822, "loss/core": 0.10577340424060822, "rewards/accuracies": 0.9375, "rewards/chosen": 1.866857886314392, "rewards/margins": 1.1371958255767822, "rewards/rejected": 0.7296620607376099, "step": 245 }, { "epoch": 0.23873470605789315, "grad_norm": 44.5, "ht_mnpo/logit": 0.12031557410955429, "ht_mnpo/residual": 0.11281557381153107, "ht_mnpo/residual_abs": 0.1145239844918251, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.5395031701104303e-07, "logits/chosen": -2.05730938911438, "logits/rejected": -2.210094928741455, "logps/chosen": -0.2798473536968231, "logps/rejected": -0.292803019285202, "loss": 0.09960635006427765, "loss/core": 0.09960635006427765, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 1.7082903385162354, "rewards/margins": 1.203155755996704, "rewards/rejected": 0.505134642124176, "step": 250 }, { "epoch": 0.24350940017905104, "grad_norm": 4.5625, "ht_mnpo/logit": 0.2883337438106537, "ht_mnpo/residual": 0.28083378076553345, "ht_mnpo/residual_abs": 0.28313925862312317, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.511083097731555e-07, "logits/chosen": -1.9648189544677734, "logits/rejected": -2.2407779693603516, "logps/chosen": -0.3144630789756775, "logps/rejected": -0.3002500832080841, "loss": 1.097538709640503, "loss/core": 1.097538709640503, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.546959400177002, "rewards/margins": 2.8833374977111816, "rewards/rejected": 0.6636216640472412, "step": 255 }, { "epoch": 0.2482840943002089, "grad_norm": 108.5, "ht_mnpo/logit": 0.2983548939228058, "ht_mnpo/residual": 0.29085490107536316, "ht_mnpo/residual_abs": 0.2929629981517792, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.481906739246894e-07, "logits/chosen": -1.9292614459991455, "logits/rejected": -2.2683722972869873, "logps/chosen": -0.2792973816394806, "logps/rejected": -0.310756653547287, "loss": 1.0083863735198975, "loss/core": 1.0083863735198975, "rewards/accuracies": 0.9375, "rewards/chosen": 3.546018600463867, "rewards/margins": 2.983549118041992, "rewards/rejected": 0.5624698400497437, "step": 260 }, { "epoch": 0.25305878842136675, "grad_norm": 29.375, "ht_mnpo/logit": 0.22387555241584778, "ht_mnpo/residual": 0.21637554466724396, "ht_mnpo/residual_abs": 0.2206393927335739, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.451985066691648e-07, "logits/chosen": -2.173464059829712, "logits/rejected": -2.459009885787964, "logps/chosen": -0.2751771807670593, "logps/rejected": -0.2682764530181885, "loss": 0.4876701831817627, "loss/core": 0.4876701831817627, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.012019634246826, "rewards/margins": 2.238755464553833, "rewards/rejected": 0.7732640504837036, "step": 265 }, { "epoch": 0.25783348254252464, "grad_norm": 888.0, "ht_mnpo/logit": 0.4514991343021393, "ht_mnpo/residual": 0.44399914145469666, "ht_mnpo/residual_abs": 0.44612884521484375, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.421329332383158e-07, "logits/chosen": -1.8616456985473633, "logits/rejected": -2.184025764465332, "logps/chosen": -0.2934468686580658, "logps/rejected": -0.2970571517944336, "loss": 2.1650872230529785, "loss/core": 2.1650872230529785, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 5.505021572113037, "rewards/margins": 4.514991283416748, "rewards/rejected": 0.9900302886962891, "step": 270 }, { "epoch": 0.26260817666368247, "grad_norm": 2.640625, "ht_mnpo/logit": 0.14674557745456696, "ht_mnpo/residual": 0.13924556970596313, "ht_mnpo/residual_abs": 0.14186374843120575, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.3899510646893696e-07, "logits/chosen": -2.0834450721740723, "logits/rejected": -2.3231890201568604, "logps/chosen": -0.27442002296447754, "logps/rejected": -0.27742353081703186, "loss": 0.23988012969493866, "loss/core": 0.23988012969493866, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.230260133743286, "rewards/margins": 1.4674557447433472, "rewards/rejected": 0.7628042101860046, "step": 275 }, { "epoch": 0.26738287078484035, "grad_norm": 8.1875, "ht_mnpo/logit": 0.2650841772556305, "ht_mnpo/residual": 0.2575841546058655, "ht_mnpo/residual_abs": 0.2594790458679199, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.357862063693485e-07, "logits/chosen": -2.0836455821990967, "logits/rejected": -2.3042664527893066, "logps/chosen": -0.32237258553504944, "logps/rejected": -0.3145085573196411, "loss": 1.0589872598648071, "loss/core": 1.0589872598648071, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.507110118865967, "rewards/margins": 2.650841474533081, "rewards/rejected": 0.8562685251235962, "step": 280 }, { "epoch": 0.2721575649059982, "grad_norm": 1.4765625, "ht_mnpo/logit": 0.10140925645828247, "ht_mnpo/residual": 0.09390926361083984, "ht_mnpo/residual_abs": 0.09555725753307343, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.3250743967564364e-07, "logits/chosen": -1.9129667282104492, "logits/rejected": -2.1948471069335938, "logps/chosen": -0.2781878709793091, "logps/rejected": -0.2911220192909241, "loss": 0.03205167502164841, "loss/core": 0.03205167502164841, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 1.7170699834823608, "rewards/margins": 1.0140926837921143, "rewards/rejected": 0.7029772996902466, "step": 285 }, { "epoch": 0.27693225902715607, "grad_norm": 3.875, "ht_mnpo/logit": 0.30378761887550354, "ht_mnpo/residual": 0.2962876260280609, "ht_mnpo/residual_abs": 0.29867178201675415, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.29160039397884e-07, "logits/chosen": -1.9081875085830688, "logits/rejected": -2.3261563777923584, "logps/chosen": -0.2825949788093567, "logps/rejected": -0.2706306278705597, "loss": 1.074646234512329, "loss/core": 1.074646234512329, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.7699265480041504, "rewards/margins": 3.0378763675689697, "rewards/rejected": 0.7320500016212463, "step": 290 }, { "epoch": 0.28170695314831395, "grad_norm": 1760.0, "ht_mnpo/logit": 0.3823835849761963, "ht_mnpo/residual": 0.37488359212875366, "ht_mnpo/residual_abs": 0.3766990602016449, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.2574526435641546e-07, "logits/chosen": -1.8789278268814087, "logits/rejected": -2.110046625137329, "logps/chosen": -0.28207749128341675, "logps/rejected": -0.2834545373916626, "loss": 1.690529465675354, "loss/core": 1.690529465675354, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 4.7853851318359375, "rewards/margins": 3.823835849761963, "rewards/rejected": 0.961548924446106, "step": 295 }, { "epoch": 0.2864816472694718, "grad_norm": 1.203125, "ht_mnpo/logit": 0.1465030014514923, "ht_mnpo/residual": 0.13900300860404968, "ht_mnpo/residual_abs": 0.14254584908485413, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.22264398708477e-07, "logits/chosen": -1.9352973699569702, "logits/rejected": -2.1947338581085205, "logps/chosen": -0.32196083664894104, "logps/rejected": -0.3307940363883972, "loss": 0.3272203505039215, "loss/core": 0.3272203505039215, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.2242655754089355, "rewards/margins": 1.4650301933288574, "rewards/rejected": 0.7592355608940125, "step": 300 }, { "epoch": 0.29125634139062967, "grad_norm": 99.5, "ht_mnpo/logit": 0.13784225285053253, "ht_mnpo/residual": 0.1303422749042511, "ht_mnpo/residual_abs": 0.1337766945362091, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.187187514652819e-07, "logits/chosen": -2.080651044845581, "logits/rejected": -2.253840684890747, "logps/chosen": -0.30505985021591187, "logps/rejected": -0.32858914136886597, "loss": 0.3612667918205261, "loss/core": 0.3612667918205261, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.7895835638046265, "rewards/margins": 1.378422498703003, "rewards/rejected": 0.4111608862876892, "step": 305 }, { "epoch": 0.2960310355117875, "grad_norm": 640.0, "ht_mnpo/logit": 0.2148059904575348, "ht_mnpo/residual": 0.20730595290660858, "ht_mnpo/residual_abs": 0.21532678604125977, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.151096559997519e-07, "logits/chosen": -2.0484871864318848, "logits/rejected": -2.299739360809326, "logps/chosen": -0.2571372389793396, "logps/rejected": -0.2654029428958893, "loss": 0.6419556736946106, "loss/core": 0.6419556736946106, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.3131752014160156, "rewards/margins": 2.148059368133545, "rewards/rejected": 1.1651160717010498, "step": 310 }, { "epoch": 0.3008057296329454, "grad_norm": 6.25, "ht_mnpo/logit": 0.26450738310813904, "ht_mnpo/residual": 0.2570074200630188, "ht_mnpo/residual_abs": 0.2596099078655243, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.114384695450905e-07, "logits/chosen": -1.8389809131622314, "logits/rejected": -2.1140105724334717, "logps/chosen": -0.289066880941391, "logps/rejected": -0.2824665904045105, "loss": 0.9795834422111511, "loss/core": 0.9795834422111511, "rewards/accuracies": 0.9375, "rewards/chosen": 3.305142641067505, "rewards/margins": 2.6450741291046143, "rewards/rejected": 0.6600683331489563, "step": 315 }, { "epoch": 0.30558042375410327, "grad_norm": 2.671875, "ht_mnpo/logit": 0.22446127235889435, "ht_mnpo/residual": 0.21696126461029053, "ht_mnpo/residual_abs": 0.2195734977722168, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.077065726843828e-07, "logits/chosen": -2.0431623458862305, "logits/rejected": -2.303877830505371, "logps/chosen": -0.2965102195739746, "logps/rejected": -0.2953088879585266, "loss": 0.659471333026886, "loss/core": 0.659471333026886, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.1237876415252686, "rewards/margins": 2.244612693786621, "rewards/rejected": 0.8791753053665161, "step": 320 }, { "epoch": 0.3103551178752611, "grad_norm": 528.0, "ht_mnpo/logit": 0.4989927411079407, "ht_mnpo/residual": 0.49149274826049805, "ht_mnpo/residual_abs": 0.49372735619544983, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.039153688314145e-07, "logits/chosen": -1.8436063528060913, "logits/rejected": -2.1103482246398926, "logps/chosen": -0.2776651084423065, "logps/rejected": -0.2627367377281189, "loss": 2.5948891639709473, "loss/core": 2.5948891639709473, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 6.395821571350098, "rewards/margins": 4.989927768707275, "rewards/rejected": 1.4058938026428223, "step": 325 }, { "epoch": 0.315129811996419, "grad_norm": 84.5, "ht_mnpo/logit": 0.11850211769342422, "ht_mnpo/residual": 0.1110021248459816, "ht_mnpo/residual_abs": 0.11589847505092621, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.0006628370290613e-07, "logits/chosen": -2.046609401702881, "logits/rejected": -2.265373945236206, "logps/chosen": -0.27779677510261536, "logps/rejected": -0.27654415369033813, "loss": 0.10753259807825089, "loss/core": 0.10753259807825089, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.0452699661254883, "rewards/margins": 1.185021162033081, "rewards/rejected": 0.8602487444877625, "step": 330 }, { "epoch": 0.31990450611757687, "grad_norm": 26.375, "ht_mnpo/logit": 0.22499430179595947, "ht_mnpo/residual": 0.21749427914619446, "ht_mnpo/residual_abs": 0.21924762427806854, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.9616076478235826e-07, "logits/chosen": -2.0356459617614746, "logits/rejected": -2.332686424255371, "logps/chosen": -0.26844361424446106, "logps/rejected": -0.2597638666629791, "loss": 0.8870676755905151, "loss/core": 0.8870676755905151, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.095395088195801, "rewards/margins": 2.2499425411224365, "rewards/rejected": 0.8454524874687195, "step": 335 }, { "epoch": 0.3246792002387347, "grad_norm": 5.34375, "ht_mnpo/logit": 0.1895650327205658, "ht_mnpo/residual": 0.18206503987312317, "ht_mnpo/residual_abs": 0.18503603339195251, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.922002807757129e-07, "logits/chosen": -1.8680092096328735, "logits/rejected": -2.092318296432495, "logps/chosen": -0.3079831302165985, "logps/rejected": -0.28415459394454956, "loss": 0.19596347212791443, "loss/core": 0.19596347212791443, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 2.789630651473999, "rewards/margins": 1.8956505060195923, "rewards/rejected": 0.8939803242683411, "step": 340 }, { "epoch": 0.3294538943598926, "grad_norm": 70.0, "ht_mnpo/logit": 0.19111791253089905, "ht_mnpo/residual": 0.18361787497997284, "ht_mnpo/residual_abs": 0.1863330900669098, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.8818632105903315e-07, "logits/chosen": -2.109628915786743, "logits/rejected": -2.291822910308838, "logps/chosen": -0.3013684153556824, "logps/rejected": -0.2833828628063202, "loss": 0.2935481667518616, "loss/core": 0.2935481667518616, "rewards/accuracies": 0.9375, "rewards/chosen": 3.4318795204162598, "rewards/margins": 1.9111789464950562, "rewards/rejected": 1.5207008123397827, "step": 345 }, { "epoch": 0.3342285884810504, "grad_norm": 21.375, "ht_mnpo/logit": 0.17235636711120605, "ht_mnpo/residual": 0.16485637426376343, "ht_mnpo/residual_abs": 0.16569381952285767, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.841203951184094e-07, "logits/chosen": -2.0808799266815186, "logits/rejected": -2.3465583324432373, "logps/chosen": -0.2933531403541565, "logps/rejected": -0.29250794649124146, "loss": 0.18648609519004822, "loss/core": 0.18648609519004822, "rewards/accuracies": 1.0, "rewards/chosen": 2.39286470413208, "rewards/margins": 1.723563551902771, "rewards/rejected": 0.6693009734153748, "step": 350 }, { "epoch": 0.3390032826022083, "grad_norm": 8.8125, "ht_mnpo/logit": 0.083734892308712, "ht_mnpo/residual": 0.07623489201068878, "ht_mnpo/residual_abs": 0.08302940428256989, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.800040319823038e-07, "logits/chosen": -1.957427978515625, "logits/rejected": -2.2546541690826416, "logps/chosen": -0.2971518635749817, "logps/rejected": -0.29797545075416565, "loss": 0.02712792530655861, "loss/core": 0.02712792530655861, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.441888451576233, "rewards/margins": 0.8373489379882812, "rewards/rejected": 0.6045395135879517, "step": 355 }, { "epoch": 0.3437779767233662, "grad_norm": 12.0, "ht_mnpo/logit": 0.13458336889743805, "ht_mnpo/residual": 0.12708337604999542, "ht_mnpo/residual_abs": 0.12896767258644104, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.75838779646545e-07, "logits/chosen": -1.9576762914657593, "logits/rejected": -2.2406909465789795, "logps/chosen": -0.3130984306335449, "logps/rejected": -0.3136448860168457, "loss": 0.10641677677631378, "loss/core": 0.10641677677631378, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.3418941497802734, "rewards/margins": 1.3458335399627686, "rewards/rejected": 0.9960603713989258, "step": 360 }, { "epoch": 0.348552670844524, "grad_norm": 37.25, "ht_mnpo/logit": 0.25376901030540466, "ht_mnpo/residual": 0.24626903235912323, "ht_mnpo/residual_abs": 0.24840883910655975, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.7162620449218993e-07, "logits/chosen": -1.9573538303375244, "logits/rejected": -2.191314458847046, "logps/chosen": -0.28709596395492554, "logps/rejected": -0.28582268953323364, "loss": 0.5417178273200989, "loss/core": 0.5417178273200989, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.668320894241333, "rewards/margins": 2.5376901626586914, "rewards/rejected": 1.1306307315826416, "step": 365 }, { "epoch": 0.3533273649656819, "grad_norm": 23.5, "ht_mnpo/logit": 0.1474151760339737, "ht_mnpo/residual": 0.13991518318653107, "ht_mnpo/residual_abs": 0.14576002955436707, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.673678906964727e-07, "logits/chosen": -2.0737175941467285, "logits/rejected": -2.3785977363586426, "logps/chosen": -0.303182452917099, "logps/rejected": -0.3058184087276459, "loss": 0.16569259762763977, "loss/core": 0.16569259762763977, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.3934152126312256, "rewards/margins": 1.474151611328125, "rewards/rejected": 0.9192636609077454, "step": 370 }, { "epoch": 0.35810205908683973, "grad_norm": 19.0, "ht_mnpo/logit": 0.2867982089519501, "ht_mnpo/residual": 0.27929818630218506, "ht_mnpo/residual_abs": 0.2815331816673279, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.6306543963705936e-07, "logits/chosen": -1.7822672128677368, "logits/rejected": -2.033956289291382, "logps/chosen": -0.2669016718864441, "logps/rejected": -0.2721244990825653, "loss": 0.7982349991798401, "loss/core": 0.7982349991798401, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.7743923664093018, "rewards/margins": 2.8679816722869873, "rewards/rejected": 0.9064106941223145, "step": 375 }, { "epoch": 0.3628767532079976, "grad_norm": 12.4375, "ht_mnpo/logit": 0.22812096774578094, "ht_mnpo/residual": 0.22062095999717712, "ht_mnpo/residual_abs": 0.22334913909435272, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.5872046928983623e-07, "logits/chosen": -1.9232280254364014, "logits/rejected": -2.178192615509033, "logps/chosen": -0.29933086037635803, "logps/rejected": -0.28768283128738403, "loss": 0.6772836446762085, "loss/core": 0.6772836446762085, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.9951553344726562, "rewards/margins": 2.281209945678711, "rewards/rejected": 0.713945746421814, "step": 380 }, { "epoch": 0.3676514473291555, "grad_norm": 3.25, "ht_mnpo/logit": 0.21591122448444366, "ht_mnpo/residual": 0.20841124653816223, "ht_mnpo/residual_abs": 0.21050401031970978, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.5433461362045447e-07, "logits/chosen": -1.847633719444275, "logits/rejected": -2.065316677093506, "logps/chosen": -0.28148138523101807, "logps/rejected": -0.2986345589160919, "loss": 0.47226905822753906, "loss/core": 0.47226905822753906, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.5782718658447266, "rewards/margins": 2.1591124534606934, "rewards/rejected": 0.4191594123840332, "step": 385 }, { "epoch": 0.37242614145031333, "grad_norm": 404.0, "ht_mnpo/logit": 0.4722256064414978, "ht_mnpo/residual": 0.46472564339637756, "ht_mnpo/residual_abs": 0.46695786714553833, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.4990952196986305e-07, "logits/chosen": -1.9401696920394897, "logits/rejected": -2.1430230140686035, "logps/chosen": -0.3209291100502014, "logps/rejected": -0.29362010955810547, "loss": 2.3541719913482666, "loss/core": 2.3541719913482666, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 5.9721999168396, "rewards/margins": 4.722256183624268, "rewards/rejected": 1.2499440908432007, "step": 390 }, { "epoch": 0.3772008355714712, "grad_norm": 28.75, "ht_mnpo/logit": 0.2606656849384308, "ht_mnpo/residual": 0.25316569209098816, "ht_mnpo/residual_abs": 0.2560683488845825, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.4544685843405875e-07, "logits/chosen": -1.8940608501434326, "logits/rejected": -2.079465389251709, "logps/chosen": -0.2855537533760071, "logps/rejected": -0.2773713767528534, "loss": 0.5645236372947693, "loss/core": 0.5645236372947693, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.6368789672851562, "rewards/margins": 2.606656789779663, "rewards/rejected": 1.030221700668335, "step": 395 }, { "epoch": 0.38197552969262905, "grad_norm": 187.0, "ht_mnpo/logit": 0.26339191198349, "ht_mnpo/residual": 0.25589194893836975, "ht_mnpo/residual_abs": 0.2572072148323059, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.4094830123828786e-07, "logits/chosen": -2.0253453254699707, "logits/rejected": -2.415069818496704, "logps/chosen": -0.2948891520500183, "logps/rejected": -0.28987759351730347, "loss": 0.494838148355484, "loss/core": 0.494838148355484, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.4587039947509766, "rewards/margins": 2.6339192390441895, "rewards/rejected": 0.824785053730011, "step": 400 }, { "epoch": 0.38675022381378693, "grad_norm": 796.0, "ht_mnpo/logit": 0.519372820854187, "ht_mnpo/residual": 0.5118728876113892, "ht_mnpo/residual_abs": 0.5148923993110657, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.3641554210593414e-07, "logits/chosen": -1.5729597806930542, "logits/rejected": -1.9059674739837646, "logps/chosen": -0.324828565120697, "logps/rejected": -0.3063935339450836, "loss": 2.9073703289031982, "loss/core": 2.9073703289031982, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 6.127835750579834, "rewards/margins": 5.193729400634766, "rewards/rejected": 0.9341069459915161, "step": 405 }, { "epoch": 0.3915249179349448, "grad_norm": 428.0, "ht_mnpo/logit": 0.33313632011413574, "ht_mnpo/residual": 0.3256363272666931, "ht_mnpo/residual_abs": 0.3284788727760315, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.3185028562233107e-07, "logits/chosen": -1.9473098516464233, "logits/rejected": -2.265756130218506, "logps/chosen": -0.28903454542160034, "logps/rejected": -0.29171279072761536, "loss": 1.68734610080719, "loss/core": 1.68734610080719, "rewards/accuracies": 0.9375, "rewards/chosen": 4.016785621643066, "rewards/margins": 3.3313636779785156, "rewards/rejected": 0.6854226589202881, "step": 410 }, { "epoch": 0.39629961205610265, "grad_norm": 4.96875, "ht_mnpo/logit": 0.14160475134849548, "ht_mnpo/residual": 0.13410475850105286, "ht_mnpo/residual_abs": 0.1367226243019104, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.272542485937368e-07, "logits/chosen": -2.0103304386138916, "logits/rejected": -2.196492910385132, "logps/chosen": -0.2817983031272888, "logps/rejected": -0.30435535311698914, "loss": 0.11378359794616699, "loss/core": 0.11378359794616699, "rewards/accuracies": 0.9375, "rewards/chosen": 2.1162495613098145, "rewards/margins": 1.41604745388031, "rewards/rejected": 0.7002020478248596, "step": 415 }, { "epoch": 0.40107430617726053, "grad_norm": 792.0, "ht_mnpo/logit": 0.2766280174255371, "ht_mnpo/residual": 0.2691280245780945, "ht_mnpo/residual_abs": 0.27129045128822327, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.226291594017137e-07, "logits/chosen": -1.7459360361099243, "logits/rejected": -1.9585685729980469, "logps/chosen": -0.2827063500881195, "logps/rejected": -0.2677227854728699, "loss": 0.8373321294784546, "loss/core": 0.8373321294784546, "rewards/accuracies": 0.9375, "rewards/chosen": 4.167675495147705, "rewards/margins": 2.766279935836792, "rewards/rejected": 1.401395320892334, "step": 420 }, { "epoch": 0.40584900029841836, "grad_norm": 676.0, "ht_mnpo/logit": 0.22410964965820312, "ht_mnpo/residual": 0.2166096419095993, "ht_mnpo/residual_abs": 0.22375044226646423, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.1797675735315454e-07, "logits/chosen": -1.9364029169082642, "logits/rejected": -2.2243618965148926, "logps/chosen": -0.29613256454467773, "logps/rejected": -0.2813289761543274, "loss": 0.44919532537460327, "loss/core": 0.44919532537460327, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.634718418121338, "rewards/margins": 2.2410967350006104, "rewards/rejected": 1.3936221599578857, "step": 425 }, { "epoch": 0.41062369441957625, "grad_norm": 2.953125, "ht_mnpo/logit": 0.11446923017501831, "ht_mnpo/residual": 0.10696922242641449, "ht_mnpo/residual_abs": 0.10941126197576523, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.1329879202620047e-07, "logits/chosen": -2.0317301750183105, "logits/rejected": -2.2824201583862305, "logps/chosen": -0.28097397089004517, "logps/rejected": -0.2934049665927887, "loss": 0.04260837286710739, "loss/core": 0.04260837286710739, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.770204782485962, "rewards/margins": 1.1446921825408936, "rewards/rejected": 0.6255124807357788, "step": 430 }, { "epoch": 0.41539838854073413, "grad_norm": 6.75, "ht_mnpo/logit": 0.17668896913528442, "ht_mnpo/residual": 0.1691889464855194, "ht_mnpo/residual_abs": 0.17058250308036804, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.0859702261229613e-07, "logits/chosen": -2.0280332565307617, "logits/rejected": -2.2525267601013184, "logps/chosen": -0.2677651345729828, "logps/rejected": -0.2582930326461792, "loss": 0.2649860978126526, "loss/core": 0.2649860978126526, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 2.587427854537964, "rewards/margins": 1.7668898105621338, "rewards/rejected": 0.8205384016036987, "step": 435 }, { "epoch": 0.42017308266189196, "grad_norm": 18.25, "ht_mnpo/logit": 0.146823450922966, "ht_mnpo/residual": 0.13932344317436218, "ht_mnpo/residual_abs": 0.1414867490530014, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.0387321725463e-07, "logits/chosen": -1.963629961013794, "logits/rejected": -2.183755397796631, "logps/chosen": -0.2657316327095032, "logps/rejected": -0.2711542248725891, "loss": 0.07426687330007553, "loss/core": 0.07426687330007553, "rewards/accuracies": 0.9375, "rewards/chosen": 2.3465564250946045, "rewards/margins": 1.4682343006134033, "rewards/rejected": 0.878322422504425, "step": 440 }, { "epoch": 0.42494777678304985, "grad_norm": 3.625, "ht_mnpo/logit": 0.2173123061656952, "ht_mnpo/residual": 0.20981231331825256, "ht_mnpo/residual_abs": 0.21300652623176575, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.991291523832075e-07, "logits/chosen": -1.9978408813476562, "logits/rejected": -2.254274606704712, "logps/chosen": -0.3106047511100769, "logps/rejected": -0.28541427850723267, "loss": 1.295969843864441, "loss/core": 1.295969843864441, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.9272522926330566, "rewards/margins": 2.1731228828430176, "rewards/rejected": 0.7541292905807495, "step": 445 }, { "epoch": 0.4297224709042077, "grad_norm": 1128.0, "ht_mnpo/logit": 0.2916508913040161, "ht_mnpo/residual": 0.2841508984565735, "ht_mnpo/residual_abs": 0.28610268235206604, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.943666120468088e-07, "logits/chosen": -1.9271923303604126, "logits/rejected": -2.2027246952056885, "logps/chosen": -0.3097872734069824, "logps/rejected": -0.28542861342430115, "loss": 0.8190312385559082, "loss/core": 0.8190312385559082, "rewards/accuracies": 0.9375, "rewards/chosen": 3.744999408721924, "rewards/margins": 2.9165091514587402, "rewards/rejected": 0.8284900784492493, "step": 450 }, { "epoch": 0.43449716502536556, "grad_norm": 6.40625, "ht_mnpo/logit": 0.21481645107269287, "ht_mnpo/residual": 0.20731644332408905, "ht_mnpo/residual_abs": 0.2103075236082077, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.8958738724208073e-07, "logits/chosen": -1.8237874507904053, "logits/rejected": -2.129227638244629, "logps/chosen": -0.30449140071868896, "logps/rejected": -0.27840521931648254, "loss": 0.42009902000427246, "loss/core": 0.42009902000427246, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.241947889328003, "rewards/margins": 2.1481645107269287, "rewards/rejected": 1.0937836170196533, "step": 455 }, { "epoch": 0.43927185914652345, "grad_norm": 167.0, "ht_mnpo/logit": 0.29557928442955017, "ht_mnpo/residual": 0.28807929158210754, "ht_mnpo/residual_abs": 0.2902158796787262, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.8479327524001633e-07, "logits/chosen": -1.9467674493789673, "logits/rejected": -2.1150288581848145, "logps/chosen": -0.25731417536735535, "logps/rejected": -0.2749153673648834, "loss": 1.0729789733886719, "loss/core": 1.0729789733886719, "rewards/accuracies": 0.9375, "rewards/chosen": 3.7630081176757812, "rewards/margins": 2.9557926654815674, "rewards/rejected": 0.8072150945663452, "step": 460 }, { "epoch": 0.4440465532676813, "grad_norm": 5.28125, "ht_mnpo/logit": 0.1232837587594986, "ht_mnpo/residual": 0.11578376591205597, "ht_mnpo/residual_abs": 0.11795320361852646, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.7998607891007493e-07, "logits/chosen": -1.9201545715332031, "logits/rejected": -2.2683331966400146, "logps/chosen": -0.3000238239765167, "logps/rejected": -0.3087528347969055, "loss": 0.0906161218881607, "loss/core": 0.0906161218881607, "rewards/accuracies": 0.9375, "rewards/chosen": 1.7928030490875244, "rewards/margins": 1.2328376770019531, "rewards/rejected": 0.5599652528762817, "step": 465 }, { "epoch": 0.44882124738883916, "grad_norm": 55.5, "ht_mnpo/logit": 0.3084077537059784, "ht_mnpo/residual": 0.3009077310562134, "ht_mnpo/residual_abs": 0.3186038136482239, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.7516760604219616e-07, "logits/chosen": -1.9410159587860107, "logits/rejected": -2.1513659954071045, "logps/chosen": -0.3083992600440979, "logps/rejected": -0.3033507764339447, "loss": 1.498625636100769, "loss/core": 1.498625636100769, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 4.285151481628418, "rewards/margins": 3.084077835083008, "rewards/rejected": 1.2010738849639893, "step": 470 }, { "epoch": 0.453595941509997, "grad_norm": 352.0, "ht_mnpo/logit": 0.36416906118392944, "ht_mnpo/residual": 0.35666903853416443, "ht_mnpo/residual_abs": 0.3577561676502228, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.703396686669646e-07, "logits/chosen": -2.192099094390869, "logits/rejected": -2.485234498977661, "logps/chosen": -0.29125604033470154, "logps/rejected": -0.3030247092247009, "loss": 1.63297438621521, "loss/core": 1.63297438621521, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 4.850213050842285, "rewards/margins": 3.641690492630005, "rewards/rejected": 1.2085225582122803, "step": 475 }, { "epoch": 0.4583706356311549, "grad_norm": 15.0, "ht_mnpo/logit": 0.3097284436225891, "ht_mnpo/residual": 0.30222851037979126, "ht_mnpo/residual_abs": 0.3041606545448303, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.6550408237417884e-07, "logits/chosen": -1.9110229015350342, "logits/rejected": -2.150383472442627, "logps/chosen": -0.29265767335891724, "logps/rejected": -0.28647303581237793, "loss": 0.8241177797317505, "loss/core": 0.8241177797317505, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 4.338518142700195, "rewards/margins": 3.0972847938537598, "rewards/rejected": 1.241233229637146, "step": 480 }, { "epoch": 0.46314532975231276, "grad_norm": 238.0, "ht_mnpo/logit": 0.3325381875038147, "ht_mnpo/residual": 0.32503825426101685, "ht_mnpo/residual_abs": 0.3343810439109802, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.6066266563008265e-07, "logits/chosen": -1.8738826513290405, "logits/rejected": -2.16226863861084, "logps/chosen": -0.3215123116970062, "logps/rejected": -0.3182523846626282, "loss": 1.0384169816970825, "loss/core": 1.0384169816970825, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.18947696685791, "rewards/margins": 3.3253822326660156, "rewards/rejected": 0.8640945553779602, "step": 485 }, { "epoch": 0.4679200238734706, "grad_norm": 12.5, "ht_mnpo/logit": 0.14976148307323456, "ht_mnpo/residual": 0.14226147532463074, "ht_mnpo/residual_abs": 0.14404740929603577, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.5581723909351404e-07, "logits/chosen": -2.215057849884033, "logits/rejected": -2.4560017585754395, "logps/chosen": -0.2996280789375305, "logps/rejected": -0.30207791924476624, "loss": 0.15293726325035095, "loss/core": 0.15293726325035095, "rewards/accuracies": 0.9375, "rewards/chosen": 2.62994384765625, "rewards/margins": 1.497614860534668, "rewards/rejected": 1.1323288679122925, "step": 490 }, { "epoch": 0.4726947179946285, "grad_norm": 1.109375, "ht_mnpo/logit": 0.1131284236907959, "ht_mnpo/residual": 0.10562840849161148, "ht_mnpo/residual_abs": 0.10809159278869629, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.509696249312301e-07, "logits/chosen": -2.2300727367401123, "logits/rejected": -2.570033550262451, "logps/chosen": -0.2869029641151428, "logps/rejected": -0.2894020676612854, "loss": 0.0645766630768776, "loss/core": 0.0645766630768776, "rewards/accuracies": 0.9375, "rewards/chosen": 2.0836386680603027, "rewards/margins": 1.1312841176986694, "rewards/rejected": 0.9523545503616333, "step": 495 }, { "epoch": 0.4774694121157863, "grad_norm": 384.0, "ht_mnpo/logit": 0.1857949197292328, "ht_mnpo/residual": 0.17829494178295135, "ht_mnpo/residual_abs": 0.1815871000289917, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.461216461326642e-07, "logits/chosen": -1.7350391149520874, "logits/rejected": -2.03250789642334, "logps/chosen": -0.30827873945236206, "logps/rejected": -0.30130141973495483, "loss": 0.2996276617050171, "loss/core": 0.2996276617050171, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.6126999855041504, "rewards/margins": 1.8579492568969727, "rewards/rejected": 0.7547506093978882, "step": 500 }, { "epoch": 0.4822441062369442, "grad_norm": 11.75, "ht_mnpo/logit": 0.196578711271286, "ht_mnpo/residual": 0.18907873332500458, "ht_mnpo/residual_abs": 0.19180910289287567, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.412751258243748e-07, "logits/chosen": -1.9780470132827759, "logits/rejected": -2.1910805702209473, "logps/chosen": -0.31306812167167664, "logps/rejected": -0.31634530425071716, "loss": 0.27675363421440125, "loss/core": 0.27675363421440125, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.940455913543701, "rewards/margins": 1.9657869338989258, "rewards/rejected": 0.9746688008308411, "step": 505 }, { "epoch": 0.4870188003581021, "grad_norm": 11.6875, "ht_mnpo/logit": 0.29334843158721924, "ht_mnpo/residual": 0.2858484089374542, "ht_mnpo/residual_abs": 0.28805553913116455, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.3643188658444158e-07, "logits/chosen": -1.839543104171753, "logits/rejected": -2.139441728591919, "logps/chosen": -0.2953009009361267, "logps/rejected": -0.27880555391311646, "loss": 0.7938116788864136, "loss/core": 0.7938116788864136, "rewards/accuracies": 0.9375, "rewards/chosen": 3.985734462738037, "rewards/margins": 2.9334845542907715, "rewards/rejected": 1.0522500276565552, "step": 510 }, { "epoch": 0.4917934944792599, "grad_norm": 2.59375, "ht_mnpo/logit": 0.11522988229990005, "ht_mnpo/residual": 0.10772988945245743, "ht_mnpo/residual_abs": 0.11010618507862091, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.315937497570688e-07, "logits/chosen": -1.898514747619629, "logits/rejected": -2.2502241134643555, "logps/chosen": -0.2737577557563782, "logps/rejected": -0.2691265940666199, "loss": 0.06796002388000488, "loss/core": 0.06796002388000488, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 1.908595323562622, "rewards/margins": 1.1522986888885498, "rewards/rejected": 0.7562967538833618, "step": 515 }, { "epoch": 0.4965681886004178, "grad_norm": 2.53125, "ht_mnpo/logit": 0.13243892788887024, "ht_mnpo/residual": 0.12493894249200821, "ht_mnpo/residual_abs": 0.13076035678386688, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.2676253476765194e-07, "logits/chosen": -1.9941246509552002, "logits/rejected": -2.2544379234313965, "logps/chosen": -0.2522723078727722, "logps/rejected": -0.2937101721763611, "loss": 0.08826544135808945, "loss/core": 0.08826544135808945, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.3944621086120605, "rewards/margins": 1.3243893384933472, "rewards/rejected": 1.0700726509094238, "step": 520 }, { "epoch": 0.5013428827215757, "grad_norm": 5.6875, "ht_mnpo/logit": 0.12501785159111023, "ht_mnpo/residual": 0.117517851293087, "ht_mnpo/residual_abs": 0.11929130554199219, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.2194005843856633e-07, "logits/chosen": -2.2246017456054688, "logits/rejected": -2.410853624343872, "logps/chosen": -0.28422626852989197, "logps/rejected": -0.28308025002479553, "loss": 0.10617043823003769, "loss/core": 0.10617043823003769, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.355593681335449, "rewards/margins": 1.2501784563064575, "rewards/rejected": 1.1054149866104126, "step": 525 }, { "epoch": 0.5061175768427335, "grad_norm": 34.25, "ht_mnpo/logit": 0.17451244592666626, "ht_mnpo/residual": 0.16701245307922363, "ht_mnpo/residual_abs": 0.16843153536319733, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.1712813430593434e-07, "logits/chosen": -2.0963895320892334, "logits/rejected": -2.3358659744262695, "logps/chosen": -0.28125277161598206, "logps/rejected": -0.2825583219528198, "loss": 0.38109883666038513, "loss/core": 0.38109883666038513, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.497371196746826, "rewards/margins": 1.7451244592666626, "rewards/rejected": 0.752246618270874, "step": 530 }, { "epoch": 0.5108922709638913, "grad_norm": 68.5, "ht_mnpo/logit": 0.2884078621864319, "ht_mnpo/residual": 0.28090786933898926, "ht_mnpo/residual_abs": 0.2828427851200104, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.123285719376292e-07, "logits/chosen": -1.8652563095092773, "logits/rejected": -2.1189205646514893, "logps/chosen": -0.3185581564903259, "logps/rejected": -0.3078560531139374, "loss": 0.5867989659309387, "loss/core": 0.5867989659309387, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.6412525177001953, "rewards/margins": 2.8840785026550293, "rewards/rejected": 0.7571739554405212, "step": 535 }, { "epoch": 0.5156669650850493, "grad_norm": 6.09375, "ht_mnpo/logit": 0.10569143295288086, "ht_mnpo/residual": 0.09819144755601883, "ht_mnpo/residual_abs": 0.10155366361141205, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.0754317625276982e-07, "logits/chosen": -2.0045483112335205, "logits/rejected": -2.270784854888916, "logps/chosen": -0.31180429458618164, "logps/rejected": -0.3156093955039978, "loss": 0.04411657899618149, "loss/core": 0.04411657899618149, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 1.5413364171981812, "rewards/margins": 1.0569144487380981, "rewards/rejected": 0.48442205786705017, "step": 540 }, { "epoch": 0.5204416592062071, "grad_norm": 580.0, "ht_mnpo/logit": 0.19196265935897827, "ht_mnpo/residual": 0.18446265161037445, "ht_mnpo/residual_abs": 0.186975359916687, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.0277374684296498e-07, "logits/chosen": -2.0267083644866943, "logits/rejected": -2.356262683868408, "logps/chosen": -0.29003921151161194, "logps/rejected": -0.3435795307159424, "loss": 0.3793261647224426, "loss/core": 0.3793261647224426, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.015592575073242, "rewards/margins": 1.9196265935897827, "rewards/rejected": 1.095966100692749, "step": 545 }, { "epoch": 0.5252163533273649, "grad_norm": 7.8125, "ht_mnpo/logit": 0.23898443579673767, "ht_mnpo/residual": 0.23148444294929504, "ht_mnpo/residual_abs": 0.2341163158416748, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.980220772955602e-07, "logits/chosen": -1.8253529071807861, "logits/rejected": -2.06758713722229, "logps/chosen": -0.27169501781463623, "logps/rejected": -0.2784484326839447, "loss": 0.5674008131027222, "loss/core": 0.5674008131027222, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 3.5318641662597656, "rewards/margins": 2.3898441791534424, "rewards/rejected": 1.142019510269165, "step": 550 }, { "epoch": 0.5299910474485229, "grad_norm": 2.96875, "ht_mnpo/logit": 0.19051536917686462, "ht_mnpo/residual": 0.183015376329422, "ht_mnpo/residual_abs": 0.18519142270088196, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.932899545191433e-07, "logits/chosen": -2.0828499794006348, "logits/rejected": -2.2646288871765137, "logps/chosen": -0.31175822019577026, "logps/rejected": -0.28370457887649536, "loss": 0.45988917350769043, "loss/core": 0.45988917350769043, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.769913911819458, "rewards/margins": 1.9051536321640015, "rewards/rejected": 0.8647602200508118, "step": 555 }, { "epoch": 0.5347657415696807, "grad_norm": 2.28125, "ht_mnpo/logit": 0.17307063937187195, "ht_mnpo/residual": 0.16557064652442932, "ht_mnpo/residual_abs": 0.16744212806224823, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.885791580715609e-07, "logits/chosen": -1.9716126918792725, "logits/rejected": -2.2994656562805176, "logps/chosen": -0.301216185092926, "logps/rejected": -0.27583974599838257, "loss": 0.45512303709983826, "loss/core": 0.45512303709983826, "rewards/accuracies": 1.0, "rewards/chosen": 2.420431137084961, "rewards/margins": 1.7307065725326538, "rewards/rejected": 0.6897248029708862, "step": 560 }, { "epoch": 0.5395404356908385, "grad_norm": 4.4375, "ht_mnpo/logit": 0.34825602173805237, "ht_mnpo/residual": 0.34075599908828735, "ht_mnpo/residual_abs": 0.34296515583992004, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.8389145949069951e-07, "logits/chosen": -2.1434173583984375, "logits/rejected": -2.416426420211792, "logps/chosen": -0.3156982362270355, "logps/rejected": -0.2996530830860138, "loss": 2.206251382827759, "loss/core": 2.206251382827759, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.403435707092285, "rewards/margins": 3.4825599193573, "rewards/rejected": 0.9208753705024719, "step": 565 }, { "epoch": 0.5443151298119964, "grad_norm": 60.0, "ht_mnpo/logit": 0.24878111481666565, "ht_mnpo/residual": 0.24128107726573944, "ht_mnpo/residual_abs": 0.246558278799057, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.792286216282824e-07, "logits/chosen": -1.9773708581924438, "logits/rejected": -2.3607583045959473, "logps/chosen": -0.28530240058898926, "logps/rejected": -0.27613160014152527, "loss": 0.4744301736354828, "loss/core": 0.4744301736354828, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.3544468879699707, "rewards/margins": 2.487811326980591, "rewards/rejected": 0.8666356801986694, "step": 570 }, { "epoch": 0.5490898239331543, "grad_norm": 2.078125, "ht_mnpo/logit": 0.31033894419670105, "ht_mnpo/residual": 0.3028389513492584, "ht_mnpo/residual_abs": 0.30536240339279175, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.745923979869336e-07, "logits/chosen": -2.0291919708251953, "logits/rejected": -2.281064510345459, "logps/chosen": -0.32846799492836, "logps/rejected": -0.3118685483932495, "loss": 1.0607415437698364, "loss/core": 1.0607415437698364, "rewards/accuracies": 0.9375, "rewards/chosen": 4.091516971588135, "rewards/margins": 3.1033897399902344, "rewards/rejected": 0.9881270527839661, "step": 575 }, { "epoch": 0.5538645180543121, "grad_norm": 3.6875, "ht_mnpo/logit": 0.4819414019584656, "ht_mnpo/residual": 0.47444137930870056, "ht_mnpo/residual_abs": 0.4764309823513031, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.6998453206075708e-07, "logits/chosen": -1.7918148040771484, "logits/rejected": -1.9814106225967407, "logps/chosen": -0.30701881647109985, "logps/rejected": -0.25892844796180725, "loss": 2.3531346321105957, "loss/core": 2.3531346321105957, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 6.004878520965576, "rewards/margins": 4.819413661956787, "rewards/rejected": 1.18546462059021, "step": 580 }, { "epoch": 0.55863921217547, "grad_norm": 50.75, "ht_mnpo/logit": 0.156865194439888, "ht_mnpo/residual": 0.14936518669128418, "ht_mnpo/residual_abs": 0.15104547142982483, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.6540675667967973e-07, "logits/chosen": -2.02787709236145, "logits/rejected": -2.223013401031494, "logps/chosen": -0.29150041937828064, "logps/rejected": -0.2901824414730072, "loss": 0.09419430792331696, "loss/core": 0.09419430792331696, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.5935516357421875, "rewards/margins": 1.5686519145965576, "rewards/rejected": 1.0248996019363403, "step": 585 }, { "epoch": 0.5634139062966279, "grad_norm": 888.0, "ht_mnpo/logit": 0.2369958609342575, "ht_mnpo/residual": 0.2294958084821701, "ht_mnpo/residual_abs": 0.2326071560382843, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.60860793357805e-07, "logits/chosen": -1.8807750940322876, "logits/rejected": -2.03248929977417, "logps/chosen": -0.29985108971595764, "logps/rejected": -0.3031771183013916, "loss": 0.7982430458068848, "loss/core": 0.7982430458068848, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 3.11862850189209, "rewards/margins": 2.3699584007263184, "rewards/rejected": 0.7486701011657715, "step": 590 }, { "epoch": 0.5681886004177857, "grad_norm": 2.515625, "ht_mnpo/logit": 0.4987797141075134, "ht_mnpo/residual": 0.4912796914577484, "ht_mnpo/residual_abs": 0.4936184287071228, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.5634835164602196e-07, "logits/chosen": -1.8091611862182617, "logits/rejected": -2.182551622390747, "logps/chosen": -0.3149603307247162, "logps/rejected": -0.2931450605392456, "loss": 2.4295339584350586, "loss/core": 2.4295339584350586, "rewards/accuracies": 0.9375, "rewards/chosen": 5.907982349395752, "rewards/margins": 4.987797260284424, "rewards/rejected": 0.9201849102973938, "step": 595 }, { "epoch": 0.5729632945389436, "grad_norm": 2.09375, "ht_mnpo/logit": 0.24387526512145996, "ht_mnpo/residual": 0.23637528717517853, "ht_mnpo/residual_abs": 0.23978765308856964, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.518711284891132e-07, "logits/chosen": -1.8736908435821533, "logits/rejected": -2.1749606132507324, "logps/chosen": -0.29759305715560913, "logps/rejected": -0.2989031970500946, "loss": 0.559294581413269, "loss/core": 0.559294581413269, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.058932304382324, "rewards/margins": 2.4387524127960205, "rewards/rejected": 1.6201797723770142, "step": 600 }, { "epoch": 0.5777379886601015, "grad_norm": 16.375, "ht_mnpo/logit": 0.27251681685447693, "ht_mnpo/residual": 0.2650168240070343, "ht_mnpo/residual_abs": 0.2667366862297058, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.47430807587603e-07, "logits/chosen": -1.8784462213516235, "logits/rejected": -2.1159400939941406, "logps/chosen": -0.29141777753829956, "logps/rejected": -0.3019111752510071, "loss": 0.662787914276123, "loss/core": 0.662787914276123, "rewards/accuracies": 0.9375, "rewards/chosen": 3.4891610145568848, "rewards/margins": 2.725167989730835, "rewards/rejected": 0.7639929056167603, "step": 605 }, { "epoch": 0.5825126827812593, "grad_norm": 3.5625, "ht_mnpo/logit": 0.23614993691444397, "ht_mnpo/residual": 0.22864992916584015, "ht_mnpo/residual_abs": 0.23106643557548523, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.430290587645865e-07, "logits/chosen": -1.9317668676376343, "logits/rejected": -2.19699764251709, "logps/chosen": -0.301189124584198, "logps/rejected": -0.2828231751918793, "loss": 0.7685016393661499, "loss/core": 0.7685016393661499, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.8618009090423584, "rewards/margins": 2.361499309539795, "rewards/rejected": 0.5003016591072083, "step": 610 }, { "epoch": 0.5872873769024172, "grad_norm": 11.1875, "ht_mnpo/logit": 0.3057871460914612, "ht_mnpo/residual": 0.2982870936393738, "ht_mnpo/residual_abs": 0.30613571405410767, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.3866753733777765e-07, "logits/chosen": -1.7794278860092163, "logits/rejected": -2.1755802631378174, "logps/chosen": -0.2987704873085022, "logps/rejected": -0.2674489915370941, "loss": 1.0454192161560059, "loss/core": 1.0454192161560059, "rewards/accuracies": 0.9375, "rewards/chosen": 4.152740001678467, "rewards/margins": 3.0578713417053223, "rewards/rejected": 1.094868779182434, "step": 615 }, { "epoch": 0.592062071023575, "grad_norm": 7.90625, "ht_mnpo/logit": 0.16309209167957306, "ht_mnpo/residual": 0.15559211373329163, "ht_mnpo/residual_abs": 0.1573321521282196, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.343478834970121e-07, "logits/chosen": -2.0946414470672607, "logits/rejected": -2.3167364597320557, "logps/chosen": -0.2965165972709656, "logps/rejected": -0.2895964980125427, "loss": 0.220075324177742, "loss/core": 0.220075324177742, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.6535112857818604, "rewards/margins": 1.6309210062026978, "rewards/rejected": 1.0225903987884521, "step": 620 }, { "epoch": 0.5968367651447329, "grad_norm": 168.0, "ht_mnpo/logit": 0.26800763607025146, "ht_mnpo/residual": 0.26050764322280884, "ht_mnpo/residual_abs": 0.26258984208106995, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.3007172168743852e-07, "logits/chosen": -1.8491462469100952, "logits/rejected": -2.066373586654663, "logps/chosen": -0.2881318926811218, "logps/rejected": -0.30786803364753723, "loss": 0.5212100744247437, "loss/core": 0.5212100744247437, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.430001735687256, "rewards/margins": 2.6800761222839355, "rewards/rejected": 0.7499252557754517, "step": 625 }, { "epoch": 0.6016114592658908, "grad_norm": 190.0, "ht_mnpo/logit": 0.3151053786277771, "ht_mnpo/residual": 0.3076053857803345, "ht_mnpo/residual_abs": 0.3096749186515808, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.25840659998631e-07, "logits/chosen": -1.9944003820419312, "logits/rejected": -2.25925350189209, "logps/chosen": -0.33351221680641174, "logps/rejected": -0.30969151854515076, "loss": 1.3126461505889893, "loss/core": 1.3126461505889893, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 4.146966457366943, "rewards/margins": 3.1510536670684814, "rewards/rejected": 0.9959124326705933, "step": 630 }, { "epoch": 0.6063861533870486, "grad_norm": 2.15625, "ht_mnpo/logit": 0.16642612218856812, "ht_mnpo/residual": 0.15892614424228668, "ht_mnpo/residual_abs": 0.1607307344675064, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.2165628955985313e-07, "logits/chosen": -2.1082916259765625, "logits/rejected": -2.336918354034424, "logps/chosen": -0.28919678926467896, "logps/rejected": -0.2757853865623474, "loss": 0.32006698846817017, "loss/core": 0.32006698846817017, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.2848565578460693, "rewards/margins": 1.6642612218856812, "rewards/rejected": 0.6205950975418091, "step": 635 }, { "epoch": 0.6111608475082065, "grad_norm": 188.0, "ht_mnpo/logit": 0.13086314499378204, "ht_mnpo/residual": 0.12336313724517822, "ht_mnpo/residual_abs": 0.1262132078409195, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.175201839416988e-07, "logits/chosen": -2.047877550125122, "logits/rejected": -2.198028564453125, "logps/chosen": -0.30865031480789185, "logps/rejected": -0.29963168501853943, "loss": 0.12744709849357605, "loss/core": 0.12744709849357605, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.1879143714904785, "rewards/margins": 1.308631420135498, "rewards/rejected": 0.87928307056427, "step": 640 }, { "epoch": 0.6159355416293644, "grad_norm": 12.75, "ht_mnpo/logit": 0.35795003175735474, "ht_mnpo/residual": 0.3504500985145569, "ht_mnpo/residual_abs": 0.3530126214027405, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.1343389856433658e-07, "logits/chosen": -1.7279043197631836, "logits/rejected": -2.076632022857666, "logps/chosen": -0.309658944606781, "logps/rejected": -0.2867974638938904, "loss": 1.6379516124725342, "loss/core": 1.6379516124725342, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 5.011632919311523, "rewards/margins": 3.579500675201416, "rewards/rejected": 1.4321321249008179, "step": 645 }, { "epoch": 0.6207102357505222, "grad_norm": 78.5, "ht_mnpo/logit": 0.2575874626636505, "ht_mnpo/residual": 0.2500874698162079, "ht_mnpo/residual_abs": 0.25217220187187195, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.0939897011258e-07, "logits/chosen": -1.9400348663330078, "logits/rejected": -2.297685146331787, "logps/chosen": -0.2735014259815216, "logps/rejected": -0.2798306345939636, "loss": 0.5699511170387268, "loss/core": 0.5699511170387268, "rewards/accuracies": 0.9375, "rewards/chosen": 3.7315871715545654, "rewards/margins": 2.5758748054504395, "rewards/rejected": 1.1557127237319946, "step": 650 }, { "epoch": 0.6254849298716801, "grad_norm": 4.4375, "ht_mnpo/logit": 0.11060944944620132, "ht_mnpo/residual": 0.1031094565987587, "ht_mnpo/residual_abs": 0.10582210123538971, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.0541691595800336e-07, "logits/chosen": -2.1760499477386475, "logits/rejected": -2.4464058876037598, "logps/chosen": -0.263247013092041, "logps/rejected": -0.2721119225025177, "loss": 0.043946050107479095, "loss/core": 0.043946050107479095, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.126974105834961, "rewards/margins": 1.1060945987701416, "rewards/rejected": 1.0208795070648193, "step": 655 }, { "epoch": 0.630259623992838, "grad_norm": 286.0, "ht_mnpo/logit": 0.3428904116153717, "ht_mnpo/residual": 0.33539044857025146, "ht_mnpo/residual_abs": 0.33829253911972046, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.0148923358832021e-07, "logits/chosen": -1.8339145183563232, "logits/rejected": -2.0472755432128906, "logps/chosen": -0.33507493138313293, "logps/rejected": -0.31190431118011475, "loss": 1.6195424795150757, "loss/core": 1.6195424795150757, "rewards/accuracies": 0.9375, "rewards/chosen": 4.361727237701416, "rewards/margins": 3.4289040565490723, "rewards/rejected": 0.9328228831291199, "step": 660 }, { "epoch": 0.6350343181139958, "grad_norm": 3.875, "ht_mnpo/logit": 0.3288431465625763, "ht_mnpo/residual": 0.3213431239128113, "ht_mnpo/residual_abs": 0.323604017496109, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 9.761740004423926e-08, "logits/chosen": -1.9675363302230835, "logits/rejected": -2.2226428985595703, "logps/chosen": -0.33497220277786255, "logps/rejected": -0.2824486792087555, "loss": 1.085584044456482, "loss/core": 1.085584044456482, "rewards/accuracies": 0.9375, "rewards/chosen": 4.039675712585449, "rewards/margins": 3.288431167602539, "rewards/rejected": 0.7512443661689758, "step": 665 }, { "epoch": 0.6398090122351537, "grad_norm": 49.5, "ht_mnpo/logit": 0.26619061827659607, "ht_mnpo/residual": 0.25869062542915344, "ht_mnpo/residual_abs": 0.2621300220489502, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 9.380287136400999e-08, "logits/chosen": -1.9468028545379639, "logits/rejected": -2.044259548187256, "logps/chosen": -0.32372915744781494, "logps/rejected": -0.32650211453437805, "loss": 1.0328400135040283, "loss/core": 1.0328400135040283, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 4.102653503417969, "rewards/margins": 2.6619060039520264, "rewards/rejected": 1.440747618675232, "step": 670 }, { "epoch": 0.6445837063563116, "grad_norm": 155.0, "ht_mnpo/logit": 0.1919471025466919, "ht_mnpo/residual": 0.18444707989692688, "ht_mnpo/residual_abs": 0.18674826622009277, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 9.004708203586628e-08, "logits/chosen": -2.0553970336914062, "logits/rejected": -2.277501344680786, "logps/chosen": -0.30528566241264343, "logps/rejected": -0.30857595801353455, "loss": 0.44716134667396545, "loss/core": 0.44716134667396545, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.826671838760376, "rewards/margins": 1.9194707870483398, "rewards/rejected": 0.9072009325027466, "step": 675 }, { "epoch": 0.6493584004774694, "grad_norm": 1408.0, "ht_mnpo/logit": 0.5156992077827454, "ht_mnpo/residual": 0.5081992149353027, "ht_mnpo/residual_abs": 0.5100849270820618, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 8.635144445857407e-08, "logits/chosen": -1.9224817752838135, "logits/rejected": -2.295820713043213, "logps/chosen": -0.3082597255706787, "logps/rejected": -0.28871044516563416, "loss": 2.1461853981018066, "loss/core": 2.1461853981018066, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 6.397282600402832, "rewards/margins": 5.1569929122924805, "rewards/rejected": 1.2402900457382202, "step": 680 }, { "epoch": 0.6541330945986272, "grad_norm": 11.875, "ht_mnpo/logit": 0.2202470600605011, "ht_mnpo/residual": 0.21274705231189728, "ht_mnpo/residual_abs": 0.21405646204948425, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 8.271734841028552e-08, "logits/chosen": -1.833820104598999, "logits/rejected": -2.1766836643218994, "logps/chosen": -0.2879013121128082, "logps/rejected": -0.2877139449119568, "loss": 0.39130714535713196, "loss/core": 0.39130714535713196, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.120842456817627, "rewards/margins": 2.202470541000366, "rewards/rejected": 0.9183722734451294, "step": 685 }, { "epoch": 0.6589077887197852, "grad_norm": 2.484375, "ht_mnpo/logit": 0.33171364665031433, "ht_mnpo/residual": 0.3242136240005493, "ht_mnpo/residual_abs": 0.32709458470344543, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 7.91461605259007e-08, "logits/chosen": -1.9676786661148071, "logits/rejected": -2.1830246448516846, "logps/chosen": -0.31910890340805054, "logps/rejected": -0.2641559839248657, "loss": 1.0723878145217896, "loss/core": 1.0723878145217896, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.9963231086730957, "rewards/margins": 3.317136287689209, "rewards/rejected": 0.6791867613792419, "step": 690 }, { "epoch": 0.663682482840943, "grad_norm": 54.75, "ht_mnpo/logit": 0.35695138573646545, "ht_mnpo/residual": 0.3494513928890228, "ht_mnpo/residual_abs": 0.35167449712753296, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 7.563922378313217e-08, "logits/chosen": -1.966403603553772, "logits/rejected": -2.119166851043701, "logps/chosen": -0.3139706552028656, "logps/rejected": -0.3095807731151581, "loss": 1.3685941696166992, "loss/core": 1.3685941696166992, "rewards/accuracies": 0.9375, "rewards/chosen": 4.306251525878906, "rewards/margins": 3.569514513015747, "rewards/rejected": 0.7367372512817383, "step": 695 }, { "epoch": 0.6684571769621008, "grad_norm": 828.0, "ht_mnpo/logit": 0.3428606390953064, "ht_mnpo/residual": 0.335360586643219, "ht_mnpo/residual_abs": 0.337501585483551, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 7.219785699746572e-08, "logits/chosen": -1.8964332342147827, "logits/rejected": -2.292275905609131, "logps/chosen": -0.30034637451171875, "logps/rejected": -0.2862694263458252, "loss": 1.527578592300415, "loss/core": 1.527578592300415, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 4.155306816101074, "rewards/margins": 3.428605556488037, "rewards/rejected": 0.7267010807991028, "step": 700 }, { "epoch": 0.6732318710832588, "grad_norm": 14.6875, "ht_mnpo/logit": 0.2360476702451706, "ht_mnpo/residual": 0.22854766249656677, "ht_mnpo/residual_abs": 0.2305394411087036, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 6.882335432620779e-08, "logits/chosen": -1.8612133264541626, "logits/rejected": -2.1879470348358154, "logps/chosen": -0.29452234506607056, "logps/rejected": -0.2882519066333771, "loss": 0.5280643701553345, "loss/core": 0.5280643701553345, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.631542205810547, "rewards/margins": 2.3604767322540283, "rewards/rejected": 1.271065592765808, "step": 705 }, { "epoch": 0.6780065652044166, "grad_norm": 496.0, "ht_mnpo/logit": 0.27381792664527893, "ht_mnpo/residual": 0.2663179337978363, "ht_mnpo/residual_abs": 0.2707618176937103, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 6.551698478180589e-08, "logits/chosen": -1.8524887561798096, "logits/rejected": -2.209779739379883, "logps/chosen": -0.3093031048774719, "logps/rejected": -0.28702372312545776, "loss": 1.0192804336547852, "loss/core": 1.0192804336547852, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.5826401710510254, "rewards/margins": 2.7381794452667236, "rewards/rejected": 0.8444606065750122, "step": 710 }, { "epoch": 0.6827812593255744, "grad_norm": 13.5, "ht_mnpo/logit": 0.17260591685771942, "ht_mnpo/residual": 0.1651059240102768, "ht_mnpo/residual_abs": 0.16933459043502808, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 6.22799917546252e-08, "logits/chosen": -1.8810266256332397, "logits/rejected": -2.1536669731140137, "logps/chosen": -0.3034297525882721, "logps/rejected": -0.29463475942611694, "loss": 0.3258531093597412, "loss/core": 0.3258531093597412, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 3.0421574115753174, "rewards/margins": 1.7260591983795166, "rewards/rejected": 1.3160979747772217, "step": 715 }, { "epoch": 0.6875559534467324, "grad_norm": 79.5, "ht_mnpo/logit": 0.241688534617424, "ht_mnpo/residual": 0.234188511967659, "ht_mnpo/residual_abs": 0.23640303313732147, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 5.9113592545359944e-08, "logits/chosen": -2.146207332611084, "logits/rejected": -2.2886502742767334, "logps/chosen": -0.309707373380661, "logps/rejected": -0.309203565120697, "loss": 0.458344042301178, "loss/core": 0.458344042301178, "rewards/accuracies": 0.9375, "rewards/chosen": 3.303243637084961, "rewards/margins": 2.4168851375579834, "rewards/rejected": 0.886358380317688, "step": 720 }, { "epoch": 0.6923306475678902, "grad_norm": 26.25, "ht_mnpo/logit": 0.16217423975467682, "ht_mnpo/residual": 0.15467426180839539, "ht_mnpo/residual_abs": 0.15796169638633728, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 5.601897790725643e-08, "logits/chosen": -1.9593452215194702, "logits/rejected": -2.2494146823883057, "logps/chosen": -0.275308758020401, "logps/rejected": -0.27825742959976196, "loss": 0.21573790907859802, "loss/core": 0.21573790907859802, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 2.375974178314209, "rewards/margins": 1.621742606163025, "rewards/rejected": 0.7542315125465393, "step": 725 }, { "epoch": 0.697105341689048, "grad_norm": 332.0, "ht_mnpo/logit": 0.381821870803833, "ht_mnpo/residual": 0.374321848154068, "ht_mnpo/residual_abs": 0.3764488399028778, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 5.299731159831952e-08, "logits/chosen": -2.0782310962677, "logits/rejected": -2.2460265159606934, "logps/chosen": -0.31326770782470703, "logps/rejected": -0.3076656758785248, "loss": 1.6463673114776611, "loss/core": 1.6463673114776611, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.482171535491943, "rewards/margins": 3.81821870803833, "rewards/rejected": 0.6639531254768372, "step": 730 }, { "epoch": 0.7018800358102059, "grad_norm": 5.34375, "ht_mnpo/logit": 0.15911749005317688, "ht_mnpo/residual": 0.15161748230457306, "ht_mnpo/residual_abs": 0.17043578624725342, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 5.0049729943671013e-08, "logits/chosen": -1.9329124689102173, "logits/rejected": -2.1531574726104736, "logps/chosen": -0.32668760418891907, "logps/rejected": -0.3379434049129486, "loss": 0.3836316466331482, "loss/core": 0.3836316466331482, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.7718982696533203, "rewards/margins": 1.591174840927124, "rewards/rejected": 1.1807234287261963, "step": 735 }, { "epoch": 0.7066547299313638, "grad_norm": 187.0, "ht_mnpo/logit": 0.21543124318122864, "ht_mnpo/residual": 0.20793123543262482, "ht_mnpo/residual_abs": 0.21036949753761292, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.7177341408223994e-08, "logits/chosen": -1.9197518825531006, "logits/rejected": -2.1591908931732178, "logps/chosen": -0.2565429210662842, "logps/rejected": -0.2755061984062195, "loss": 0.30916422605514526, "loss/core": 0.30916422605514526, "rewards/accuracies": 0.9375, "rewards/chosen": 3.222672939300537, "rewards/margins": 2.1543123722076416, "rewards/rejected": 1.068360686302185, "step": 740 }, { "epoch": 0.7114294240525216, "grad_norm": 500.0, "ht_mnpo/logit": 0.26379165053367615, "ht_mnpo/residual": 0.2562916874885559, "ht_mnpo/residual_abs": 0.25983983278274536, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.438122617983442e-08, "logits/chosen": -1.9132686853408813, "logits/rejected": -2.159632921218872, "logps/chosen": -0.26915454864501953, "logps/rejected": -0.26063108444213867, "loss": 0.9998035430908203, "loss/core": 0.9998035430908203, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.6523070335388184, "rewards/margins": 2.6379168033599854, "rewards/rejected": 1.0143901109695435, "step": 745 }, { "epoch": 0.7162041181736795, "grad_norm": 3.765625, "ht_mnpo/logit": 0.2811947762966156, "ht_mnpo/residual": 0.273694783449173, "ht_mnpo/residual_abs": 0.2760910987854004, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.1662435763087114e-08, "logits/chosen": -2.0892138481140137, "logits/rejected": -2.2944881916046143, "logps/chosen": -0.2737082839012146, "logps/rejected": -0.259973406791687, "loss": 1.173396110534668, "loss/core": 1.173396110534668, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.8202099800109863, "rewards/margins": 2.8119475841522217, "rewards/rejected": 1.0082619190216064, "step": 750 }, { "epoch": 0.7209788122948374, "grad_norm": 2.140625, "ht_mnpo/logit": 0.2567535936832428, "ht_mnpo/residual": 0.24925358593463898, "ht_mnpo/residual_abs": 0.25218337774276733, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.902199258386732e-08, "logits/chosen": -2.019989252090454, "logits/rejected": -2.1991806030273438, "logps/chosen": -0.3405812978744507, "logps/rejected": -0.31987202167510986, "loss": 1.182205319404602, "loss/core": 1.182205319404602, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.823350429534912, "rewards/margins": 2.567535877227783, "rewards/rejected": 1.255814790725708, "step": 755 }, { "epoch": 0.7257535064159952, "grad_norm": 82.5, "ht_mnpo/logit": 0.40269652009010315, "ht_mnpo/residual": 0.39519649744033813, "ht_mnpo/residual_abs": 0.39851316809654236, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.646088960486862e-08, "logits/chosen": -1.9100509881973267, "logits/rejected": -2.2545628547668457, "logps/chosen": -0.31178393959999084, "logps/rejected": -0.28806573152542114, "loss": 1.7591040134429932, "loss/core": 1.7591040134429932, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 5.450934410095215, "rewards/margins": 4.026965141296387, "rewards/rejected": 1.4239691495895386, "step": 760 }, { "epoch": 0.7305282005371531, "grad_norm": 5.9375, "ht_mnpo/logit": 0.24737465381622314, "ht_mnpo/residual": 0.23987463116645813, "ht_mnpo/residual_abs": 0.24287545680999756, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.398008995217988e-08, "logits/chosen": -1.9502792358398438, "logits/rejected": -2.0989506244659424, "logps/chosen": -0.2815241813659668, "logps/rejected": -0.29228392243385315, "loss": 0.46429914236068726, "loss/core": 0.46429914236068726, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.6968235969543457, "rewards/margins": 2.4737465381622314, "rewards/rejected": 1.2230769395828247, "step": 765 }, { "epoch": 0.735302894658311, "grad_norm": 88.0, "ht_mnpo/logit": 0.365441232919693, "ht_mnpo/residual": 0.357941210269928, "ht_mnpo/residual_abs": 0.3601204752922058, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.1580526553093315e-08, "logits/chosen": -1.895281195640564, "logits/rejected": -2.2292673587799072, "logps/chosen": -0.3453708291053772, "logps/rejected": -0.3260127305984497, "loss": 1.5065122842788696, "loss/core": 1.5065122842788696, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.30310583114624, "rewards/margins": 3.6544125080108643, "rewards/rejected": 0.6486933827400208, "step": 770 }, { "epoch": 0.7400775887794688, "grad_norm": 10.25, "ht_mnpo/logit": 0.21786002814769745, "ht_mnpo/residual": 0.21036005020141602, "ht_mnpo/residual_abs": 0.2132169008255005, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.9263101785268252e-08, "logits/chosen": -1.9512803554534912, "logits/rejected": -2.2073264122009277, "logps/chosen": -0.3291786313056946, "logps/rejected": -0.31446442008018494, "loss": 0.46908074617385864, "loss/core": 0.46908074617385864, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 3.122516393661499, "rewards/margins": 2.178600549697876, "rewards/rejected": 0.9439161419868469, "step": 775 }, { "epoch": 0.7448522829006267, "grad_norm": 5.03125, "ht_mnpo/logit": 0.2005649358034134, "ht_mnpo/residual": 0.19306494295597076, "ht_mnpo/residual_abs": 0.19553616642951965, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.7028687137384264e-08, "logits/chosen": -2.147618293762207, "logits/rejected": -2.329775333404541, "logps/chosen": -0.2661969065666199, "logps/rejected": -0.2787913680076599, "loss": 0.28196778893470764, "loss/core": 0.28196778893470764, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.99574613571167, "rewards/margins": 2.0056493282318115, "rewards/rejected": 0.9900972247123718, "step": 780 }, { "epoch": 0.7496269770217845, "grad_norm": 5.625, "ht_mnpo/logit": 0.23682251572608948, "ht_mnpo/residual": 0.22932252287864685, "ht_mnpo/residual_abs": 0.23593783378601074, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.4878122881409447e-08, "logits/chosen": -1.9615819454193115, "logits/rejected": -2.3150079250335693, "logps/chosen": -0.3067627549171448, "logps/rejected": -0.2893941402435303, "loss": 0.7686271667480469, "loss/core": 0.7686271667480469, "rewards/accuracies": 0.9375, "rewards/chosen": 3.3052401542663574, "rewards/margins": 2.36822509765625, "rewards/rejected": 0.9370150566101074, "step": 785 }, { "epoch": 0.7544016711429424, "grad_norm": 91.5, "ht_mnpo/logit": 0.26232343912124634, "ht_mnpo/residual": 0.2548234462738037, "ht_mnpo/residual_abs": 0.25668424367904663, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.2812217756608937e-08, "logits/chosen": -1.9566657543182373, "logits/rejected": -2.261164426803589, "logps/chosen": -0.32755106687545776, "logps/rejected": -0.3085654377937317, "loss": 0.5578528642654419, "loss/core": 0.5578528642654419, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.1903016567230225, "rewards/margins": 2.623234272003174, "rewards/rejected": 0.5670672059059143, "step": 790 }, { "epoch": 0.7591763652641003, "grad_norm": 724.0, "ht_mnpo/logit": 0.31201422214508057, "ht_mnpo/residual": 0.30451422929763794, "ht_mnpo/residual_abs": 0.30678969621658325, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.0831748665410763e-08, "logits/chosen": -2.053274393081665, "logits/rejected": -2.2694168090820312, "logps/chosen": -0.3137114942073822, "logps/rejected": -0.29724258184432983, "loss": 0.9311115145683289, "loss/core": 0.9311115145683289, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 4.055318832397461, "rewards/margins": 3.1201424598693848, "rewards/rejected": 0.9351763725280762, "step": 795 }, { "epoch": 0.7639510593852581, "grad_norm": 23.625, "ht_mnpo/logit": 0.20176801085472107, "ht_mnpo/residual": 0.19426801800727844, "ht_mnpo/residual_abs": 0.19569215178489685, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.8937460381244967e-08, "logits/chosen": -2.0264482498168945, "logits/rejected": -2.240051746368408, "logps/chosen": -0.2773614525794983, "logps/rejected": -0.2761766314506531, "loss": 0.31296461820602417, "loss/core": 0.31296461820602417, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 3.1320345401763916, "rewards/margins": 2.0176799297332764, "rewards/rejected": 1.1143543720245361, "step": 800 }, { "epoch": 0.768725753506416, "grad_norm": 338.0, "ht_mnpo/logit": 0.31727129220962524, "ht_mnpo/residual": 0.3097712993621826, "ht_mnpo/residual_abs": 0.3111569285392761, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.713006526846439e-08, "logits/chosen": -2.038196086883545, "logits/rejected": -2.4574263095855713, "logps/chosen": -0.30098554491996765, "logps/rejected": -0.29372772574424744, "loss": 0.8073376417160034, "loss/core": 0.8073376417160034, "rewards/accuracies": 0.987500011920929, "rewards/chosen": 4.151384353637695, "rewards/margins": 3.172712802886963, "rewards/rejected": 0.9786712527275085, "step": 805 }, { "epoch": 0.7735004476275739, "grad_norm": 540.0, "ht_mnpo/logit": 0.21397161483764648, "ht_mnpo/residual": 0.20647163689136505, "ht_mnpo/residual_abs": 0.21008792519569397, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.541024301445404e-08, "logits/chosen": -1.9568876028060913, "logits/rejected": -2.162205457687378, "logps/chosen": -0.32081520557403564, "logps/rejected": -0.28566688299179077, "loss": 0.5576066970825195, "loss/core": 0.5576066970825195, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 3.346921443939209, "rewards/margins": 2.139716386795044, "rewards/rejected": 1.2072051763534546, "step": 810 }, { "epoch": 0.7782751417487317, "grad_norm": 120.5, "ht_mnpo/logit": 0.34778210520744324, "ht_mnpo/residual": 0.34028205275535583, "ht_mnpo/residual_abs": 0.34224414825439453, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.3778640374027983e-08, "logits/chosen": -1.81075119972229, "logits/rejected": -2.0091278553009033, "logps/chosen": -0.28327834606170654, "logps/rejected": -0.2862473726272583, "loss": 0.8907841444015503, "loss/core": 0.8907841444015503, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.21142578125, "rewards/margins": 3.477820873260498, "rewards/rejected": 0.7336045503616333, "step": 815 }, { "epoch": 0.7830498358698896, "grad_norm": 14.1875, "ht_mnpo/logit": 0.15176530182361603, "ht_mnpo/residual": 0.1442653089761734, "ht_mnpo/residual_abs": 0.14901027083396912, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.2235870926211616e-08, "logits/chosen": -1.9298378229141235, "logits/rejected": -2.1080105304718018, "logps/chosen": -0.2902945578098297, "logps/rejected": -0.309215247631073, "loss": 0.4051854610443115, "loss/core": 0.4051854610443115, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 2.702075481414795, "rewards/margins": 1.517652988433838, "rewards/rejected": 1.184422492980957, "step": 820 }, { "epoch": 0.7878245299910475, "grad_norm": 1312.0, "ht_mnpo/logit": 0.3178713321685791, "ht_mnpo/residual": 0.3103713393211365, "ht_mnpo/residual_abs": 0.3130297064781189, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.0782514843499652e-08, "logits/chosen": -2.0223171710968018, "logits/rejected": -2.2735869884490967, "logps/chosen": -0.34258824586868286, "logps/rejected": -0.2813429832458496, "loss": 0.8753134608268738, "loss/core": 0.8753134608268738, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 3.9592788219451904, "rewards/margins": 3.178713321685791, "rewards/rejected": 0.7805658578872681, "step": 825 }, { "epoch": 0.7925992241122053, "grad_norm": 29.625, "ht_mnpo/logit": 0.232127383351326, "ht_mnpo/residual": 0.22462734580039978, "ht_mnpo/residual_abs": 0.22618567943572998, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 9.419118673676924e-09, "logits/chosen": -2.1135377883911133, "logits/rejected": -2.3391060829162598, "logps/chosen": -0.27336445450782776, "logps/rejected": -0.28452619910240173, "loss": 0.5279203653335571, "loss/core": 0.5279203653335571, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 3.090513229370117, "rewards/margins": 2.3212738037109375, "rewards/rejected": 0.7692393660545349, "step": 830 }, { "epoch": 0.7973739182333631, "grad_norm": 31.75, "ht_mnpo/logit": 0.23250722885131836, "ht_mnpo/residual": 0.22500717639923096, "ht_mnpo/residual_abs": 0.22754648327827454, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 8.14619513428405e-09, "logits/chosen": -1.7653303146362305, "logits/rejected": -1.9559898376464844, "logps/chosen": -0.2960887551307678, "logps/rejected": -0.3074343502521515, "loss": 0.5747458934783936, "loss/core": 0.5747458934783936, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.5430405139923096, "rewards/margins": 2.3250718116760254, "rewards/rejected": 1.217968463897705, "step": 835 }, { "epoch": 0.8021486123545211, "grad_norm": 434.0, "ht_mnpo/logit": 0.22585304081439972, "ht_mnpo/residual": 0.2183530032634735, "ht_mnpo/residual_abs": 0.2201291024684906, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 6.96422291980539e-09, "logits/chosen": -2.053584337234497, "logits/rejected": -2.245476722717285, "logps/chosen": -0.3269621729850769, "logps/rejected": -0.26414385437965393, "loss": 0.5430350303649902, "loss/core": 0.5430350303649902, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 3.252704620361328, "rewards/margins": 2.2585301399230957, "rewards/rejected": 0.9941746592521667, "step": 840 }, { "epoch": 0.8069233064756789, "grad_norm": 112.5, "ht_mnpo/logit": 0.2180916965007782, "ht_mnpo/residual": 0.21059170365333557, "ht_mnpo/residual_abs": 0.21333935856819153, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 5.8736465216517594e-09, "logits/chosen": -1.9658924341201782, "logits/rejected": -2.281589984893799, "logps/chosen": -0.30036720633506775, "logps/rejected": -0.2967119812965393, "loss": 0.3782276511192322, "loss/core": 0.3782276511192322, "rewards/accuracies": 0.925000011920929, "rewards/chosen": 2.758944511413574, "rewards/margins": 2.1809170246124268, "rewards/rejected": 0.5780273079872131, "step": 845 }, { "epoch": 0.8116980005968367, "grad_norm": 2.140625, "ht_mnpo/logit": 0.2448008507490158, "ht_mnpo/residual": 0.23730087280273438, "ht_mnpo/residual_abs": 0.23997625708580017, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.874876061005173e-09, "logits/chosen": -1.8780618906021118, "logits/rejected": -2.181715488433838, "logps/chosen": -0.2794502377510071, "logps/rejected": -0.26828446984291077, "loss": 0.8545548319816589, "loss/core": 0.8545548319816589, "rewards/accuracies": 0.9375, "rewards/chosen": 3.7230312824249268, "rewards/margins": 2.4480087757110596, "rewards/rejected": 1.2750225067138672, "step": 850 }, { "epoch": 0.8164726947179947, "grad_norm": 11.0625, "ht_mnpo/logit": 0.1709764301776886, "ht_mnpo/residual": 0.16347643733024597, "ht_mnpo/residual_abs": 0.1738419234752655, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.968287134589188e-09, "logits/chosen": -2.113243579864502, "logits/rejected": -2.4351394176483154, "logps/chosen": -0.26266947388648987, "logps/rejected": -0.2733474373817444, "loss": 0.12632545828819275, "loss/core": 0.12632545828819275, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 2.9499731063842773, "rewards/margins": 1.7097641229629517, "rewards/rejected": 1.2402087450027466, "step": 855 }, { "epoch": 0.8212473888391525, "grad_norm": 3.296875, "ht_mnpo/logit": 0.14397737383842468, "ht_mnpo/residual": 0.13647738099098206, "ht_mnpo/residual_abs": 0.13811519742012024, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 3.154220673422192e-09, "logits/chosen": -1.9785963296890259, "logits/rejected": -2.251053810119629, "logps/chosen": -0.27199432253837585, "logps/rejected": -0.27827268838882446, "loss": 0.15808339416980743, "loss/core": 0.15808339416980743, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 2.0167627334594727, "rewards/margins": 1.4397737979888916, "rewards/rejected": 0.5769888758659363, "step": 860 }, { "epoch": 0.8260220829603103, "grad_norm": 161.0, "ht_mnpo/logit": 0.3069954812526703, "ht_mnpo/residual": 0.2994953989982605, "ht_mnpo/residual_abs": 0.3049822747707367, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.4329828146074096e-09, "logits/chosen": -1.8511345386505127, "logits/rejected": -2.2174289226531982, "logps/chosen": -0.3130944073200226, "logps/rejected": -0.3080425560474396, "loss": 1.1950414180755615, "loss/core": 1.1950414180755615, "rewards/accuracies": 0.862500011920929, "rewards/chosen": 3.894388198852539, "rewards/margins": 3.0699546337127686, "rewards/rejected": 0.8244336247444153, "step": 865 }, { "epoch": 0.8307967770814683, "grad_norm": 29.75, "ht_mnpo/logit": 0.09861830621957779, "ht_mnpo/residual": 0.09111831337213516, "ht_mnpo/residual_abs": 0.09437834471464157, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.8048447862070714e-09, "logits/chosen": -2.1397294998168945, "logits/rejected": -2.3984568119049072, "logps/chosen": -0.28324785828590393, "logps/rejected": -0.28805121779441833, "loss": 0.06719032675027847, "loss/core": 0.06719032675027847, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": 1.770001769065857, "rewards/margins": 0.9861831665039062, "rewards/rejected": 0.7838188409805298, "step": 870 }, { "epoch": 0.8355714712026261, "grad_norm": 3.078125, "ht_mnpo/logit": 0.11863557994365692, "ht_mnpo/residual": 0.1111355796456337, "ht_mnpo/residual_abs": 0.11609260737895966, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.2700428052447033e-09, "logits/chosen": -1.9720087051391602, "logits/rejected": -2.288149356842041, "logps/chosen": -0.2549072802066803, "logps/rejected": -0.27585774660110474, "loss": 0.08893564343452454, "loss/core": 0.08893564343452454, "rewards/accuracies": 0.949999988079071, "rewards/chosen": 1.8031947612762451, "rewards/margins": 1.1863558292388916, "rewards/rejected": 0.6168391108512878, "step": 875 }, { "epoch": 0.8403461653237839, "grad_norm": 50.75, "ht_mnpo/logit": 0.13304957747459412, "ht_mnpo/residual": 0.1255495846271515, "ht_mnpo/residual_abs": 0.12660455703735352, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 8.287779888734858e-10, "logits/chosen": -2.260289430618286, "logits/rejected": -2.4413468837738037, "logps/chosen": -0.304781973361969, "logps/rejected": -0.30884766578674316, "loss": 0.14404062926769257, "loss/core": 0.14404062926769257, "rewards/accuracies": 1.0, "rewards/chosen": 1.8423153162002563, "rewards/margins": 1.3304959535598755, "rewards/rejected": 0.5118193626403809, "step": 880 }, { "epoch": 0.8451208594449419, "grad_norm": 2.796875, "ht_mnpo/logit": 0.2581387162208557, "ht_mnpo/residual": 0.2506386935710907, "ht_mnpo/residual_abs": 0.2524881660938263, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 4.812162787445062e-10, "logits/chosen": -1.9322468042373657, "logits/rejected": -2.3333423137664795, "logps/chosen": -0.319403737783432, "logps/rejected": -0.3051341474056244, "loss": 0.6756513714790344, "loss/core": 0.6756513714790344, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 3.50068736076355, "rewards/margins": 2.5813872814178467, "rewards/rejected": 0.9193001985549927, "step": 885 }, { "epoch": 0.8498955535660997, "grad_norm": 12.8125, "ht_mnpo/logit": 0.22012737393379211, "ht_mnpo/residual": 0.2126273661851883, "ht_mnpo/residual_abs": 0.2149299830198288, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 2.2748837860270265e-10, "logits/chosen": -1.9026530981063843, "logits/rejected": -2.1633448600769043, "logps/chosen": -0.29086828231811523, "logps/rejected": -0.27807530760765076, "loss": 0.35099947452545166, "loss/core": 0.35099947452545166, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 2.9746203422546387, "rewards/margins": 2.2012736797332764, "rewards/rejected": 0.7733467817306519, "step": 890 }, { "epoch": 0.8546702476872575, "grad_norm": 15.8125, "ht_mnpo/logit": 0.3148799538612366, "ht_mnpo/residual": 0.3073800206184387, "ht_mnpo/residual_abs": 0.3098318874835968, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 6.768970513457151e-11, "logits/chosen": -1.93404221534729, "logits/rejected": -2.2042999267578125, "logps/chosen": -0.32174813747406006, "logps/rejected": -0.2906070351600647, "loss": 0.975996196269989, "loss/core": 0.975996196269989, "rewards/accuracies": 0.9375, "rewards/chosen": 4.408227443695068, "rewards/margins": 3.1487996578216553, "rewards/rejected": 1.259427785873413, "step": 895 }, { "epoch": 0.8594449418084154, "grad_norm": 16.5, "ht_mnpo/logit": 0.22211775183677673, "ht_mnpo/residual": 0.2146177738904953, "ht_mnpo/residual_abs": 0.26752087473869324, "ht_mnpo/target": 0.007499999366700649, "ht_mnpo/target_abs": 0.007499999366700649, "learning_rate": 1.8803520859811406e-12, "logits/chosen": -1.908017873764038, "logits/rejected": -2.071946382522583, "logps/chosen": -0.30949825048446655, "logps/rejected": -0.33156439661979675, "loss": 0.5480181574821472, "loss/core": 0.5480181574821472, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": 4.108867645263672, "rewards/margins": 2.221177577972412, "rewards/rejected": 1.8876899480819702, "step": 900 }, { "epoch": 0.8594449418084154, "step": 900, "total_flos": 0.0, "train_loss": 0.7468786084651947, "train_runtime": 6934.2752, "train_samples_per_second": 2.077, "train_steps_per_second": 0.13 } ], "logging_steps": 5, "max_steps": 900, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 900, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }