{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.999244142101285, "eval_steps": 200, "global_step": 661, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "KL/chosen_KL_mean": 0.02867889404296875, "KL/mean": 0.029354453086853027, "KL/rejected_KL_mean": 0.030029296875, "KL/std": 0.2071000635623932, "epoch": 0.0015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.0013532638549804688, "fcm_dpo/q_t": 0.5000336766242981, "grad_norm": 28.22066879272461, "learning_rate": 0.0, "logits/chosen": 0.13337239623069763, "logits/rejected": 0.12492949515581131, "logps/chosen": -64.5841293334961, "logps/ref_chosen": -64.61280822753906, "logps/ref_rejected": -64.17195129394531, "logps/rejected": -64.14192199707031, "loss": 1.3866, "margin_dpo/margin_mean": -0.0013527870178222656, "margin_dpo/margin_std": 0.2561596930027008, "step": 1 }, { "KL/chosen_KL_mean": -0.00289154052734375, "KL/mean": -0.021616414189338684, "KL/rejected_KL_mean": -0.04033660888671875, "KL/std": 0.19624735414981842, "epoch": 0.0030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.037450045347213745, "fcm_dpo/q_t": 0.4990639090538025, "grad_norm": 27.826658248901367, "learning_rate": 7.462686567164179e-09, "logits/chosen": 0.09414851665496826, "logits/rejected": 0.07363267242908478, "logps/chosen": -56.101890563964844, "logps/ref_chosen": -56.0989990234375, "logps/ref_rejected": -66.59971618652344, "logps/rejected": -66.64006042480469, "loss": 1.3828, "margin_dpo/margin_mean": 0.03744968771934509, "margin_dpo/margin_std": 0.27811938524246216, "step": 2 }, { "KL/chosen_KL_mean": 0.04264068603515625, "KL/mean": 0.053322210907936096, "KL/rejected_KL_mean": 0.06399917602539062, "KL/std": 0.2611844837665558, "epoch": 0.0045351473922902496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.021352916955947876, "fcm_dpo/q_t": 0.5005335211753845, "grad_norm": 31.28985023498535, "learning_rate": 1.4925373134328357e-08, "logits/chosen": 0.09941151738166809, "logits/rejected": 0.061459895223379135, "logps/chosen": -65.41461944580078, "logps/ref_chosen": -65.45726013183594, "logps/ref_rejected": -90.82853698730469, "logps/rejected": -90.76453399658203, "loss": 1.3888, "margin_dpo/margin_mean": -0.021352916955947876, "margin_dpo/margin_std": 0.36793452501296997, "step": 3 }, { "KL/chosen_KL_mean": 0.02747344970703125, "KL/mean": 0.011432260274887085, "KL/rejected_KL_mean": -0.00461578369140625, "KL/std": 0.216526597738266, "epoch": 0.006046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.032090187072753906, "fcm_dpo/q_t": 0.4991982579231262, "grad_norm": 34.291229248046875, "learning_rate": 2.2388059701492534e-08, "logits/chosen": 0.1126030832529068, "logits/rejected": 0.09631362557411194, "logps/chosen": -76.83271026611328, "logps/ref_chosen": -76.86018371582031, "logps/ref_rejected": -79.91523742675781, "logps/rejected": -79.91985321044922, "loss": 1.3833, "margin_dpo/margin_mean": 0.03208950161933899, "margin_dpo/margin_std": 0.3081591725349426, "step": 4 }, { "KL/chosen_KL_mean": -0.019184112548828125, "KL/mean": 0.0005896538496017456, "KL/rejected_KL_mean": 0.020366668701171875, "KL/std": 0.24410542845726013, "epoch": 0.007558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03955063223838806, "fcm_dpo/q_t": 0.5009881854057312, "grad_norm": 29.674455642700195, "learning_rate": 2.9850746268656714e-08, "logits/chosen": 0.07852072268724442, "logits/rejected": 0.039799898862838745, "logps/chosen": -62.99052429199219, "logps/ref_chosen": -62.97134017944336, "logps/ref_rejected": -79.9192123413086, "logps/rejected": -79.89884948730469, "loss": 1.3905, "margin_dpo/margin_mean": -0.03954946994781494, "margin_dpo/margin_std": 0.3197075128555298, "step": 5 }, { "KL/chosen_KL_mean": -0.0473175048828125, "KL/mean": -0.04120214283466339, "KL/rejected_KL_mean": -0.03508567810058594, "KL/std": 0.24611763656139374, "epoch": 0.009070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.012232571840286255, "fcm_dpo/q_t": 0.5003060698509216, "grad_norm": 29.636795043945312, "learning_rate": 3.731343283582089e-08, "logits/chosen": 0.16185824573040009, "logits/rejected": 0.12125864624977112, "logps/chosen": -51.35468292236328, "logps/ref_chosen": -51.30736541748047, "logps/ref_rejected": -82.77239227294922, "logps/rejected": -82.80747985839844, "loss": 1.3878, "margin_dpo/margin_mean": -0.012232691049575806, "margin_dpo/margin_std": 0.35705843567848206, "step": 6 }, { "KL/chosen_KL_mean": 0.027551651000976562, "KL/mean": 0.02225738763809204, "KL/rejected_KL_mean": 0.016962051391601562, "KL/std": 0.2204289585351944, "epoch": 0.010582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01059180498123169, "fcm_dpo/q_t": 0.49973538517951965, "grad_norm": 27.118959426879883, "learning_rate": 4.477611940298507e-08, "logits/chosen": 0.01888515241444111, "logits/rejected": -0.025128159672021866, "logps/chosen": -51.431861877441406, "logps/ref_chosen": -51.45941162109375, "logps/ref_rejected": -66.3828125, "logps/rejected": -66.3658447265625, "loss": 1.3855, "margin_dpo/margin_mean": 0.0105915367603302, "margin_dpo/margin_std": 0.29007700085639954, "step": 7 }, { "KL/chosen_KL_mean": -0.00579833984375, "KL/mean": -0.006937950849533081, "KL/rejected_KL_mean": -0.008075714111328125, "KL/std": 0.2304605394601822, "epoch": 0.012093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0022736191749572754, "fcm_dpo/q_t": 0.4999435245990753, "grad_norm": 28.647260665893555, "learning_rate": 5.223880597014925e-08, "logits/chosen": 0.06418946385383606, "logits/rejected": 0.042302586138248444, "logps/chosen": -62.203346252441406, "logps/ref_chosen": -62.197547912597656, "logps/ref_rejected": -74.66180419921875, "logps/rejected": -74.66988372802734, "loss": 1.3863, "margin_dpo/margin_mean": 0.002272665500640869, "margin_dpo/margin_std": 0.29995080828666687, "step": 8 }, { "KL/chosen_KL_mean": -0.039676666259765625, "KL/mean": -0.03592316806316376, "KL/rejected_KL_mean": -0.03217315673828125, "KL/std": 0.22302508354187012, "epoch": 0.013605442176870748, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.007506042718887329, "fcm_dpo/q_t": 0.5001875162124634, "grad_norm": 31.57317352294922, "learning_rate": 5.970149253731343e-08, "logits/chosen": 0.17316150665283203, "logits/rejected": 0.11361009627580643, "logps/chosen": -55.669395446777344, "logps/ref_chosen": -55.629722595214844, "logps/ref_rejected": -86.21221923828125, "logps/rejected": -86.24440002441406, "loss": 1.3873, "margin_dpo/margin_mean": -0.007505506277084351, "margin_dpo/margin_std": 0.32757535576820374, "step": 9 }, { "KL/chosen_KL_mean": -0.015594482421875, "KL/mean": 0.0028478801250457764, "KL/rejected_KL_mean": 0.021289825439453125, "KL/std": 0.2507847547531128, "epoch": 0.015117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.03688693046569824, "fcm_dpo/q_t": 0.5009225010871887, "grad_norm": 29.66307830810547, "learning_rate": 6.71641791044776e-08, "logits/chosen": 0.15237724781036377, "logits/rejected": 0.12025925517082214, "logps/chosen": -62.70619583129883, "logps/ref_chosen": -62.69060134887695, "logps/ref_rejected": -90.610107421875, "logps/rejected": -90.58882141113281, "loss": 1.3903, "margin_dpo/margin_mean": -0.03688687086105347, "margin_dpo/margin_std": 0.34781166911125183, "step": 10 }, { "KL/chosen_KL_mean": -0.04038047790527344, "KL/mean": -0.027389228343963623, "KL/rejected_KL_mean": -0.014400482177734375, "KL/std": 0.18189536035060883, "epoch": 0.016628873771730914, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.025982022285461426, "fcm_dpo/q_t": 0.5006492733955383, "grad_norm": 29.398035049438477, "learning_rate": 7.462686567164178e-08, "logits/chosen": 0.12077412009239197, "logits/rejected": 0.11373384296894073, "logps/chosen": -65.8074951171875, "logps/ref_chosen": -65.76712036132812, "logps/ref_rejected": -72.4764633178711, "logps/rejected": -72.49085998535156, "loss": 1.3891, "margin_dpo/margin_mean": -0.025981128215789795, "margin_dpo/margin_std": 0.24610239267349243, "step": 11 }, { "KL/chosen_KL_mean": -0.018270492553710938, "KL/mean": -0.028908535838127136, "KL/rejected_KL_mean": -0.03954887390136719, "KL/std": 0.2320721447467804, "epoch": 0.018140589569160998, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.02127799391746521, "fcm_dpo/q_t": 0.49946796894073486, "grad_norm": 28.293529510498047, "learning_rate": 8.208955223880596e-08, "logits/chosen": 0.00675106979906559, "logits/rejected": -0.008964652195572853, "logps/chosen": -60.72315979003906, "logps/ref_chosen": -60.704891204833984, "logps/ref_rejected": -69.41564178466797, "logps/rejected": -69.45519256591797, "loss": 1.3844, "margin_dpo/margin_mean": 0.021277964115142822, "margin_dpo/margin_std": 0.2855193614959717, "step": 12 }, { "KL/chosen_KL_mean": -0.014070510864257812, "KL/mean": 0.011509418487548828, "KL/rejected_KL_mean": 0.0370941162109375, "KL/std": 0.2110683023929596, "epoch": 0.019652305366591082, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.051162004470825195, "fcm_dpo/q_t": 0.5012787580490112, "grad_norm": 29.074474334716797, "learning_rate": 8.955223880597014e-08, "logits/chosen": 0.11439329385757446, "logits/rejected": 0.05134594440460205, "logps/chosen": -49.92332458496094, "logps/ref_chosen": -49.90925598144531, "logps/ref_rejected": -92.37818145751953, "logps/rejected": -92.3410873413086, "loss": 1.3916, "margin_dpo/margin_mean": -0.051162540912628174, "margin_dpo/margin_std": 0.29525551199913025, "step": 13 }, { "KL/chosen_KL_mean": 0.0183868408203125, "KL/mean": -0.0022123605012893677, "KL/rejected_KL_mean": -0.022808074951171875, "KL/std": 0.20953845977783203, "epoch": 0.021164021164021163, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04119846224784851, "fcm_dpo/q_t": 0.49897050857543945, "grad_norm": 29.494983673095703, "learning_rate": 9.701492537313432e-08, "logits/chosen": 0.09920904040336609, "logits/rejected": 0.0811566635966301, "logps/chosen": -60.600406646728516, "logps/ref_chosen": -60.61879348754883, "logps/ref_rejected": -71.79306030273438, "logps/rejected": -71.81587219238281, "loss": 1.3824, "margin_dpo/margin_mean": 0.041198402643203735, "margin_dpo/margin_std": 0.2794230878353119, "step": 14 }, { "KL/chosen_KL_mean": -0.027385711669921875, "KL/mean": -0.029895320534706116, "KL/rejected_KL_mean": -0.03240966796875, "KL/std": 0.22818677127361298, "epoch": 0.022675736961451247, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.005019158124923706, "fcm_dpo/q_t": 0.49987471103668213, "grad_norm": 33.24843215942383, "learning_rate": 1.044776119402985e-07, "logits/chosen": 0.08923092484474182, "logits/rejected": 0.04480152577161789, "logps/chosen": -63.496925354003906, "logps/ref_chosen": -63.46953582763672, "logps/ref_rejected": -88.88951110839844, "logps/rejected": -88.92192077636719, "loss": 1.3861, "margin_dpo/margin_mean": 0.005019336938858032, "margin_dpo/margin_std": 0.3223441541194916, "step": 15 }, { "KL/chosen_KL_mean": -0.00457763671875, "KL/mean": -0.01490524411201477, "KL/rejected_KL_mean": -0.025234222412109375, "KL/std": 0.20558756589889526, "epoch": 0.02418745275888133, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.020651936531066895, "fcm_dpo/q_t": 0.4994843006134033, "grad_norm": 26.84224510192871, "learning_rate": 1.1194029850746268e-07, "logits/chosen": 0.121390700340271, "logits/rejected": 0.08392910659313202, "logps/chosen": -46.5368766784668, "logps/ref_chosen": -46.53229904174805, "logps/ref_rejected": -74.27533721923828, "logps/rejected": -74.30056762695312, "loss": 1.3845, "margin_dpo/margin_mean": 0.020652025938034058, "margin_dpo/margin_std": 0.29893872141838074, "step": 16 }, { "KL/chosen_KL_mean": -0.013319015502929688, "KL/mean": -0.0209941565990448, "KL/rejected_KL_mean": -0.028667449951171875, "KL/std": 0.22113090753555298, "epoch": 0.025699168556311415, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.015350341796875, "fcm_dpo/q_t": 0.4996163547039032, "grad_norm": 32.962730407714844, "learning_rate": 1.1940298507462686e-07, "logits/chosen": 0.06988856196403503, "logits/rejected": 0.05055886507034302, "logps/chosen": -64.09115600585938, "logps/ref_chosen": -64.07783508300781, "logps/ref_rejected": -86.40876770019531, "logps/rejected": -86.43743896484375, "loss": 1.385, "margin_dpo/margin_mean": 0.015350043773651123, "margin_dpo/margin_std": 0.3078957796096802, "step": 17 }, { "KL/chosen_KL_mean": 0.03248405456542969, "KL/mean": 0.027805477380752563, "KL/rejected_KL_mean": 0.02312469482421875, "KL/std": 0.17294234037399292, "epoch": 0.027210884353741496, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.009357690811157227, "fcm_dpo/q_t": 0.4997663199901581, "grad_norm": 27.813247680664062, "learning_rate": 1.2686567164179106e-07, "logits/chosen": 0.0905240923166275, "logits/rejected": 0.04524059593677521, "logps/chosen": -44.84185028076172, "logps/ref_chosen": -44.87433624267578, "logps/ref_rejected": -70.97604370117188, "logps/rejected": -70.95292663574219, "loss": 1.3855, "margin_dpo/margin_mean": 0.009358137845993042, "margin_dpo/margin_std": 0.2632210850715637, "step": 18 }, { "KL/chosen_KL_mean": 0.03167915344238281, "KL/mean": 0.002976052463054657, "KL/rejected_KL_mean": -0.025730133056640625, "KL/std": 0.2518116235733032, "epoch": 0.02872260015117158, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.057412102818489075, "fcm_dpo/q_t": 0.498563677072525, "grad_norm": 30.920751571655273, "learning_rate": 1.343283582089552e-07, "logits/chosen": 0.06809468567371368, "logits/rejected": 0.05450137332081795, "logps/chosen": -68.12812805175781, "logps/ref_chosen": -68.1598129272461, "logps/ref_rejected": -81.17138671875, "logps/rejected": -81.1971206665039, "loss": 1.3809, "margin_dpo/margin_mean": 0.057412490248680115, "margin_dpo/margin_std": 0.3499699831008911, "step": 19 }, { "KL/chosen_KL_mean": 0.024566650390625, "KL/mean": 0.012526951730251312, "KL/rejected_KL_mean": 0.000492095947265625, "KL/std": 0.18837958574295044, "epoch": 0.030234315948601664, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.024075254797935486, "fcm_dpo/q_t": 0.49939823150634766, "grad_norm": 29.136823654174805, "learning_rate": 1.4179104477611938e-07, "logits/chosen": 0.1479886770248413, "logits/rejected": 0.12440957129001617, "logps/chosen": -53.653995513916016, "logps/ref_chosen": -53.67856216430664, "logps/ref_rejected": -74.16911315917969, "logps/rejected": -74.16862487792969, "loss": 1.384, "margin_dpo/margin_mean": 0.02407585084438324, "margin_dpo/margin_std": 0.24943657219409943, "step": 20 }, { "KL/chosen_KL_mean": 0.007982254028320312, "KL/mean": 0.0034023597836494446, "KL/rejected_KL_mean": -0.001178741455078125, "KL/std": 0.23807811737060547, "epoch": 0.031746031746031744, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.009163036942481995, "fcm_dpo/q_t": 0.4997709393501282, "grad_norm": 29.345401763916016, "learning_rate": 1.4925373134328355e-07, "logits/chosen": 0.1197953000664711, "logits/rejected": 0.09367487579584122, "logps/chosen": -64.69357299804688, "logps/ref_chosen": -64.70155334472656, "logps/ref_rejected": -81.02095031738281, "logps/rejected": -81.02212524414062, "loss": 1.3856, "margin_dpo/margin_mean": 0.00916224718093872, "margin_dpo/margin_std": 0.2760716676712036, "step": 21 }, { "KL/chosen_KL_mean": -0.021139144897460938, "KL/mean": -0.024111106991767883, "KL/rejected_KL_mean": -0.027080535888671875, "KL/std": 0.21007117629051208, "epoch": 0.03325774754346183, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.005945473909378052, "fcm_dpo/q_t": 0.4998513460159302, "grad_norm": 29.009286880493164, "learning_rate": 1.5671641791044775e-07, "logits/chosen": 0.00022973224986344576, "logits/rejected": -0.020442631095647812, "logps/chosen": -58.05712890625, "logps/ref_chosen": -58.03599166870117, "logps/ref_rejected": -80.72721862792969, "logps/rejected": -80.75430297851562, "loss": 1.3859, "margin_dpo/margin_mean": 0.005945086479187012, "margin_dpo/margin_std": 0.2577439546585083, "step": 22 }, { "KL/chosen_KL_mean": -0.01317596435546875, "KL/mean": -0.014801204204559326, "KL/rejected_KL_mean": -0.016429901123046875, "KL/std": 0.24061861634254456, "epoch": 0.03476946334089191, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.003259122371673584, "fcm_dpo/q_t": 0.49991869926452637, "grad_norm": 33.08818054199219, "learning_rate": 1.6417910447761193e-07, "logits/chosen": 0.12761254608631134, "logits/rejected": 0.10276039689779282, "logps/chosen": -66.3692626953125, "logps/ref_chosen": -66.35608673095703, "logps/ref_rejected": -93.02769470214844, "logps/rejected": -93.04412078857422, "loss": 1.3863, "margin_dpo/margin_mean": 0.0032591521739959717, "margin_dpo/margin_std": 0.3656204640865326, "step": 23 }, { "KL/chosen_KL_mean": 0.0018863677978515625, "KL/mean": -0.009600698947906494, "KL/rejected_KL_mean": -0.021089553833007812, "KL/std": 0.19692258536815643, "epoch": 0.036281179138321996, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.022979021072387695, "fcm_dpo/q_t": 0.49942547082901, "grad_norm": 26.260557174682617, "learning_rate": 1.716417910447761e-07, "logits/chosen": 0.1274389773607254, "logits/rejected": 0.09479451179504395, "logps/chosen": -54.4593505859375, "logps/ref_chosen": -54.461238861083984, "logps/ref_rejected": -68.33817291259766, "logps/rejected": -68.35926818847656, "loss": 1.3842, "margin_dpo/margin_mean": 0.02297845482826233, "margin_dpo/margin_std": 0.25866520404815674, "step": 24 }, { "KL/chosen_KL_mean": -0.04363059997558594, "KL/mean": -0.03292663395404816, "KL/rejected_KL_mean": -0.02222442626953125, "KL/std": 0.22737306356430054, "epoch": 0.03779289493575208, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.021410435438156128, "fcm_dpo/q_t": 0.5005350112915039, "grad_norm": 29.813453674316406, "learning_rate": 1.7910447761194027e-07, "logits/chosen": 0.09910149872303009, "logits/rejected": 0.04798390716314316, "logps/chosen": -60.04783630371094, "logps/ref_chosen": -60.00420379638672, "logps/ref_rejected": -90.47376251220703, "logps/rejected": -90.49598693847656, "loss": 1.3886, "margin_dpo/margin_mean": -0.021410197019577026, "margin_dpo/margin_std": 0.2856305241584778, "step": 25 }, { "KL/chosen_KL_mean": -0.03212165832519531, "KL/mean": -0.010377638041973114, "KL/rejected_KL_mean": 0.0113677978515625, "KL/std": 0.23722830414772034, "epoch": 0.039304610733182165, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.04349200427532196, "fcm_dpo/q_t": 0.5010876655578613, "grad_norm": 29.885282516479492, "learning_rate": 1.8656716417910447e-07, "logits/chosen": 0.10826554894447327, "logits/rejected": 0.08984041213989258, "logps/chosen": -56.85127639770508, "logps/ref_chosen": -56.81915283203125, "logps/ref_rejected": -77.84333038330078, "logps/rejected": -77.83196258544922, "loss": 1.3909, "margin_dpo/margin_mean": -0.043492019176483154, "margin_dpo/margin_std": 0.3361806273460388, "step": 26 }, { "KL/chosen_KL_mean": -0.0045871734619140625, "KL/mean": -0.008165434002876282, "KL/rejected_KL_mean": -0.011743545532226562, "KL/std": 0.19913233816623688, "epoch": 0.04081632653061224, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.00715222954750061, "fcm_dpo/q_t": 0.49982118606567383, "grad_norm": 28.897428512573242, "learning_rate": 1.9402985074626865e-07, "logits/chosen": 0.08622437715530396, "logits/rejected": 0.062031831592321396, "logps/chosen": -62.881614685058594, "logps/ref_chosen": -62.87702560424805, "logps/ref_rejected": -71.34437561035156, "logps/rejected": -71.35611724853516, "loss": 1.3858, "margin_dpo/margin_mean": 0.007152438163757324, "margin_dpo/margin_std": 0.29086369276046753, "step": 27 }, { "KL/chosen_KL_mean": -0.029260635375976562, "KL/mean": -0.03680327534675598, "KL/rejected_KL_mean": -0.044345855712890625, "KL/std": 0.19967246055603027, "epoch": 0.042328042328042326, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.01508396863937378, "fcm_dpo/q_t": 0.49962306022644043, "grad_norm": 27.587074279785156, "learning_rate": 2.0149253731343282e-07, "logits/chosen": 0.04726283252239227, "logits/rejected": 0.03866342082619667, "logps/chosen": -59.86263656616211, "logps/ref_chosen": -59.8333740234375, "logps/ref_rejected": -70.39804077148438, "logps/rejected": -70.4423828125, "loss": 1.385, "margin_dpo/margin_mean": 0.015084385871887207, "margin_dpo/margin_std": 0.2913114130496979, "step": 28 }, { "KL/chosen_KL_mean": -0.01448822021484375, "KL/mean": -0.02990015596151352, "KL/rejected_KL_mean": -0.045307159423828125, "KL/std": 0.2235536277294159, "epoch": 0.04383975812547241, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03082270920276642, "fcm_dpo/q_t": 0.4992297887802124, "grad_norm": 32.74290084838867, "learning_rate": 2.08955223880597e-07, "logits/chosen": 0.11370386928319931, "logits/rejected": 0.09676564484834671, "logps/chosen": -74.13468933105469, "logps/ref_chosen": -74.12020111083984, "logps/ref_rejected": -83.33099365234375, "logps/rejected": -83.37629699707031, "loss": 1.3835, "margin_dpo/margin_mean": 0.03082290291786194, "margin_dpo/margin_std": 0.3283058702945709, "step": 29 }, { "KL/chosen_KL_mean": -0.05242156982421875, "KL/mean": -0.056987062096595764, "KL/rejected_KL_mean": -0.061553955078125, "KL/std": 0.22280657291412354, "epoch": 0.045351473922902494, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.009135037660598755, "fcm_dpo/q_t": 0.4997720718383789, "grad_norm": 30.316654205322266, "learning_rate": 2.1641791044776117e-07, "logits/chosen": 0.11641520261764526, "logits/rejected": 0.06267019361257553, "logps/chosen": -50.8037109375, "logps/ref_chosen": -50.75128936767578, "logps/ref_rejected": -89.29063415527344, "logps/rejected": -89.35218811035156, "loss": 1.3856, "margin_dpo/margin_mean": 0.009134829044342041, "margin_dpo/margin_std": 0.3106568455696106, "step": 30 }, { "KL/chosen_KL_mean": -0.05117988586425781, "KL/mean": -0.05755278468132019, "KL/rejected_KL_mean": -0.06392669677734375, "KL/std": 0.2555512487888336, "epoch": 0.04686318972033258, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.012740731239318848, "fcm_dpo/q_t": 0.49968260526657104, "grad_norm": 34.35220718383789, "learning_rate": 2.2388059701492537e-07, "logits/chosen": 0.12674255669116974, "logits/rejected": 0.07923215627670288, "logps/chosen": -65.38793182373047, "logps/ref_chosen": -65.33675384521484, "logps/ref_rejected": -100.76666259765625, "logps/rejected": -100.8305892944336, "loss": 1.3854, "margin_dpo/margin_mean": 0.01274147629737854, "margin_dpo/margin_std": 0.3628019690513611, "step": 31 }, { "KL/chosen_KL_mean": -0.0181884765625, "KL/mean": -0.0361301451921463, "KL/rejected_KL_mean": -0.054073333740234375, "KL/std": 0.23413747549057007, "epoch": 0.04837490551776266, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03588786721229553, "fcm_dpo/q_t": 0.49910300970077515, "grad_norm": 30.009159088134766, "learning_rate": 2.3134328358208954e-07, "logits/chosen": 0.08153019845485687, "logits/rejected": 0.0736636072397232, "logps/chosen": -67.20152282714844, "logps/ref_chosen": -67.18333435058594, "logps/ref_rejected": -82.80763244628906, "logps/rejected": -82.86170959472656, "loss": 1.383, "margin_dpo/margin_mean": 0.0358881950378418, "margin_dpo/margin_std": 0.35985904932022095, "step": 32 }, { "KL/chosen_KL_mean": -0.034152984619140625, "KL/mean": -0.05544004589319229, "KL/rejected_KL_mean": -0.07672500610351562, "KL/std": 0.24409998953342438, "epoch": 0.049886621315192746, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.04257179796695709, "fcm_dpo/q_t": 0.498935729265213, "grad_norm": 30.951351165771484, "learning_rate": 2.388059701492537e-07, "logits/chosen": 0.027346061542630196, "logits/rejected": 0.0015142709016799927, "logps/chosen": -64.0736312866211, "logps/ref_chosen": -64.03948211669922, "logps/ref_rejected": -75.68357849121094, "logps/rejected": -75.76030731201172, "loss": 1.3823, "margin_dpo/margin_mean": 0.04257209599018097, "margin_dpo/margin_std": 0.30183154344558716, "step": 33 }, { "KL/chosen_KL_mean": -0.048999786376953125, "KL/mean": -0.06053346395492554, "KL/rejected_KL_mean": -0.07206916809082031, "KL/std": 0.19695493578910828, "epoch": 0.05139833711262283, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.023070037364959717, "fcm_dpo/q_t": 0.49942296743392944, "grad_norm": 28.55535125732422, "learning_rate": 2.4626865671641786e-07, "logits/chosen": 0.08082367479801178, "logits/rejected": 0.05176318809390068, "logps/chosen": -53.713294982910156, "logps/ref_chosen": -53.6642951965332, "logps/ref_rejected": -65.77989959716797, "logps/rejected": -65.85196685791016, "loss": 1.3842, "margin_dpo/margin_mean": 0.023069947957992554, "margin_dpo/margin_std": 0.2683805227279663, "step": 34 }, { "KL/chosen_KL_mean": -0.08650398254394531, "KL/mean": -0.08371259272098541, "KL/rejected_KL_mean": -0.0809173583984375, "KL/std": 0.2462875247001648, "epoch": 0.05291005291005291, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.005585581064224243, "fcm_dpo/q_t": 0.500140905380249, "grad_norm": 27.881855010986328, "learning_rate": 2.537313432835821e-07, "logits/chosen": 0.04871156066656113, "logits/rejected": 0.02646300010383129, "logps/chosen": -61.10336685180664, "logps/ref_chosen": -61.01686096191406, "logps/ref_rejected": -72.78598022460938, "logps/rejected": -72.86689758300781, "loss": 1.3872, "margin_dpo/margin_mean": -0.005585342645645142, "margin_dpo/margin_std": 0.3802899122238159, "step": 35 }, { "KL/chosen_KL_mean": -0.08986663818359375, "KL/mean": -0.08186689019203186, "KL/rejected_KL_mean": -0.0738677978515625, "KL/std": 0.25683659315109253, "epoch": 0.05442176870748299, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": -0.01600050926208496, "fcm_dpo/q_t": 0.5004006624221802, "grad_norm": 28.70569610595703, "learning_rate": 2.611940298507462e-07, "logits/chosen": 0.08580096811056137, "logits/rejected": 0.03410476818680763, "logps/chosen": -50.627227783203125, "logps/ref_chosen": -50.53736114501953, "logps/ref_rejected": -78.11678314208984, "logps/rejected": -78.1906509399414, "loss": 1.3883, "margin_dpo/margin_mean": -0.01600012183189392, "margin_dpo/margin_std": 0.3842473030090332, "step": 36 }, { "KL/chosen_KL_mean": -0.028474807739257812, "KL/mean": -0.09227629005908966, "KL/rejected_KL_mean": -0.1560821533203125, "KL/std": 0.26934385299682617, "epoch": 0.055933484504913075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.12760671973228455, "fcm_dpo/q_t": 0.49681222438812256, "grad_norm": 37.727169036865234, "learning_rate": 2.686567164179104e-07, "logits/chosen": 0.12197470664978027, "logits/rejected": 0.03995547443628311, "logps/chosen": -59.582420349121094, "logps/ref_chosen": -59.55394744873047, "logps/ref_rejected": -108.27702331542969, "logps/rejected": -108.43310546875, "loss": 1.3739, "margin_dpo/margin_mean": 0.12760648131370544, "margin_dpo/margin_std": 0.37096428871154785, "step": 37 }, { "KL/chosen_KL_mean": -0.0984039306640625, "KL/mean": -0.10117024183273315, "KL/rejected_KL_mean": -0.10393905639648438, "KL/std": 0.2519524097442627, "epoch": 0.05744520030234316, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.005530297756195068, "fcm_dpo/q_t": 0.4998621940612793, "grad_norm": 29.83066177368164, "learning_rate": 2.761194029850746e-07, "logits/chosen": 0.0732765644788742, "logits/rejected": 0.05932066589593887, "logps/chosen": -65.88676452636719, "logps/ref_chosen": -65.78836059570312, "logps/ref_rejected": -76.1619873046875, "logps/rejected": -76.26593017578125, "loss": 1.3861, "margin_dpo/margin_mean": 0.005530297756195068, "margin_dpo/margin_std": 0.3729862570762634, "step": 38 }, { "KL/chosen_KL_mean": -0.09945297241210938, "KL/mean": -0.11698070168495178, "KL/rejected_KL_mean": -0.13450241088867188, "KL/std": 0.25591549277305603, "epoch": 0.05895691609977324, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.03504788875579834, "fcm_dpo/q_t": 0.4991235136985779, "grad_norm": 29.30242347717285, "learning_rate": 2.8358208955223876e-07, "logits/chosen": 0.2040865123271942, "logits/rejected": 0.1760084629058838, "logps/chosen": -57.27626037597656, "logps/ref_chosen": -57.17681121826172, "logps/ref_rejected": -79.486328125, "logps/rejected": -79.62083435058594, "loss": 1.3831, "margin_dpo/margin_mean": 0.03504854440689087, "margin_dpo/margin_std": 0.3559015691280365, "step": 39 }, { "KL/chosen_KL_mean": -0.08912086486816406, "KL/mean": -0.10761454701423645, "KL/rejected_KL_mean": -0.12611007690429688, "KL/std": 0.22716698050498962, "epoch": 0.06046863189720333, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.036985158920288086, "fcm_dpo/q_t": 0.49907582998275757, "grad_norm": 31.3107967376709, "learning_rate": 2.9104477611940296e-07, "logits/chosen": 0.10985475778579712, "logits/rejected": 0.0602511391043663, "logps/chosen": -61.42328643798828, "logps/ref_chosen": -61.33416748046875, "logps/ref_rejected": -79.10697174072266, "logps/rejected": -79.23307800292969, "loss": 1.3829, "margin_dpo/margin_mean": 0.03698521852493286, "margin_dpo/margin_std": 0.31080809235572815, "step": 40 }, { "KL/chosen_KL_mean": -0.1392059326171875, "KL/mean": -0.1555485725402832, "KL/rejected_KL_mean": -0.17188453674316406, "KL/std": 0.27494168281555176, "epoch": 0.06198034769463341, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.032675087451934814, "fcm_dpo/q_t": 0.4991832971572876, "grad_norm": 30.506553649902344, "learning_rate": 2.985074626865671e-07, "logits/chosen": 0.04268798977136612, "logits/rejected": 0.022809015586972237, "logps/chosen": -67.6859359741211, "logps/ref_chosen": -67.5467300415039, "logps/ref_rejected": -83.87788391113281, "logps/rejected": -84.04977416992188, "loss": 1.3833, "margin_dpo/margin_mean": 0.03267523646354675, "margin_dpo/margin_std": 0.3265340030193329, "step": 41 }, { "KL/chosen_KL_mean": -0.11741065979003906, "KL/mean": -0.15568827092647552, "KL/rejected_KL_mean": -0.19396591186523438, "KL/std": 0.30766791105270386, "epoch": 0.06349206349206349, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07655629515647888, "fcm_dpo/q_t": 0.4980867803096771, "grad_norm": 29.174734115600586, "learning_rate": 3.059701492537313e-07, "logits/chosen": 0.05423213541507721, "logits/rejected": 0.032556917518377304, "logps/chosen": -61.38227081298828, "logps/ref_chosen": -61.26485824584961, "logps/ref_rejected": -76.3629150390625, "logps/rejected": -76.556884765625, "loss": 1.3791, "margin_dpo/margin_mean": 0.07655629515647888, "margin_dpo/margin_std": 0.42006781697273254, "step": 42 }, { "KL/chosen_KL_mean": -0.09850692749023438, "KL/mean": -0.1341993361711502, "KL/rejected_KL_mean": -0.169891357421875, "KL/std": 0.29504144191741943, "epoch": 0.06500377928949358, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.07138493657112122, "fcm_dpo/q_t": 0.49821680784225464, "grad_norm": 34.448570251464844, "learning_rate": 3.134328358208955e-07, "logits/chosen": 0.08338065445423126, "logits/rejected": 0.07241372764110565, "logps/chosen": -71.90753173828125, "logps/ref_chosen": -71.80902862548828, "logps/ref_rejected": -81.12464141845703, "logps/rejected": -81.2945327758789, "loss": 1.3796, "margin_dpo/margin_mean": 0.07138559222221375, "margin_dpo/margin_std": 0.41163527965545654, "step": 43 }, { "KL/chosen_KL_mean": -0.13344192504882812, "KL/mean": -0.16079673171043396, "KL/rejected_KL_mean": -0.18815231323242188, "KL/std": 0.26067036390304565, "epoch": 0.06651549508692366, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05471104383468628, "fcm_dpo/q_t": 0.49863290786743164, "grad_norm": 32.42856216430664, "learning_rate": 3.2089552238805965e-07, "logits/chosen": 0.05968674272298813, "logits/rejected": 0.028688661754131317, "logps/chosen": -66.68386840820312, "logps/ref_chosen": -66.55043029785156, "logps/ref_rejected": -85.06198120117188, "logps/rejected": -85.25013732910156, "loss": 1.3812, "margin_dpo/margin_mean": 0.054710716009140015, "margin_dpo/margin_std": 0.40378284454345703, "step": 44 }, { "KL/chosen_KL_mean": -0.13708114624023438, "KL/mean": -0.18379130959510803, "KL/rejected_KL_mean": -0.23050308227539062, "KL/std": 0.2648080289363861, "epoch": 0.06802721088435375, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.09342862665653229, "fcm_dpo/q_t": 0.49766480922698975, "grad_norm": 31.722583770751953, "learning_rate": 3.2835820895522385e-07, "logits/chosen": 0.130683034658432, "logits/rejected": 0.07715224474668503, "logps/chosen": -62.38093566894531, "logps/ref_chosen": -62.24385452270508, "logps/ref_rejected": -92.96665954589844, "logps/rejected": -93.1971664428711, "loss": 1.3773, "margin_dpo/margin_mean": 0.09342983365058899, "margin_dpo/margin_std": 0.346953809261322, "step": 45 }, { "KL/chosen_KL_mean": -0.10871696472167969, "KL/mean": -0.19501355290412903, "KL/rejected_KL_mean": -0.2813072204589844, "KL/std": 0.3256569504737854, "epoch": 0.06953892668178382, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.17258453369140625, "fcm_dpo/q_t": 0.4956880807876587, "grad_norm": 30.854923248291016, "learning_rate": 3.3582089552238805e-07, "logits/chosen": 0.12547335028648376, "logits/rejected": 0.0794464647769928, "logps/chosen": -61.60762023925781, "logps/ref_chosen": -61.498905181884766, "logps/ref_rejected": -78.91172790527344, "logps/rejected": -79.19303894042969, "loss": 1.3696, "margin_dpo/margin_mean": 0.17258423566818237, "margin_dpo/margin_std": 0.4200759530067444, "step": 46 }, { "KL/chosen_KL_mean": -0.13623046875, "KL/mean": -0.2177979201078415, "KL/rejected_KL_mean": -0.2993621826171875, "KL/std": 0.27024394273757935, "epoch": 0.0710506424792139, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.16313031315803528, "fcm_dpo/q_t": 0.4959232807159424, "grad_norm": 28.427127838134766, "learning_rate": 3.432835820895522e-07, "logits/chosen": 0.03357366845011711, "logits/rejected": -0.009209276176989079, "logps/chosen": -51.714576721191406, "logps/ref_chosen": -51.578346252441406, "logps/ref_rejected": -68.2215576171875, "logps/rejected": -68.52091979980469, "loss": 1.3704, "margin_dpo/margin_mean": 0.16313022375106812, "margin_dpo/margin_std": 0.3930588364601135, "step": 47 }, { "KL/chosen_KL_mean": -0.22270774841308594, "KL/mean": -0.24841207265853882, "KL/rejected_KL_mean": -0.27411460876464844, "KL/std": 0.31731051206588745, "epoch": 0.07256235827664399, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05140644311904907, "fcm_dpo/q_t": 0.498715877532959, "grad_norm": 26.81363296508789, "learning_rate": 3.507462686567164e-07, "logits/chosen": 0.14631465077400208, "logits/rejected": 0.11654843389987946, "logps/chosen": -52.016357421875, "logps/ref_chosen": -51.79365158081055, "logps/ref_rejected": -64.22503662109375, "logps/rejected": -64.49915313720703, "loss": 1.3817, "margin_dpo/margin_mean": 0.0514070987701416, "margin_dpo/margin_std": 0.4587850868701935, "step": 48 }, { "KL/chosen_KL_mean": -0.20112228393554688, "KL/mean": -0.2560715079307556, "KL/rejected_KL_mean": -0.3110179901123047, "KL/std": 0.3602464199066162, "epoch": 0.07407407407407407, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10989488661289215, "fcm_dpo/q_t": 0.4972541630268097, "grad_norm": 27.207046508789062, "learning_rate": 3.5820895522388055e-07, "logits/chosen": 0.05348002910614014, "logits/rejected": 0.03150768578052521, "logps/chosen": -58.335723876953125, "logps/ref_chosen": -58.13460159301758, "logps/ref_rejected": -64.63206481933594, "logps/rejected": -64.94308471679688, "loss": 1.3759, "margin_dpo/margin_mean": 0.10989552736282349, "margin_dpo/margin_std": 0.48010891675949097, "step": 49 }, { "KL/chosen_KL_mean": -0.3021965026855469, "KL/mean": -0.3455643653869629, "KL/rejected_KL_mean": -0.3889350891113281, "KL/std": 0.34557676315307617, "epoch": 0.07558578987150416, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.0867350697517395, "fcm_dpo/q_t": 0.4978324770927429, "grad_norm": 28.065460205078125, "learning_rate": 3.6567164179104475e-07, "logits/chosen": 0.11386396735906601, "logits/rejected": 0.08404330164194107, "logps/chosen": -53.15863800048828, "logps/ref_chosen": -52.85643768310547, "logps/ref_rejected": -72.17460632324219, "logps/rejected": -72.56353759765625, "loss": 1.3781, "margin_dpo/margin_mean": 0.08673503994941711, "margin_dpo/margin_std": 0.4149363040924072, "step": 50 }, { "KL/chosen_KL_mean": -0.31117820739746094, "KL/mean": -0.42082780599594116, "KL/rejected_KL_mean": -0.5304794311523438, "KL/std": 0.4126695394515991, "epoch": 0.07709750566893424, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21929967403411865, "fcm_dpo/q_t": 0.4945247769355774, "grad_norm": 30.938974380493164, "learning_rate": 3.7313432835820895e-07, "logits/chosen": 0.07800806313753128, "logits/rejected": 0.05045904591679573, "logps/chosen": -63.967620849609375, "logps/ref_chosen": -63.65644073486328, "logps/ref_rejected": -86.13229370117188, "logps/rejected": -86.66278076171875, "loss": 1.3652, "margin_dpo/margin_mean": 0.2192995846271515, "margin_dpo/margin_std": 0.5256927609443665, "step": 51 }, { "KL/chosen_KL_mean": -0.3040885925292969, "KL/mean": -0.39921391010284424, "KL/rejected_KL_mean": -0.4943389892578125, "KL/std": 0.4276493191719055, "epoch": 0.07860922146636433, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19024460017681122, "fcm_dpo/q_t": 0.49524950981140137, "grad_norm": 32.57892990112305, "learning_rate": 3.805970149253731e-07, "logits/chosen": 0.06836412101984024, "logits/rejected": 0.020395169034600258, "logps/chosen": -68.14430236816406, "logps/ref_chosen": -67.8402099609375, "logps/ref_rejected": -96.97090911865234, "logps/rejected": -97.46524810791016, "loss": 1.3681, "margin_dpo/margin_mean": 0.19024433195590973, "margin_dpo/margin_std": 0.5287501811981201, "step": 52 }, { "KL/chosen_KL_mean": -0.322906494140625, "KL/mean": -0.42890429496765137, "KL/rejected_KL_mean": -0.5349006652832031, "KL/std": 0.40694308280944824, "epoch": 0.0801209372637944, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.21199661493301392, "fcm_dpo/q_t": 0.49470412731170654, "grad_norm": 27.37774658203125, "learning_rate": 3.880597014925373e-07, "logits/chosen": 0.09767497330904007, "logits/rejected": 0.08682269603013992, "logps/chosen": -57.201045989990234, "logps/ref_chosen": -56.87813949584961, "logps/ref_rejected": -60.75569152832031, "logps/rejected": -61.29059600830078, "loss": 1.3659, "margin_dpo/margin_mean": 0.2119971513748169, "margin_dpo/margin_std": 0.5070023536682129, "step": 53 }, { "KL/chosen_KL_mean": -0.3383960723876953, "KL/mean": -0.4334535002708435, "KL/rejected_KL_mean": -0.5285091400146484, "KL/std": 0.39724022150039673, "epoch": 0.08163265306122448, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19011196494102478, "fcm_dpo/q_t": 0.49525079131126404, "grad_norm": 26.38343048095703, "learning_rate": 3.9552238805970144e-07, "logits/chosen": 0.05778650939464569, "logits/rejected": 0.04246218502521515, "logps/chosen": -47.605316162109375, "logps/ref_chosen": -47.26692199707031, "logps/ref_rejected": -62.19426727294922, "logps/rejected": -62.722774505615234, "loss": 1.3679, "margin_dpo/margin_mean": 0.19011151790618896, "margin_dpo/margin_std": 0.46603357791900635, "step": 54 }, { "KL/chosen_KL_mean": -0.36114501953125, "KL/mean": -0.502538800239563, "KL/rejected_KL_mean": -0.6439323425292969, "KL/std": 0.4991796016693115, "epoch": 0.08314436885865457, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.28278931975364685, "fcm_dpo/q_t": 0.4929451048374176, "grad_norm": 31.28098487854004, "learning_rate": 4.0298507462686564e-07, "logits/chosen": 0.047343235462903976, "logits/rejected": -0.030307874083518982, "logps/chosen": -50.68733596801758, "logps/ref_chosen": -50.32619094848633, "logps/ref_rejected": -92.44389343261719, "logps/rejected": -93.08782196044922, "loss": 1.3593, "margin_dpo/margin_mean": 0.2827894985675812, "margin_dpo/margin_std": 0.6369022727012634, "step": 55 }, { "KL/chosen_KL_mean": -0.3630104064941406, "KL/mean": -0.45767152309417725, "KL/rejected_KL_mean": -0.5523262023925781, "KL/std": 0.4814542829990387, "epoch": 0.08465608465608465, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.18931812047958374, "fcm_dpo/q_t": 0.49526914954185486, "grad_norm": 27.53534698486328, "learning_rate": 4.1044776119402984e-07, "logits/chosen": 0.1598096489906311, "logits/rejected": 0.13690778613090515, "logps/chosen": -57.129981994628906, "logps/ref_chosen": -56.766971588134766, "logps/ref_rejected": -66.30504608154297, "logps/rejected": -66.85737609863281, "loss": 1.3684, "margin_dpo/margin_mean": 0.18931809067726135, "margin_dpo/margin_std": 0.6055810451507568, "step": 56 }, { "KL/chosen_KL_mean": -0.47999000549316406, "KL/mean": -0.6331826448440552, "KL/rejected_KL_mean": -0.7863693237304688, "KL/std": 0.5945008993148804, "epoch": 0.08616780045351474, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.30638235807418823, "fcm_dpo/q_t": 0.49235400557518005, "grad_norm": 30.548871994018555, "learning_rate": 4.17910447761194e-07, "logits/chosen": 0.09187550097703934, "logits/rejected": 0.028308037668466568, "logps/chosen": -58.24773406982422, "logps/ref_chosen": -57.76774597167969, "logps/ref_rejected": -82.75698852539062, "logps/rejected": -83.5433578491211, "loss": 1.3572, "margin_dpo/margin_mean": 0.30638229846954346, "margin_dpo/margin_std": 0.6942629814147949, "step": 57 }, { "KL/chosen_KL_mean": -0.5946998596191406, "KL/mean": -0.6939823031425476, "KL/rejected_KL_mean": -0.7932624816894531, "KL/std": 0.6381080746650696, "epoch": 0.08767951625094482, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.19855886697769165, "fcm_dpo/q_t": 0.49508431553840637, "grad_norm": 30.27342987060547, "learning_rate": 4.253731343283582e-07, "logits/chosen": 0.05209419131278992, "logits/rejected": 0.03686722368001938, "logps/chosen": -73.3587875366211, "logps/ref_chosen": -72.76408386230469, "logps/ref_rejected": -84.49275207519531, "logps/rejected": -85.2860107421875, "loss": 1.3689, "margin_dpo/margin_mean": 0.19855844974517822, "margin_dpo/margin_std": 0.9104675054550171, "step": 58 }, { "KL/chosen_KL_mean": -0.5438365936279297, "KL/mean": -0.6774130463600159, "KL/rejected_KL_mean": -0.8109931945800781, "KL/std": 0.5832891464233398, "epoch": 0.08919123204837491, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.26715219020843506, "fcm_dpo/q_t": 0.493341326713562, "grad_norm": 26.679256439208984, "learning_rate": 4.3283582089552234e-07, "logits/chosen": 0.1515531837940216, "logits/rejected": 0.08378548175096512, "logps/chosen": -50.3646125793457, "logps/ref_chosen": -49.820777893066406, "logps/ref_rejected": -77.14368438720703, "logps/rejected": -77.95468139648438, "loss": 1.3614, "margin_dpo/margin_mean": 0.26715266704559326, "margin_dpo/margin_std": 0.7313680052757263, "step": 59 }, { "KL/chosen_KL_mean": -0.6181087493896484, "KL/mean": -0.6719140410423279, "KL/rejected_KL_mean": -0.7257213592529297, "KL/std": 0.618611216545105, "epoch": 0.09070294784580499, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.10761374235153198, "fcm_dpo/q_t": 0.49730825424194336, "grad_norm": 29.533267974853516, "learning_rate": 4.4029850746268654e-07, "logits/chosen": 0.09845545142889023, "logits/rejected": 0.09702516347169876, "logps/chosen": -63.84288024902344, "logps/ref_chosen": -63.22477340698242, "logps/ref_rejected": -61.360477447509766, "logps/rejected": -62.08619689941406, "loss": 1.3771, "margin_dpo/margin_mean": 0.10761353373527527, "margin_dpo/margin_std": 0.7893075346946716, "step": 60 }, { "KL/chosen_KL_mean": -0.7608661651611328, "KL/mean": -0.786099374294281, "KL/rejected_KL_mean": -0.8113327026367188, "KL/std": 0.7570456862449646, "epoch": 0.09221466364323508, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.05046442151069641, "fcm_dpo/q_t": 0.49873921275138855, "grad_norm": 28.125015258789062, "learning_rate": 4.4776119402985074e-07, "logits/chosen": 0.15135915577411652, "logits/rejected": 0.11800111830234528, "logps/chosen": -49.77766418457031, "logps/ref_chosen": -49.01679992675781, "logps/ref_rejected": -74.90817260742188, "logps/rejected": -75.71949768066406, "loss": 1.384, "margin_dpo/margin_mean": 0.05046433210372925, "margin_dpo/margin_std": 1.048170804977417, "step": 61 }, { "KL/chosen_KL_mean": -0.7540721893310547, "KL/mean": -0.8673438429832458, "KL/rejected_KL_mean": -0.9806175231933594, "KL/std": 0.6291642189025879, "epoch": 0.09372637944066516, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.2265465259552002, "fcm_dpo/q_t": 0.4943479299545288, "grad_norm": 28.553312301635742, "learning_rate": 4.552238805970149e-07, "logits/chosen": 0.11329515278339386, "logits/rejected": 0.07428479939699173, "logps/chosen": -63.505943298339844, "logps/ref_chosen": -62.751869201660156, "logps/ref_rejected": -78.93360900878906, "logps/rejected": -79.91423034667969, "loss": 1.3657, "margin_dpo/margin_mean": 0.22654661536216736, "margin_dpo/margin_std": 0.8588752150535583, "step": 62 }, { "KL/chosen_KL_mean": -0.5554847717285156, "KL/mean": -0.8419216871261597, "KL/rejected_KL_mean": -1.1283645629882812, "KL/std": 0.7228051424026489, "epoch": 0.09523809523809523, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5728802680969238, "fcm_dpo/q_t": 0.4857173562049866, "grad_norm": 31.28562355041504, "learning_rate": 4.626865671641791e-07, "logits/chosen": 0.18709728121757507, "logits/rejected": 0.1622365117073059, "logps/chosen": -61.07073211669922, "logps/ref_chosen": -60.51525115966797, "logps/ref_rejected": -85.11021423339844, "logps/rejected": -86.23857879638672, "loss": 1.3318, "margin_dpo/margin_mean": 0.5728808641433716, "margin_dpo/margin_std": 0.8949563503265381, "step": 63 }, { "KL/chosen_KL_mean": -0.7727451324462891, "KL/mean": -0.830580472946167, "KL/rejected_KL_mean": -0.8884201049804688, "KL/std": 0.6900042295455933, "epoch": 0.09674981103552532, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.11567190289497375, "fcm_dpo/q_t": 0.4971042275428772, "grad_norm": 26.15057373046875, "learning_rate": 4.701492537313433e-07, "logits/chosen": 0.1170622780919075, "logits/rejected": 0.09124461561441422, "logps/chosen": -51.979591369628906, "logps/ref_chosen": -51.20684814453125, "logps/ref_rejected": -66.93081665039062, "logps/rejected": -67.81924438476562, "loss": 1.3766, "margin_dpo/margin_mean": 0.11567208170890808, "margin_dpo/margin_std": 0.8656256794929504, "step": 64 }, { "KL/chosen_KL_mean": -0.8728599548339844, "KL/mean": -1.1317873001098633, "KL/rejected_KL_mean": -1.3907203674316406, "KL/std": 0.8693119287490845, "epoch": 0.0982615268329554, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5178593397140503, "fcm_dpo/q_t": 0.48711270093917847, "grad_norm": 30.82215118408203, "learning_rate": 4.776119402985074e-07, "logits/chosen": 0.20676767826080322, "logits/rejected": 0.17682614922523499, "logps/chosen": -68.16155242919922, "logps/ref_chosen": -67.2886962890625, "logps/ref_rejected": -74.44281005859375, "logps/rejected": -75.83352661132812, "loss": 1.3385, "margin_dpo/margin_mean": 0.5178589224815369, "margin_dpo/margin_std": 1.1341583728790283, "step": 65 }, { "KL/chosen_KL_mean": -0.9256954193115234, "KL/mean": -1.107551097869873, "KL/rejected_KL_mean": -1.2894096374511719, "KL/std": 0.8482298851013184, "epoch": 0.09977324263038549, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3637177050113678, "fcm_dpo/q_t": 0.4909314811229706, "grad_norm": 29.322872161865234, "learning_rate": 4.850746268656717e-07, "logits/chosen": 0.10713790357112885, "logits/rejected": 0.08299659192562103, "logps/chosen": -71.66910552978516, "logps/ref_chosen": -70.743408203125, "logps/ref_rejected": -77.26499938964844, "logps/rejected": -78.55441284179688, "loss": 1.3536, "margin_dpo/margin_mean": 0.363717257976532, "margin_dpo/margin_std": 1.1283318996429443, "step": 66 }, { "KL/chosen_KL_mean": -0.7712230682373047, "KL/mean": -0.9370452165603638, "KL/rejected_KL_mean": -1.1028671264648438, "KL/std": 0.7597904205322266, "epoch": 0.10128495842781557, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.33164212107658386, "fcm_dpo/q_t": 0.49173396825790405, "grad_norm": 29.111217498779297, "learning_rate": 4.925373134328357e-07, "logits/chosen": 0.08660893142223358, "logits/rejected": 0.031195688992738724, "logps/chosen": -61.37382507324219, "logps/ref_chosen": -60.60260009765625, "logps/ref_rejected": -75.22235870361328, "logps/rejected": -76.32522583007812, "loss": 1.3555, "margin_dpo/margin_mean": 0.3316424787044525, "margin_dpo/margin_std": 0.921684980392456, "step": 67 }, { "KL/chosen_KL_mean": -1.242034912109375, "KL/mean": -1.413798451423645, "KL/rejected_KL_mean": -1.5855636596679688, "KL/std": 0.9572592377662659, "epoch": 0.10279667422524566, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.34353235363960266, "fcm_dpo/q_t": 0.49143853783607483, "grad_norm": 31.30362892150879, "learning_rate": 5e-07, "logits/chosen": 0.0596737265586853, "logits/rejected": 0.03012073040008545, "logps/chosen": -78.77040100097656, "logps/ref_chosen": -77.52836608886719, "logps/ref_rejected": -93.17778015136719, "logps/rejected": -94.76333618164062, "loss": 1.3562, "margin_dpo/margin_mean": 0.3435323238372803, "margin_dpo/margin_std": 1.236339807510376, "step": 68 }, { "KL/chosen_KL_mean": -1.1912288665771484, "KL/mean": -1.477524995803833, "KL/rejected_KL_mean": -1.7638206481933594, "KL/std": 1.001812219619751, "epoch": 0.10430839002267574, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.5725945234298706, "fcm_dpo/q_t": 0.4857825040817261, "grad_norm": 31.007518768310547, "learning_rate": 4.999965034812934e-07, "logits/chosen": 0.09101719409227371, "logits/rejected": 0.04851025342941284, "logps/chosen": -67.13427734375, "logps/ref_chosen": -65.94305419921875, "logps/ref_rejected": -89.7735595703125, "logps/rejected": -91.53738403320312, "loss": 1.3337, "margin_dpo/margin_mean": 0.5725944638252258, "margin_dpo/margin_std": 1.2039427757263184, "step": 69 }, { "KL/chosen_KL_mean": -1.2000751495361328, "KL/mean": -1.3958642482757568, "KL/rejected_KL_mean": -1.5916500091552734, "KL/std": 0.91285240650177, "epoch": 0.10582010582010581, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.39157044887542725, "fcm_dpo/q_t": 0.49025198817253113, "grad_norm": 28.630355834960938, "learning_rate": 4.999860140229787e-07, "logits/chosen": 0.14131593704223633, "logits/rejected": 0.11839009821414948, "logps/chosen": -63.157989501953125, "logps/ref_chosen": -61.95791244506836, "logps/ref_rejected": -75.80945587158203, "logps/rejected": -77.40110778808594, "loss": 1.3515, "margin_dpo/margin_mean": 0.3915710151195526, "margin_dpo/margin_std": 1.252969741821289, "step": 70 }, { "KL/chosen_KL_mean": -1.4675350189208984, "KL/mean": -1.5703108310699463, "KL/rejected_KL_mean": -1.6730918884277344, "KL/std": 1.0348397493362427, "epoch": 0.1073318216175359, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.20555351674556732, "fcm_dpo/q_t": 0.4948495030403137, "grad_norm": 28.58488655090332, "learning_rate": 4.999685319184688e-07, "logits/chosen": 0.09156721830368042, "logits/rejected": 0.07582792639732361, "logps/chosen": -64.81510925292969, "logps/ref_chosen": -63.34757995605469, "logps/ref_rejected": -67.49658203125, "logps/rejected": -69.16967010498047, "loss": 1.3713, "margin_dpo/margin_mean": 0.20555375516414642, "margin_dpo/margin_std": 1.4667941331863403, "step": 71 }, { "KL/chosen_KL_mean": -1.237548828125, "KL/mean": -1.5851967334747314, "KL/rejected_KL_mean": -1.9328422546386719, "KL/std": 1.1038551330566406, "epoch": 0.10884353741496598, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6952927112579346, "fcm_dpo/q_t": 0.4827180504798889, "grad_norm": 30.270689010620117, "learning_rate": 4.999440576567755e-07, "logits/chosen": 0.10874535143375397, "logits/rejected": 0.04621289297938347, "logps/chosen": -57.09684753417969, "logps/ref_chosen": -55.85929870605469, "logps/ref_rejected": -68.45423889160156, "logps/rejected": -70.3870849609375, "loss": 1.3231, "margin_dpo/margin_mean": 0.6952927112579346, "margin_dpo/margin_std": 1.4010509252548218, "step": 72 }, { "KL/chosen_KL_mean": -1.8302345275878906, "KL/mean": -1.8590284585952759, "KL/rejected_KL_mean": -1.887826919555664, "KL/std": 1.2061991691589355, "epoch": 0.11035525321239607, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.057596683502197266, "fcm_dpo/q_t": 0.49866846203804016, "grad_norm": 32.27928924560547, "learning_rate": 4.999125919224965e-07, "logits/chosen": 0.10941021144390106, "logits/rejected": 0.0953536182641983, "logps/chosen": -70.96903991699219, "logps/ref_chosen": -69.13880920410156, "logps/ref_rejected": -79.04586791992188, "logps/rejected": -80.93370056152344, "loss": 1.3869, "margin_dpo/margin_mean": 0.05759665369987488, "margin_dpo/margin_std": 1.5423662662506104, "step": 73 }, { "KL/chosen_KL_mean": -1.2537212371826172, "KL/mean": -1.6008504629135132, "KL/rejected_KL_mean": -1.9479789733886719, "KL/std": 1.0375707149505615, "epoch": 0.11186696900982615, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.6942564249038696, "fcm_dpo/q_t": 0.48278287053108215, "grad_norm": 27.944766998291016, "learning_rate": 4.998741355957963e-07, "logits/chosen": 0.15747080743312836, "logits/rejected": 0.10257548838853836, "logps/chosen": -51.177459716796875, "logps/ref_chosen": -49.923736572265625, "logps/ref_rejected": -81.73213958740234, "logps/rejected": -83.68011474609375, "loss": 1.323, "margin_dpo/margin_mean": 0.6942560076713562, "margin_dpo/margin_std": 1.3548740148544312, "step": 74 }, { "KL/chosen_KL_mean": -1.3986186981201172, "KL/mean": -1.7732503414154053, "KL/rejected_KL_mean": -2.147878646850586, "KL/std": 1.1023242473602295, "epoch": 0.11337868480725624, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7492611408233643, "fcm_dpo/q_t": 0.4813886284828186, "grad_norm": 26.229400634765625, "learning_rate": 4.998286897523808e-07, "logits/chosen": 0.1198577880859375, "logits/rejected": 0.08726748824119568, "logps/chosen": -47.467369079589844, "logps/ref_chosen": -46.06875228881836, "logps/ref_rejected": -66.1181411743164, "logps/rejected": -68.26602172851562, "loss": 1.3174, "margin_dpo/margin_mean": 0.7492610812187195, "margin_dpo/margin_std": 1.3630497455596924, "step": 75 }, { "KL/chosen_KL_mean": -1.5863704681396484, "KL/mean": -1.7453091144561768, "KL/rejected_KL_mean": -1.9042491912841797, "KL/std": 1.1934852600097656, "epoch": 0.11489040060468632, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.3178822100162506, "fcm_dpo/q_t": 0.49208852648735046, "grad_norm": 29.335805892944336, "learning_rate": 4.997762556634679e-07, "logits/chosen": 0.1410784125328064, "logits/rejected": 0.09613093733787537, "logps/chosen": -55.64912033081055, "logps/ref_chosen": -54.06275177001953, "logps/ref_rejected": -74.87464141845703, "logps/rejected": -76.77889251708984, "loss": 1.3609, "margin_dpo/margin_mean": 0.31788232922554016, "margin_dpo/margin_std": 1.566576600074768, "step": 76 }, { "KL/chosen_KL_mean": -1.64678955078125, "KL/mean": -2.0163772106170654, "KL/rejected_KL_mean": -2.3859634399414062, "KL/std": 1.2249342203140259, "epoch": 0.1164021164021164, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7391700744628906, "fcm_dpo/q_t": 0.48164504766464233, "grad_norm": 29.228374481201172, "learning_rate": 4.99716834795752e-07, "logits/chosen": 0.14172329008579254, "logits/rejected": 0.10119108855724335, "logps/chosen": -54.72288131713867, "logps/ref_chosen": -53.07609176635742, "logps/ref_rejected": -74.45601654052734, "logps/rejected": -76.84197998046875, "loss": 1.3191, "margin_dpo/margin_mean": 0.7391701936721802, "margin_dpo/margin_std": 1.426438331604004, "step": 77 }, { "KL/chosen_KL_mean": -1.7951850891113281, "KL/mean": -2.0447793006896973, "KL/rejected_KL_mean": -2.29437255859375, "KL/std": 1.2816565036773682, "epoch": 0.11791383219954649, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4991890788078308, "fcm_dpo/q_t": 0.4875733256340027, "grad_norm": 29.23097038269043, "learning_rate": 4.996504288113623e-07, "logits/chosen": 0.0910310223698616, "logits/rejected": 0.07102985680103302, "logps/chosen": -69.5206069946289, "logps/ref_chosen": -67.72541809082031, "logps/ref_rejected": -79.03926849365234, "logps/rejected": -81.3336410522461, "loss": 1.3421, "margin_dpo/margin_mean": 0.49918922781944275, "margin_dpo/margin_std": 1.4283866882324219, "step": 78 }, { "KL/chosen_KL_mean": -1.7279739379882812, "KL/mean": -2.2947750091552734, "KL/rejected_KL_mean": -2.8615760803222656, "KL/std": 1.5286821126937866, "epoch": 0.11942554799697656, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1336019039154053, "fcm_dpo/q_t": 0.472137987613678, "grad_norm": 30.839723587036133, "learning_rate": 4.995770395678171e-07, "logits/chosen": 0.17144104838371277, "logits/rejected": 0.11132954061031342, "logps/chosen": -53.88861846923828, "logps/ref_chosen": -52.16064453125, "logps/ref_rejected": -83.31062316894531, "logps/rejected": -86.17219543457031, "loss": 1.2855, "margin_dpo/margin_mean": 1.1336017847061157, "margin_dpo/margin_std": 1.944055199623108, "step": 79 }, { "KL/chosen_KL_mean": -2.089902877807617, "KL/mean": -2.3355894088745117, "KL/rejected_KL_mean": -2.5812759399414062, "KL/std": 1.3842788934707642, "epoch": 0.12093726379440665, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.4913783669471741, "fcm_dpo/q_t": 0.4878585636615753, "grad_norm": 28.682737350463867, "learning_rate": 4.994966691179711e-07, "logits/chosen": 0.14792883396148682, "logits/rejected": 0.08779671043157578, "logps/chosen": -63.500465393066406, "logps/ref_chosen": -61.410560607910156, "logps/ref_rejected": -78.66004943847656, "logps/rejected": -81.24132537841797, "loss": 1.3476, "margin_dpo/margin_mean": 0.4913783073425293, "margin_dpo/margin_std": 1.9447224140167236, "step": 80 }, { "KL/chosen_KL_mean": -2.1452388763427734, "KL/mean": -2.5972180366516113, "KL/rejected_KL_mean": -3.0491905212402344, "KL/std": 1.4756536483764648, "epoch": 0.12244897959183673, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.903956413269043, "fcm_dpo/q_t": 0.4777379631996155, "grad_norm": 28.927122116088867, "learning_rate": 4.994093197099587e-07, "logits/chosen": 0.13018305599689484, "logits/rejected": 0.09627491235733032, "logps/chosen": -65.9496078491211, "logps/ref_chosen": -63.80437088012695, "logps/ref_rejected": -79.3484115600586, "logps/rejected": -82.39759826660156, "loss": 1.3065, "margin_dpo/margin_mean": 0.903956413269043, "margin_dpo/margin_std": 1.7683978080749512, "step": 81 }, { "KL/chosen_KL_mean": -1.7878913879394531, "KL/mean": -2.3377861976623535, "KL/rejected_KL_mean": -2.8876800537109375, "KL/std": 1.384407877922058, "epoch": 0.12396069538926682, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0997854471206665, "fcm_dpo/q_t": 0.47270792722702026, "grad_norm": 26.443857192993164, "learning_rate": 4.993149937871306e-07, "logits/chosen": 0.10511986166238785, "logits/rejected": 0.04272346943616867, "logps/chosen": -50.60578155517578, "logps/ref_chosen": -48.817893981933594, "logps/ref_rejected": -70.31497955322266, "logps/rejected": -73.2026596069336, "loss": 1.2857, "margin_dpo/margin_mean": 1.099785566329956, "margin_dpo/margin_std": 1.5717735290527344, "step": 82 }, { "KL/chosen_KL_mean": -2.0469188690185547, "KL/mean": -2.6421515941619873, "KL/rejected_KL_mean": -3.2373886108398438, "KL/std": 1.444145679473877, "epoch": 0.1254724111866969, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1904712915420532, "fcm_dpo/q_t": 0.4705899953842163, "grad_norm": 29.20498275756836, "learning_rate": 4.992136939879856e-07, "logits/chosen": 0.2080458104610443, "logits/rejected": 0.15732426941394806, "logps/chosen": -59.19769287109375, "logps/ref_chosen": -57.15077209472656, "logps/ref_rejected": -75.1710205078125, "logps/rejected": -78.40841674804688, "loss": 1.2784, "margin_dpo/margin_mean": 1.1904711723327637, "margin_dpo/margin_std": 1.746569037437439, "step": 83 }, { "KL/chosen_KL_mean": -2.537485122680664, "KL/mean": -2.9616825580596924, "KL/rejected_KL_mean": -3.385883331298828, "KL/std": 1.8511030673980713, "epoch": 0.12698412698412698, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8484005928039551, "fcm_dpo/q_t": 0.47899675369262695, "grad_norm": 30.805755615234375, "learning_rate": 4.991054231460969e-07, "logits/chosen": 0.13806471228599548, "logits/rejected": 0.0984412282705307, "logps/chosen": -67.31478881835938, "logps/ref_chosen": -64.77729797363281, "logps/ref_rejected": -84.71949768066406, "logps/rejected": -88.10537719726562, "loss": 1.3126, "margin_dpo/margin_mean": 0.848400354385376, "margin_dpo/margin_std": 1.9342408180236816, "step": 84 }, { "KL/chosen_KL_mean": -2.259206771850586, "KL/mean": -2.916848659515381, "KL/rejected_KL_mean": -3.5744857788085938, "KL/std": 1.7699023485183716, "epoch": 0.12849584278155707, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.3152821063995361, "fcm_dpo/q_t": 0.4675264358520508, "grad_norm": 26.661396026611328, "learning_rate": 4.989901842900325e-07, "logits/chosen": 0.16218605637550354, "logits/rejected": 0.11834333837032318, "logps/chosen": -52.510902404785156, "logps/ref_chosen": -50.25169372558594, "logps/ref_rejected": -66.55439758300781, "logps/rejected": -70.12887573242188, "loss": 1.2695, "margin_dpo/margin_mean": 1.3152821063995361, "margin_dpo/margin_std": 1.9742646217346191, "step": 85 }, { "KL/chosen_KL_mean": -2.825664520263672, "KL/mean": -3.325305700302124, "KL/rejected_KL_mean": -3.8249435424804688, "KL/std": 1.9348211288452148, "epoch": 0.13000755857898716, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9992844462394714, "fcm_dpo/q_t": 0.4753116965293884, "grad_norm": 27.131023406982422, "learning_rate": 4.988679806432711e-07, "logits/chosen": 0.23128435015678406, "logits/rejected": 0.21146416664123535, "logps/chosen": -63.55484390258789, "logps/ref_chosen": -60.72917938232422, "logps/ref_rejected": -72.30961608886719, "logps/rejected": -76.13455200195312, "loss": 1.2995, "margin_dpo/margin_mean": 0.9992850422859192, "margin_dpo/margin_std": 2.070314884185791, "step": 86 }, { "KL/chosen_KL_mean": -2.9881515502929688, "KL/mean": -3.519070625305176, "KL/rejected_KL_mean": -4.04998779296875, "KL/std": 2.076875925064087, "epoch": 0.13151927437641722, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.061842441558838, "fcm_dpo/q_t": 0.473818302154541, "grad_norm": 30.4871883392334, "learning_rate": 4.987388156241114e-07, "logits/chosen": 0.13899703323841095, "logits/rejected": 0.08255505561828613, "logps/chosen": -68.74612426757812, "logps/ref_chosen": -65.75796508789062, "logps/ref_rejected": -84.81159973144531, "logps/rejected": -88.86158752441406, "loss": 1.2989, "margin_dpo/margin_mean": 1.061842441558838, "margin_dpo/margin_std": 2.512241840362549, "step": 87 }, { "KL/chosen_KL_mean": -3.012208938598633, "KL/mean": -3.4354000091552734, "KL/rejected_KL_mean": -3.8585853576660156, "KL/std": 2.2582128047943115, "epoch": 0.1330309901738473, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8463754057884216, "fcm_dpo/q_t": 0.47936731576919556, "grad_norm": 30.181737899780273, "learning_rate": 4.986026928455767e-07, "logits/chosen": 0.2531355917453766, "logits/rejected": 0.22557546198368073, "logps/chosen": -65.83623504638672, "logps/ref_chosen": -62.82402801513672, "logps/ref_rejected": -74.9607162475586, "logps/rejected": -78.81930541992188, "loss": 1.3251, "margin_dpo/margin_mean": 0.846375584602356, "margin_dpo/margin_std": 2.8723127841949463, "step": 88 }, { "KL/chosen_KL_mean": -2.5241851806640625, "KL/mean": -3.2427046298980713, "KL/rejected_KL_mean": -3.961223602294922, "KL/std": 2.128784418106079, "epoch": 0.1345427059712774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4370373487472534, "fcm_dpo/q_t": 0.4648668169975281, "grad_norm": 29.15933609008789, "learning_rate": 4.984596161153135e-07, "logits/chosen": 0.273225873708725, "logits/rejected": 0.18827831745147705, "logps/chosen": -43.71562194824219, "logps/ref_chosen": -41.191436767578125, "logps/ref_rejected": -85.44769287109375, "logps/rejected": -89.40892028808594, "loss": 1.2629, "margin_dpo/margin_mean": 1.4370378255844116, "margin_dpo/margin_std": 2.407123565673828, "step": 89 }, { "KL/chosen_KL_mean": -3.1085948944091797, "KL/mean": -3.753403663635254, "KL/rejected_KL_mean": -4.398212432861328, "KL/std": 2.442763566970825, "epoch": 0.1360544217687075, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.2896150350570679, "fcm_dpo/q_t": 0.4683343172073364, "grad_norm": 29.212316513061523, "learning_rate": 4.983095894354857e-07, "logits/chosen": 0.14692363142967224, "logits/rejected": 0.09251993149518967, "logps/chosen": -59.6925048828125, "logps/ref_chosen": -56.58390808105469, "logps/ref_rejected": -86.86978149414062, "logps/rejected": -91.26799011230469, "loss": 1.277, "margin_dpo/margin_mean": 1.2896147966384888, "margin_dpo/margin_std": 2.4425294399261475, "step": 90 }, { "KL/chosen_KL_mean": -3.105762481689453, "KL/mean": -3.701371192932129, "KL/rejected_KL_mean": -4.296977996826172, "KL/std": 2.3072173595428467, "epoch": 0.13756613756613756, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.1912150382995605, "fcm_dpo/q_t": 0.4711948037147522, "grad_norm": 25.63971710205078, "learning_rate": 4.98152617002662e-07, "logits/chosen": 0.15437094867229462, "logits/rejected": 0.1109311506152153, "logps/chosen": -55.48810577392578, "logps/ref_chosen": -52.38234329223633, "logps/ref_rejected": -72.17642211914062, "logps/rejected": -76.47340393066406, "loss": 1.2899, "margin_dpo/margin_mean": 1.191214680671692, "margin_dpo/margin_std": 2.780487060546875, "step": 91 }, { "KL/chosen_KL_mean": -3.237283706665039, "KL/mean": -4.0282511711120605, "KL/rejected_KL_mean": -4.819221496582031, "KL/std": 2.670938014984131, "epoch": 0.13907785336356765, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.581930160522461, "fcm_dpo/q_t": 0.4618622064590454, "grad_norm": 27.040842056274414, "learning_rate": 4.979887032076988e-07, "logits/chosen": 0.21197925508022308, "logits/rejected": 0.17208608984947205, "logps/chosen": -56.24598693847656, "logps/ref_chosen": -53.00870132446289, "logps/ref_rejected": -79.77812957763672, "logps/rejected": -84.59735107421875, "loss": 1.2589, "margin_dpo/margin_mean": 1.5819299221038818, "margin_dpo/margin_std": 3.144960403442383, "step": 92 }, { "KL/chosen_KL_mean": -3.615896224975586, "KL/mean": -4.108669757843018, "KL/rejected_KL_mean": -4.601442337036133, "KL/std": 2.6333351135253906, "epoch": 0.14058956916099774, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9855465888977051, "fcm_dpo/q_t": 0.4764956831932068, "grad_norm": 24.519123077392578, "learning_rate": 4.978178526356172e-07, "logits/chosen": 0.19642697274684906, "logits/rejected": 0.1676049828529358, "logps/chosen": -48.52294921875, "logps/ref_chosen": -44.90705108642578, "logps/ref_rejected": -58.7879524230957, "logps/rejected": -63.38939666748047, "loss": 1.3158, "margin_dpo/margin_mean": 0.9855464696884155, "margin_dpo/margin_std": 3.2435593605041504, "step": 93 }, { "KL/chosen_KL_mean": -3.2522754669189453, "KL/mean": -4.209831237792969, "KL/rejected_KL_mean": -5.167385101318359, "KL/std": 3.2133913040161133, "epoch": 0.1421012849584278, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.9151129722595215, "fcm_dpo/q_t": 0.4539569616317749, "grad_norm": 27.537464141845703, "learning_rate": 4.976400700654751e-07, "logits/chosen": 0.2658318877220154, "logits/rejected": 0.22487923502922058, "logps/chosen": -63.19004821777344, "logps/ref_chosen": -59.93777084350586, "logps/ref_rejected": -79.3138427734375, "logps/rejected": -84.4812240600586, "loss": 1.2425, "margin_dpo/margin_mean": 1.9151129722595215, "margin_dpo/margin_std": 3.9685182571411133, "step": 94 }, { "KL/chosen_KL_mean": -4.282629013061523, "KL/mean": -5.0815019607543945, "KL/rejected_KL_mean": -5.880374908447266, "KL/std": 2.7092788219451904, "epoch": 0.1436130007558579, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5977485179901123, "fcm_dpo/q_t": 0.46140381693840027, "grad_norm": 29.245241165161133, "learning_rate": 4.974553604702332e-07, "logits/chosen": 0.18501371145248413, "logits/rejected": 0.11809428036212921, "logps/chosen": -64.45111846923828, "logps/ref_chosen": -60.168487548828125, "logps/ref_rejected": -90.73665618896484, "logps/rejected": -96.61703491210938, "loss": 1.2618, "margin_dpo/margin_mean": 1.597748041152954, "margin_dpo/margin_std": 3.4290518760681152, "step": 95 }, { "KL/chosen_KL_mean": -4.180746078491211, "KL/mean": -5.110323905944824, "KL/rejected_KL_mean": -6.0399017333984375, "KL/std": 3.4257893562316895, "epoch": 0.14512471655328799, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8591567277908325, "fcm_dpo/q_t": 0.4567599296569824, "grad_norm": 27.415103912353516, "learning_rate": 4.972637290166157e-07, "logits/chosen": 0.19027361273765564, "logits/rejected": 0.14786377549171448, "logps/chosen": -64.84952545166016, "logps/ref_chosen": -60.66877746582031, "logps/ref_rejected": -88.30673217773438, "logps/rejected": -94.34663391113281, "loss": 1.2588, "margin_dpo/margin_mean": 1.8591563701629639, "margin_dpo/margin_std": 4.5050578117370605, "step": 96 }, { "KL/chosen_KL_mean": -4.967893600463867, "KL/mean": -5.493868827819824, "KL/rejected_KL_mean": -6.019844055175781, "KL/std": 3.6154160499572754, "epoch": 0.14663643235071808, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.0519614219665527, "fcm_dpo/q_t": 0.4750325083732605, "grad_norm": 36.224552154541016, "learning_rate": 4.970651810649666e-07, "logits/chosen": 0.1339898705482483, "logits/rejected": 0.09054124355316162, "logps/chosen": -70.01201629638672, "logps/ref_chosen": -65.04412078857422, "logps/ref_rejected": -78.42092895507812, "logps/rejected": -84.4407730102539, "loss": 1.3439, "margin_dpo/margin_mean": 1.0519613027572632, "margin_dpo/margin_std": 4.982733726501465, "step": 97 }, { "KL/chosen_KL_mean": -4.659694671630859, "KL/mean": -5.020186901092529, "KL/rejected_KL_mean": -5.380680084228516, "KL/std": 3.311429738998413, "epoch": 0.14814814814814814, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.7209818363189697, "fcm_dpo/q_t": 0.4820060729980469, "grad_norm": 30.29183578491211, "learning_rate": 4.968597221690985e-07, "logits/chosen": 0.21896392107009888, "logits/rejected": 0.19200772047042847, "logps/chosen": -60.162925720214844, "logps/ref_chosen": -55.503231048583984, "logps/ref_rejected": -72.81553649902344, "logps/rejected": -78.19622039794922, "loss": 1.3425, "margin_dpo/margin_mean": 0.720981240272522, "margin_dpo/margin_std": 3.2679855823516846, "step": 98 }, { "KL/chosen_KL_mean": -4.7128143310546875, "KL/mean": -5.198019981384277, "KL/rejected_KL_mean": -5.683235168457031, "KL/std": 3.49711012840271, "epoch": 0.14965986394557823, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.9704172015190125, "fcm_dpo/q_t": 0.476929634809494, "grad_norm": 32.677608489990234, "learning_rate": 4.966473580761389e-07, "logits/chosen": 0.25504788756370544, "logits/rejected": 0.21502363681793213, "logps/chosen": -63.2884521484375, "logps/ref_chosen": -58.57563781738281, "logps/ref_rejected": -78.693603515625, "logps/rejected": -84.37684631347656, "loss": 1.3429, "margin_dpo/margin_mean": 0.9704174399375916, "margin_dpo/margin_std": 4.577587604522705, "step": 99 }, { "KL/chosen_KL_mean": -5.356479644775391, "KL/mean": -6.064589500427246, "KL/rejected_KL_mean": -6.772693634033203, "KL/std": 4.213325023651123, "epoch": 0.15117157974300832, "fcm_dpo/beta": 0.10000000149011612, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.4162156581878662, "fcm_dpo/q_t": 0.46854132413864136, "grad_norm": 33.35466766357422, "learning_rate": 4.964280947263676e-07, "logits/chosen": 0.2241612672805786, "logits/rejected": 0.21632839739322662, "logps/chosen": -84.93991088867188, "logps/ref_chosen": -79.58343505859375, "logps/ref_rejected": -92.152587890625, "logps/rejected": -98.92527770996094, "loss": 1.3376, "margin_dpo/margin_mean": 1.416215419769287, "margin_dpo/margin_std": 5.909281253814697, "step": 100 }, { "KL/chosen_KL_mean": -4.311086654663086, "KL/mean": -5.3412017822265625, "KL/rejected_KL_mean": -6.371318817138672, "KL/std": 3.681835174560547, "epoch": 0.15268329554043839, "fcm_dpo/beta": 0.10105337202548981, "fcm_dpo/delta": 0.10424260795116425, "fcm_dpo/margin": 2.0602338314056396, "fcm_dpo/q_t": 0.45020347833633423, "grad_norm": 26.12833595275879, "learning_rate": 4.96201938253052e-07, "logits/chosen": 0.22986358404159546, "logits/rejected": 0.19068977236747742, "logps/chosen": -56.64387512207031, "logps/ref_chosen": -52.332786560058594, "logps/ref_rejected": -69.55589294433594, "logps/rejected": -75.92721557617188, "loss": 1.233, "margin_dpo/margin_mean": 2.0602331161499023, "margin_dpo/margin_std": 4.143397331237793, "step": 101 }, { "KL/chosen_KL_mean": -5.332441329956055, "KL/mean": -5.749258995056152, "KL/rejected_KL_mean": -6.166080474853516, "KL/std": 3.662102699279785, "epoch": 0.15419501133786848, "fcm_dpo/beta": 0.1021067351102829, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.833631157875061, "fcm_dpo/q_t": 0.48263412714004517, "grad_norm": 33.488887786865234, "learning_rate": 4.959688949822748e-07, "logits/chosen": 0.1631137728691101, "logits/rejected": 0.12416748702526093, "logps/chosen": -70.075927734375, "logps/ref_chosen": -64.74348449707031, "logps/ref_rejected": -69.06132507324219, "logps/rejected": -75.22740936279297, "loss": 1.3757, "margin_dpo/margin_mean": 0.833631694316864, "margin_dpo/margin_std": 5.453669548034668, "step": 102 }, { "KL/chosen_KL_mean": -4.815326690673828, "KL/mean": -5.606250762939453, "KL/rejected_KL_mean": -6.397182464599609, "KL/std": 3.9247231483459473, "epoch": 0.15570672713529857, "fcm_dpo/beta": 0.10332540422677994, "fcm_dpo/delta": 0.11795026063919067, "fcm_dpo/margin": 1.5818533897399902, "fcm_dpo/q_t": 0.4614016115665436, "grad_norm": 30.380908966064453, "learning_rate": 4.957289714327572e-07, "logits/chosen": 0.2731260359287262, "logits/rejected": 0.23873889446258545, "logps/chosen": -68.65196990966797, "logps/ref_chosen": -63.83664321899414, "logps/ref_rejected": -79.32362365722656, "logps/rejected": -85.72080993652344, "loss": 1.2652, "margin_dpo/margin_mean": 1.5818531513214111, "margin_dpo/margin_std": 3.693053960800171, "step": 103 }, { "KL/chosen_KL_mean": -4.689603805541992, "KL/mean": -5.855137825012207, "KL/rejected_KL_mean": -7.020671844482422, "KL/std": 5.013128280639648, "epoch": 0.15721844293272866, "fcm_dpo/beta": 0.10536164045333862, "fcm_dpo/delta": 0.07759787887334824, "fcm_dpo/margin": 2.3310675621032715, "fcm_dpo/q_t": 0.44842541217803955, "grad_norm": 32.38325500488281, "learning_rate": 4.954821743156767e-07, "logits/chosen": 0.27137792110443115, "logits/rejected": 0.1822788119316101, "logps/chosen": -65.68881225585938, "logps/ref_chosen": -60.99920654296875, "logps/ref_rejected": -98.84645080566406, "logps/rejected": -105.86712646484375, "loss": 1.2606, "margin_dpo/margin_mean": 2.331068754196167, "margin_dpo/margin_std": 6.3999152183532715, "step": 104 }, { "KL/chosen_KL_mean": -5.333820343017578, "KL/mean": -6.264369010925293, "KL/rejected_KL_mean": -7.194919586181641, "KL/std": 3.99896240234375, "epoch": 0.15873015873015872, "fcm_dpo/beta": 0.10617920756340027, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8611037731170654, "fcm_dpo/q_t": 0.4552581012248993, "grad_norm": 32.53849792480469, "learning_rate": 4.952285105344791e-07, "logits/chosen": 0.18711794912815094, "logits/rejected": 0.13286931812763214, "logps/chosen": -76.28408813476562, "logps/ref_chosen": -70.95027160644531, "logps/ref_rejected": -87.88340759277344, "logps/rejected": -95.07833099365234, "loss": 1.2644, "margin_dpo/margin_mean": 1.8611032962799072, "margin_dpo/margin_std": 4.930830955505371, "step": 105 }, { "KL/chosen_KL_mean": -5.013969421386719, "KL/mean": -5.812124252319336, "KL/rejected_KL_mean": -6.6102752685546875, "KL/std": 3.922760486602783, "epoch": 0.1602418745275888, "fcm_dpo/beta": 0.10617920756340027, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.5963062047958374, "fcm_dpo/q_t": 0.45990967750549316, "grad_norm": 33.27928924560547, "learning_rate": 4.949679871846857e-07, "logits/chosen": 0.22435937821865082, "logits/rejected": 0.21065881848335266, "logps/chosen": -67.47329711914062, "logps/ref_chosen": -62.45933151245117, "logps/ref_rejected": -67.00595092773438, "logps/rejected": -73.61622619628906, "loss": 1.3007, "margin_dpo/margin_mean": 1.5963062047958374, "margin_dpo/margin_std": 5.327633380889893, "step": 106 }, { "KL/chosen_KL_mean": -6.503017425537109, "KL/mean": -6.911505699157715, "KL/rejected_KL_mean": -7.3199920654296875, "KL/std": 4.925832748413086, "epoch": 0.1617535903250189, "fcm_dpo/beta": 0.10617920756340027, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.8169774413108826, "fcm_dpo/q_t": 0.4772740602493286, "grad_norm": 45.139347076416016, "learning_rate": 4.947006115536947e-07, "logits/chosen": 0.1692843735218048, "logits/rejected": 0.1457558125257492, "logps/chosen": -82.34098815917969, "logps/ref_chosen": -75.83796691894531, "logps/ref_rejected": -87.74038696289062, "logps/rejected": -95.06037902832031, "loss": 1.4032, "margin_dpo/margin_mean": 0.8169773817062378, "margin_dpo/margin_std": 6.143199920654297, "step": 107 }, { "KL/chosen_KL_mean": -5.4889373779296875, "KL/mean": -6.39979362487793, "KL/rejected_KL_mean": -7.310649871826172, "KL/std": 4.071392059326172, "epoch": 0.16326530612244897, "fcm_dpo/beta": 0.10617920756340027, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 1.8217132091522217, "fcm_dpo/q_t": 0.4561387300491333, "grad_norm": 30.513721466064453, "learning_rate": 4.944263911205772e-07, "logits/chosen": 0.19126787781715393, "logits/rejected": 0.15949223935604095, "logps/chosen": -73.88217163085938, "logps/ref_chosen": -68.39323425292969, "logps/ref_rejected": -83.24267578125, "logps/rejected": -90.5533218383789, "loss": 1.2797, "margin_dpo/margin_mean": 1.8217132091522217, "margin_dpo/margin_std": 5.339081287384033, "step": 108 }, { "KL/chosen_KL_mean": -4.884004592895508, "KL/mean": -6.358839988708496, "KL/rejected_KL_mean": -7.833675384521484, "KL/std": 4.998663425445557, "epoch": 0.16477702191987906, "fcm_dpo/beta": 0.10841409116983414, "fcm_dpo/delta": 0.13440188765525818, "fcm_dpo/margin": 2.949669361114502, "fcm_dpo/q_t": 0.43165239691734314, "grad_norm": 30.32049560546875, "learning_rate": 4.941453335558681e-07, "logits/chosen": 0.21224114298820496, "logits/rejected": 0.15585273504257202, "logps/chosen": -60.41148376464844, "logps/ref_chosen": -55.52748107910156, "logps/ref_rejected": -83.55218505859375, "logps/rejected": -91.3858642578125, "loss": 1.168, "margin_dpo/margin_mean": 2.949669361114502, "margin_dpo/margin_std": 5.457679271697998, "step": 109 }, { "KL/chosen_KL_mean": -6.235057830810547, "KL/mean": -6.359735488891602, "KL/rejected_KL_mean": -6.484409332275391, "KL/std": 4.0676727294921875, "epoch": 0.16628873771730915, "fcm_dpo/beta": 0.10907205194234848, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 0.24935531616210938, "fcm_dpo/q_t": 0.4908309876918793, "grad_norm": 49.313865661621094, "learning_rate": 4.938574467213517e-07, "logits/chosen": 0.1845201551914215, "logits/rejected": 0.19402091205120087, "logps/chosen": -87.393798828125, "logps/ref_chosen": -81.15874481201172, "logps/ref_rejected": -72.56021118164062, "logps/rejected": -79.04461669921875, "loss": 1.4428, "margin_dpo/margin_mean": 0.24935469031333923, "margin_dpo/margin_std": 5.4945878982543945, "step": 110 }, { "KL/chosen_KL_mean": -5.652936935424805, "KL/mean": -6.612331390380859, "KL/rejected_KL_mean": -7.571727752685547, "KL/std": 4.178367614746094, "epoch": 0.16780045351473924, "fcm_dpo/beta": 0.11027145385742188, "fcm_dpo/delta": 0.10877230763435364, "fcm_dpo/margin": 1.918793797492981, "fcm_dpo/q_t": 0.45316487550735474, "grad_norm": 28.390825271606445, "learning_rate": 4.935627386698418e-07, "logits/chosen": 0.26181134581565857, "logits/rejected": 0.22537365555763245, "logps/chosen": -58.01192092895508, "logps/ref_chosen": -52.358985900878906, "logps/ref_rejected": -77.06150817871094, "logps/rejected": -84.63323211669922, "loss": 1.2593, "margin_dpo/margin_mean": 1.91879403591156, "margin_dpo/margin_std": 5.053813934326172, "step": 111 }, { "KL/chosen_KL_mean": -5.305566787719727, "KL/mean": -6.52008581161499, "KL/rejected_KL_mean": -7.7346038818359375, "KL/std": 4.221207141876221, "epoch": 0.1693121693121693, "fcm_dpo/beta": 0.11457160115242004, "fcm_dpo/delta": 0.17640449106693268, "fcm_dpo/margin": 2.429037570953369, "fcm_dpo/q_t": 0.4356866478919983, "grad_norm": 36.389381408691406, "learning_rate": 4.932612176449559e-07, "logits/chosen": 0.1818389594554901, "logits/rejected": 0.11763059347867966, "logps/chosen": -68.32563018798828, "logps/ref_chosen": -63.02006530761719, "logps/ref_rejected": -111.36941528320312, "logps/rejected": -119.10401916503906, "loss": 1.1964, "margin_dpo/margin_mean": 2.429037570953369, "margin_dpo/margin_std": 4.7098541259765625, "step": 112 }, { "KL/chosen_KL_mean": -5.34063720703125, "KL/mean": -6.032615661621094, "KL/rejected_KL_mean": -6.7245941162109375, "KL/std": 3.539608955383301, "epoch": 0.1708238851095994, "fcm_dpo/beta": 0.1167660802602768, "fcm_dpo/delta": 0.11067280918359756, "fcm_dpo/margin": 1.3839517831802368, "fcm_dpo/q_t": 0.4618106484413147, "grad_norm": 41.924644470214844, "learning_rate": 4.929528920808854e-07, "logits/chosen": 0.20727001130580902, "logits/rejected": 0.1677381694316864, "logps/chosen": -61.14830017089844, "logps/ref_chosen": -55.80766296386719, "logps/ref_rejected": -69.84014129638672, "logps/rejected": -76.56473541259766, "loss": 1.3001, "margin_dpo/margin_mean": 1.3839521408081055, "margin_dpo/margin_std": 4.557351112365723, "step": 113 }, { "KL/chosen_KL_mean": -5.004667282104492, "KL/mean": -6.154642105102539, "KL/rejected_KL_mean": -7.304622650146484, "KL/std": 3.8118882179260254, "epoch": 0.17233560090702948, "fcm_dpo/beta": 0.12111715972423553, "fcm_dpo/delta": 0.1764792948961258, "fcm_dpo/margin": 2.2999496459960938, "fcm_dpo/q_t": 0.43808525800704956, "grad_norm": 35.51799392700195, "learning_rate": 4.92637770602159e-07, "logits/chosen": 0.2466518133878708, "logits/rejected": 0.18633544445037842, "logps/chosen": -71.33744049072266, "logps/ref_chosen": -66.33277130126953, "logps/ref_rejected": -71.61489868164062, "logps/rejected": -78.91951751708984, "loss": 1.1996, "margin_dpo/margin_mean": 2.2999496459960938, "margin_dpo/margin_std": 4.60300350189209, "step": 114 }, { "KL/chosen_KL_mean": -5.393440246582031, "KL/mean": -6.3272271156311035, "KL/rejected_KL_mean": -7.261011123657227, "KL/std": 3.5061001777648926, "epoch": 0.17384731670445955, "fcm_dpo/beta": 0.12467402219772339, "fcm_dpo/delta": 0.096140056848526, "fcm_dpo/margin": 1.8675713539123535, "fcm_dpo/q_t": 0.4473262131214142, "grad_norm": 35.43161392211914, "learning_rate": 4.923158620234019e-07, "logits/chosen": 0.24343647062778473, "logits/rejected": 0.17968247830867767, "logps/chosen": -61.142478942871094, "logps/ref_chosen": -55.74903869628906, "logps/ref_rejected": -79.59849548339844, "logps/rejected": -86.85951232910156, "loss": 1.2327, "margin_dpo/margin_mean": 1.8675715923309326, "margin_dpo/margin_std": 4.200431823730469, "step": 115 }, { "KL/chosen_KL_mean": -5.084573745727539, "KL/mean": -6.464913845062256, "KL/rejected_KL_mean": -7.845256805419922, "KL/std": 3.6170547008514404, "epoch": 0.17535903250188964, "fcm_dpo/beta": 0.1270686388015747, "fcm_dpo/delta": 0.10212881863117218, "fcm_dpo/margin": 2.760686159133911, "fcm_dpo/q_t": 0.4189320206642151, "grad_norm": 31.042070388793945, "learning_rate": 4.91987175349089e-07, "logits/chosen": 0.2448294758796692, "logits/rejected": 0.17530521750450134, "logps/chosen": -54.449737548828125, "logps/ref_chosen": -49.36516571044922, "logps/ref_rejected": -72.84671020507812, "logps/rejected": -80.69196319580078, "loss": 1.1271, "margin_dpo/margin_mean": 2.7606863975524902, "margin_dpo/margin_std": 3.9437413215637207, "step": 116 }, { "KL/chosen_KL_mean": -5.042119979858398, "KL/mean": -6.001665115356445, "KL/rejected_KL_mean": -6.961215972900391, "KL/std": 3.4388794898986816, "epoch": 0.17687074829931973, "fcm_dpo/beta": 0.1273379921913147, "fcm_dpo/delta": 0.0035832971334457397, "fcm_dpo/margin": 1.9190866947174072, "fcm_dpo/q_t": 0.4429383873939514, "grad_norm": 36.0718879699707, "learning_rate": 4.916517197732933e-07, "logits/chosen": 0.21628046035766602, "logits/rejected": 0.18022674322128296, "logps/chosen": -62.75301742553711, "logps/ref_chosen": -57.710899353027344, "logps/ref_rejected": -69.77253723144531, "logps/rejected": -76.73374938964844, "loss": 1.235, "margin_dpo/margin_mean": 1.9190870523452759, "margin_dpo/margin_std": 4.254258155822754, "step": 117 }, { "KL/chosen_KL_mean": -4.869361877441406, "KL/mean": -6.127657413482666, "KL/rejected_KL_mean": -7.385948181152344, "KL/std": 3.6793594360351562, "epoch": 0.17838246409674982, "fcm_dpo/beta": 0.12952576577663422, "fcm_dpo/delta": 0.1283983290195465, "fcm_dpo/margin": 2.516590118408203, "fcm_dpo/q_t": 0.4246635138988495, "grad_norm": 34.91251754760742, "learning_rate": 4.913095046794281e-07, "logits/chosen": 0.29132065176963806, "logits/rejected": 0.24994787573814392, "logps/chosen": -57.34925842285156, "logps/ref_chosen": -52.479896545410156, "logps/ref_rejected": -81.359130859375, "logps/rejected": -88.74507141113281, "loss": 1.1443, "margin_dpo/margin_mean": 2.516590118408203, "margin_dpo/margin_std": 3.754570484161377, "step": 118 }, { "KL/chosen_KL_mean": -6.241527557373047, "KL/mean": -7.111940860748291, "KL/rejected_KL_mean": -7.982353210449219, "KL/std": 4.2465362548828125, "epoch": 0.17989417989417988, "fcm_dpo/beta": 0.131746307015419, "fcm_dpo/delta": 0.08318955451250076, "fcm_dpo/margin": 1.7408270835876465, "fcm_dpo/q_t": 0.4494980275630951, "grad_norm": 39.3913688659668, "learning_rate": 4.909605396399855e-07, "logits/chosen": 0.21818503737449646, "logits/rejected": 0.17873303592205048, "logps/chosen": -67.59919738769531, "logps/ref_chosen": -61.35767364501953, "logps/ref_rejected": -75.71510314941406, "logps/rejected": -83.69744873046875, "loss": 1.278, "margin_dpo/margin_mean": 1.7408275604248047, "margin_dpo/margin_std": 5.219765663146973, "step": 119 }, { "KL/chosen_KL_mean": -5.114025115966797, "KL/mean": -6.457116603851318, "KL/rejected_KL_mean": -7.800205230712891, "KL/std": 3.6140894889831543, "epoch": 0.18140589569160998, "fcm_dpo/beta": 0.1348700374364853, "fcm_dpo/delta": 0.09081363677978516, "fcm_dpo/margin": 2.6861793994903564, "fcm_dpo/q_t": 0.41728341579437256, "grad_norm": 36.61309051513672, "learning_rate": 4.906048344162676e-07, "logits/chosen": 0.23546835780143738, "logits/rejected": 0.17328256368637085, "logps/chosen": -65.02159118652344, "logps/ref_chosen": -59.907569885253906, "logps/ref_rejected": -79.6910629272461, "logps/rejected": -87.49127197265625, "loss": 1.1157, "margin_dpo/margin_mean": 2.6861801147460938, "margin_dpo/margin_std": 3.675182342529297, "step": 120 }, { "KL/chosen_KL_mean": -5.602396011352539, "KL/mean": -7.010382652282715, "KL/rejected_KL_mean": -8.41836929321289, "KL/std": 4.133810043334961, "epoch": 0.18291761148904007, "fcm_dpo/beta": 0.13647235929965973, "fcm_dpo/delta": 0.06837910413742065, "fcm_dpo/margin": 2.815971851348877, "fcm_dpo/q_t": 0.41775327920913696, "grad_norm": 34.387046813964844, "learning_rate": 4.902423989581143e-07, "logits/chosen": 0.3467385172843933, "logits/rejected": 0.2564208209514618, "logps/chosen": -61.26844024658203, "logps/ref_chosen": -55.66604232788086, "logps/ref_rejected": -101.56233978271484, "logps/rejected": -109.980712890625, "loss": 1.1232, "margin_dpo/margin_mean": 2.815971851348877, "margin_dpo/margin_std": 4.585853576660156, "step": 121 }, { "KL/chosen_KL_mean": -5.983097076416016, "KL/mean": -7.31648063659668, "KL/rejected_KL_mean": -8.649864196777344, "KL/std": 4.461297512054443, "epoch": 0.18442932728647016, "fcm_dpo/beta": 0.13917195796966553, "fcm_dpo/delta": 0.0815592035651207, "fcm_dpo/margin": 2.6667704582214355, "fcm_dpo/q_t": 0.4239156246185303, "grad_norm": 42.97147750854492, "learning_rate": 4.898732434036243e-07, "logits/chosen": 0.22524026036262512, "logits/rejected": 0.1896163374185562, "logps/chosen": -69.31747436523438, "logps/ref_chosen": -63.334373474121094, "logps/ref_rejected": -73.67523193359375, "logps/rejected": -82.32510375976562, "loss": 1.168, "margin_dpo/margin_mean": 2.6667704582214355, "margin_dpo/margin_std": 5.3650221824646, "step": 122 }, { "KL/chosen_KL_mean": -5.680210113525391, "KL/mean": -6.947849273681641, "KL/rejected_KL_mean": -8.215492248535156, "KL/std": 3.915778636932373, "epoch": 0.18594104308390022, "fcm_dpo/beta": 0.14164075255393982, "fcm_dpo/delta": 0.09402373433113098, "fcm_dpo/margin": 2.5352840423583984, "fcm_dpo/q_t": 0.4203948676586151, "grad_norm": 36.68033981323242, "learning_rate": 4.894973780788722e-07, "logits/chosen": 0.23389171063899994, "logits/rejected": 0.19273313879966736, "logps/chosen": -62.57895278930664, "logps/ref_chosen": -56.89874267578125, "logps/ref_rejected": -78.97028350830078, "logps/rejected": -87.18577575683594, "loss": 1.1451, "margin_dpo/margin_mean": 2.5352840423583984, "margin_dpo/margin_std": 4.32481575012207, "step": 123 }, { "KL/chosen_KL_mean": -6.0447540283203125, "KL/mean": -7.777586460113525, "KL/rejected_KL_mean": -9.510417938232422, "KL/std": 4.207749366760254, "epoch": 0.1874527588813303, "fcm_dpo/beta": 0.1409628838300705, "fcm_dpo/delta": -0.040624044835567474, "fcm_dpo/margin": 3.465663194656372, "fcm_dpo/q_t": 0.3909822106361389, "grad_norm": 36.187339782714844, "learning_rate": 4.89114813497619e-07, "logits/chosen": 0.28463006019592285, "logits/rejected": 0.22122982144355774, "logps/chosen": -63.16083908081055, "logps/ref_chosen": -57.116085052490234, "logps/ref_rejected": -87.93074035644531, "logps/rejected": -97.441162109375, "loss": 1.0433, "margin_dpo/margin_mean": 3.465662717819214, "margin_dpo/margin_std": 4.411318778991699, "step": 124 }, { "KL/chosen_KL_mean": -6.541475296020508, "KL/mean": -8.008241653442383, "KL/rejected_KL_mean": -9.47500228881836, "KL/std": 4.520659923553467, "epoch": 0.1889644746787604, "fcm_dpo/beta": 0.14088676869869232, "fcm_dpo/delta": 0.0378384031355381, "fcm_dpo/margin": 2.933529853820801, "fcm_dpo/q_t": 0.4101593494415283, "grad_norm": 39.248573303222656, "learning_rate": 4.887255603610184e-07, "logits/chosen": 0.2627220153808594, "logits/rejected": 0.20320525765419006, "logps/chosen": -72.24765014648438, "logps/ref_chosen": -65.7061767578125, "logps/ref_rejected": -91.72711944580078, "logps/rejected": -101.20211791992188, "loss": 1.1161, "margin_dpo/margin_mean": 2.933529853820801, "margin_dpo/margin_std": 4.710192680358887, "step": 125 }, { "KL/chosen_KL_mean": -6.104684829711914, "KL/mean": -6.989722728729248, "KL/rejected_KL_mean": -7.874761581420898, "KL/std": 4.20753288269043, "epoch": 0.19047619047619047, "fcm_dpo/beta": 0.14634212851524353, "fcm_dpo/delta": 0.19579944014549255, "fcm_dpo/margin": 1.7700788974761963, "fcm_dpo/q_t": 0.44312962889671326, "grad_norm": 41.04850769042969, "learning_rate": 4.883296295573176e-07, "logits/chosen": 0.11527390033006668, "logits/rejected": 0.10950449109077454, "logps/chosen": -74.28077697753906, "logps/ref_chosen": -68.17608642578125, "logps/ref_rejected": -65.1175537109375, "logps/rejected": -72.99231719970703, "loss": 1.2469, "margin_dpo/margin_mean": 1.7700791358947754, "margin_dpo/margin_std": 4.583360195159912, "step": 126 }, { "KL/chosen_KL_mean": -6.241485595703125, "KL/mean": -7.84971809387207, "KL/rejected_KL_mean": -9.457956314086914, "KL/std": 3.7725934982299805, "epoch": 0.19198790627362056, "fcm_dpo/beta": 0.14626064896583557, "fcm_dpo/delta": -0.022328916937112808, "fcm_dpo/margin": 3.2164688110351562, "fcm_dpo/q_t": 0.39330053329467773, "grad_norm": 36.990318298339844, "learning_rate": 4.87927032161552e-07, "logits/chosen": 0.1813475489616394, "logits/rejected": 0.15173006057739258, "logps/chosen": -68.12171936035156, "logps/ref_chosen": -61.88023376464844, "logps/ref_rejected": -68.46012878417969, "logps/rejected": -77.91808319091797, "loss": 1.0528, "margin_dpo/margin_mean": 3.216468334197998, "margin_dpo/margin_std": 3.9844183921813965, "step": 127 }, { "KL/chosen_KL_mean": -6.738676071166992, "KL/mean": -8.41079330444336, "KL/rejected_KL_mean": -10.08291244506836, "KL/std": 4.916608810424805, "epoch": 0.19349962207105065, "fcm_dpo/beta": 0.14551366865634918, "fcm_dpo/delta": -0.039026957005262375, "fcm_dpo/margin": 3.344231605529785, "fcm_dpo/q_t": 0.398078978061676, "grad_norm": 37.41531753540039, "learning_rate": 4.875177794352363e-07, "logits/chosen": 0.2204674333333969, "logits/rejected": 0.16373318433761597, "logps/chosen": -73.44766235351562, "logps/ref_chosen": -66.708984375, "logps/ref_rejected": -94.97969055175781, "logps/rejected": -105.06260681152344, "loss": 1.118, "margin_dpo/margin_mean": 3.344231367111206, "margin_dpo/margin_std": 5.8806328773498535, "step": 128 }, { "KL/chosen_KL_mean": -7.556180953979492, "KL/mean": -8.538894653320312, "KL/rejected_KL_mean": -9.5216064453125, "KL/std": 4.42452335357666, "epoch": 0.19501133786848074, "fcm_dpo/beta": 0.1494940221309662, "fcm_dpo/delta": 0.16076956689357758, "fcm_dpo/margin": 1.9654262065887451, "fcm_dpo/q_t": 0.43633008003234863, "grad_norm": 52.058353424072266, "learning_rate": 4.871018828260491e-07, "logits/chosen": 0.21647898852825165, "logits/rejected": 0.2104889303445816, "logps/chosen": -72.89500427246094, "logps/ref_chosen": -65.33882904052734, "logps/ref_rejected": -68.06109619140625, "logps/rejected": -77.58270263671875, "loss": 1.2569, "margin_dpo/margin_mean": 1.9654266834259033, "margin_dpo/margin_std": 5.319171905517578, "step": 129 }, { "KL/chosen_KL_mean": -7.091085433959961, "KL/mean": -8.634103775024414, "KL/rejected_KL_mean": -10.1771240234375, "KL/std": 4.412557125091553, "epoch": 0.1965230536659108, "fcm_dpo/beta": 0.14928355813026428, "fcm_dpo/delta": -0.01228487491607666, "fcm_dpo/margin": 3.0860366821289062, "fcm_dpo/q_t": 0.39678919315338135, "grad_norm": 43.6436882019043, "learning_rate": 4.866793539675126e-07, "logits/chosen": 0.17643359303474426, "logits/rejected": 0.12553930282592773, "logps/chosen": -65.7518310546875, "logps/ref_chosen": -58.660743713378906, "logps/ref_rejected": -79.24510192871094, "logps/rejected": -89.42222595214844, "loss": 1.0744, "margin_dpo/margin_mean": 3.0860366821289062, "margin_dpo/margin_std": 4.253152847290039, "step": 130 }, { "KL/chosen_KL_mean": -6.78155517578125, "KL/mean": -8.716371536254883, "KL/rejected_KL_mean": -10.651187896728516, "KL/std": 5.087396621704102, "epoch": 0.1980347694633409, "fcm_dpo/beta": 0.14708811044692993, "fcm_dpo/delta": -0.12676924467086792, "fcm_dpo/margin": 3.8696370124816895, "fcm_dpo/q_t": 0.38106900453567505, "grad_norm": 36.27388381958008, "learning_rate": 4.86250204678667e-07, "logits/chosen": 0.19298642873764038, "logits/rejected": 0.12822577357292175, "logps/chosen": -59.29608917236328, "logps/ref_chosen": -52.51453399658203, "logps/ref_rejected": -85.18299865722656, "logps/rejected": -95.83418273925781, "loss": 1.0539, "margin_dpo/margin_mean": 3.8696374893188477, "margin_dpo/margin_std": 5.73309326171875, "step": 131 }, { "KL/chosen_KL_mean": -7.4026336669921875, "KL/mean": -9.053910255432129, "KL/rejected_KL_mean": -10.705190658569336, "KL/std": 4.851495742797852, "epoch": 0.19954648526077098, "fcm_dpo/beta": 0.14569681882858276, "fcm_dpo/delta": -0.032763972878456116, "fcm_dpo/margin": 3.3025588989257812, "fcm_dpo/q_t": 0.3902255892753601, "grad_norm": 38.206172943115234, "learning_rate": 4.858144469637408e-07, "logits/chosen": 0.2730848789215088, "logits/rejected": 0.24042408168315887, "logps/chosen": -73.0877685546875, "logps/ref_chosen": -65.68513488769531, "logps/ref_rejected": -69.54120635986328, "logps/rejected": -80.24639892578125, "loss": 1.0744, "margin_dpo/margin_mean": 3.3025588989257812, "margin_dpo/margin_std": 4.778876304626465, "step": 132 }, { "KL/chosen_KL_mean": -7.787448883056641, "KL/mean": -9.092580795288086, "KL/rejected_KL_mean": -10.397708892822266, "KL/std": 4.733419418334961, "epoch": 0.20105820105820105, "fcm_dpo/beta": 0.1470513641834259, "fcm_dpo/delta": 0.06877677142620087, "fcm_dpo/margin": 2.6102614402770996, "fcm_dpo/q_t": 0.41596511006355286, "grad_norm": 45.837135314941406, "learning_rate": 4.853720930118138e-07, "logits/chosen": 0.183448925614357, "logits/rejected": 0.1737731397151947, "logps/chosen": -71.38555908203125, "logps/ref_chosen": -63.598114013671875, "logps/ref_rejected": -73.72798156738281, "logps/rejected": -84.12568664550781, "loss": 1.1678, "margin_dpo/margin_mean": 2.6102609634399414, "margin_dpo/margin_std": 5.107699394226074, "step": 133 }, { "KL/chosen_KL_mean": -7.144420623779297, "KL/mean": -9.359694480895996, "KL/rejected_KL_mean": -11.574962615966797, "KL/std": 5.464275360107422, "epoch": 0.20256991685563114, "fcm_dpo/beta": 0.1433096081018448, "fcm_dpo/delta": -0.1974298059940338, "fcm_dpo/margin": 4.430543422698975, "fcm_dpo/q_t": 0.36382484436035156, "grad_norm": 33.925498962402344, "learning_rate": 4.849231551964771e-07, "logits/chosen": 0.29932132363319397, "logits/rejected": 0.24174457788467407, "logps/chosen": -60.93899154663086, "logps/ref_chosen": -53.79457092285156, "logps/ref_rejected": -74.16741943359375, "logps/rejected": -85.74238586425781, "loss": 0.9776, "margin_dpo/margin_mean": 4.430543899536133, "margin_dpo/margin_std": 5.441107749938965, "step": 134 }, { "KL/chosen_KL_mean": -7.194000244140625, "KL/mean": -8.616719245910645, "KL/rejected_KL_mean": -10.039440155029297, "KL/std": 5.098479270935059, "epoch": 0.20408163265306123, "fcm_dpo/beta": 0.1429201066493988, "fcm_dpo/delta": 0.04509545862674713, "fcm_dpo/margin": 2.84544038772583, "fcm_dpo/q_t": 0.41076767444610596, "grad_norm": 34.94743347167969, "learning_rate": 4.844676460754862e-07, "logits/chosen": 0.21902255713939667, "logits/rejected": 0.18694475293159485, "logps/chosen": -56.63507843017578, "logps/ref_chosen": -49.441078186035156, "logps/ref_rejected": -65.96878051757812, "logps/rejected": -76.00822448730469, "loss": 1.1456, "margin_dpo/margin_mean": 2.845439910888672, "margin_dpo/margin_std": 5.2115278244018555, "step": 135 }, { "KL/chosen_KL_mean": -8.81231689453125, "KL/mean": -10.606294631958008, "KL/rejected_KL_mean": -12.400272369384766, "KL/std": 5.5866851806640625, "epoch": 0.20559334845049132, "fcm_dpo/beta": 0.14106975495815277, "fcm_dpo/delta": -0.06005358695983887, "fcm_dpo/margin": 3.5879569053649902, "fcm_dpo/q_t": 0.39729416370391846, "grad_norm": 43.33884048461914, "learning_rate": 4.840055783904106e-07, "logits/chosen": 0.21669332683086395, "logits/rejected": 0.14369165897369385, "logps/chosen": -75.57157897949219, "logps/ref_chosen": -66.75926208496094, "logps/ref_rejected": -94.61787414550781, "logps/rejected": -107.01814270019531, "loss": 1.1576, "margin_dpo/margin_mean": 3.5879576206207275, "margin_dpo/margin_std": 7.040832996368408, "step": 136 }, { "KL/chosen_KL_mean": -8.229362487792969, "KL/mean": -10.075550079345703, "KL/rejected_KL_mean": -11.921737670898438, "KL/std": 5.315681457519531, "epoch": 0.20710506424792138, "fcm_dpo/beta": 0.14049124717712402, "fcm_dpo/delta": -0.07258465886116028, "fcm_dpo/margin": 3.692375659942627, "fcm_dpo/q_t": 0.3897295892238617, "grad_norm": 34.87916564941406, "learning_rate": 4.835369650662767e-07, "logits/chosen": 0.25548824667930603, "logits/rejected": 0.22579577565193176, "logps/chosen": -65.0131607055664, "logps/ref_chosen": -56.78379821777344, "logps/ref_rejected": -69.89952087402344, "logps/rejected": -81.82125854492188, "loss": 1.0721, "margin_dpo/margin_mean": 3.692375659942627, "margin_dpo/margin_std": 5.650632858276367, "step": 137 }, { "KL/chosen_KL_mean": -9.61297607421875, "KL/mean": -11.213253021240234, "KL/rejected_KL_mean": -12.813533782958984, "KL/std": 5.492915630340576, "epoch": 0.20861678004535147, "fcm_dpo/beta": 0.13947460055351257, "fcm_dpo/delta": 0.003768614027649164, "fcm_dpo/margin": 3.200556993484497, "fcm_dpo/q_t": 0.4026796221733093, "grad_norm": 37.49756622314453, "learning_rate": 4.830618192112065e-07, "logits/chosen": 0.2518809735774994, "logits/rejected": 0.21509861946105957, "logps/chosen": -68.37899017333984, "logps/ref_chosen": -58.766014099121094, "logps/ref_rejected": -68.12371826171875, "logps/rejected": -80.937255859375, "loss": 1.1513, "margin_dpo/margin_mean": 3.200556755065918, "margin_dpo/margin_std": 6.079831123352051, "step": 138 }, { "KL/chosen_KL_mean": -10.039913177490234, "KL/mean": -11.939605712890625, "KL/rejected_KL_mean": -13.83929443359375, "KL/std": 5.361264228820801, "epoch": 0.21012849584278157, "fcm_dpo/beta": 0.13811233639717102, "fcm_dpo/delta": -0.0788048505783081, "fcm_dpo/margin": 3.7993788719177246, "fcm_dpo/q_t": 0.38475942611694336, "grad_norm": 40.01036071777344, "learning_rate": 4.825801541160509e-07, "logits/chosen": 0.1889549195766449, "logits/rejected": 0.15961438417434692, "logps/chosen": -81.2655029296875, "logps/ref_chosen": -71.2255859375, "logps/ref_rejected": -82.1834716796875, "logps/rejected": -96.02276611328125, "loss": 1.0565, "margin_dpo/margin_mean": 3.7993783950805664, "margin_dpo/margin_std": 5.455878734588623, "step": 139 }, { "KL/chosen_KL_mean": -9.19911003112793, "KL/mean": -11.774042129516602, "KL/rejected_KL_mean": -14.348968505859375, "KL/std": 7.151411056518555, "epoch": 0.21164021164021163, "fcm_dpo/beta": 0.13198350369930267, "fcm_dpo/delta": -0.24726887047290802, "fcm_dpo/margin": 5.149857044219971, "fcm_dpo/q_t": 0.35907793045043945, "grad_norm": 40.80632781982422, "learning_rate": 4.820919832540181e-07, "logits/chosen": 0.21919316053390503, "logits/rejected": 0.17110946774482727, "logps/chosen": -72.47677612304688, "logps/ref_chosen": -63.27766418457031, "logps/ref_rejected": -83.30647277832031, "logps/rejected": -97.65544128417969, "loss": 1.0468, "margin_dpo/margin_mean": 5.149857044219971, "margin_dpo/margin_std": 7.859900951385498, "step": 140 }, { "KL/chosen_KL_mean": -10.046802520751953, "KL/mean": -12.450661659240723, "KL/rejected_KL_mean": -14.854522705078125, "KL/std": 5.955142974853516, "epoch": 0.21315192743764172, "fcm_dpo/beta": 0.12623918056488037, "fcm_dpo/delta": -0.17030926048755646, "fcm_dpo/margin": 4.8077168464660645, "fcm_dpo/q_t": 0.37628281116485596, "grad_norm": 35.90546417236328, "learning_rate": 4.815973202802966e-07, "logits/chosen": 0.25997358560562134, "logits/rejected": 0.21349510550498962, "logps/chosen": -71.81356811523438, "logps/ref_chosen": -61.76676940917969, "logps/ref_rejected": -88.60601806640625, "logps/rejected": -103.46054077148438, "loss": 1.0647, "margin_dpo/margin_mean": 4.807717800140381, "margin_dpo/margin_std": 7.479946613311768, "step": 141 }, { "KL/chosen_KL_mean": -9.879066467285156, "KL/mean": -11.788759231567383, "KL/rejected_KL_mean": -13.698455810546875, "KL/std": 5.767460823059082, "epoch": 0.2146636432350718, "fcm_dpo/beta": 0.12553216516971588, "fcm_dpo/delta": -0.031014427542686462, "fcm_dpo/margin": 3.819387435913086, "fcm_dpo/q_t": 0.3950141668319702, "grad_norm": 33.704952239990234, "learning_rate": 4.810961790316729e-07, "logits/chosen": 0.252850204706192, "logits/rejected": 0.22739627957344055, "logps/chosen": -75.15383911132812, "logps/ref_chosen": -65.2747802734375, "logps/ref_rejected": -81.1378173828125, "logps/rejected": -94.83627319335938, "loss": 1.0841, "margin_dpo/margin_mean": 3.819387197494507, "margin_dpo/margin_std": 5.84207010269165, "step": 142 }, { "KL/chosen_KL_mean": -10.517995834350586, "KL/mean": -12.156927108764648, "KL/rejected_KL_mean": -13.795856475830078, "KL/std": 6.709386825561523, "epoch": 0.2161753590325019, "fcm_dpo/beta": 0.12634995579719543, "fcm_dpo/delta": 0.037314772605895996, "fcm_dpo/margin": 3.2778611183166504, "fcm_dpo/q_t": 0.40795421600341797, "grad_norm": 46.63773727416992, "learning_rate": 4.805885735261454e-07, "logits/chosen": 0.27107226848602295, "logits/rejected": 0.25340479612350464, "logps/chosen": -73.13581848144531, "logps/ref_chosen": -62.617828369140625, "logps/ref_rejected": -70.39239501953125, "logps/rejected": -84.18824768066406, "loss": 1.2333, "margin_dpo/margin_mean": 3.2778611183166504, "margin_dpo/margin_std": 7.917219161987305, "step": 143 }, { "KL/chosen_KL_mean": -10.777961730957031, "KL/mean": -12.884668350219727, "KL/rejected_KL_mean": -14.991374969482422, "KL/std": 6.686635971069336, "epoch": 0.21768707482993196, "fcm_dpo/beta": 0.12440179288387299, "fcm_dpo/delta": -0.07902223616838455, "fcm_dpo/margin": 4.21340799331665, "fcm_dpo/q_t": 0.39374351501464844, "grad_norm": 37.01242446899414, "learning_rate": 4.800745179625307e-07, "logits/chosen": 0.2540838122367859, "logits/rejected": 0.2218245565891266, "logps/chosen": -71.58065032958984, "logps/ref_chosen": -60.80268859863281, "logps/ref_rejected": -79.07284545898438, "logps/rejected": -94.06421661376953, "loss": 1.1251, "margin_dpo/margin_mean": 4.21340799331665, "margin_dpo/margin_std": 7.684472560882568, "step": 144 }, { "KL/chosen_KL_mean": -9.943706512451172, "KL/mean": -12.170425415039062, "KL/rejected_KL_mean": -14.397136688232422, "KL/std": 6.7663373947143555, "epoch": 0.21919879062736206, "fcm_dpo/beta": 0.12235713750123978, "fcm_dpo/delta": -0.10066936910152435, "fcm_dpo/margin": 4.453435897827148, "fcm_dpo/q_t": 0.3854507803916931, "grad_norm": 39.809226989746094, "learning_rate": 4.795540267200686e-07, "logits/chosen": 0.20279008150100708, "logits/rejected": 0.21891656517982483, "logps/chosen": -84.55516815185547, "logps/ref_chosen": -74.61146545410156, "logps/ref_rejected": -83.24461364746094, "logps/rejected": -97.64175415039062, "loss": 1.1188, "margin_dpo/margin_mean": 4.453435897827148, "margin_dpo/margin_std": 8.038009643554688, "step": 145 }, { "KL/chosen_KL_mean": -9.593629837036133, "KL/mean": -11.913301467895508, "KL/rejected_KL_mean": -14.232978820800781, "KL/std": 6.6814470291137695, "epoch": 0.22071050642479215, "fcm_dpo/beta": 0.12027982622385025, "fcm_dpo/delta": -0.11424677819013596, "fcm_dpo/margin": 4.6393537521362305, "fcm_dpo/q_t": 0.38348451256752014, "grad_norm": 34.77975845336914, "learning_rate": 4.790271143580173e-07, "logits/chosen": 0.19675567746162415, "logits/rejected": 0.18253400921821594, "logps/chosen": -67.43461608886719, "logps/ref_chosen": -57.84098434448242, "logps/ref_rejected": -67.47422790527344, "logps/rejected": -81.70720672607422, "loss": 1.0785, "margin_dpo/margin_mean": 4.6393537521362305, "margin_dpo/margin_std": 7.411983489990234, "step": 146 }, { "KL/chosen_KL_mean": -11.185792922973633, "KL/mean": -13.020978927612305, "KL/rejected_KL_mean": -14.85616683959961, "KL/std": 7.158926963806152, "epoch": 0.2222222222222222, "fcm_dpo/beta": 0.11947156488895416, "fcm_dpo/delta": 0.012014790438115597, "fcm_dpo/margin": 3.6703786849975586, "fcm_dpo/q_t": 0.4066588580608368, "grad_norm": 42.58670425415039, "learning_rate": 4.784937956152489e-07, "logits/chosen": 0.22439086437225342, "logits/rejected": 0.18046727776527405, "logps/chosen": -77.999267578125, "logps/ref_chosen": -66.81346893310547, "logps/ref_rejected": -81.1796875, "logps/rejected": -96.03585815429688, "loss": 1.1787, "margin_dpo/margin_mean": 3.6703782081604004, "margin_dpo/margin_std": 7.668435096740723, "step": 147 }, { "KL/chosen_KL_mean": -10.262016296386719, "KL/mean": -12.560417175292969, "KL/rejected_KL_mean": -14.858814239501953, "KL/std": 6.569252967834473, "epoch": 0.2237339380196523, "fcm_dpo/beta": 0.11826850473880768, "fcm_dpo/delta": -0.09899365901947021, "fcm_dpo/margin": 4.596801280975342, "fcm_dpo/q_t": 0.3836548328399658, "grad_norm": 28.631181716918945, "learning_rate": 4.779540854098347e-07, "logits/chosen": 0.3569459319114685, "logits/rejected": 0.2824467420578003, "logps/chosen": -58.949771881103516, "logps/ref_chosen": -48.6877555847168, "logps/ref_rejected": -67.50503540039062, "logps/rejected": -82.36384582519531, "loss": 1.0861, "margin_dpo/margin_mean": 4.596801280975342, "margin_dpo/margin_std": 7.328686237335205, "step": 148 }, { "KL/chosen_KL_mean": -12.381685256958008, "KL/mean": -14.980354309082031, "KL/rejected_KL_mean": -17.579021453857422, "KL/std": 7.072737693786621, "epoch": 0.2252456538170824, "fcm_dpo/beta": 0.11462083458900452, "fcm_dpo/delta": -0.1548178791999817, "fcm_dpo/margin": 5.197340965270996, "fcm_dpo/q_t": 0.3695276379585266, "grad_norm": 29.586246490478516, "learning_rate": 4.774079988386296e-07, "logits/chosen": 0.21398887038230896, "logits/rejected": 0.16681325435638428, "logps/chosen": -67.52546691894531, "logps/ref_chosen": -55.143775939941406, "logps/ref_rejected": -64.79888916015625, "logps/rejected": -82.37791442871094, "loss": 1.0049, "margin_dpo/margin_mean": 5.19734001159668, "margin_dpo/margin_std": 6.64594841003418, "step": 149 }, { "KL/chosen_KL_mean": -10.293771743774414, "KL/mean": -13.691690444946289, "KL/rejected_KL_mean": -17.089603424072266, "KL/std": 7.609349250793457, "epoch": 0.22675736961451248, "fcm_dpo/beta": 0.10836917161941528, "fcm_dpo/delta": -0.3096678853034973, "fcm_dpo/margin": 6.79583740234375, "fcm_dpo/q_t": 0.34543052315711975, "grad_norm": 27.946935653686523, "learning_rate": 4.768555511768486e-07, "logits/chosen": 0.24622797966003418, "logits/rejected": 0.20110848546028137, "logps/chosen": -77.76451873779297, "logps/ref_chosen": -67.47074890136719, "logps/ref_rejected": -89.21170806884766, "logps/rejected": -106.30131530761719, "loss": 0.933, "margin_dpo/margin_mean": 6.795836925506592, "margin_dpo/margin_std": 7.71773624420166, "step": 150 }, { "KL/chosen_KL_mean": -9.049201965332031, "KL/mean": -12.828123092651367, "KL/rejected_KL_mean": -16.607044219970703, "KL/std": 8.092508316040039, "epoch": 0.22826908541194255, "fcm_dpo/beta": 0.10158209502696991, "fcm_dpo/delta": -0.3438608646392822, "fcm_dpo/margin": 7.557843208312988, "fcm_dpo/q_t": 0.33743754029273987, "grad_norm": 24.914169311523438, "learning_rate": 4.762967578776406e-07, "logits/chosen": 0.2584247887134552, "logits/rejected": 0.20215699076652527, "logps/chosen": -61.50874328613281, "logps/ref_chosen": -52.45954132080078, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -95.67005920410156, "loss": 0.901, "margin_dpo/margin_mean": 7.55784273147583, "margin_dpo/margin_std": 8.19579029083252, "step": 151 }, { "KL/chosen_KL_mean": -11.547626495361328, "KL/mean": -14.018619537353516, "KL/rejected_KL_mean": -16.489608764648438, "KL/std": 7.6490159034729, "epoch": 0.22978080120937264, "fcm_dpo/beta": 0.09971933811903, "fcm_dpo/delta": -0.045774128288030624, "fcm_dpo/margin": 4.941987037658691, "fcm_dpo/q_t": 0.3965153694152832, "grad_norm": 29.130266189575195, "learning_rate": 4.757316345716553e-07, "logits/chosen": 0.2998914420604706, "logits/rejected": 0.24673616886138916, "logps/chosen": -68.10145568847656, "logps/ref_chosen": -56.5538330078125, "logps/ref_rejected": -76.55074310302734, "logps/rejected": -93.04035186767578, "loss": 1.1252, "margin_dpo/margin_mean": 4.941986560821533, "margin_dpo/margin_std": 8.686779975891113, "step": 152 }, { "KL/chosen_KL_mean": -10.736824035644531, "KL/mean": -13.64626693725586, "KL/rejected_KL_mean": -16.555709838867188, "KL/std": 7.962043762207031, "epoch": 0.23129251700680273, "fcm_dpo/beta": 0.09694686532020569, "fcm_dpo/delta": -0.12102583050727844, "fcm_dpo/margin": 5.818882942199707, "fcm_dpo/q_t": 0.375736802816391, "grad_norm": 26.531309127807617, "learning_rate": 4.751601970666064e-07, "logits/chosen": 0.22037890553474426, "logits/rejected": 0.18365280330181122, "logps/chosen": -78.74371337890625, "logps/ref_chosen": -68.00689697265625, "logps/ref_rejected": -74.83482360839844, "logps/rejected": -91.39053344726562, "loss": 1.0171, "margin_dpo/margin_mean": 5.818883895874023, "margin_dpo/margin_std": 7.457840919494629, "step": 153 }, { "KL/chosen_KL_mean": -11.988504409790039, "KL/mean": -14.088945388793945, "KL/rejected_KL_mean": -16.189388275146484, "KL/std": 8.195691108703613, "epoch": 0.2328042328042328, "fcm_dpo/beta": 0.09695246070623398, "fcm_dpo/delta": 0.04455181211233139, "fcm_dpo/margin": 4.2008795738220215, "fcm_dpo/q_t": 0.4105023741722107, "grad_norm": 28.0793399810791, "learning_rate": 4.745824613468292e-07, "logits/chosen": 0.31881213188171387, "logits/rejected": 0.31584903597831726, "logps/chosen": -71.21104431152344, "logps/ref_chosen": -59.222537994384766, "logps/ref_rejected": -64.19131469726562, "logps/rejected": -80.38070678710938, "loss": 1.1732, "margin_dpo/margin_mean": 4.2008795738220215, "margin_dpo/margin_std": 8.427075386047363, "step": 154 }, { "KL/chosen_KL_mean": -12.549442291259766, "KL/mean": -15.515789031982422, "KL/rejected_KL_mean": -18.482131958007812, "KL/std": 8.112979888916016, "epoch": 0.23431594860166288, "fcm_dpo/beta": 0.09494648873806, "fcm_dpo/delta": -0.12154512107372284, "fcm_dpo/margin": 5.932689666748047, "fcm_dpo/q_t": 0.3839681148529053, "grad_norm": 28.116670608520508, "learning_rate": 4.7399844357283393e-07, "logits/chosen": 0.30328816175460815, "logits/rejected": 0.28287360072135925, "logps/chosen": -81.00413513183594, "logps/ref_chosen": -68.45469665527344, "logps/ref_rejected": -77.91763305664062, "logps/rejected": -96.39976501464844, "loss": 1.1091, "margin_dpo/margin_mean": 5.932689666748047, "margin_dpo/margin_std": 10.310593605041504, "step": 155 }, { "KL/chosen_KL_mean": -12.504823684692383, "KL/mean": -16.008132934570312, "KL/rejected_KL_mean": -19.51144027709961, "KL/std": 9.073400497436523, "epoch": 0.23582766439909297, "fcm_dpo/beta": 0.09212834388017654, "fcm_dpo/delta": -0.2088165283203125, "fcm_dpo/margin": 7.006613731384277, "fcm_dpo/q_t": 0.3617513179779053, "grad_norm": 26.31268882751465, "learning_rate": 4.7340816008085305e-07, "logits/chosen": 0.2535175681114197, "logits/rejected": 0.20776960253715515, "logps/chosen": -79.77442169189453, "logps/ref_chosen": -67.26959991455078, "logps/ref_rejected": -86.95914459228516, "logps/rejected": -106.4705810546875, "loss": 0.9756, "margin_dpo/margin_mean": 7.006614685058594, "margin_dpo/margin_std": 8.517118453979492, "step": 156 }, { "KL/chosen_KL_mean": -11.273056030273438, "KL/mean": -14.099236488342285, "KL/rejected_KL_mean": -16.925411224365234, "KL/std": 8.791933059692383, "epoch": 0.23733938019652306, "fcm_dpo/beta": 0.08914068341255188, "fcm_dpo/delta": -0.06050056219100952, "fcm_dpo/margin": 5.652355194091797, "fcm_dpo/q_t": 0.3902330994606018, "grad_norm": 25.38435173034668, "learning_rate": 4.728116273823847e-07, "logits/chosen": 0.2619755268096924, "logits/rejected": 0.24316802620887756, "logps/chosen": -66.04592895507812, "logps/ref_chosen": -54.77287292480469, "logps/ref_rejected": -63.87866973876953, "logps/rejected": -80.8040771484375, "loss": 1.0866, "margin_dpo/margin_mean": 5.652355194091797, "margin_dpo/margin_std": 8.495977401733398, "step": 157 }, { "KL/chosen_KL_mean": -12.521953582763672, "KL/mean": -15.536789894104004, "KL/rejected_KL_mean": -18.55162811279297, "KL/std": 8.959171295166016, "epoch": 0.23885109599395313, "fcm_dpo/beta": 0.08881160616874695, "fcm_dpo/delta": -0.09029243886470795, "fcm_dpo/margin": 6.029675483703613, "fcm_dpo/q_t": 0.3862513601779938, "grad_norm": 26.304529190063477, "learning_rate": 4.7220886216373085e-07, "logits/chosen": 0.29904159903526306, "logits/rejected": 0.2606283724308014, "logps/chosen": -77.44467163085938, "logps/ref_chosen": -64.92271423339844, "logps/ref_rejected": -82.23789978027344, "logps/rejected": -100.78952026367188, "loss": 1.0768, "margin_dpo/margin_mean": 6.029675483703613, "margin_dpo/margin_std": 9.455682754516602, "step": 158 }, { "KL/chosen_KL_mean": -12.712642669677734, "KL/mean": -16.16486358642578, "KL/rejected_KL_mean": -19.617088317871094, "KL/std": 9.580606460571289, "epoch": 0.24036281179138322, "fcm_dpo/beta": 0.08682575821876526, "fcm_dpo/delta": -0.1588217169046402, "fcm_dpo/margin": 6.904443264007568, "fcm_dpo/q_t": 0.37174341082572937, "grad_norm": 27.926536560058594, "learning_rate": 4.715998812855304e-07, "logits/chosen": 0.3188456892967224, "logits/rejected": 0.2785356044769287, "logps/chosen": -69.75963592529297, "logps/ref_chosen": -57.046993255615234, "logps/ref_rejected": -73.32441711425781, "logps/rejected": -92.9415054321289, "loss": 1.0438, "margin_dpo/margin_mean": 6.90444278717041, "margin_dpo/margin_std": 10.084371566772461, "step": 159 }, { "KL/chosen_KL_mean": -14.375518798828125, "KL/mean": -17.316848754882812, "KL/rejected_KL_mean": -20.258174896240234, "KL/std": 8.959368705749512, "epoch": 0.2418745275888133, "fcm_dpo/beta": 0.0849989801645279, "fcm_dpo/delta": -0.052619896829128265, "fcm_dpo/margin": 5.882652759552002, "fcm_dpo/q_t": 0.3973754644393921, "grad_norm": 24.18868064880371, "learning_rate": 4.7098470178228755e-07, "logits/chosen": 0.16954955458641052, "logits/rejected": 0.1237107664346695, "logps/chosen": -64.18243408203125, "logps/ref_chosen": -49.806915283203125, "logps/ref_rejected": -68.3370132446289, "logps/rejected": -88.59518432617188, "loss": 1.1145, "margin_dpo/margin_mean": 5.882652282714844, "margin_dpo/margin_std": 10.348878860473633, "step": 160 }, { "KL/chosen_KL_mean": -14.157938003540039, "KL/mean": -17.39502716064453, "KL/rejected_KL_mean": -20.63211441040039, "KL/std": 8.806609153747559, "epoch": 0.24338624338624337, "fcm_dpo/beta": 0.08354272693395615, "fcm_dpo/delta": -0.09600973129272461, "fcm_dpo/margin": 6.474179267883301, "fcm_dpo/q_t": 0.3830728530883789, "grad_norm": 23.179370880126953, "learning_rate": 4.703633408618955e-07, "logits/chosen": 0.30207502841949463, "logits/rejected": 0.2627732753753662, "logps/chosen": -66.6584243774414, "logps/ref_chosen": -52.50048828125, "logps/ref_rejected": -66.04540252685547, "logps/rejected": -86.67752075195312, "loss": 1.0708, "margin_dpo/margin_mean": 6.474179267883301, "margin_dpo/margin_std": 9.89106559753418, "step": 161 }, { "KL/chosen_KL_mean": -15.079242706298828, "KL/mean": -19.381000518798828, "KL/rejected_KL_mean": -23.682758331298828, "KL/std": 9.969080924987793, "epoch": 0.24489795918367346, "fcm_dpo/beta": 0.07994553446769714, "fcm_dpo/delta": -0.2555304765701294, "fcm_dpo/margin": 8.603517532348633, "fcm_dpo/q_t": 0.3500090539455414, "grad_norm": 24.3117733001709, "learning_rate": 4.697358159051549e-07, "logits/chosen": 0.34407973289489746, "logits/rejected": 0.29311051964759827, "logps/chosen": -84.54843139648438, "logps/ref_chosen": -69.46919250488281, "logps/ref_rejected": -92.00952911376953, "logps/rejected": -115.69229125976562, "loss": 0.9533, "margin_dpo/margin_mean": 8.603517532348633, "margin_dpo/margin_std": 10.078315734863281, "step": 162 }, { "KL/chosen_KL_mean": -13.317777633666992, "KL/mean": -17.590572357177734, "KL/rejected_KL_mean": -21.86336898803711, "KL/std": 9.784802436828613, "epoch": 0.24640967498110355, "fcm_dpo/beta": 0.0772738978266716, "fcm_dpo/delta": -0.22547154128551483, "fcm_dpo/margin": 8.545589447021484, "fcm_dpo/q_t": 0.35784658789634705, "grad_norm": 22.079208374023438, "learning_rate": 4.691021444652876e-07, "logits/chosen": 0.2510836720466614, "logits/rejected": 0.20358552038669586, "logps/chosen": -63.931610107421875, "logps/ref_chosen": -50.613834381103516, "logps/ref_rejected": -74.62033081054688, "logps/rejected": -96.48369598388672, "loss": 0.9873, "margin_dpo/margin_mean": 8.5455904006958, "margin_dpo/margin_std": 10.263286590576172, "step": 163 }, { "KL/chosen_KL_mean": -14.311813354492188, "KL/mean": -18.52021026611328, "KL/rejected_KL_mean": -22.72860336303711, "KL/std": 10.371530532836914, "epoch": 0.24792139077853365, "fcm_dpo/beta": 0.07386015355587006, "fcm_dpo/delta": -0.18267156183719635, "fcm_dpo/margin": 8.416790008544922, "fcm_dpo/q_t": 0.3670443892478943, "grad_norm": 20.7839298248291, "learning_rate": 4.6846234426744624e-07, "logits/chosen": 0.2554076611995697, "logits/rejected": 0.1913755238056183, "logps/chosen": -69.15992736816406, "logps/ref_chosen": -54.848114013671875, "logps/ref_rejected": -79.0630111694336, "logps/rejected": -101.79161071777344, "loss": 1.0178, "margin_dpo/margin_mean": 8.416790008544922, "margin_dpo/margin_std": 11.19088363647461, "step": 164 }, { "KL/chosen_KL_mean": -15.9600830078125, "KL/mean": -19.80939483642578, "KL/rejected_KL_mean": -23.658706665039062, "KL/std": 10.531667709350586, "epoch": 0.2494331065759637, "fcm_dpo/beta": 0.071999192237854, "fcm_dpo/delta": -0.1104959100484848, "fcm_dpo/margin": 7.698622226715088, "fcm_dpo/q_t": 0.3786641061306, "grad_norm": 21.82950782775879, "learning_rate": 4.678164332082175e-07, "logits/chosen": 0.33307212591171265, "logits/rejected": 0.27663278579711914, "logps/chosen": -67.0492935180664, "logps/ref_chosen": -51.089210510253906, "logps/ref_rejected": -71.23370361328125, "logps/rejected": -94.89241027832031, "loss": 1.0631, "margin_dpo/margin_mean": 7.69862174987793, "margin_dpo/margin_std": 11.216785430908203, "step": 165 }, { "KL/chosen_KL_mean": -15.295942306518555, "KL/mean": -18.283523559570312, "KL/rejected_KL_mean": -21.271102905273438, "KL/std": 10.442094802856445, "epoch": 0.2509448223733938, "fcm_dpo/beta": 0.0712810531258583, "fcm_dpo/delta": 0.025124577805399895, "fcm_dpo/margin": 5.975157737731934, "fcm_dpo/q_t": 0.40849870443344116, "grad_norm": 24.805681228637695, "learning_rate": 4.6716442935512214e-07, "logits/chosen": 0.27621328830718994, "logits/rejected": 0.1836838275194168, "logps/chosen": -78.48675537109375, "logps/ref_chosen": -63.19081115722656, "logps/ref_rejected": -93.8402099609375, "logps/rejected": -115.11131286621094, "loss": 1.1214, "margin_dpo/margin_mean": 5.975157260894775, "margin_dpo/margin_std": 10.045669555664062, "step": 166 }, { "KL/chosen_KL_mean": -13.135995864868164, "KL/mean": -17.774768829345703, "KL/rejected_KL_mean": -22.41353988647461, "KL/std": 10.714015007019043, "epoch": 0.25245653817082386, "fcm_dpo/beta": 0.06891494989395142, "fcm_dpo/delta": -0.20458835363388062, "fcm_dpo/margin": 9.277545928955078, "fcm_dpo/q_t": 0.3598324656486511, "grad_norm": 20.088937759399414, "learning_rate": 4.6650635094610966e-07, "logits/chosen": 0.24189813435077667, "logits/rejected": 0.20661483705043793, "logps/chosen": -72.06027221679688, "logps/ref_chosen": -58.92427062988281, "logps/ref_rejected": -72.97377014160156, "logps/rejected": -95.38731384277344, "loss": 0.9602, "margin_dpo/margin_mean": 9.277545928955078, "margin_dpo/margin_std": 10.376180648803711, "step": 167 }, { "KL/chosen_KL_mean": -15.331819534301758, "KL/mean": -18.971294403076172, "KL/rejected_KL_mean": -22.610767364501953, "KL/std": 11.420768737792969, "epoch": 0.25396825396825395, "fcm_dpo/beta": 0.06804715096950531, "fcm_dpo/delta": -0.04781053215265274, "fcm_dpo/margin": 7.278944969177246, "fcm_dpo/q_t": 0.39334410429000854, "grad_norm": 22.273143768310547, "learning_rate": 4.6584221638904767e-07, "logits/chosen": 0.25680267810821533, "logits/rejected": 0.2203751504421234, "logps/chosen": -80.98320007324219, "logps/ref_chosen": -65.65138244628906, "logps/ref_rejected": -79.71418762207031, "logps/rejected": -102.324951171875, "loss": 1.0675, "margin_dpo/margin_mean": 7.27894401550293, "margin_dpo/margin_std": 10.532588958740234, "step": 168 }, { "KL/chosen_KL_mean": -12.636564254760742, "KL/mean": -16.88669776916504, "KL/rejected_KL_mean": -21.13683319091797, "KL/std": 11.466968536376953, "epoch": 0.25547996976568405, "fcm_dpo/beta": 0.06674225628376007, "fcm_dpo/delta": -0.12439984083175659, "fcm_dpo/margin": 8.500267028808594, "fcm_dpo/q_t": 0.38215646147727966, "grad_norm": 22.956363677978516, "learning_rate": 4.651720442612075e-07, "logits/chosen": 0.319760799407959, "logits/rejected": 0.28525394201278687, "logps/chosen": -74.06243133544922, "logps/ref_chosen": -61.425865173339844, "logps/ref_rejected": -76.09590148925781, "logps/rejected": -97.23272705078125, "loss": 1.0698, "margin_dpo/margin_mean": 8.500267028808594, "margin_dpo/margin_std": 13.569889068603516, "step": 169 }, { "KL/chosen_KL_mean": -12.162647247314453, "KL/mean": -16.37779998779297, "KL/rejected_KL_mean": -20.592947006225586, "KL/std": 11.154409408569336, "epoch": 0.25699168556311414, "fcm_dpo/beta": 0.06543616950511932, "fcm_dpo/delta": -0.10744147002696991, "fcm_dpo/margin": 8.43030071258545, "fcm_dpo/q_t": 0.3819403052330017, "grad_norm": 18.930606842041016, "learning_rate": 4.6449585330874425e-07, "logits/chosen": 0.27050989866256714, "logits/rejected": 0.26806920766830444, "logps/chosen": -68.81584167480469, "logps/ref_chosen": -56.65319061279297, "logps/ref_rejected": -63.45965576171875, "logps/rejected": -84.05259704589844, "loss": 1.0897, "margin_dpo/margin_mean": 8.43030071258545, "margin_dpo/margin_std": 13.604389190673828, "step": 170 }, { "KL/chosen_KL_mean": -13.745929718017578, "KL/mean": -18.389129638671875, "KL/rejected_KL_mean": -23.032333374023438, "KL/std": 12.820600509643555, "epoch": 0.2585034013605442, "fcm_dpo/beta": 0.06286909431219101, "fcm_dpo/delta": -0.14584212005138397, "fcm_dpo/margin": 9.286405563354492, "fcm_dpo/q_t": 0.3753986656665802, "grad_norm": 20.48279571533203, "learning_rate": 4.6381366244617224e-07, "logits/chosen": 0.3456369638442993, "logits/rejected": 0.2936851382255554, "logps/chosen": -77.4806900024414, "logps/ref_chosen": -63.73476028442383, "logps/ref_rejected": -78.50328063964844, "logps/rejected": -101.53561401367188, "loss": 1.0752, "margin_dpo/margin_mean": 9.286405563354492, "margin_dpo/margin_std": 14.497032165527344, "step": 171 }, { "KL/chosen_KL_mean": -15.124223709106445, "KL/mean": -19.514453887939453, "KL/rejected_KL_mean": -23.904678344726562, "KL/std": 12.908832550048828, "epoch": 0.2600151171579743, "fcm_dpo/beta": 0.06263021379709244, "fcm_dpo/delta": -0.10646422207355499, "fcm_dpo/margin": 8.780457496643066, "fcm_dpo/q_t": 0.38289976119995117, "grad_norm": 20.70548439025879, "learning_rate": 4.631254907558365e-07, "logits/chosen": 0.3301047384738922, "logits/rejected": 0.27112317085266113, "logps/chosen": -67.32598114013672, "logps/ref_chosen": -52.201759338378906, "logps/ref_rejected": -82.85285949707031, "logps/rejected": -106.75753784179688, "loss": 1.0691, "margin_dpo/margin_mean": 8.78045654296875, "margin_dpo/margin_std": 13.278366088867188, "step": 172 }, { "KL/chosen_KL_mean": -14.149677276611328, "KL/mean": -18.881351470947266, "KL/rejected_KL_mean": -23.613025665283203, "KL/std": 13.220224380493164, "epoch": 0.2615268329554044, "fcm_dpo/beta": 0.059835657477378845, "fcm_dpo/delta": -0.12774161994457245, "fcm_dpo/margin": 9.463345527648926, "fcm_dpo/q_t": 0.3847862184047699, "grad_norm": 20.46826171875, "learning_rate": 4.624313574873786e-07, "logits/chosen": 0.37014636397361755, "logits/rejected": 0.27701422572135925, "logps/chosen": -69.58440399169922, "logps/ref_chosen": -55.434722900390625, "logps/ref_rejected": -77.81967163085938, "logps/rejected": -101.43270111083984, "loss": 1.1185, "margin_dpo/margin_mean": 9.463346481323242, "margin_dpo/margin_std": 16.405101776123047, "step": 173 }, { "KL/chosen_KL_mean": -15.740091323852539, "KL/mean": -20.807640075683594, "KL/rejected_KL_mean": -25.87519073486328, "KL/std": 13.291604995727539, "epoch": 0.26303854875283444, "fcm_dpo/beta": 0.058659717440605164, "fcm_dpo/delta": -0.15415163338184357, "fcm_dpo/margin": 10.135098457336426, "fcm_dpo/q_t": 0.3727102279663086, "grad_norm": 20.58899688720703, "learning_rate": 4.61731282057198e-07, "logits/chosen": 0.34955164790153503, "logits/rejected": 0.2790573239326477, "logps/chosen": -72.91204833984375, "logps/ref_chosen": -57.17195129394531, "logps/ref_rejected": -85.47578430175781, "logps/rejected": -111.35096740722656, "loss": 1.0354, "margin_dpo/margin_mean": 10.135099411010742, "margin_dpo/margin_std": 14.499110221862793, "step": 174 }, { "KL/chosen_KL_mean": -13.907922744750977, "KL/mean": -19.6348819732666, "KL/rejected_KL_mean": -25.36184310913086, "KL/std": 13.645920753479004, "epoch": 0.26455026455026454, "fcm_dpo/beta": 0.05664439499378204, "fcm_dpo/delta": -0.21225407719612122, "fcm_dpo/margin": 11.453914642333984, "fcm_dpo/q_t": 0.36655163764953613, "grad_norm": 19.182777404785156, "learning_rate": 4.6102528404790965e-07, "logits/chosen": 0.368941068649292, "logits/rejected": 0.3376131057739258, "logps/chosen": -81.57354736328125, "logps/ref_chosen": -67.6656265258789, "logps/ref_rejected": -84.36766815185547, "logps/rejected": -109.72950744628906, "loss": 1.0132, "margin_dpo/margin_mean": 11.453914642333984, "margin_dpo/margin_std": 15.843772888183594, "step": 175 }, { "KL/chosen_KL_mean": -17.130245208740234, "KL/mean": -20.843347549438477, "KL/rejected_KL_mean": -24.556453704833984, "KL/std": 14.491414070129395, "epoch": 0.2660619803476946, "fcm_dpo/beta": 0.05589904636144638, "fcm_dpo/delta": 0.03595317155122757, "fcm_dpo/margin": 7.426205158233643, "fcm_dpo/q_t": 0.4137015640735626, "grad_norm": 24.373594284057617, "learning_rate": 4.603133832077953e-07, "logits/chosen": 0.32078686356544495, "logits/rejected": 0.29332348704338074, "logps/chosen": -94.98900604248047, "logps/ref_chosen": -77.8587646484375, "logps/ref_rejected": -81.08732604980469, "logps/rejected": -105.64378356933594, "loss": 1.1976, "margin_dpo/margin_mean": 7.426205635070801, "margin_dpo/margin_std": 16.123355865478516, "step": 176 }, { "KL/chosen_KL_mean": -13.158845901489258, "KL/mean": -20.39859390258789, "KL/rejected_KL_mean": -27.638347625732422, "KL/std": 14.79086685180664, "epoch": 0.2675736961451247, "fcm_dpo/beta": 0.05322066694498062, "fcm_dpo/delta": -0.34956711530685425, "fcm_dpo/margin": 14.479496955871582, "fcm_dpo/q_t": 0.33395838737487793, "grad_norm": 23.84845733642578, "learning_rate": 4.5959559945025183e-07, "logits/chosen": 0.42620840668678284, "logits/rejected": 0.3288993239402771, "logps/chosen": -68.37924194335938, "logps/ref_chosen": -55.22039794921875, "logps/ref_rejected": -92.54973602294922, "logps/rejected": -120.18807983398438, "loss": 0.8946, "margin_dpo/margin_mean": 14.479496002197266, "margin_dpo/margin_std": 15.118330955505371, "step": 177 }, { "KL/chosen_KL_mean": -15.544981002807617, "KL/mean": -19.833009719848633, "KL/rejected_KL_mean": -24.12103271484375, "KL/std": 13.912124633789062, "epoch": 0.2690854119425548, "fcm_dpo/beta": 0.05207317695021629, "fcm_dpo/delta": 0.002033442258834839, "fcm_dpo/margin": 8.576053619384766, "fcm_dpo/q_t": 0.40217965841293335, "grad_norm": 21.546463012695312, "learning_rate": 4.588719528532341e-07, "logits/chosen": 0.2882809042930603, "logits/rejected": 0.23839804530143738, "logps/chosen": -76.35546875, "logps/ref_chosen": -60.81049346923828, "logps/ref_rejected": -81.12973022460938, "logps/rejected": -105.25076293945312, "loss": 1.1183, "margin_dpo/margin_mean": 8.576054573059082, "margin_dpo/margin_std": 14.08738899230957, "step": 178 }, { "KL/chosen_KL_mean": -16.099651336669922, "KL/mean": -21.50237464904785, "KL/rejected_KL_mean": -26.905094146728516, "KL/std": 14.630666732788086, "epoch": 0.2705971277399849, "fcm_dpo/beta": 0.05162374675273895, "fcm_dpo/delta": -0.11425016075372696, "fcm_dpo/margin": 10.805438995361328, "fcm_dpo/q_t": 0.3815948963165283, "grad_norm": 21.03357696533203, "learning_rate": 4.581424636586928e-07, "logits/chosen": 0.3580857515335083, "logits/rejected": 0.34163224697113037, "logps/chosen": -81.7713623046875, "logps/ref_chosen": -65.67171478271484, "logps/ref_rejected": -75.32586669921875, "logps/rejected": -102.23095703125, "loss": 1.0771, "margin_dpo/margin_mean": 10.805438995361328, "margin_dpo/margin_std": 17.263511657714844, "step": 179 }, { "KL/chosen_KL_mean": -12.344501495361328, "KL/mean": -16.842422485351562, "KL/rejected_KL_mean": -21.3403377532959, "KL/std": 14.285127639770508, "epoch": 0.272108843537415, "fcm_dpo/beta": 0.05138145387172699, "fcm_dpo/delta": -0.01291588507592678, "fcm_dpo/margin": 8.995841026306152, "fcm_dpo/q_t": 0.40272700786590576, "grad_norm": 17.912363052368164, "learning_rate": 4.5740715227200897e-07, "logits/chosen": 0.1993003785610199, "logits/rejected": 0.1796572208404541, "logps/chosen": -69.02731323242188, "logps/ref_chosen": -56.68280792236328, "logps/ref_rejected": -64.94414520263672, "logps/rejected": -86.28448486328125, "loss": 1.1388, "margin_dpo/margin_mean": 8.995841026306152, "margin_dpo/margin_std": 16.66576385498047, "step": 180 }, { "KL/chosen_KL_mean": -10.670660018920898, "KL/mean": -17.220378875732422, "KL/rejected_KL_mean": -23.770092010498047, "KL/std": 15.256805419921875, "epoch": 0.273620559334845, "fcm_dpo/beta": 0.049501724541187286, "fcm_dpo/delta": -0.21285748481750488, "fcm_dpo/margin": 13.099435806274414, "fcm_dpo/q_t": 0.3588384985923767, "grad_norm": 18.757675170898438, "learning_rate": 4.566660392614228e-07, "logits/chosen": 0.3928632140159607, "logits/rejected": 0.35001248121261597, "logps/chosen": -71.44670867919922, "logps/ref_chosen": -60.77604675292969, "logps/ref_rejected": -83.98361206054688, "logps/rejected": -107.75370788574219, "loss": 0.9532, "margin_dpo/margin_mean": 13.099435806274414, "margin_dpo/margin_std": 14.686405181884766, "step": 181 }, { "KL/chosen_KL_mean": -14.897209167480469, "KL/mean": -21.812625885009766, "KL/rejected_KL_mean": -28.728042602539062, "KL/std": 16.35493278503418, "epoch": 0.2751322751322751, "fcm_dpo/beta": 0.04746267572045326, "fcm_dpo/delta": -0.22113311290740967, "fcm_dpo/margin": 13.830833435058594, "fcm_dpo/q_t": 0.36115947365760803, "grad_norm": 19.716379165649414, "learning_rate": 4.5591914535745817e-07, "logits/chosen": 0.3558586835861206, "logits/rejected": 0.269753634929657, "logps/chosen": -75.15099334716797, "logps/ref_chosen": -60.2537841796875, "logps/ref_rejected": -89.7706298828125, "logps/rejected": -118.49867248535156, "loss": 1.0055, "margin_dpo/margin_mean": 13.830833435058594, "margin_dpo/margin_std": 18.645549774169922, "step": 182 }, { "KL/chosen_KL_mean": -18.52231788635254, "KL/mean": -21.377840042114258, "KL/rejected_KL_mean": -24.233356475830078, "KL/std": 15.700705528259277, "epoch": 0.2766439909297052, "fcm_dpo/beta": 0.04812941700220108, "fcm_dpo/delta": 0.1803903579711914, "fcm_dpo/margin": 5.7110395431518555, "fcm_dpo/q_t": 0.43941351771354675, "grad_norm": 19.41120719909668, "learning_rate": 4.551664914523433e-07, "logits/chosen": 0.3424264192581177, "logits/rejected": 0.31929779052734375, "logps/chosen": -80.28373718261719, "logps/ref_chosen": -61.76142120361328, "logps/ref_rejected": -72.54627990722656, "logps/rejected": -96.7796401977539, "loss": 1.2632, "margin_dpo/margin_mean": 5.7110395431518555, "margin_dpo/margin_std": 15.732259750366211, "step": 183 }, { "KL/chosen_KL_mean": -11.37504768371582, "KL/mean": -16.473596572875977, "KL/rejected_KL_mean": -21.5721435546875, "KL/std": 13.631113052368164, "epoch": 0.2781557067271353, "fcm_dpo/beta": 0.04807819426059723, "fcm_dpo/delta": -0.04334007948637009, "fcm_dpo/margin": 10.197093963623047, "fcm_dpo/q_t": 0.39357906579971313, "grad_norm": 18.226768493652344, "learning_rate": 4.544080985994258e-07, "logits/chosen": 0.45362773537635803, "logits/rejected": 0.3854234218597412, "logps/chosen": -58.21576690673828, "logps/ref_chosen": -46.840721130371094, "logps/ref_rejected": -69.3609390258789, "logps/rejected": -90.9330825805664, "loss": 1.0616, "margin_dpo/margin_mean": 10.197093963623047, "margin_dpo/margin_std": 14.069075584411621, "step": 184 }, { "KL/chosen_KL_mean": -14.680137634277344, "KL/mean": -20.519908905029297, "KL/rejected_KL_mean": -26.359676361083984, "KL/std": 16.397075653076172, "epoch": 0.2796674225245654, "fcm_dpo/beta": 0.04713146388530731, "fcm_dpo/delta": -0.10833575576543808, "fcm_dpo/margin": 11.679544448852539, "fcm_dpo/q_t": 0.3850805163383484, "grad_norm": 17.211332321166992, "learning_rate": 4.5364398801258394e-07, "logits/chosen": 0.3641235828399658, "logits/rejected": 0.3162262439727783, "logps/chosen": -67.00128173828125, "logps/ref_chosen": -52.32114028930664, "logps/ref_rejected": -68.3885726928711, "logps/rejected": -94.74824523925781, "loss": 1.1116, "margin_dpo/margin_mean": 11.679544448852539, "margin_dpo/margin_std": 20.055240631103516, "step": 185 }, { "KL/chosen_KL_mean": -12.910173416137695, "KL/mean": -19.38536834716797, "KL/rejected_KL_mean": -25.860557556152344, "KL/std": 16.375133514404297, "epoch": 0.2811791383219955, "fcm_dpo/beta": 0.046077094972133636, "fcm_dpo/delta": -0.15629881620407104, "fcm_dpo/margin": 12.950380325317383, "fcm_dpo/q_t": 0.3747071325778961, "grad_norm": 21.335391998291016, "learning_rate": 4.5287418106563354e-07, "logits/chosen": 0.2941705584526062, "logits/rejected": 0.25050365924835205, "logps/chosen": -80.33030700683594, "logps/ref_chosen": -67.42012786865234, "logps/ref_rejected": -82.50968933105469, "logps/rejected": -108.37024688720703, "loss": 1.0573, "margin_dpo/margin_mean": 12.950380325317383, "margin_dpo/margin_std": 19.893539428710938, "step": 186 }, { "KL/chosen_KL_mean": -16.437469482421875, "KL/mean": -22.146421432495117, "KL/rejected_KL_mean": -27.85538101196289, "KL/std": 17.256420135498047, "epoch": 0.28269085411942557, "fcm_dpo/beta": 0.045174386352300644, "fcm_dpo/delta": -0.06973277032375336, "fcm_dpo/margin": 11.417915344238281, "fcm_dpo/q_t": 0.39325594902038574, "grad_norm": 20.540571212768555, "learning_rate": 4.520986992917297e-07, "logits/chosen": 0.3017156720161438, "logits/rejected": 0.24813276529312134, "logps/chosen": -91.96296691894531, "logps/ref_chosen": -75.52549743652344, "logps/ref_rejected": -94.76289367675781, "logps/rejected": -122.61827087402344, "loss": 1.1177, "margin_dpo/margin_mean": 11.417915344238281, "margin_dpo/margin_std": 20.136951446533203, "step": 187 }, { "KL/chosen_KL_mean": -14.43658447265625, "KL/mean": -20.95711898803711, "KL/rejected_KL_mean": -27.47765350341797, "KL/std": 17.697364807128906, "epoch": 0.2842025699168556, "fcm_dpo/beta": 0.044272612780332565, "fcm_dpo/delta": -0.1350875049829483, "fcm_dpo/margin": 13.041068077087402, "fcm_dpo/q_t": 0.3793148994445801, "grad_norm": 17.25244903564453, "learning_rate": 4.5131756438276466e-07, "logits/chosen": 0.3734307885169983, "logits/rejected": 0.3284209966659546, "logps/chosen": -85.95991516113281, "logps/ref_chosen": -71.52333068847656, "logps/ref_rejected": -78.29949951171875, "logps/rejected": -105.77715301513672, "loss": 1.043, "margin_dpo/margin_mean": 13.041067123413086, "margin_dpo/margin_std": 19.054115295410156, "step": 188 }, { "KL/chosen_KL_mean": -14.128122329711914, "KL/mean": -20.039127349853516, "KL/rejected_KL_mean": -25.950130462646484, "KL/std": 17.645959854125977, "epoch": 0.2857142857142857, "fcm_dpo/beta": 0.04308847337961197, "fcm_dpo/delta": -0.06519506126642227, "fcm_dpo/margin": 11.822007179260254, "fcm_dpo/q_t": 0.3899165391921997, "grad_norm": 19.810237884521484, "learning_rate": 4.5053079818876096e-07, "logits/chosen": 0.39445924758911133, "logits/rejected": 0.4088705778121948, "logps/chosen": -86.30438995361328, "logps/ref_chosen": -72.17626953125, "logps/ref_rejected": -75.26313781738281, "logps/rejected": -101.21326446533203, "loss": 1.0893, "margin_dpo/margin_mean": 11.82200813293457, "margin_dpo/margin_std": 18.43564224243164, "step": 189 }, { "KL/chosen_KL_mean": -13.363594055175781, "KL/mean": -21.484352111816406, "KL/rejected_KL_mean": -29.605113983154297, "KL/std": 18.145763397216797, "epoch": 0.2872260015117158, "fcm_dpo/beta": 0.04177084565162659, "fcm_dpo/delta": -0.24536725878715515, "fcm_dpo/margin": 16.241512298583984, "fcm_dpo/q_t": 0.355669766664505, "grad_norm": 23.54590606689453, "learning_rate": 4.4973842271726024e-07, "logits/chosen": 0.39284735918045044, "logits/rejected": 0.24727825820446014, "logps/chosen": -67.98786926269531, "logps/ref_chosen": -54.624271392822266, "logps/ref_rejected": -101.47068786621094, "logps/rejected": -131.0758056640625, "loss": 0.9601, "margin_dpo/margin_mean": 16.241514205932617, "margin_dpo/margin_std": 19.464500427246094, "step": 190 }, { "KL/chosen_KL_mean": -19.313480377197266, "KL/mean": -25.634521484375, "KL/rejected_KL_mean": -31.95555877685547, "KL/std": 19.023698806762695, "epoch": 0.2887377173091459, "fcm_dpo/beta": 0.04085142910480499, "fcm_dpo/delta": -0.07008600234985352, "fcm_dpo/margin": 12.642083168029785, "fcm_dpo/q_t": 0.3872701823711395, "grad_norm": 20.005186080932617, "learning_rate": 4.48940460132708e-07, "logits/chosen": 0.44901180267333984, "logits/rejected": 0.4179879426956177, "logps/chosen": -92.2459945678711, "logps/ref_chosen": -72.93251037597656, "logps/ref_rejected": -89.95103454589844, "logps/rejected": -121.9065933227539, "loss": 1.0917, "margin_dpo/margin_mean": 12.642083168029785, "margin_dpo/margin_std": 20.432785034179688, "step": 191 }, { "KL/chosen_KL_mean": -20.968889236450195, "KL/mean": -25.023563385009766, "KL/rejected_KL_mean": -29.078235626220703, "KL/std": 19.83245849609375, "epoch": 0.29024943310657597, "fcm_dpo/beta": 0.04154283553361893, "fcm_dpo/delta": 0.11658979952335358, "fcm_dpo/margin": 8.109346389770508, "fcm_dpo/q_t": 0.42591214179992676, "grad_norm": 16.941612243652344, "learning_rate": 4.481369327558329e-07, "logits/chosen": 0.4174021780490875, "logits/rejected": 0.38975629210472107, "logps/chosen": -74.97000885009766, "logps/ref_chosen": -54.001121520996094, "logps/ref_rejected": -63.531551361083984, "logps/rejected": -92.60978698730469, "loss": 1.195, "margin_dpo/margin_mean": 8.109346389770508, "margin_dpo/margin_std": 17.20096206665039, "step": 192 }, { "KL/chosen_KL_mean": -16.72517967224121, "KL/mean": -24.319087982177734, "KL/rejected_KL_mean": -31.912994384765625, "KL/std": 18.46902084350586, "epoch": 0.29176114890400606, "fcm_dpo/beta": 0.04037639498710632, "fcm_dpo/delta": -0.17553646862506866, "fcm_dpo/margin": 15.187812805175781, "fcm_dpo/q_t": 0.367713987827301, "grad_norm": 15.405920028686523, "learning_rate": 4.47327863063023e-07, "logits/chosen": 0.33871471881866455, "logits/rejected": 0.31610235571861267, "logps/chosen": -73.47445678710938, "logps/ref_chosen": -56.74927520751953, "logps/ref_rejected": -58.80629348754883, "logps/rejected": -90.71928405761719, "loss": 0.9951, "margin_dpo/margin_mean": 15.187812805175781, "margin_dpo/margin_std": 19.09872817993164, "step": 193 }, { "KL/chosen_KL_mean": -17.941669464111328, "KL/mean": -24.060016632080078, "KL/rejected_KL_mean": -30.178367614746094, "KL/std": 18.968008041381836, "epoch": 0.29327286470143615, "fcm_dpo/beta": 0.039627574384212494, "fcm_dpo/delta": -0.038923852145671844, "fcm_dpo/margin": 12.236695289611816, "fcm_dpo/q_t": 0.3964529037475586, "grad_norm": 19.522842407226562, "learning_rate": 4.4651327368569684e-07, "logits/chosen": 0.4086693525314331, "logits/rejected": 0.37831252813339233, "logps/chosen": -74.59111022949219, "logps/ref_chosen": -56.64944076538086, "logps/ref_rejected": -69.98954772949219, "logps/rejected": -100.16790771484375, "loss": 1.1441, "margin_dpo/margin_mean": 12.2366943359375, "margin_dpo/margin_std": 22.420700073242188, "step": 194 }, { "KL/chosen_KL_mean": -21.67102813720703, "KL/mean": -29.539018630981445, "KL/rejected_KL_mean": -37.40701675415039, "KL/std": 21.462011337280273, "epoch": 0.2947845804988662, "fcm_dpo/beta": 0.038745589554309845, "fcm_dpo/delta": -0.17129415273666382, "fcm_dpo/margin": 15.735990524291992, "fcm_dpo/q_t": 0.3701290488243103, "grad_norm": 19.54598045349121, "learning_rate": 4.4569318740967043e-07, "logits/chosen": 0.3189467787742615, "logits/rejected": 0.32533496618270874, "logps/chosen": -92.080810546875, "logps/ref_chosen": -70.40977478027344, "logps/ref_rejected": -74.39448547363281, "logps/rejected": -111.80149841308594, "loss": 1.0249, "margin_dpo/margin_mean": 15.735990524291992, "margin_dpo/margin_std": 21.78451156616211, "step": 195 }, { "KL/chosen_KL_mean": -21.09269905090332, "KL/mean": -27.36833953857422, "KL/rejected_KL_mean": -33.643978118896484, "KL/std": 21.385547637939453, "epoch": 0.2962962962962963, "fcm_dpo/beta": 0.0385778471827507, "fcm_dpo/delta": -0.036321524530649185, "fcm_dpo/margin": 12.551275253295898, "fcm_dpo/q_t": 0.3921698331832886, "grad_norm": 19.41595458984375, "learning_rate": 4.448676271745197e-07, "logits/chosen": 0.4123719036579132, "logits/rejected": 0.368742972612381, "logps/chosen": -80.32027435302734, "logps/ref_chosen": -59.227577209472656, "logps/ref_rejected": -83.54757690429688, "logps/rejected": -117.1915512084961, "loss": 1.0808, "margin_dpo/margin_mean": 12.551275253295898, "margin_dpo/margin_std": 18.66332244873047, "step": 196 }, { "KL/chosen_KL_mean": -18.31484031677246, "KL/mean": -26.058391571044922, "KL/rejected_KL_mean": -33.80194854736328, "KL/std": 20.686565399169922, "epoch": 0.29780801209372637, "fcm_dpo/beta": 0.03779051452875137, "fcm_dpo/delta": -0.14380398392677307, "fcm_dpo/margin": 15.487101554870605, "fcm_dpo/q_t": 0.3769325911998749, "grad_norm": 18.967296600341797, "learning_rate": 4.440366160729392e-07, "logits/chosen": 0.5108134150505066, "logits/rejected": 0.456866979598999, "logps/chosen": -69.8439712524414, "logps/ref_chosen": -51.52912902832031, "logps/ref_rejected": -73.70631408691406, "logps/rejected": -107.50825500488281, "loss": 1.099, "margin_dpo/margin_mean": 15.487101554870605, "margin_dpo/margin_std": 25.65450668334961, "step": 197 }, { "KL/chosen_KL_mean": -20.055164337158203, "KL/mean": -28.295562744140625, "KL/rejected_KL_mean": -36.53596115112305, "KL/std": 20.62320327758789, "epoch": 0.29931972789115646, "fcm_dpo/beta": 0.0365363247692585, "fcm_dpo/delta": -0.1616281270980835, "fcm_dpo/margin": 16.480792999267578, "fcm_dpo/q_t": 0.36763936281204224, "grad_norm": 17.987558364868164, "learning_rate": 4.432001773500957e-07, "logits/chosen": 0.4926878809928894, "logits/rejected": 0.4490898847579956, "logps/chosen": -79.83784484863281, "logps/ref_chosen": -59.78268051147461, "logps/ref_rejected": -72.24533081054688, "logps/rejected": -108.78128814697266, "loss": 1.0007, "margin_dpo/margin_mean": 16.48079490661621, "margin_dpo/margin_std": 20.50601577758789, "step": 198 }, { "KL/chosen_KL_mean": -24.607725143432617, "KL/mean": -31.082088470458984, "KL/rejected_KL_mean": -37.556453704833984, "KL/std": 21.534305572509766, "epoch": 0.30083144368858655, "fcm_dpo/beta": 0.03629198670387268, "fcm_dpo/delta": -0.022943876683712006, "fcm_dpo/margin": 12.948731422424316, "fcm_dpo/q_t": 0.3984564244747162, "grad_norm": 16.96387481689453, "learning_rate": 4.4235833440297856e-07, "logits/chosen": 0.4463760256767273, "logits/rejected": 0.3538878560066223, "logps/chosen": -80.99449157714844, "logps/ref_chosen": -56.38677215576172, "logps/ref_rejected": -74.56779479980469, "logps/rejected": -112.12425231933594, "loss": 1.1585, "margin_dpo/margin_mean": 12.948731422424316, "margin_dpo/margin_std": 24.406784057617188, "step": 199 }, { "KL/chosen_KL_mean": -19.250396728515625, "KL/mean": -28.578182220458984, "KL/rejected_KL_mean": -37.905967712402344, "KL/std": 22.57465362548828, "epoch": 0.30234315948601664, "fcm_dpo/beta": 0.03459034115076065, "fcm_dpo/delta": -0.21037332713603973, "fcm_dpo/margin": 18.655567169189453, "fcm_dpo/q_t": 0.36464548110961914, "grad_norm": 16.139589309692383, "learning_rate": 4.415111107797445e-07, "logits/chosen": 0.4873886704444885, "logits/rejected": 0.40935078263282776, "logps/chosen": -77.07472229003906, "logps/ref_chosen": -57.82432556152344, "logps/ref_rejected": -89.28246307373047, "logps/rejected": -127.18843078613281, "loss": 1.0284, "margin_dpo/margin_mean": 18.655567169189453, "margin_dpo/margin_std": 26.809616088867188, "step": 200 }, { "KL/chosen_KL_mean": -27.423805236816406, "KL/mean": -34.75459289550781, "KL/rejected_KL_mean": -42.08538055419922, "KL/std": 23.84355926513672, "epoch": 0.30385487528344673, "fcm_dpo/beta": 0.033915258944034576, "fcm_dpo/delta": -0.0508701354265213, "fcm_dpo/margin": 14.661575317382812, "fcm_dpo/q_t": 0.3947141766548157, "grad_norm": 18.19431495666504, "learning_rate": 4.4065853017905953e-07, "logits/chosen": 0.5265877842903137, "logits/rejected": 0.4765250086784363, "logps/chosen": -86.42356872558594, "logps/ref_chosen": -58.999759674072266, "logps/ref_rejected": -84.67575073242188, "logps/rejected": -126.76112365722656, "loss": 1.1112, "margin_dpo/margin_mean": 14.661576271057129, "margin_dpo/margin_std": 24.935222625732422, "step": 201 }, { "KL/chosen_KL_mean": -22.689090728759766, "KL/mean": -31.702587127685547, "KL/rejected_KL_mean": -40.7160758972168, "KL/std": 22.56459617614746, "epoch": 0.30536659108087677, "fcm_dpo/beta": 0.033367373049259186, "fcm_dpo/delta": -0.1608964502811432, "fcm_dpo/margin": 18.026988983154297, "fcm_dpo/q_t": 0.36734747886657715, "grad_norm": 19.372838973999023, "learning_rate": 4.3980061644943575e-07, "logits/chosen": 0.42092597484588623, "logits/rejected": 0.34300437569618225, "logps/chosen": -70.34973907470703, "logps/ref_chosen": -47.660648345947266, "logps/ref_rejected": -73.63249969482422, "logps/rejected": -114.34857177734375, "loss": 1.0262, "margin_dpo/margin_mean": 18.026988983154297, "margin_dpo/margin_std": 24.56505012512207, "step": 202 }, { "KL/chosen_KL_mean": -28.01001739501953, "KL/mean": -36.113922119140625, "KL/rejected_KL_mean": -44.21782684326172, "KL/std": 23.278791427612305, "epoch": 0.30687830687830686, "fcm_dpo/beta": 0.03262447193264961, "fcm_dpo/delta": -0.08308503031730652, "fcm_dpo/margin": 16.207809448242188, "fcm_dpo/q_t": 0.3873659372329712, "grad_norm": 19.63701820373535, "learning_rate": 4.3893739358856455e-07, "logits/chosen": 0.4732247591018677, "logits/rejected": 0.3917619287967682, "logps/chosen": -90.33555603027344, "logps/ref_chosen": -62.32553482055664, "logps/ref_rejected": -99.37226104736328, "logps/rejected": -143.590087890625, "loss": 1.0772, "margin_dpo/margin_mean": 16.207809448242188, "margin_dpo/margin_std": 25.38452911376953, "step": 203 }, { "KL/chosen_KL_mean": -25.417871475219727, "KL/mean": -34.3574104309082, "KL/rejected_KL_mean": -43.29694366455078, "KL/std": 24.221799850463867, "epoch": 0.30839002267573695, "fcm_dpo/beta": 0.031458836048841476, "fcm_dpo/delta": -0.1234700083732605, "fcm_dpo/margin": 17.87906837463379, "fcm_dpo/q_t": 0.37869489192962646, "grad_norm": 17.177324295043945, "learning_rate": 4.380688857426449e-07, "logits/chosen": 0.45654842257499695, "logits/rejected": 0.3831368684768677, "logps/chosen": -76.04718780517578, "logps/ref_chosen": -50.62931823730469, "logps/ref_rejected": -66.60475158691406, "logps/rejected": -109.90169525146484, "loss": 1.0412, "margin_dpo/margin_mean": 17.87906837463379, "margin_dpo/margin_std": 24.354957580566406, "step": 204 }, { "KL/chosen_KL_mean": -29.891223907470703, "KL/mean": -38.5206298828125, "KL/rejected_KL_mean": -47.150020599365234, "KL/std": 25.599483489990234, "epoch": 0.30990173847316704, "fcm_dpo/beta": 0.03112097829580307, "fcm_dpo/delta": -0.0924113467335701, "fcm_dpo/margin": 17.258800506591797, "fcm_dpo/q_t": 0.38659146428108215, "grad_norm": 20.992780685424805, "learning_rate": 4.3719511720570814e-07, "logits/chosen": 0.49310895800590515, "logits/rejected": 0.4214407801628113, "logps/chosen": -100.24740600585938, "logps/ref_chosen": -70.3561782836914, "logps/ref_rejected": -93.39848327636719, "logps/rejected": -140.54849243164062, "loss": 1.0924, "margin_dpo/margin_mean": 17.258800506591797, "margin_dpo/margin_std": 28.509033203125, "step": 205 }, { "KL/chosen_KL_mean": -32.124385833740234, "KL/mean": -38.33163070678711, "KL/rejected_KL_mean": -44.53887176513672, "KL/std": 24.964656829833984, "epoch": 0.31141345427059713, "fcm_dpo/beta": 0.03151599317789078, "fcm_dpo/delta": 0.0600641593337059, "fcm_dpo/margin": 12.414488792419434, "fcm_dpo/q_t": 0.419567734003067, "grad_norm": 18.595619201660156, "learning_rate": 4.363161124189387e-07, "logits/chosen": 0.5298315286636353, "logits/rejected": 0.5128300189971924, "logps/chosen": -99.76986694335938, "logps/ref_chosen": -67.64547729492188, "logps/ref_rejected": -79.89584350585938, "logps/rejected": -124.4347152709961, "loss": 1.2328, "margin_dpo/margin_mean": 12.414488792419434, "margin_dpo/margin_std": 29.71973419189453, "step": 206 }, { "KL/chosen_KL_mean": -38.06774139404297, "KL/mean": -46.9461784362793, "KL/rejected_KL_mean": -55.82461929321289, "KL/std": 26.254493713378906, "epoch": 0.3129251700680272, "fcm_dpo/beta": 0.030902881175279617, "fcm_dpo/delta": -0.1046326532959938, "fcm_dpo/margin": 17.756877899169922, "fcm_dpo/q_t": 0.38568490743637085, "grad_norm": 16.643341064453125, "learning_rate": 4.3543189596998986e-07, "logits/chosen": 0.4498439431190491, "logits/rejected": 0.3794988989830017, "logps/chosen": -105.73193359375, "logps/ref_chosen": -67.66419219970703, "logps/ref_rejected": -85.10249328613281, "logps/rejected": -140.92710876464844, "loss": 1.0656, "margin_dpo/margin_mean": 17.756879806518555, "margin_dpo/margin_std": 27.332908630371094, "step": 207 }, { "KL/chosen_KL_mean": -30.73422622680664, "KL/mean": -35.53645706176758, "KL/rejected_KL_mean": -40.33868408203125, "KL/std": 24.153095245361328, "epoch": 0.3144368858654573, "fcm_dpo/beta": 0.031466640532016754, "fcm_dpo/delta": 0.152464359998703, "fcm_dpo/margin": 9.60445499420166, "fcm_dpo/q_t": 0.4369770884513855, "grad_norm": 23.85514259338379, "learning_rate": 4.3454249259229664e-07, "logits/chosen": 0.4734978973865509, "logits/rejected": 0.4461069107055664, "logps/chosen": -88.4659423828125, "logps/ref_chosen": -57.731712341308594, "logps/ref_rejected": -74.19276428222656, "logps/rejected": -114.53144836425781, "loss": 1.2801, "margin_dpo/margin_mean": 9.604454040527344, "margin_dpo/margin_std": 27.349924087524414, "step": 208 }, { "KL/chosen_KL_mean": -31.75300407409668, "KL/mean": -42.757266998291016, "KL/rejected_KL_mean": -53.761539459228516, "KL/std": 27.966007232666016, "epoch": 0.31594860166288735, "fcm_dpo/beta": 0.03062412329018116, "fcm_dpo/delta": -0.24015069007873535, "fcm_dpo/margin": 22.008533477783203, "fcm_dpo/q_t": 0.3586593270301819, "grad_norm": 19.86554718017578, "learning_rate": 4.336479271643833e-07, "logits/chosen": 0.4041667580604553, "logits/rejected": 0.34923887252807617, "logps/chosen": -100.30308532714844, "logps/ref_chosen": -68.55007934570312, "logps/ref_rejected": -87.90541076660156, "logps/rejected": -141.66696166992188, "loss": 1.0325, "margin_dpo/margin_mean": 22.008533477783203, "margin_dpo/margin_std": 32.60284423828125, "step": 209 }, { "KL/chosen_KL_mean": -31.51213264465332, "KL/mean": -42.13514709472656, "KL/rejected_KL_mean": -52.75815200805664, "KL/std": 29.606407165527344, "epoch": 0.31746031746031744, "fcm_dpo/beta": 0.029575884342193604, "fcm_dpo/delta": -0.18992537260055542, "fcm_dpo/margin": 21.24602508544922, "fcm_dpo/q_t": 0.369351863861084, "grad_norm": 18.382568359375, "learning_rate": 4.327482247091679e-07, "logits/chosen": 0.5498272776603699, "logits/rejected": 0.443499356508255, "logps/chosen": -88.78041076660156, "logps/ref_chosen": -57.268272399902344, "logps/ref_rejected": -85.72807312011719, "logps/rejected": -138.48623657226562, "loss": 1.0245, "margin_dpo/margin_mean": 21.24602508544922, "margin_dpo/margin_std": 29.765714645385742, "step": 210 }, { "KL/chosen_KL_mean": -28.97467613220215, "KL/mean": -39.192665100097656, "KL/rejected_KL_mean": -49.41065216064453, "KL/std": 26.596416473388672, "epoch": 0.31897203325774753, "fcm_dpo/beta": 0.028674710541963577, "fcm_dpo/delta": -0.14435598254203796, "fcm_dpo/margin": 20.435976028442383, "fcm_dpo/q_t": 0.37782740592956543, "grad_norm": 21.482284545898438, "learning_rate": 4.3184341039326217e-07, "logits/chosen": 0.5518479347229004, "logits/rejected": 0.44849225878715515, "logps/chosen": -82.61538696289062, "logps/ref_chosen": -53.640708923339844, "logps/ref_rejected": -93.0387954711914, "logps/rejected": -142.44944763183594, "loss": 1.0265, "margin_dpo/margin_mean": 20.435976028442383, "margin_dpo/margin_std": 28.254350662231445, "step": 211 }, { "KL/chosen_KL_mean": -33.499977111816406, "KL/mean": -44.172698974609375, "KL/rejected_KL_mean": -54.845420837402344, "KL/std": 29.302959442138672, "epoch": 0.3204837490551776, "fcm_dpo/beta": 0.02764958329498768, "fcm_dpo/delta": -0.14908015727996826, "fcm_dpo/margin": 21.345449447631836, "fcm_dpo/q_t": 0.37234535813331604, "grad_norm": 14.397629737854004, "learning_rate": 4.309335095262675e-07, "logits/chosen": 0.5605090856552124, "logits/rejected": 0.48153620958328247, "logps/chosen": -90.86671447753906, "logps/ref_chosen": -57.36674499511719, "logps/ref_rejected": -79.89643096923828, "logps/rejected": -134.74185180664062, "loss": 1.0431, "margin_dpo/margin_mean": 21.345447540283203, "margin_dpo/margin_std": 30.819881439208984, "step": 212 }, { "KL/chosen_KL_mean": -26.205650329589844, "KL/mean": -38.57976531982422, "KL/rejected_KL_mean": -50.953880310058594, "KL/std": 29.666786193847656, "epoch": 0.3219954648526077, "fcm_dpo/beta": 0.026420973241329193, "fcm_dpo/delta": -0.22006627917289734, "fcm_dpo/margin": 24.748226165771484, "fcm_dpo/q_t": 0.3609957993030548, "grad_norm": 13.625341415405273, "learning_rate": 4.3001854756006724e-07, "logits/chosen": 0.49518492817878723, "logits/rejected": 0.47064149379730225, "logps/chosen": -91.42676544189453, "logps/ref_chosen": -65.22111511230469, "logps/ref_rejected": -80.1810302734375, "logps/rejected": -131.13490295410156, "loss": 1.0003, "margin_dpo/margin_mean": 24.748226165771484, "margin_dpo/margin_std": 33.10297775268555, "step": 213 }, { "KL/chosen_KL_mean": -31.163930892944336, "KL/mean": -43.70262145996094, "KL/rejected_KL_mean": -56.24131774902344, "KL/std": 29.9415340423584, "epoch": 0.3235071806500378, "fcm_dpo/beta": 0.025676624849438667, "fcm_dpo/delta": -0.2068711519241333, "fcm_dpo/margin": 25.077390670776367, "fcm_dpo/q_t": 0.3648650646209717, "grad_norm": 16.90061378479004, "learning_rate": 4.290985500881143e-07, "logits/chosen": 0.3979254364967346, "logits/rejected": 0.3791281282901764, "logps/chosen": -92.45626068115234, "logps/ref_chosen": -61.292327880859375, "logps/ref_rejected": -67.69841003417969, "logps/rejected": -123.93972778320312, "loss": 1.0147, "margin_dpo/margin_mean": 25.077390670776367, "margin_dpo/margin_std": 33.935569763183594, "step": 214 }, { "KL/chosen_KL_mean": -36.773006439208984, "KL/mean": -49.322052001953125, "KL/rejected_KL_mean": -61.87109375, "KL/std": 31.468637466430664, "epoch": 0.3250188964474679, "fcm_dpo/beta": 0.024757249280810356, "fcm_dpo/delta": -0.18273316323757172, "fcm_dpo/margin": 25.09808349609375, "fcm_dpo/q_t": 0.3704993426799774, "grad_norm": 15.784854888916016, "learning_rate": 4.281735428447157e-07, "logits/chosen": 0.41309452056884766, "logits/rejected": 0.29789355397224426, "logps/chosen": -100.64214324951172, "logps/ref_chosen": -63.869136810302734, "logps/ref_rejected": -98.7657241821289, "logps/rejected": -160.63681030273438, "loss": 1.0395, "margin_dpo/margin_mean": 25.09808349609375, "margin_dpo/margin_std": 35.794097900390625, "step": 215 }, { "KL/chosen_KL_mean": -31.85125732421875, "KL/mean": -45.10109329223633, "KL/rejected_KL_mean": -58.35092544555664, "KL/std": 30.88085174560547, "epoch": 0.32653061224489793, "fcm_dpo/beta": 0.02361372672021389, "fcm_dpo/delta": -0.18767762184143066, "fcm_dpo/margin": 26.49966812133789, "fcm_dpo/q_t": 0.36168473958969116, "grad_norm": 23.61011505126953, "learning_rate": 4.2724355170431247e-07, "logits/chosen": 0.5278521180152893, "logits/rejected": 0.4376213550567627, "logps/chosen": -99.67620849609375, "logps/ref_chosen": -67.824951171875, "logps/ref_rejected": -96.40231323242188, "logps/rejected": -154.75323486328125, "loss": 0.9764, "margin_dpo/margin_mean": 26.49966812133789, "margin_dpo/margin_std": 31.30062484741211, "step": 216 }, { "KL/chosen_KL_mean": -35.986595153808594, "KL/mean": -49.955936431884766, "KL/rejected_KL_mean": -63.92528533935547, "KL/std": 32.65003204345703, "epoch": 0.328042328042328, "fcm_dpo/beta": 0.022649064660072327, "fcm_dpo/delta": -0.1961820423603058, "fcm_dpo/margin": 27.938690185546875, "fcm_dpo/q_t": 0.3633325695991516, "grad_norm": 13.823315620422363, "learning_rate": 4.26308602680756e-07, "logits/chosen": 0.5091556310653687, "logits/rejected": 0.4011778235435486, "logps/chosen": -96.49159240722656, "logps/ref_chosen": -60.5049934387207, "logps/ref_rejected": -84.26618194580078, "logps/rejected": -148.19146728515625, "loss": 0.9818, "margin_dpo/margin_mean": 27.938690185546875, "margin_dpo/margin_std": 34.23504638671875, "step": 217 }, { "KL/chosen_KL_mean": -39.407283782958984, "KL/mean": -47.98320007324219, "KL/rejected_KL_mean": -56.559120178222656, "KL/std": 31.762195587158203, "epoch": 0.3295540438397581, "fcm_dpo/beta": 0.022890662774443626, "fcm_dpo/delta": 0.05783740431070328, "fcm_dpo/margin": 17.151836395263672, "fcm_dpo/q_t": 0.41548776626586914, "grad_norm": 16.205947875976562, "learning_rate": 4.253687219265803e-07, "logits/chosen": 0.37078291177749634, "logits/rejected": 0.370650053024292, "logps/chosen": -110.00160217285156, "logps/ref_chosen": -70.59431457519531, "logps/ref_rejected": -73.89038848876953, "logps/rejected": -130.4495086669922, "loss": 1.2118, "margin_dpo/margin_mean": 17.151836395263672, "margin_dpo/margin_std": 37.95496368408203, "step": 218 }, { "KL/chosen_KL_mean": -37.35448455810547, "KL/mean": -46.43778991699219, "KL/rejected_KL_mean": -55.521095275878906, "KL/std": 32.03618621826172, "epoch": 0.3310657596371882, "fcm_dpo/beta": 0.022805888205766678, "fcm_dpo/delta": 0.03707420825958252, "fcm_dpo/margin": 18.166610717773438, "fcm_dpo/q_t": 0.40925899147987366, "grad_norm": 15.538980484008789, "learning_rate": 4.2442393573227043e-07, "logits/chosen": 0.4556928873062134, "logits/rejected": 0.4128846526145935, "logps/chosen": -97.84542846679688, "logps/ref_chosen": -60.490943908691406, "logps/ref_rejected": -75.85001373291016, "logps/rejected": -131.37110900878906, "loss": 1.1334, "margin_dpo/margin_mean": 18.166610717773438, "margin_dpo/margin_std": 31.397647857666016, "step": 219 }, { "KL/chosen_KL_mean": -31.5102596282959, "KL/mean": -42.39979553222656, "KL/rejected_KL_mean": -53.28932571411133, "KL/std": 33.013694763183594, "epoch": 0.3325774754346183, "fcm_dpo/beta": 0.022611908614635468, "fcm_dpo/delta": -0.046144720166921616, "fcm_dpo/margin": 21.7790584564209, "fcm_dpo/q_t": 0.3967788815498352, "grad_norm": 15.905135154724121, "learning_rate": 4.234742705255272e-07, "logits/chosen": 0.5297170877456665, "logits/rejected": 0.4632262587547302, "logps/chosen": -76.5236587524414, "logps/ref_chosen": -45.013397216796875, "logps/ref_rejected": -70.49369812011719, "logps/rejected": -123.78302764892578, "loss": 1.1211, "margin_dpo/margin_mean": 21.77906036376953, "margin_dpo/margin_std": 37.682254791259766, "step": 220 }, { "KL/chosen_KL_mean": -31.15220069885254, "KL/mean": -43.39739990234375, "KL/rejected_KL_mean": -55.642601013183594, "KL/std": 32.60444641113281, "epoch": 0.3340891912320484, "fcm_dpo/beta": 0.02241317182779312, "fcm_dpo/delta": -0.10453681647777557, "fcm_dpo/margin": 24.490406036376953, "fcm_dpo/q_t": 0.384391188621521, "grad_norm": 17.505321502685547, "learning_rate": 4.22519752870528e-07, "logits/chosen": 0.49240219593048096, "logits/rejected": 0.4145415425300598, "logps/chosen": -90.24803924560547, "logps/ref_chosen": -59.09584045410156, "logps/ref_rejected": -88.64388275146484, "logps/rejected": -144.28648376464844, "loss": 1.0602, "margin_dpo/margin_mean": 24.490407943725586, "margin_dpo/margin_std": 36.9871940612793, "step": 221 }, { "KL/chosen_KL_mean": -32.00490951538086, "KL/mean": -48.416385650634766, "KL/rejected_KL_mean": -64.8278579711914, "KL/std": 37.11773681640625, "epoch": 0.3356009070294785, "fcm_dpo/beta": 0.02136814594268799, "fcm_dpo/delta": -0.2705325782299042, "fcm_dpo/margin": 32.82295227050781, "fcm_dpo/q_t": 0.35026657581329346, "grad_norm": 14.998271942138672, "learning_rate": 4.2156040946718343e-07, "logits/chosen": 0.5728538036346436, "logits/rejected": 0.47882080078125, "logps/chosen": -88.00260162353516, "logps/ref_chosen": -55.9976921081543, "logps/ref_rejected": -111.94727325439453, "logps/rejected": -176.77513122558594, "loss": 0.9547, "margin_dpo/margin_mean": 32.82295227050781, "margin_dpo/margin_std": 39.01953125, "step": 222 }, { "KL/chosen_KL_mean": -37.9762077331543, "KL/mean": -51.26795959472656, "KL/rejected_KL_mean": -64.55970764160156, "KL/std": 35.981632232666016, "epoch": 0.3371126228269085, "fcm_dpo/beta": 0.020683202892541885, "fcm_dpo/delta": -0.10624384880065918, "fcm_dpo/margin": 26.58349609375, "fcm_dpo/q_t": 0.3813319802284241, "grad_norm": 15.701749801635742, "learning_rate": 4.2059626715039065e-07, "logits/chosen": 0.5513671636581421, "logits/rejected": 0.49275338649749756, "logps/chosen": -97.86763000488281, "logps/ref_chosen": -59.891422271728516, "logps/ref_rejected": -86.28954315185547, "logps/rejected": -150.8492431640625, "loss": 1.0349, "margin_dpo/margin_mean": 26.58349609375, "margin_dpo/margin_std": 36.03096008300781, "step": 223 }, { "KL/chosen_KL_mean": -43.705589294433594, "KL/mean": -51.3037109375, "KL/rejected_KL_mean": -58.901832580566406, "KL/std": 35.61591339111328, "epoch": 0.3386243386243386, "fcm_dpo/beta": 0.02102397009730339, "fcm_dpo/delta": 0.13499858975410461, "fcm_dpo/margin": 15.196237564086914, "fcm_dpo/q_t": 0.43189573287963867, "grad_norm": 20.207576751708984, "learning_rate": 4.1962735288928304e-07, "logits/chosen": 0.5725216269493103, "logits/rejected": 0.5493526458740234, "logps/chosen": -107.75022888183594, "logps/ref_chosen": -64.04463195800781, "logps/ref_rejected": -75.05450439453125, "logps/rejected": -133.95632934570312, "loss": 1.2345, "margin_dpo/margin_mean": 15.196236610412598, "margin_dpo/margin_std": 37.5325927734375, "step": 224 }, { "KL/chosen_KL_mean": -41.65528869628906, "KL/mean": -56.287723541259766, "KL/rejected_KL_mean": -70.92013549804688, "KL/std": 39.70849609375, "epoch": 0.3401360544217687, "fcm_dpo/beta": 0.02054654248058796, "fcm_dpo/delta": -0.16328230500221252, "fcm_dpo/margin": 29.264860153198242, "fcm_dpo/q_t": 0.3704851269721985, "grad_norm": 15.133810043334961, "learning_rate": 4.186536937864752e-07, "logits/chosen": 0.5531373620033264, "logits/rejected": 0.4349825978279114, "logps/chosen": -107.75111389160156, "logps/ref_chosen": -66.0958251953125, "logps/ref_rejected": -97.68675231933594, "logps/rejected": -168.6068878173828, "loss": 1.0044, "margin_dpo/margin_mean": 29.26485824584961, "margin_dpo/margin_std": 37.1306266784668, "step": 225 }, { "KL/chosen_KL_mean": -38.91139221191406, "KL/mean": -50.595481872558594, "KL/rejected_KL_mean": -62.27956771850586, "KL/std": 36.265830993652344, "epoch": 0.3416477702191988, "fcm_dpo/beta": 0.020526161417365074, "fcm_dpo/delta": -0.03146500885486603, "fcm_dpo/margin": 23.368175506591797, "fcm_dpo/q_t": 0.3965521454811096, "grad_norm": 15.107815742492676, "learning_rate": 4.176753170773052e-07, "logits/chosen": 0.6142418384552002, "logits/rejected": 0.5654819011688232, "logps/chosen": -90.32826232910156, "logps/ref_chosen": -51.4168701171875, "logps/ref_rejected": -66.30068969726562, "logps/rejected": -128.58026123046875, "loss": 1.1383, "margin_dpo/margin_mean": 23.368173599243164, "margin_dpo/margin_std": 42.59581756591797, "step": 226 }, { "KL/chosen_KL_mean": -41.923866271972656, "KL/mean": -54.000038146972656, "KL/rejected_KL_mean": -66.07620239257812, "KL/std": 39.64067840576172, "epoch": 0.3431594860166289, "fcm_dpo/beta": 0.02022862620651722, "fcm_dpo/delta": -0.04073462262749672, "fcm_dpo/margin": 24.15234375, "fcm_dpo/q_t": 0.39708375930786133, "grad_norm": 17.119104385375977, "learning_rate": 4.166922501290729e-07, "logits/chosen": 0.62053382396698, "logits/rejected": 0.5810697078704834, "logps/chosen": -99.91365051269531, "logps/ref_chosen": -57.989776611328125, "logps/ref_rejected": -75.05464172363281, "logps/rejected": -141.130859375, "loss": 1.1322, "margin_dpo/margin_mean": 24.152341842651367, "margin_dpo/margin_std": 43.917694091796875, "step": 227 }, { "KL/chosen_KL_mean": -44.68891525268555, "KL/mean": -58.21568298339844, "KL/rejected_KL_mean": -71.7424545288086, "KL/std": 37.54546356201172, "epoch": 0.34467120181405897, "fcm_dpo/beta": 0.020018167793750763, "fcm_dpo/delta": -0.09673406183719635, "fcm_dpo/margin": 27.053539276123047, "fcm_dpo/q_t": 0.38233309984207153, "grad_norm": 15.651515007019043, "learning_rate": 4.1570452044027405e-07, "logits/chosen": 0.6193653345108032, "logits/rejected": 0.5345925688743591, "logps/chosen": -100.24828338623047, "logps/ref_chosen": -55.55936813354492, "logps/ref_rejected": -77.02364349365234, "logps/rejected": -148.76609802246094, "loss": 1.052, "margin_dpo/margin_mean": 27.053539276123047, "margin_dpo/margin_std": 38.620765686035156, "step": 228 }, { "KL/chosen_KL_mean": -37.900203704833984, "KL/mean": -49.77465057373047, "KL/rejected_KL_mean": -61.64910125732422, "KL/std": 36.71519470214844, "epoch": 0.34618291761148906, "fcm_dpo/beta": 0.019772969186306, "fcm_dpo/delta": -0.020564941689372063, "fcm_dpo/margin": 23.748897552490234, "fcm_dpo/q_t": 0.3996862471103668, "grad_norm": 23.846460342407227, "learning_rate": 4.147121556398312e-07, "logits/chosen": 0.6539832353591919, "logits/rejected": 0.5783480405807495, "logps/chosen": -88.69486999511719, "logps/ref_chosen": -50.79466247558594, "logps/ref_rejected": -78.4474105834961, "logps/rejected": -140.0965118408203, "loss": 1.1464, "margin_dpo/margin_mean": 23.748897552490234, "margin_dpo/margin_std": 44.864593505859375, "step": 229 }, { "KL/chosen_KL_mean": -41.51421356201172, "KL/mean": -55.32987976074219, "KL/rejected_KL_mean": -69.14554595947266, "KL/std": 40.503692626953125, "epoch": 0.3476946334089191, "fcm_dpo/beta": 0.019660290330648422, "fcm_dpo/delta": -0.09952552616596222, "fcm_dpo/margin": 27.631336212158203, "fcm_dpo/q_t": 0.3826754689216614, "grad_norm": 14.859014511108398, "learning_rate": 4.137151834863213e-07, "logits/chosen": 0.569249153137207, "logits/rejected": 0.5702674388885498, "logps/chosen": -98.24343872070312, "logps/ref_chosen": -56.729225158691406, "logps/ref_rejected": -62.99180603027344, "logps/rejected": -132.13735961914062, "loss": 1.0567, "margin_dpo/margin_mean": 27.631336212158203, "margin_dpo/margin_std": 39.70343017578125, "step": 230 }, { "KL/chosen_KL_mean": -47.226043701171875, "KL/mean": -67.83246612548828, "KL/rejected_KL_mean": -88.43889617919922, "KL/std": 39.61866760253906, "epoch": 0.3492063492063492, "fcm_dpo/beta": 0.018344787880778313, "fcm_dpo/delta": -0.33336055278778076, "fcm_dpo/margin": 41.21284484863281, "fcm_dpo/q_t": 0.33213451504707336, "grad_norm": 15.280308723449707, "learning_rate": 4.1271363186719835e-07, "logits/chosen": 0.5038709044456482, "logits/rejected": 0.499245822429657, "logps/chosen": -119.8231430053711, "logps/ref_chosen": -72.59709930419922, "logps/ref_rejected": -86.2322998046875, "logps/rejected": -174.67120361328125, "loss": 0.8998, "margin_dpo/margin_mean": 41.21284484863281, "margin_dpo/margin_std": 41.88604736328125, "step": 231 }, { "KL/chosen_KL_mean": -47.655906677246094, "KL/mean": -61.479305267333984, "KL/rejected_KL_mean": -75.30270385742188, "KL/std": 43.63159942626953, "epoch": 0.3507180650037793, "fcm_dpo/beta": 0.017985675483942032, "fcm_dpo/delta": -0.049727872014045715, "fcm_dpo/margin": 27.646800994873047, "fcm_dpo/q_t": 0.39663565158843994, "grad_norm": 15.373372077941895, "learning_rate": 4.1170752879801436e-07, "logits/chosen": 0.49574434757232666, "logits/rejected": 0.46651044487953186, "logps/chosen": -115.7744369506836, "logps/ref_chosen": -68.1185302734375, "logps/ref_rejected": -83.79415893554688, "logps/rejected": -159.09686279296875, "loss": 1.1249, "margin_dpo/margin_mean": 27.646799087524414, "margin_dpo/margin_std": 49.709197998046875, "step": 232 }, { "KL/chosen_KL_mean": -54.42681121826172, "KL/mean": -64.93189239501953, "KL/rejected_KL_mean": -75.43698120117188, "KL/std": 40.570587158203125, "epoch": 0.35222978080120937, "fcm_dpo/beta": 0.017682647332549095, "fcm_dpo/delta": -0.07306541502475739, "fcm_dpo/margin": 21.010177612304688, "fcm_dpo/q_t": 0.4198026657104492, "grad_norm": 15.090434074401855, "learning_rate": 4.106969024216348e-07, "logits/chosen": 0.6033905148506165, "logits/rejected": 0.5428017377853394, "logps/chosen": -109.49696350097656, "logps/ref_chosen": -55.070152282714844, "logps/ref_rejected": -66.61845397949219, "logps/rejected": -142.05543518066406, "loss": 1.1757, "margin_dpo/margin_mean": 21.010177612304688, "margin_dpo/margin_std": 40.30328369140625, "step": 233 }, { "KL/chosen_KL_mean": -48.80119323730469, "KL/mean": -59.234046936035156, "KL/rejected_KL_mean": -69.66690063476562, "KL/std": 39.58076477050781, "epoch": 0.35374149659863946, "fcm_dpo/beta": 0.017503011971712112, "fcm_dpo/delta": -0.10263454169034958, "fcm_dpo/margin": 20.865713119506836, "fcm_dpo/q_t": 0.4213663339614868, "grad_norm": 20.658870697021484, "learning_rate": 4.09681781007452e-07, "logits/chosen": 0.5194283723831177, "logits/rejected": 0.5139452815055847, "logps/chosen": -104.72708892822266, "logps/ref_chosen": -55.92589569091797, "logps/ref_rejected": -51.11608123779297, "logps/rejected": -120.7829818725586, "loss": 1.2047, "margin_dpo/margin_mean": 20.865711212158203, "margin_dpo/margin_std": 42.82243347167969, "step": 234 }, { "KL/chosen_KL_mean": -43.279945373535156, "KL/mean": -61.363121032714844, "KL/rejected_KL_mean": -79.44630432128906, "KL/std": 42.8857421875, "epoch": 0.35525321239606955, "fcm_dpo/beta": 0.016922399401664734, "fcm_dpo/delta": -0.17231330275535583, "fcm_dpo/margin": 36.166358947753906, "fcm_dpo/q_t": 0.36375778913497925, "grad_norm": 16.162338256835938, "learning_rate": 4.08662192950594e-07, "logits/chosen": 0.5850652456283569, "logits/rejected": 0.571670651435852, "logps/chosen": -107.81967163085938, "logps/ref_chosen": -64.53972625732422, "logps/ref_rejected": -77.69151306152344, "logps/rejected": -157.1378173828125, "loss": 0.9736, "margin_dpo/margin_mean": 36.166358947753906, "margin_dpo/margin_std": 41.02494812011719, "step": 235 }, { "KL/chosen_KL_mean": -64.5616683959961, "KL/mean": -78.20289611816406, "KL/rejected_KL_mean": -91.8441390991211, "KL/std": 41.81065368652344, "epoch": 0.35676492819349964, "fcm_dpo/beta": 0.01667531207203865, "fcm_dpo/delta": -0.005363345146179199, "fcm_dpo/margin": 27.282466888427734, "fcm_dpo/q_t": 0.40045133233070374, "grad_norm": 14.700178146362305, "learning_rate": 4.076381667711306e-07, "logits/chosen": 0.5476399660110474, "logits/rejected": 0.5346208810806274, "logps/chosen": -135.71640014648438, "logps/ref_chosen": -71.15473937988281, "logps/ref_rejected": -84.88541412353516, "logps/rejected": -176.72955322265625, "loss": 1.1252, "margin_dpo/margin_mean": 27.282466888427734, "margin_dpo/margin_std": 47.215057373046875, "step": 236 }, { "KL/chosen_KL_mean": -53.720584869384766, "KL/mean": -70.24567413330078, "KL/rejected_KL_mean": -86.77075958251953, "KL/std": 42.25255584716797, "epoch": 0.35827664399092973, "fcm_dpo/beta": 0.016512542963027954, "fcm_dpo/delta": -0.10114513337612152, "fcm_dpo/margin": 33.0501708984375, "fcm_dpo/q_t": 0.3799697458744049, "grad_norm": 17.659883499145508, "learning_rate": 4.066097311132753e-07, "logits/chosen": 0.5982592105865479, "logits/rejected": 0.5887188911437988, "logps/chosen": -129.8625946044922, "logps/ref_chosen": -76.14201354980469, "logps/ref_rejected": -80.88479614257812, "logps/rejected": -167.65554809570312, "loss": 1.0614, "margin_dpo/margin_mean": 33.0501708984375, "margin_dpo/margin_std": 48.059844970703125, "step": 237 }, { "KL/chosen_KL_mean": -48.632911682128906, "KL/mean": -64.87059020996094, "KL/rejected_KL_mean": -81.1082763671875, "KL/std": 43.56510925292969, "epoch": 0.35978835978835977, "fcm_dpo/beta": 0.016101723536849022, "fcm_dpo/delta": -0.0780644491314888, "fcm_dpo/margin": 32.475364685058594, "fcm_dpo/q_t": 0.38529616594314575, "grad_norm": 19.395490646362305, "learning_rate": 4.0557691474458414e-07, "logits/chosen": 0.5503574013710022, "logits/rejected": 0.5416954755783081, "logps/chosen": -117.51776123046875, "logps/ref_chosen": -68.88484954833984, "logps/ref_rejected": -75.8946304321289, "logps/rejected": -157.00289916992188, "loss": 1.0578, "margin_dpo/margin_mean": 32.475364685058594, "margin_dpo/margin_std": 45.807884216308594, "step": 238 }, { "KL/chosen_KL_mean": -59.37183380126953, "KL/mean": -75.34651947021484, "KL/rejected_KL_mean": -91.32120513916016, "KL/std": 47.33335876464844, "epoch": 0.36130007558578986, "fcm_dpo/beta": 0.016072561964392662, "fcm_dpo/delta": -0.06735628843307495, "fcm_dpo/margin": 31.949378967285156, "fcm_dpo/q_t": 0.38884609937667847, "grad_norm": 18.898448944091797, "learning_rate": 4.045397465551513e-07, "logits/chosen": 0.6873798966407776, "logits/rejected": 0.5454525947570801, "logps/chosen": -116.14366149902344, "logps/ref_chosen": -56.771827697753906, "logps/ref_rejected": -116.23050689697266, "logps/rejected": -207.5517120361328, "loss": 1.0838, "margin_dpo/margin_mean": 31.949382781982422, "margin_dpo/margin_std": 49.25836944580078, "step": 239 }, { "KL/chosen_KL_mean": -56.67302322387695, "KL/mean": -77.18383026123047, "KL/rejected_KL_mean": -97.69464111328125, "KL/std": 47.6438102722168, "epoch": 0.36281179138321995, "fcm_dpo/beta": 0.01546771451830864, "fcm_dpo/delta": -0.19688355922698975, "fcm_dpo/margin": 41.02162170410156, "fcm_dpo/q_t": 0.3613903522491455, "grad_norm": 13.303434371948242, "learning_rate": 4.0349825555680045e-07, "logits/chosen": 0.6007834672927856, "logits/rejected": 0.50172358751297, "logps/chosen": -110.02713012695312, "logps/ref_chosen": -53.35411071777344, "logps/ref_rejected": -80.12019348144531, "logps/rejected": -177.81484985351562, "loss": 0.9792, "margin_dpo/margin_mean": 41.02162170410156, "margin_dpo/margin_std": 49.39841842651367, "step": 240 }, { "KL/chosen_KL_mean": -53.514156341552734, "KL/mean": -66.90748596191406, "KL/rejected_KL_mean": -80.30081176757812, "KL/std": 44.207427978515625, "epoch": 0.36432350718065004, "fcm_dpo/beta": 0.015427043661475182, "fcm_dpo/delta": 0.03810626268386841, "fcm_dpo/margin": 26.786663055419922, "fcm_dpo/q_t": 0.4107113480567932, "grad_norm": 19.00992774963379, "learning_rate": 4.0245247088227377e-07, "logits/chosen": 0.5664269924163818, "logits/rejected": 0.5286130905151367, "logps/chosen": -125.4095687866211, "logps/ref_chosen": -71.89541625976562, "logps/ref_rejected": -83.03492736816406, "logps/rejected": -163.3357391357422, "loss": 1.1461, "margin_dpo/margin_mean": 26.786663055419922, "margin_dpo/margin_std": 48.930908203125, "step": 241 }, { "KL/chosen_KL_mean": -55.324440002441406, "KL/mean": -72.56484985351562, "KL/rejected_KL_mean": -89.80526733398438, "KL/std": 47.05958557128906, "epoch": 0.36583522297808013, "fcm_dpo/beta": 0.015138762071728706, "fcm_dpo/delta": -0.07740029692649841, "fcm_dpo/margin": 34.4808349609375, "fcm_dpo/q_t": 0.3867493271827698, "grad_norm": 13.885424613952637, "learning_rate": 4.0140242178441665e-07, "logits/chosen": 0.5691415071487427, "logits/rejected": 0.5489025115966797, "logps/chosen": -113.25187683105469, "logps/ref_chosen": -57.927433013916016, "logps/ref_rejected": -67.838623046875, "logps/rejected": -157.64389038085938, "loss": 1.0625, "margin_dpo/margin_mean": 34.4808349609375, "margin_dpo/margin_std": 50.14107894897461, "step": 242 }, { "KL/chosen_KL_mean": -53.59979248046875, "KL/mean": -69.44566345214844, "KL/rejected_KL_mean": -85.29151153564453, "KL/std": 46.36688232421875, "epoch": 0.3673469387755102, "fcm_dpo/beta": 0.015110740438103676, "fcm_dpo/delta": -0.030339926481246948, "fcm_dpo/margin": 31.691720962524414, "fcm_dpo/q_t": 0.39408785104751587, "grad_norm": 18.434778213500977, "learning_rate": 4.003481376353596e-07, "logits/chosen": 0.541643500328064, "logits/rejected": 0.5529664754867554, "logps/chosen": -127.87646484375, "logps/ref_chosen": -74.27667236328125, "logps/ref_rejected": -73.24340057373047, "logps/rejected": -158.534912109375, "loss": 1.0824, "margin_dpo/margin_mean": 31.691722869873047, "margin_dpo/margin_std": 47.64122772216797, "step": 243 }, { "KL/chosen_KL_mean": -50.46135711669922, "KL/mean": -71.55018615722656, "KL/rejected_KL_mean": -92.63902282714844, "KL/std": 46.358036041259766, "epoch": 0.3688586545729403, "fcm_dpo/beta": 0.014749385416507721, "fcm_dpo/delta": -0.18310438096523285, "fcm_dpo/margin": 42.177650451660156, "fcm_dpo/q_t": 0.3620183765888214, "grad_norm": 13.311697959899902, "learning_rate": 3.9928964792569654e-07, "logits/chosen": 0.6441433429718018, "logits/rejected": 0.5542799830436707, "logps/chosen": -103.82526397705078, "logps/ref_chosen": -53.36390686035156, "logps/ref_rejected": -71.10276794433594, "logps/rejected": -163.74179077148438, "loss": 0.9702, "margin_dpo/margin_mean": 42.177650451660156, "margin_dpo/margin_std": 47.35203552246094, "step": 244 }, { "KL/chosen_KL_mean": -55.57624816894531, "KL/mean": -79.77105712890625, "KL/rejected_KL_mean": -103.96587371826172, "KL/std": 47.97163391113281, "epoch": 0.37037037037037035, "fcm_dpo/beta": 0.014003526419401169, "fcm_dpo/delta": -0.24466592073440552, "fcm_dpo/margin": 48.389625549316406, "fcm_dpo/q_t": 0.3471682667732239, "grad_norm": 14.802033424377441, "learning_rate": 3.982269822636601e-07, "logits/chosen": 0.6564725637435913, "logits/rejected": 0.6286360025405884, "logps/chosen": -126.77135467529297, "logps/ref_chosen": -71.19510650634766, "logps/ref_rejected": -80.76235961914062, "logps/rejected": -184.72824096679688, "loss": 0.9235, "margin_dpo/margin_mean": 48.389625549316406, "margin_dpo/margin_std": 48.512237548828125, "step": 245 }, { "KL/chosen_KL_mean": -66.88098907470703, "KL/mean": -84.69398498535156, "KL/rejected_KL_mean": -102.5069580078125, "KL/std": 47.89185333251953, "epoch": 0.37188208616780044, "fcm_dpo/beta": 0.01382741890847683, "fcm_dpo/delta": -0.04510826617479324, "fcm_dpo/margin": 35.62596893310547, "fcm_dpo/q_t": 0.39292967319488525, "grad_norm": 15.253338813781738, "learning_rate": 3.971601703742932e-07, "logits/chosen": 0.641024112701416, "logits/rejected": 0.5778420567512512, "logps/chosen": -138.50204467773438, "logps/ref_chosen": -71.62104797363281, "logps/ref_rejected": -94.03392028808594, "logps/rejected": -196.5408935546875, "loss": 1.1137, "margin_dpo/margin_mean": 35.62596893310547, "margin_dpo/margin_std": 60.41918182373047, "step": 246 }, { "KL/chosen_KL_mean": -69.6776123046875, "KL/mean": -81.42636108398438, "KL/rejected_KL_mean": -93.17510223388672, "KL/std": 47.25940704345703, "epoch": 0.37339380196523053, "fcm_dpo/beta": 0.014045486226677895, "fcm_dpo/delta": 0.12305961549282074, "fcm_dpo/margin": 23.497482299804688, "fcm_dpo/q_t": 0.4268472194671631, "grad_norm": 18.581787109375, "learning_rate": 3.960892420986177e-07, "logits/chosen": 0.6367757320404053, "logits/rejected": 0.6267802715301514, "logps/chosen": -149.70016479492188, "logps/ref_chosen": -80.02254486083984, "logps/ref_rejected": -89.22705841064453, "logps/rejected": -182.40216064453125, "loss": 1.2146, "margin_dpo/margin_mean": 23.497482299804688, "margin_dpo/margin_std": 52.682167053222656, "step": 247 }, { "KL/chosen_KL_mean": -59.59392547607422, "KL/mean": -78.8796615600586, "KL/rejected_KL_mean": -98.1654052734375, "KL/std": 51.46784210205078, "epoch": 0.3749055177626606, "fcm_dpo/beta": 0.013925114646553993, "fcm_dpo/delta": -0.0919812023639679, "fcm_dpo/margin": 38.57147979736328, "fcm_dpo/q_t": 0.3852105438709259, "grad_norm": 16.99346351623535, "learning_rate": 3.9501422739279953e-07, "logits/chosen": 0.5982637405395508, "logits/rejected": 0.6556388139724731, "logps/chosen": -124.97187805175781, "logps/ref_chosen": -65.37796020507812, "logps/ref_rejected": -61.365787506103516, "logps/rejected": -159.53118896484375, "loss": 1.0684, "margin_dpo/margin_mean": 38.57147979736328, "margin_dpo/margin_std": 58.039794921875, "step": 248 }, { "KL/chosen_KL_mean": -75.0390625, "KL/mean": -82.37818908691406, "KL/rejected_KL_mean": -89.71731567382812, "KL/std": 47.77378845214844, "epoch": 0.3764172335600907, "fcm_dpo/beta": 0.013879897072911263, "fcm_dpo/delta": 0.04884837195277214, "fcm_dpo/margin": 14.678247451782227, "fcm_dpo/q_t": 0.453724205493927, "grad_norm": 19.524457931518555, "learning_rate": 3.9393515632731094e-07, "logits/chosen": 0.623253345489502, "logits/rejected": 0.6645709872245789, "logps/chosen": -149.64053344726562, "logps/ref_chosen": -74.60145568847656, "logps/ref_rejected": -63.79338455200195, "logps/rejected": -153.51071166992188, "loss": 1.3402, "margin_dpo/margin_mean": 14.67824649810791, "margin_dpo/margin_std": 55.51128387451172, "step": 249 }, { "KL/chosen_KL_mean": -65.30694580078125, "KL/mean": -85.24577331542969, "KL/rejected_KL_mean": -105.18460845947266, "KL/std": 52.569244384765625, "epoch": 0.3779289493575208, "fcm_dpo/beta": 0.013815690763294697, "fcm_dpo/delta": -0.10696772485971451, "fcm_dpo/margin": 39.87767028808594, "fcm_dpo/q_t": 0.3786598742008209, "grad_norm": 14.615215301513672, "learning_rate": 3.9285205908608934e-07, "logits/chosen": 0.6955363750457764, "logits/rejected": 0.6532368659973145, "logps/chosen": -127.24514770507812, "logps/ref_chosen": -61.938209533691406, "logps/ref_rejected": -72.21602630615234, "logps/rejected": -177.400634765625, "loss": 1.0392, "margin_dpo/margin_mean": 39.87766647338867, "margin_dpo/margin_std": 54.281761169433594, "step": 250 }, { "KL/chosen_KL_mean": -71.13668823242188, "KL/mean": -83.47338104248047, "KL/rejected_KL_mean": -95.81005859375, "KL/std": 47.63330841064453, "epoch": 0.3794406651549509, "fcm_dpo/beta": 0.013858238235116005, "fcm_dpo/delta": 0.11203131079673767, "fcm_dpo/margin": 24.67337417602539, "fcm_dpo/q_t": 0.42451345920562744, "grad_norm": 25.10679054260254, "learning_rate": 3.9176496596569265e-07, "logits/chosen": 0.6791577339172363, "logits/rejected": 0.6360569000244141, "logps/chosen": -137.99363708496094, "logps/ref_chosen": -66.85694885253906, "logps/ref_rejected": -84.83396911621094, "logps/rejected": -180.64404296875, "loss": 1.2046, "margin_dpo/margin_mean": 24.67337417602539, "margin_dpo/margin_std": 54.10609436035156, "step": 251 }, { "KL/chosen_KL_mean": -67.5489501953125, "KL/mean": -77.78974914550781, "KL/rejected_KL_mean": -88.03054809570312, "KL/std": 50.0221061706543, "epoch": 0.38095238095238093, "fcm_dpo/beta": 0.013924909755587578, "fcm_dpo/delta": -0.02651354670524597, "fcm_dpo/margin": 20.48160171508789, "fcm_dpo/q_t": 0.43610525131225586, "grad_norm": 21.32712173461914, "learning_rate": 3.9067390737445254e-07, "logits/chosen": 0.6195981502532959, "logits/rejected": 0.5631581544876099, "logps/chosen": -123.77287292480469, "logps/ref_chosen": -56.22393035888672, "logps/ref_rejected": -77.1136245727539, "logps/rejected": -165.1441650390625, "loss": 1.2734, "margin_dpo/margin_mean": 20.48160171508789, "margin_dpo/margin_std": 55.925498962402344, "step": 252 }, { "KL/chosen_KL_mean": -65.90925598144531, "KL/mean": -80.541015625, "KL/rejected_KL_mean": -95.17279052734375, "KL/std": 48.117431640625, "epoch": 0.382464096749811, "fcm_dpo/beta": 0.013884095475077629, "fcm_dpo/delta": 0.04486459493637085, "fcm_dpo/margin": 29.263538360595703, "fcm_dpo/q_t": 0.410060316324234, "grad_norm": 15.181997299194336, "learning_rate": 3.8957891383162304e-07, "logits/chosen": 0.7083457708358765, "logits/rejected": 0.6677348017692566, "logps/chosen": -118.1192626953125, "logps/ref_chosen": -52.21001434326172, "logps/ref_rejected": -58.75764846801758, "logps/rejected": -153.93045043945312, "loss": 1.1326, "margin_dpo/margin_mean": 29.263538360595703, "margin_dpo/margin_std": 48.80119323730469, "step": 253 }, { "KL/chosen_KL_mean": -71.01615142822266, "KL/mean": -86.79163360595703, "KL/rejected_KL_mean": -102.56710815429688, "KL/std": 51.534637451171875, "epoch": 0.3839758125472411, "fcm_dpo/beta": 0.013972971588373184, "fcm_dpo/delta": 0.009229157119989395, "fcm_dpo/margin": 31.55096435546875, "fcm_dpo/q_t": 0.40487420558929443, "grad_norm": 15.08626937866211, "learning_rate": 3.884800159665276e-07, "logits/chosen": 0.628813624382019, "logits/rejected": 0.5745677947998047, "logps/chosen": -136.65248107910156, "logps/ref_chosen": -65.63632202148438, "logps/ref_rejected": -82.34425354003906, "logps/rejected": -184.91136169433594, "loss": 1.1139, "margin_dpo/margin_mean": 31.55096435546875, "margin_dpo/margin_std": 51.70745086669922, "step": 254 }, { "KL/chosen_KL_mean": -62.9200325012207, "KL/mean": -82.491943359375, "KL/rejected_KL_mean": -102.06385803222656, "KL/std": 51.66278839111328, "epoch": 0.3854875283446712, "fcm_dpo/beta": 0.01380923017859459, "fcm_dpo/delta": -0.09603819251060486, "fcm_dpo/margin": 39.14384078979492, "fcm_dpo/q_t": 0.38224583864212036, "grad_norm": 17.70724105834961, "learning_rate": 3.873772445177015e-07, "logits/chosen": 0.6162057518959045, "logits/rejected": 0.5852491855621338, "logps/chosen": -130.83111572265625, "logps/ref_chosen": -67.91108703613281, "logps/ref_rejected": -83.89114379882812, "logps/rejected": -185.95501708984375, "loss": 1.0525, "margin_dpo/margin_mean": 39.143836975097656, "margin_dpo/margin_std": 56.28773880004883, "step": 255 }, { "KL/chosen_KL_mean": -77.14630889892578, "KL/mean": -94.94715881347656, "KL/rejected_KL_mean": -112.74800872802734, "KL/std": 48.50966262817383, "epoch": 0.3869992441421013, "fcm_dpo/beta": 0.013629905879497528, "fcm_dpo/delta": -0.03769109398126602, "fcm_dpo/margin": 35.60170364379883, "fcm_dpo/q_t": 0.39596813917160034, "grad_norm": 17.2496337890625, "learning_rate": 3.862706303320329e-07, "logits/chosen": 0.6348433494567871, "logits/rejected": 0.5660309791564941, "logps/chosen": -140.64630126953125, "logps/ref_chosen": -63.49998474121094, "logps/ref_rejected": -90.77104187011719, "logps/rejected": -203.51904296875, "loss": 1.1243, "margin_dpo/margin_mean": 35.60170364379883, "margin_dpo/margin_std": 62.84724044799805, "step": 256 }, { "KL/chosen_KL_mean": -73.90077209472656, "KL/mean": -95.79080200195312, "KL/rejected_KL_mean": -117.68083953857422, "KL/std": 54.22140121459961, "epoch": 0.3885109599395314, "fcm_dpo/beta": 0.013345291838049889, "fcm_dpo/delta": -0.14312303066253662, "fcm_dpo/margin": 43.780067443847656, "fcm_dpo/q_t": 0.37517955899238586, "grad_norm": 16.31106948852539, "learning_rate": 3.851602043638994e-07, "logits/chosen": 0.5821058750152588, "logits/rejected": 0.5140093564987183, "logps/chosen": -144.50143432617188, "logps/ref_chosen": -70.60064697265625, "logps/ref_rejected": -108.58313751220703, "logps/rejected": -226.26397705078125, "loss": 1.0304, "margin_dpo/margin_mean": 43.780067443847656, "margin_dpo/margin_std": 61.248958587646484, "step": 257 }, { "KL/chosen_KL_mean": -73.88896179199219, "KL/mean": -90.58930969238281, "KL/rejected_KL_mean": -107.2896728515625, "KL/std": 47.34673309326172, "epoch": 0.3900226757369615, "fcm_dpo/beta": 0.013297569938004017, "fcm_dpo/delta": 0.006121770944446325, "fcm_dpo/margin": 33.40070343017578, "fcm_dpo/q_t": 0.3981979489326477, "grad_norm": 16.508010864257812, "learning_rate": 3.840459976743023e-07, "logits/chosen": 0.654517412185669, "logits/rejected": 0.6004564762115479, "logps/chosen": -133.14312744140625, "logps/ref_chosen": -59.25416564941406, "logps/ref_rejected": -85.58709716796875, "logps/rejected": -192.87677001953125, "loss": 1.0747, "margin_dpo/margin_mean": 33.40070343017578, "margin_dpo/margin_std": 45.45872497558594, "step": 258 }, { "KL/chosen_KL_mean": -63.81354904174805, "KL/mean": -88.93872833251953, "KL/rejected_KL_mean": -114.06390380859375, "KL/std": 53.24052429199219, "epoch": 0.3915343915343915, "fcm_dpo/beta": 0.012812048196792603, "fcm_dpo/delta": -0.2092743068933487, "fcm_dpo/margin": 50.25034713745117, "fcm_dpo/q_t": 0.3581874966621399, "grad_norm": 15.079253196716309, "learning_rate": 3.8292804142999796e-07, "logits/chosen": 0.5811412334442139, "logits/rejected": 0.47360676527023315, "logps/chosen": -129.24842834472656, "logps/ref_chosen": -65.43487548828125, "logps/ref_rejected": -95.41731262207031, "logps/rejected": -209.48121643066406, "loss": 0.9841, "margin_dpo/margin_mean": 50.25034713745117, "margin_dpo/margin_std": 61.20269775390625, "step": 259 }, { "KL/chosen_KL_mean": -63.07318115234375, "KL/mean": -83.33232116699219, "KL/rejected_KL_mean": -103.5914535522461, "KL/std": 49.821250915527344, "epoch": 0.3930461073318216, "fcm_dpo/beta": 0.012570559047162533, "fcm_dpo/delta": -0.06336915493011475, "fcm_dpo/margin": 40.51826858520508, "fcm_dpo/q_t": 0.38906094431877136, "grad_norm": 17.87150764465332, "learning_rate": 3.818063669026256e-07, "logits/chosen": 0.6366351842880249, "logits/rejected": 0.5401675701141357, "logps/chosen": -112.16276550292969, "logps/ref_chosen": -49.08958435058594, "logps/ref_rejected": -79.01708221435547, "logps/rejected": -182.60853576660156, "loss": 1.0794, "margin_dpo/margin_mean": 40.51826858520508, "margin_dpo/margin_std": 61.59083557128906, "step": 260 }, { "KL/chosen_KL_mean": -62.76991653442383, "KL/mean": -79.77311706542969, "KL/rejected_KL_mean": -96.77631378173828, "KL/std": 48.37738037109375, "epoch": 0.3945578231292517, "fcm_dpo/beta": 0.012666585855185986, "fcm_dpo/delta": 0.01990126632153988, "fcm_dpo/margin": 34.006385803222656, "fcm_dpo/q_t": 0.4056266248226166, "grad_norm": 17.378955841064453, "learning_rate": 3.806810054678331e-07, "logits/chosen": 0.5030592679977417, "logits/rejected": 0.5419148206710815, "logps/chosen": -133.64230346679688, "logps/ref_chosen": -70.87239074707031, "logps/ref_rejected": -65.01522064208984, "logps/rejected": -161.79153442382812, "loss": 1.1136, "margin_dpo/margin_mean": 34.006385803222656, "margin_dpo/margin_std": 55.04287338256836, "step": 261 }, { "KL/chosen_KL_mean": -67.97140502929688, "KL/mean": -83.72860717773438, "KL/rejected_KL_mean": -99.48582458496094, "KL/std": 48.292259216308594, "epoch": 0.3960695389266818, "fcm_dpo/beta": 0.012770213186740875, "fcm_dpo/delta": 0.04927818849682808, "fcm_dpo/margin": 31.5144100189209, "fcm_dpo/q_t": 0.4105941355228424, "grad_norm": 17.072776794433594, "learning_rate": 3.7955198860439887e-07, "logits/chosen": 0.6593347787857056, "logits/rejected": 0.5915542840957642, "logps/chosen": -135.842041015625, "logps/ref_chosen": -67.8706283569336, "logps/ref_rejected": -88.7205810546875, "logps/rejected": -188.20640563964844, "loss": 1.1254, "margin_dpo/margin_mean": 31.514413833618164, "margin_dpo/margin_std": 51.63859939575195, "step": 262 }, { "KL/chosen_KL_mean": -63.095603942871094, "KL/mean": -78.94845581054688, "KL/rejected_KL_mean": -94.80131530761719, "KL/std": 48.40270233154297, "epoch": 0.3975812547241119, "fcm_dpo/beta": 0.012869570404291153, "fcm_dpo/delta": 0.04364916682243347, "fcm_dpo/margin": 31.705705642700195, "fcm_dpo/q_t": 0.4082304537296295, "grad_norm": 15.210543632507324, "learning_rate": 3.784193478933516e-07, "logits/chosen": 0.585516095161438, "logits/rejected": 0.47373443841934204, "logps/chosen": -118.2901840209961, "logps/ref_chosen": -55.194583892822266, "logps/ref_rejected": -80.54048156738281, "logps/rejected": -175.341796875, "loss": 1.1253, "margin_dpo/margin_mean": 31.705705642700195, "margin_dpo/margin_std": 52.43214416503906, "step": 263 }, { "KL/chosen_KL_mean": -66.87875366210938, "KL/mean": -84.63905334472656, "KL/rejected_KL_mean": -102.39934539794922, "KL/std": 52.75183868408203, "epoch": 0.39909297052154197, "fcm_dpo/beta": 0.01285006757825613, "fcm_dpo/delta": -0.006798055022954941, "fcm_dpo/margin": 35.52058792114258, "fcm_dpo/q_t": 0.3998854160308838, "grad_norm": 15.444226264953613, "learning_rate": 3.7728311501708674e-07, "logits/chosen": 0.5134918093681335, "logits/rejected": 0.47192150354385376, "logps/chosen": -150.0494384765625, "logps/ref_chosen": -83.17068481445312, "logps/ref_rejected": -88.33625793457031, "logps/rejected": -190.735595703125, "loss": 1.1086, "margin_dpo/margin_mean": 35.52058792114258, "margin_dpo/margin_std": 57.95384216308594, "step": 264 }, { "KL/chosen_KL_mean": -63.74461364746094, "KL/mean": -82.62153625488281, "KL/rejected_KL_mean": -101.49846649169922, "KL/std": 52.51271057128906, "epoch": 0.40060468631897206, "fcm_dpo/beta": 0.01283263973891735, "fcm_dpo/delta": -0.03631345182657242, "fcm_dpo/margin": 37.753849029541016, "fcm_dpo/q_t": 0.39464491605758667, "grad_norm": 13.600537300109863, "learning_rate": 3.7614332175848027e-07, "logits/chosen": 0.6821668148040771, "logits/rejected": 0.6149172782897949, "logps/chosen": -115.407470703125, "logps/ref_chosen": -51.66284942626953, "logps/ref_rejected": -67.1720962524414, "logps/rejected": -168.67056274414062, "loss": 1.1192, "margin_dpo/margin_mean": 37.75384521484375, "margin_dpo/margin_std": 64.861572265625, "step": 265 }, { "KL/chosen_KL_mean": -61.073211669921875, "KL/mean": -79.75131225585938, "KL/rejected_KL_mean": -98.42941284179688, "KL/std": 52.90031433105469, "epoch": 0.4021164021164021, "fcm_dpo/beta": 0.012705964967608452, "fcm_dpo/delta": -0.025934092700481415, "fcm_dpo/margin": 37.35620880126953, "fcm_dpo/q_t": 0.3964086174964905, "grad_norm": 19.823822021484375, "learning_rate": 3.75e-07, "logits/chosen": 0.632943332195282, "logits/rejected": 0.553703784942627, "logps/chosen": -118.5237045288086, "logps/ref_chosen": -57.45049285888672, "logps/ref_rejected": -77.60826110839844, "logps/rejected": -176.03765869140625, "loss": 1.0939, "margin_dpo/margin_mean": 37.35620880126953, "margin_dpo/margin_std": 59.012001037597656, "step": 266 }, { "KL/chosen_KL_mean": -55.029483795166016, "KL/mean": -69.81303405761719, "KL/rejected_KL_mean": -84.59658813476562, "KL/std": 49.53461837768555, "epoch": 0.4036281179138322, "fcm_dpo/beta": 0.012619069777429104, "fcm_dpo/delta": -0.06738676875829697, "fcm_dpo/margin": 29.567102432250977, "fcm_dpo/q_t": 0.4169025719165802, "grad_norm": 16.014333724975586, "learning_rate": 3.738531817228131e-07, "logits/chosen": 0.6566835641860962, "logits/rejected": 0.6375699043273926, "logps/chosen": -110.06483459472656, "logps/ref_chosen": -55.03535079956055, "logps/ref_rejected": -66.0953369140625, "logps/rejected": -150.69192504882812, "loss": 1.1897, "margin_dpo/margin_mean": 29.567100524902344, "margin_dpo/margin_std": 59.041709899902344, "step": 267 }, { "KL/chosen_KL_mean": -55.67205810546875, "KL/mean": -72.71066284179688, "KL/rejected_KL_mean": -89.749267578125, "KL/std": 52.541404724121094, "epoch": 0.4051398337112623, "fcm_dpo/beta": 0.012609283439815044, "fcm_dpo/delta": 0.020995769649744034, "fcm_dpo/margin": 34.07720947265625, "fcm_dpo/q_t": 0.40545758605003357, "grad_norm": 14.707161903381348, "learning_rate": 3.7270289900589204e-07, "logits/chosen": 0.48954570293426514, "logits/rejected": 0.4764719009399414, "logps/chosen": -120.74380493164062, "logps/ref_chosen": -65.07174682617188, "logps/ref_rejected": -71.42485809326172, "logps/rejected": -161.17413330078125, "loss": 1.0974, "margin_dpo/margin_mean": 34.077205657958984, "margin_dpo/margin_std": 50.785972595214844, "step": 268 }, { "KL/chosen_KL_mean": -62.85627746582031, "KL/mean": -83.8494873046875, "KL/rejected_KL_mean": -104.84269714355469, "KL/std": 55.634765625, "epoch": 0.40665154950869237, "fcm_dpo/beta": 0.01252103503793478, "fcm_dpo/delta": -0.08028771728277206, "fcm_dpo/margin": 41.98643112182617, "fcm_dpo/q_t": 0.3840331435203552, "grad_norm": 12.968265533447266, "learning_rate": 3.7154918402511714e-07, "logits/chosen": 0.7296715974807739, "logits/rejected": 0.6805523633956909, "logps/chosen": -129.99249267578125, "logps/ref_chosen": -67.1362075805664, "logps/ref_rejected": -82.55778503417969, "logps/rejected": -187.40048217773438, "loss": 1.0429, "margin_dpo/margin_mean": 41.98643112182617, "margin_dpo/margin_std": 55.632362365722656, "step": 269 }, { "KL/chosen_KL_mean": -63.16179656982422, "KL/mean": -80.41263580322266, "KL/rejected_KL_mean": -97.66346740722656, "KL/std": 53.77967071533203, "epoch": 0.40816326530612246, "fcm_dpo/beta": 0.012402812018990517, "fcm_dpo/delta": 0.023027736693620682, "fcm_dpo/margin": 34.50166320800781, "fcm_dpo/q_t": 0.4045753479003906, "grad_norm": 15.0111665725708, "learning_rate": 3.7039206905237656e-07, "logits/chosen": 0.6593344807624817, "logits/rejected": 0.5738272070884705, "logps/chosen": -129.85049438476562, "logps/ref_chosen": -66.6886978149414, "logps/ref_rejected": -85.16129302978516, "logps/rejected": -182.82476806640625, "loss": 1.117, "margin_dpo/margin_mean": 34.50166320800781, "margin_dpo/margin_std": 56.74278259277344, "step": 270 }, { "KL/chosen_KL_mean": -64.6517333984375, "KL/mean": -78.27857971191406, "KL/rejected_KL_mean": -91.90541076660156, "KL/std": 57.019386291503906, "epoch": 0.40967498110355255, "fcm_dpo/beta": 0.012488750740885735, "fcm_dpo/delta": 0.01816648617386818, "fcm_dpo/margin": 27.25368881225586, "fcm_dpo/q_t": 0.4309072196483612, "grad_norm": 18.295251846313477, "learning_rate": 3.692315864546635e-07, "logits/chosen": 0.6561949253082275, "logits/rejected": 0.5901546478271484, "logps/chosen": -137.0592803955078, "logps/ref_chosen": -72.40754699707031, "logps/ref_rejected": -92.06311798095703, "logps/rejected": -183.96853637695312, "loss": 1.2321, "margin_dpo/margin_mean": 27.253690719604492, "margin_dpo/margin_std": 66.5860595703125, "step": 271 }, { "KL/chosen_KL_mean": -54.95947265625, "KL/mean": -80.6039810180664, "KL/rejected_KL_mean": -106.24848937988281, "KL/std": 51.89457702636719, "epoch": 0.41118669690098264, "fcm_dpo/beta": 0.012187042273581028, "fcm_dpo/delta": -0.18635287880897522, "fcm_dpo/margin": 51.28901672363281, "fcm_dpo/q_t": 0.35789382457733154, "grad_norm": 14.965229988098145, "learning_rate": 3.6806776869317067e-07, "logits/chosen": 0.6322664022445679, "logits/rejected": 0.653581976890564, "logps/chosen": -121.56086730957031, "logps/ref_chosen": -66.60140228271484, "logps/ref_rejected": -67.74340057373047, "logps/rejected": -173.99188232421875, "loss": 0.9413, "margin_dpo/margin_mean": 51.28901672363281, "margin_dpo/margin_std": 49.99754333496094, "step": 272 }, { "KL/chosen_KL_mean": -73.67694091796875, "KL/mean": -92.46715545654297, "KL/rejected_KL_mean": -111.25737762451172, "KL/std": 55.94935989379883, "epoch": 0.4126984126984127, "fcm_dpo/beta": 0.011996024288237095, "fcm_dpo/delta": -0.00102166086435318, "fcm_dpo/margin": 37.58043670654297, "fcm_dpo/q_t": 0.4026259779930115, "grad_norm": 19.010719299316406, "learning_rate": 3.669006483223828e-07, "logits/chosen": 0.6612379550933838, "logits/rejected": 0.5861459970474243, "logps/chosen": -131.0318145751953, "logps/ref_chosen": -57.35487747192383, "logps/ref_rejected": -84.17168426513672, "logps/rejected": -195.42906188964844, "loss": 1.1487, "margin_dpo/margin_mean": 37.58043670654297, "margin_dpo/margin_std": 70.39936828613281, "step": 273 }, { "KL/chosen_KL_mean": -71.5015869140625, "KL/mean": -91.9190673828125, "KL/rejected_KL_mean": -112.33653259277344, "KL/std": 55.5938835144043, "epoch": 0.41421012849584277, "fcm_dpo/beta": 0.011979161761701107, "fcm_dpo/delta": -0.041180066764354706, "fcm_dpo/margin": 40.83495330810547, "fcm_dpo/q_t": 0.3926715850830078, "grad_norm": 14.555620193481445, "learning_rate": 3.657302579891656e-07, "logits/chosen": 0.5605759024620056, "logits/rejected": 0.5425212979316711, "logps/chosen": -131.14308166503906, "logps/ref_chosen": -59.64149475097656, "logps/ref_rejected": -68.29348754882812, "logps/rejected": -180.63002014160156, "loss": 1.0978, "margin_dpo/margin_mean": 40.834957122802734, "margin_dpo/margin_std": 65.75580596923828, "step": 274 }, { "KL/chosen_KL_mean": -70.40373992919922, "KL/mean": -90.73006439208984, "KL/rejected_KL_mean": -111.05638885498047, "KL/std": 53.968345642089844, "epoch": 0.41572184429327286, "fcm_dpo/beta": 0.011903250589966774, "fcm_dpo/delta": -0.03569941222667694, "fcm_dpo/margin": 40.65265655517578, "fcm_dpo/q_t": 0.391641229391098, "grad_norm": 15.478461265563965, "learning_rate": 3.645566304318526e-07, "logits/chosen": 0.6459121704101562, "logits/rejected": 0.5600987076759338, "logps/chosen": -123.67037963867188, "logps/ref_chosen": -53.26664352416992, "logps/ref_rejected": -73.84062194824219, "logps/rejected": -184.89700317382812, "loss": 1.0633, "margin_dpo/margin_mean": 40.65265655517578, "margin_dpo/margin_std": 56.11018753051758, "step": 275 }, { "KL/chosen_KL_mean": -66.95787048339844, "KL/mean": -87.26176452636719, "KL/rejected_KL_mean": -107.56565856933594, "KL/std": 51.03001022338867, "epoch": 0.41723356009070295, "fcm_dpo/beta": 0.01180434413254261, "fcm_dpo/delta": -0.030816514045000076, "fcm_dpo/margin": 40.607791900634766, "fcm_dpo/q_t": 0.39217013120651245, "grad_norm": 15.905548095703125, "learning_rate": 3.633797984793294e-07, "logits/chosen": 0.6082146763801575, "logits/rejected": 0.5748229026794434, "logps/chosen": -119.97866821289062, "logps/ref_chosen": -53.02079772949219, "logps/ref_rejected": -61.56678771972656, "logps/rejected": -169.1324462890625, "loss": 1.0616, "margin_dpo/margin_mean": 40.607791900634766, "margin_dpo/margin_std": 55.26260757446289, "step": 276 }, { "KL/chosen_KL_mean": -77.37110137939453, "KL/mean": -88.0814208984375, "KL/rejected_KL_mean": -98.79173278808594, "KL/std": 54.884521484375, "epoch": 0.41874527588813304, "fcm_dpo/beta": 0.011947010643780231, "fcm_dpo/delta": 0.07333403825759888, "fcm_dpo/margin": 21.420637130737305, "fcm_dpo/q_t": 0.4435683786869049, "grad_norm": 20.63077163696289, "learning_rate": 3.6219979505011555e-07, "logits/chosen": 0.6831735372543335, "logits/rejected": 0.7109507322311401, "logps/chosen": -148.80409240722656, "logps/ref_chosen": -71.43299102783203, "logps/ref_rejected": -67.65852355957031, "logps/rejected": -166.45025634765625, "loss": 1.2787, "margin_dpo/margin_mean": 21.420637130737305, "margin_dpo/margin_std": 62.50048065185547, "step": 277 }, { "KL/chosen_KL_mean": -79.96455383300781, "KL/mean": -100.06394958496094, "KL/rejected_KL_mean": -120.16334533691406, "KL/std": 58.283294677734375, "epoch": 0.42025699168556313, "fcm_dpo/beta": 0.011976407840847969, "fcm_dpo/delta": -0.03391536697745323, "fcm_dpo/margin": 40.19879150390625, "fcm_dpo/q_t": 0.39265549182891846, "grad_norm": 23.727630615234375, "learning_rate": 3.6101665315144353e-07, "logits/chosen": 0.5698814392089844, "logits/rejected": 0.5110211968421936, "logps/chosen": -147.0753173828125, "logps/ref_chosen": -67.11076354980469, "logps/ref_rejected": -88.74851989746094, "logps/rejected": -208.911865234375, "loss": 1.0875, "margin_dpo/margin_mean": 40.19879150390625, "margin_dpo/margin_std": 60.55006790161133, "step": 278 }, { "KL/chosen_KL_mean": -62.668453216552734, "KL/mean": -89.84060668945312, "KL/rejected_KL_mean": -117.01275634765625, "KL/std": 56.908164978027344, "epoch": 0.4217687074829932, "fcm_dpo/beta": 0.011571118608117104, "fcm_dpo/delta": -0.19043880701065063, "fcm_dpo/margin": 54.344303131103516, "fcm_dpo/q_t": 0.35756611824035645, "grad_norm": 17.530946731567383, "learning_rate": 3.5983040587833563e-07, "logits/chosen": 0.609818696975708, "logits/rejected": 0.571174144744873, "logps/chosen": -117.16593933105469, "logps/ref_chosen": -54.49748611450195, "logps/ref_rejected": -70.42373657226562, "logps/rejected": -187.43649291992188, "loss": 0.9465, "margin_dpo/margin_mean": 54.34429931640625, "margin_dpo/margin_std": 54.498779296875, "step": 279 }, { "KL/chosen_KL_mean": -68.33837890625, "KL/mean": -96.23875427246094, "KL/rejected_KL_mean": -124.13914489746094, "KL/std": 59.32992935180664, "epoch": 0.42328042328042326, "fcm_dpo/beta": 0.011118508875370026, "fcm_dpo/delta": -0.18170782923698425, "fcm_dpo/margin": 55.8007698059082, "fcm_dpo/q_t": 0.36028069257736206, "grad_norm": 13.276687622070312, "learning_rate": 3.586410864126781e-07, "logits/chosen": 0.6702148914337158, "logits/rejected": 0.623775064945221, "logps/chosen": -128.7711944580078, "logps/ref_chosen": -60.43281173706055, "logps/ref_rejected": -78.39051818847656, "logps/rejected": -202.5296630859375, "loss": 0.9398, "margin_dpo/margin_mean": 55.80076599121094, "margin_dpo/margin_std": 54.328269958496094, "step": 280 }, { "KL/chosen_KL_mean": -72.25418853759766, "KL/mean": -95.67463684082031, "KL/rejected_KL_mean": -119.09507751464844, "KL/std": 58.33176803588867, "epoch": 0.42479213907785335, "fcm_dpo/beta": 0.010939419269561768, "fcm_dpo/delta": -0.0657629668712616, "fcm_dpo/margin": 46.84089279174805, "fcm_dpo/q_t": 0.38581421971321106, "grad_norm": 14.414909362792969, "learning_rate": 3.574487280222929e-07, "logits/chosen": 0.6432371139526367, "logits/rejected": 0.6703838109970093, "logps/chosen": -132.53628540039062, "logps/ref_chosen": -60.2820930480957, "logps/ref_rejected": -62.04009246826172, "logps/rejected": -181.13516235351562, "loss": 1.0502, "margin_dpo/margin_mean": 46.84089660644531, "margin_dpo/margin_std": 63.40924072265625, "step": 281 }, { "KL/chosen_KL_mean": -80.641845703125, "KL/mean": -102.34271240234375, "KL/rejected_KL_mean": -124.0435791015625, "KL/std": 58.330078125, "epoch": 0.42630385487528344, "fcm_dpo/beta": 0.010934704914689064, "fcm_dpo/delta": -0.027530424296855927, "fcm_dpo/margin": 43.40172576904297, "fcm_dpo/q_t": 0.396485298871994, "grad_norm": 17.353635787963867, "learning_rate": 3.562533640600075e-07, "logits/chosen": 0.583552360534668, "logits/rejected": 0.537044107913971, "logps/chosen": -141.26577758789062, "logps/ref_chosen": -60.623924255371094, "logps/ref_rejected": -68.67400360107422, "logps/rejected": -192.71759033203125, "loss": 1.0975, "margin_dpo/margin_mean": 43.40172576904297, "margin_dpo/margin_std": 66.38933563232422, "step": 282 }, { "KL/chosen_KL_mean": -80.09376525878906, "KL/mean": -101.1219711303711, "KL/rejected_KL_mean": -122.15017700195312, "KL/std": 60.03446578979492, "epoch": 0.42781557067271353, "fcm_dpo/beta": 0.010815637186169624, "fcm_dpo/delta": -0.005119716748595238, "fcm_dpo/margin": 42.05641174316406, "fcm_dpo/q_t": 0.399352490901947, "grad_norm": 15.772919654846191, "learning_rate": 3.550550279627215e-07, "logits/chosen": 0.5872868299484253, "logits/rejected": 0.4839021861553192, "logps/chosen": -147.74151611328125, "logps/ref_chosen": -67.64775085449219, "logps/ref_rejected": -99.96835327148438, "logps/rejected": -222.1185302734375, "loss": 1.104, "margin_dpo/margin_mean": 42.05641174316406, "margin_dpo/margin_std": 67.12612915039062, "step": 283 }, { "KL/chosen_KL_mean": -76.53446960449219, "KL/mean": -98.15267181396484, "KL/rejected_KL_mean": -119.7708740234375, "KL/std": 61.65085983276367, "epoch": 0.4293272864701436, "fcm_dpo/beta": 0.010724857449531555, "fcm_dpo/delta": -0.014916583895683289, "fcm_dpo/margin": 43.236419677734375, "fcm_dpo/q_t": 0.3957221508026123, "grad_norm": 13.379623413085938, "learning_rate": 3.5385375325047163e-07, "logits/chosen": 0.6814301609992981, "logits/rejected": 0.6149337291717529, "logps/chosen": -133.50189208984375, "logps/ref_chosen": -56.96742630004883, "logps/ref_rejected": -86.36236572265625, "logps/rejected": -206.13323974609375, "loss": 1.0681, "margin_dpo/margin_mean": 43.236419677734375, "margin_dpo/margin_std": 58.351688385009766, "step": 284 }, { "KL/chosen_KL_mean": -89.5875244140625, "KL/mean": -107.78976440429688, "KL/rejected_KL_mean": -125.99200439453125, "KL/std": 55.76294708251953, "epoch": 0.4308390022675737, "fcm_dpo/beta": 0.010900814086198807, "fcm_dpo/delta": 0.0547969751060009, "fcm_dpo/margin": 36.404502868652344, "fcm_dpo/q_t": 0.41238832473754883, "grad_norm": 17.87129783630371, "learning_rate": 3.5264957352549375e-07, "logits/chosen": 0.6561132073402405, "logits/rejected": 0.629011869430542, "logps/chosen": -161.24362182617188, "logps/ref_chosen": -71.65611267089844, "logps/ref_rejected": -81.63829803466797, "logps/rejected": -207.63031005859375, "loss": 1.1345, "margin_dpo/margin_mean": 36.404502868652344, "margin_dpo/margin_std": 61.58406066894531, "step": 285 }, { "KL/chosen_KL_mean": -85.88138580322266, "KL/mean": -114.774169921875, "KL/rejected_KL_mean": -143.66696166992188, "KL/std": 59.88359832763672, "epoch": 0.4323507180650038, "fcm_dpo/beta": 0.010602448135614395, "fcm_dpo/delta": -0.17343094944953918, "fcm_dpo/margin": 57.78559112548828, "fcm_dpo/q_t": 0.363902747631073, "grad_norm": 14.141740798950195, "learning_rate": 3.514425224712835e-07, "logits/chosen": 0.5710334777832031, "logits/rejected": 0.4783239960670471, "logps/chosen": -146.96090698242188, "logps/ref_chosen": -61.07952117919922, "logps/ref_rejected": -91.28128051757812, "logps/rejected": -234.9482421875, "loss": 0.9628, "margin_dpo/margin_mean": 57.78559112548828, "margin_dpo/margin_std": 63.03284454345703, "step": 286 }, { "KL/chosen_KL_mean": -70.0899887084961, "KL/mean": -98.25200653076172, "KL/rejected_KL_mean": -126.41402435302734, "KL/std": 61.41065979003906, "epoch": 0.43386243386243384, "fcm_dpo/beta": 0.010312942788004875, "fcm_dpo/delta": -0.1388087272644043, "fcm_dpo/margin": 56.32403564453125, "fcm_dpo/q_t": 0.36997073888778687, "grad_norm": 11.145482063293457, "learning_rate": 3.502326338516534e-07, "logits/chosen": 0.7032564282417297, "logits/rejected": 0.6613306999206543, "logps/chosen": -116.12577819824219, "logps/ref_chosen": -46.035789489746094, "logps/ref_rejected": -59.95293426513672, "logps/rejected": -186.36695861816406, "loss": 0.9875, "margin_dpo/margin_mean": 56.32403564453125, "margin_dpo/margin_std": 64.42481994628906, "step": 287 }, { "KL/chosen_KL_mean": -88.99663543701172, "KL/mean": -110.02481842041016, "KL/rejected_KL_mean": -131.05300903320312, "KL/std": 58.78456115722656, "epoch": 0.43537414965986393, "fcm_dpo/beta": 0.010274114087224007, "fcm_dpo/delta": 0.018622491508722305, "fcm_dpo/margin": 42.05636978149414, "fcm_dpo/q_t": 0.40322625637054443, "grad_norm": 15.26879596710205, "learning_rate": 3.490199415097892e-07, "logits/chosen": 0.5211523771286011, "logits/rejected": 0.4631347060203552, "logps/chosen": -154.38748168945312, "logps/ref_chosen": -65.3908462524414, "logps/ref_rejected": -88.53607940673828, "logps/rejected": -219.58908081054688, "loss": 1.0982, "margin_dpo/margin_mean": 42.05636978149414, "margin_dpo/margin_std": 63.13880920410156, "step": 288 }, { "KL/chosen_KL_mean": -90.59356689453125, "KL/mean": -110.605712890625, "KL/rejected_KL_mean": -130.61785888671875, "KL/std": 59.32279968261719, "epoch": 0.436885865457294, "fcm_dpo/beta": 0.010357742197811604, "fcm_dpo/delta": 0.03660314157605171, "fcm_dpo/margin": 40.02429962158203, "fcm_dpo/q_t": 0.41001832485198975, "grad_norm": 15.136282920837402, "learning_rate": 3.4780447936730247e-07, "logits/chosen": 0.7384834289550781, "logits/rejected": 0.6993527412414551, "logps/chosen": -145.18719482421875, "logps/ref_chosen": -54.5936279296875, "logps/ref_rejected": -67.20855712890625, "logps/rejected": -197.826416015625, "loss": 1.1311, "margin_dpo/margin_mean": 40.02430725097656, "margin_dpo/margin_std": 68.23431396484375, "step": 289 }, { "KL/chosen_KL_mean": -103.62715911865234, "KL/mean": -125.84639739990234, "KL/rejected_KL_mean": -148.06561279296875, "KL/std": 57.64268493652344, "epoch": 0.4383975812547241, "fcm_dpo/beta": 0.010276639834046364, "fcm_dpo/delta": -0.007583964616060257, "fcm_dpo/margin": 44.43846130371094, "fcm_dpo/q_t": 0.3985353708267212, "grad_norm": 16.58408546447754, "learning_rate": 3.465862814232821e-07, "logits/chosen": 0.7248564958572388, "logits/rejected": 0.6485689878463745, "logps/chosen": -165.01173400878906, "logps/ref_chosen": -61.38457489013672, "logps/ref_rejected": -91.92778015136719, "logps/rejected": -239.993408203125, "loss": 1.0911, "margin_dpo/margin_mean": 44.43846130371094, "margin_dpo/margin_std": 66.8131103515625, "step": 290 }, { "KL/chosen_KL_mean": -96.03206634521484, "KL/mean": -122.14167785644531, "KL/rejected_KL_mean": -148.2512969970703, "KL/std": 58.933162689208984, "epoch": 0.4399092970521542, "fcm_dpo/beta": 0.010280381888151169, "fcm_dpo/delta": -0.09266703575849533, "fcm_dpo/margin": 52.21923065185547, "fcm_dpo/q_t": 0.38349100947380066, "grad_norm": 14.520237922668457, "learning_rate": 3.4536538175334343e-07, "logits/chosen": 0.7851830720901489, "logits/rejected": 0.7087798118591309, "logps/chosen": -146.89511108398438, "logps/ref_chosen": -50.863037109375, "logps/ref_rejected": -82.20868682861328, "logps/rejected": -230.45997619628906, "loss": 1.037, "margin_dpo/margin_mean": 52.21923065185547, "margin_dpo/margin_std": 68.57379150390625, "step": 291 }, { "KL/chosen_KL_mean": -96.4028091430664, "KL/mean": -115.39314270019531, "KL/rejected_KL_mean": -134.38348388671875, "KL/std": 62.22578430175781, "epoch": 0.4414210128495843, "fcm_dpo/beta": 0.010227020829916, "fcm_dpo/delta": 0.06409165263175964, "fcm_dpo/margin": 37.98066329956055, "fcm_dpo/q_t": 0.41515278816223145, "grad_norm": 17.732135772705078, "learning_rate": 3.4414181450867465e-07, "logits/chosen": 0.693466067314148, "logits/rejected": 0.6415978670120239, "logps/chosen": -160.75169372558594, "logps/ref_chosen": -64.34888458251953, "logps/ref_rejected": -72.86434173583984, "logps/rejected": -207.24781799316406, "loss": 1.1693, "margin_dpo/margin_mean": 37.98065948486328, "margin_dpo/margin_std": 74.58848571777344, "step": 292 }, { "KL/chosen_KL_mean": -93.11393737792969, "KL/mean": -122.25959777832031, "KL/rejected_KL_mean": -151.40525817871094, "KL/std": 62.89670944213867, "epoch": 0.4429327286470144, "fcm_dpo/beta": 0.010097621008753777, "fcm_dpo/delta": -0.1470598578453064, "fcm_dpo/margin": 58.29131317138672, "fcm_dpo/q_t": 0.37115803360939026, "grad_norm": 12.535250663757324, "learning_rate": 3.4291561391508185e-07, "logits/chosen": 0.7780508995056152, "logits/rejected": 0.6862406730651855, "logps/chosen": -147.9833984375, "logps/ref_chosen": -54.869468688964844, "logps/ref_rejected": -81.858642578125, "logps/rejected": -233.263916015625, "loss": 1.0148, "margin_dpo/margin_mean": 58.29132080078125, "margin_dpo/margin_std": 75.01055908203125, "step": 293 }, { "KL/chosen_KL_mean": -97.99851989746094, "KL/mean": -117.81070709228516, "KL/rejected_KL_mean": -137.62289428710938, "KL/std": 65.31426239013672, "epoch": 0.4444444444444444, "fcm_dpo/beta": 0.010007752105593681, "fcm_dpo/delta": 0.0554986447095871, "fcm_dpo/margin": 39.62438201904297, "fcm_dpo/q_t": 0.4120730757713318, "grad_norm": 12.451395988464355, "learning_rate": 3.4168681427203153e-07, "logits/chosen": 0.6983363032341003, "logits/rejected": 0.6498584747314453, "logps/chosen": -154.66941833496094, "logps/ref_chosen": -56.670902252197266, "logps/ref_rejected": -70.32819366455078, "logps/rejected": -207.95108032226562, "loss": 1.1098, "margin_dpo/margin_mean": 39.6243782043457, "margin_dpo/margin_std": 59.10851287841797, "step": 294 }, { "KL/chosen_KL_mean": -103.488525390625, "KL/mean": -122.23589324951172, "KL/rejected_KL_mean": -140.98326110839844, "KL/std": 61.330848693847656, "epoch": 0.4459561602418745, "fcm_dpo/beta": 0.010175291448831558, "fcm_dpo/delta": 0.07125428318977356, "fcm_dpo/margin": 37.49473571777344, "fcm_dpo/q_t": 0.41606712341308594, "grad_norm": 17.44603729248047, "learning_rate": 3.4045544995169125e-07, "logits/chosen": 0.6700437664985657, "logits/rejected": 0.5628273487091064, "logps/chosen": -153.889404296875, "logps/ref_chosen": -50.40088653564453, "logps/ref_rejected": -83.43521881103516, "logps/rejected": -224.41848754882812, "loss": 1.1453, "margin_dpo/margin_mean": 37.49473571777344, "margin_dpo/margin_std": 66.1406021118164, "step": 295 }, { "KL/chosen_KL_mean": -105.82876586914062, "KL/mean": -129.0733642578125, "KL/rejected_KL_mean": -152.31796264648438, "KL/std": 65.7169418334961, "epoch": 0.4474678760393046, "fcm_dpo/beta": 0.010146657936275005, "fcm_dpo/delta": -0.023256970569491386, "fcm_dpo/margin": 46.48919677734375, "fcm_dpo/q_t": 0.39643797278404236, "grad_norm": 14.188161849975586, "learning_rate": 3.392215553979679e-07, "logits/chosen": 0.6409674882888794, "logits/rejected": 0.5902814865112305, "logps/chosen": -174.9791259765625, "logps/ref_chosen": -69.15034484863281, "logps/ref_rejected": -89.60166931152344, "logps/rejected": -241.9196319580078, "loss": 1.0983, "margin_dpo/margin_mean": 46.48919677734375, "margin_dpo/margin_std": 73.70729064941406, "step": 296 }, { "KL/chosen_KL_mean": -110.43616485595703, "KL/mean": -133.87158203125, "KL/rejected_KL_mean": -157.3070068359375, "KL/std": 57.288177490234375, "epoch": 0.4489795918367347, "fcm_dpo/beta": 0.010175475850701332, "fcm_dpo/delta": -0.028494687750935555, "fcm_dpo/margin": 46.87084197998047, "fcm_dpo/q_t": 0.39212262630462646, "grad_norm": 16.023086547851562, "learning_rate": 3.3798516512554485e-07, "logits/chosen": 0.6705678701400757, "logits/rejected": 0.6146547794342041, "logps/chosen": -168.4524688720703, "logps/ref_chosen": -58.01630401611328, "logps/ref_rejected": -69.95780944824219, "logps/rejected": -227.26483154296875, "loss": 1.0562, "margin_dpo/margin_mean": 46.87084197998047, "margin_dpo/margin_std": 61.05129623413086, "step": 297 }, { "KL/chosen_KL_mean": -106.61897277832031, "KL/mean": -124.87419891357422, "KL/rejected_KL_mean": -143.12940979003906, "KL/std": 61.50054168701172, "epoch": 0.4504913076341648, "fcm_dpo/beta": 0.010227935388684273, "fcm_dpo/delta": 0.07961219549179077, "fcm_dpo/margin": 36.51044464111328, "fcm_dpo/q_t": 0.41826608777046204, "grad_norm": 13.112458229064941, "learning_rate": 3.367463137189156e-07, "logits/chosen": 0.7707704305648804, "logits/rejected": 0.7134817838668823, "logps/chosen": -162.7882843017578, "logps/ref_chosen": -56.1693115234375, "logps/ref_rejected": -68.55052185058594, "logps/rejected": -211.679931640625, "loss": 1.1702, "margin_dpo/margin_mean": 36.51044464111328, "margin_dpo/margin_std": 70.75037384033203, "step": 298 }, { "KL/chosen_KL_mean": -111.99178314208984, "KL/mean": -126.85576629638672, "KL/rejected_KL_mean": -141.71974182128906, "KL/std": 59.19386291503906, "epoch": 0.4520030234315949, "fcm_dpo/beta": 0.010346837341785431, "fcm_dpo/delta": 0.03121235966682434, "fcm_dpo/margin": 29.727954864501953, "fcm_dpo/q_t": 0.4333716630935669, "grad_norm": 19.00541114807129, "learning_rate": 3.355050358314172e-07, "logits/chosen": 0.5875415205955505, "logits/rejected": 0.554801881313324, "logps/chosen": -174.30958557128906, "logps/ref_chosen": -62.31780242919922, "logps/ref_rejected": -72.60028839111328, "logps/rejected": -214.32003784179688, "loss": 1.243, "margin_dpo/margin_mean": 29.727954864501953, "margin_dpo/margin_std": 73.96747589111328, "step": 299 }, { "KL/chosen_KL_mean": -109.86944580078125, "KL/mean": -130.2449951171875, "KL/rejected_KL_mean": -150.62054443359375, "KL/std": 60.240020751953125, "epoch": 0.45351473922902497, "fcm_dpo/beta": 0.010411659255623817, "fcm_dpo/delta": 0.026686938479542732, "fcm_dpo/margin": 40.75108337402344, "fcm_dpo/q_t": 0.4073053002357483, "grad_norm": 15.532927513122559, "learning_rate": 3.3426136618426043e-07, "logits/chosen": 0.6936973333358765, "logits/rejected": 0.6232248544692993, "logps/chosen": -170.25103759765625, "logps/ref_chosen": -60.38157653808594, "logps/ref_rejected": -75.45442199707031, "logps/rejected": -226.074951171875, "loss": 1.1313, "margin_dpo/margin_mean": 40.75108337402344, "margin_dpo/margin_std": 70.04428100585938, "step": 300 }, { "KL/chosen_KL_mean": -110.76192474365234, "KL/mean": -127.73435974121094, "KL/rejected_KL_mean": -144.706787109375, "KL/std": 59.79651641845703, "epoch": 0.455026455026455, "fcm_dpo/beta": 0.010507562197744846, "fcm_dpo/delta": 0.09680550545454025, "fcm_dpo/margin": 33.94485855102539, "fcm_dpo/q_t": 0.4223489761352539, "grad_norm": 13.857847213745117, "learning_rate": 3.3301533956555885e-07, "logits/chosen": 0.7236491441726685, "logits/rejected": 0.693558931350708, "logps/chosen": -163.61282348632812, "logps/ref_chosen": -52.85089111328125, "logps/ref_rejected": -69.97584533691406, "logps/rejected": -214.68263244628906, "loss": 1.1847, "margin_dpo/margin_mean": 33.94485855102539, "margin_dpo/margin_std": 69.35813903808594, "step": 301 }, { "KL/chosen_KL_mean": -117.26535034179688, "KL/mean": -130.64083862304688, "KL/rejected_KL_mean": -144.016357421875, "KL/std": 61.98484802246094, "epoch": 0.4565381708238851, "fcm_dpo/beta": 0.01086408831179142, "fcm_dpo/delta": 0.16421742737293243, "fcm_dpo/margin": 26.750995635986328, "fcm_dpo/q_t": 0.4370330572128296, "grad_norm": 18.773527145385742, "learning_rate": 3.317669908293554e-07, "logits/chosen": 0.5519767999649048, "logits/rejected": 0.4936150908470154, "logps/chosen": -184.23187255859375, "logps/ref_chosen": -66.96650695800781, "logps/ref_rejected": -88.09510803222656, "logps/rejected": -232.1114501953125, "loss": 1.2423, "margin_dpo/margin_mean": 26.750995635986328, "margin_dpo/margin_std": 68.02701568603516, "step": 302 }, { "KL/chosen_KL_mean": -102.22068786621094, "KL/mean": -128.0861053466797, "KL/rejected_KL_mean": -153.9515380859375, "KL/std": 63.22712707519531, "epoch": 0.4580498866213152, "fcm_dpo/beta": 0.010783663019537926, "fcm_dpo/delta": -0.11414587497711182, "fcm_dpo/margin": 51.7308349609375, "fcm_dpo/q_t": 0.37940526008605957, "grad_norm": 12.33498764038086, "learning_rate": 3.3051635489464793e-07, "logits/chosen": 0.6826484203338623, "logits/rejected": 0.6071947813034058, "logps/chosen": -164.34222412109375, "logps/ref_chosen": -62.12152862548828, "logps/ref_rejected": -90.31204223632812, "logps/rejected": -244.26358032226562, "loss": 1.0652, "margin_dpo/margin_mean": 51.7308349609375, "margin_dpo/margin_std": 77.99957275390625, "step": 303 }, { "KL/chosen_KL_mean": -94.04334259033203, "KL/mean": -120.75105285644531, "KL/rejected_KL_mean": -147.45877075195312, "KL/std": 66.89263916015625, "epoch": 0.4595616024187453, "fcm_dpo/beta": 0.01050855778157711, "fcm_dpo/delta": -0.118191197514534, "fcm_dpo/margin": 53.41542053222656, "fcm_dpo/q_t": 0.3726978898048401, "grad_norm": 13.780329704284668, "learning_rate": 3.292634667444117e-07, "logits/chosen": 0.6737751364707947, "logits/rejected": 0.6167408227920532, "logps/chosen": -154.73843383789062, "logps/ref_chosen": -60.695091247558594, "logps/ref_rejected": -78.2525405883789, "logps/rejected": -225.7113037109375, "loss": 0.9972, "margin_dpo/margin_mean": 53.41542053222656, "margin_dpo/margin_std": 61.099151611328125, "step": 304 }, { "KL/chosen_KL_mean": -105.29306030273438, "KL/mean": -124.80269622802734, "KL/rejected_KL_mean": -144.31231689453125, "KL/std": 60.25926971435547, "epoch": 0.46107331821617537, "fcm_dpo/beta": 0.01046682707965374, "fcm_dpo/delta": 0.04289165511727333, "fcm_dpo/margin": 39.01927185058594, "fcm_dpo/q_t": 0.4094550609588623, "grad_norm": 15.33985710144043, "learning_rate": 3.280083614246217e-07, "logits/chosen": 0.6184134483337402, "logits/rejected": 0.6530435085296631, "logps/chosen": -177.9921875, "logps/ref_chosen": -72.69914245605469, "logps/ref_rejected": -65.65670776367188, "logps/rejected": -209.9690399169922, "loss": 1.1597, "margin_dpo/margin_mean": 39.0192756652832, "margin_dpo/margin_std": 73.0045166015625, "step": 305 }, { "KL/chosen_KL_mean": -99.73224639892578, "KL/mean": -119.26398468017578, "KL/rejected_KL_mean": -138.79571533203125, "KL/std": 58.238197326660156, "epoch": 0.46258503401360546, "fcm_dpo/beta": 0.010540273040533066, "fcm_dpo/delta": 0.039174970239400864, "fcm_dpo/margin": 39.063472747802734, "fcm_dpo/q_t": 0.4074612855911255, "grad_norm": 14.64108943939209, "learning_rate": 3.267510740432719e-07, "logits/chosen": 0.729286789894104, "logits/rejected": 0.6173923015594482, "logps/chosen": -153.70277404785156, "logps/ref_chosen": -53.97052764892578, "logps/ref_rejected": -71.02423095703125, "logps/rejected": -209.8199462890625, "loss": 1.1081, "margin_dpo/margin_mean": 39.063472747802734, "margin_dpo/margin_std": 57.965023040771484, "step": 306 }, { "KL/chosen_KL_mean": -92.92012786865234, "KL/mean": -104.03545379638672, "KL/rejected_KL_mean": -115.15077209472656, "KL/std": 60.11651611328125, "epoch": 0.46409674981103555, "fcm_dpo/beta": 0.010723145678639412, "fcm_dpo/delta": 0.0820213183760643, "fcm_dpo/margin": 22.230642318725586, "fcm_dpo/q_t": 0.44964367151260376, "grad_norm": 19.061439514160156, "learning_rate": 3.2549163976939285e-07, "logits/chosen": 0.6955245733261108, "logits/rejected": 0.647261381149292, "logps/chosen": -150.33323669433594, "logps/ref_chosen": -57.413108825683594, "logps/ref_rejected": -68.68010711669922, "logps/rejected": -183.83087158203125, "loss": 1.3116, "margin_dpo/margin_mean": 22.230642318725586, "margin_dpo/margin_std": 74.4197006225586, "step": 307 }, { "KL/chosen_KL_mean": -93.56331634521484, "KL/mean": -113.46038818359375, "KL/rejected_KL_mean": -133.3574676513672, "KL/std": 61.609130859375, "epoch": 0.4656084656084656, "fcm_dpo/beta": 0.010778220370411873, "fcm_dpo/delta": 0.021368850022554398, "fcm_dpo/margin": 39.79413604736328, "fcm_dpo/q_t": 0.4058694839477539, "grad_norm": 12.138362884521484, "learning_rate": 3.2423009383206874e-07, "logits/chosen": 0.6182932257652283, "logits/rejected": 0.6067421436309814, "logps/chosen": -160.162109375, "logps/ref_chosen": -66.59879302978516, "logps/ref_rejected": -74.337158203125, "logps/rejected": -207.69461059570312, "loss": 1.1387, "margin_dpo/margin_mean": 39.79413604736328, "margin_dpo/margin_std": 70.455078125, "step": 308 }, { "KL/chosen_KL_mean": -107.09249877929688, "KL/mean": -126.61676788330078, "KL/rejected_KL_mean": -146.14102172851562, "KL/std": 58.26902770996094, "epoch": 0.4671201814058957, "fcm_dpo/beta": 0.010832262225449085, "fcm_dpo/delta": 0.027536533772945404, "fcm_dpo/margin": 39.04853057861328, "fcm_dpo/q_t": 0.4072054624557495, "grad_norm": 13.843729972839355, "learning_rate": 3.229664715194511e-07, "logits/chosen": 0.7150523662567139, "logits/rejected": 0.6556819677352905, "logps/chosen": -172.48724365234375, "logps/ref_chosen": -65.39474487304688, "logps/ref_rejected": -75.70930480957031, "logps/rejected": -221.850341796875, "loss": 1.1303, "margin_dpo/margin_mean": 39.04853057861328, "margin_dpo/margin_std": 66.06800842285156, "step": 309 }, { "KL/chosen_KL_mean": -109.26652526855469, "KL/mean": -121.12366485595703, "KL/rejected_KL_mean": -132.98080444335938, "KL/std": 53.856170654296875, "epoch": 0.46863189720332576, "fcm_dpo/beta": 0.011067605577409267, "fcm_dpo/delta": 0.06834125518798828, "fcm_dpo/margin": 23.71427345275879, "fcm_dpo/q_t": 0.44445621967315674, "grad_norm": 15.862635612487793, "learning_rate": 3.2170080817777257e-07, "logits/chosen": 0.6403053998947144, "logits/rejected": 0.62421053647995, "logps/chosen": -183.93479919433594, "logps/ref_chosen": -74.66827392578125, "logps/ref_rejected": -80.5689697265625, "logps/rejected": -213.54977416992188, "loss": 1.2593, "margin_dpo/margin_mean": 23.714271545410156, "margin_dpo/margin_std": 64.26668548583984, "step": 310 }, { "KL/chosen_KL_mean": -86.25080871582031, "KL/mean": -107.97197723388672, "KL/rejected_KL_mean": -129.69314575195312, "KL/std": 68.0066146850586, "epoch": 0.47014361300075586, "fcm_dpo/beta": 0.010956868529319763, "fcm_dpo/delta": -0.02816399559378624, "fcm_dpo/margin": 43.44232940673828, "fcm_dpo/q_t": 0.39939939975738525, "grad_norm": 14.952717781066895, "learning_rate": 3.204331392103574e-07, "logits/chosen": 0.5733008980751038, "logits/rejected": 0.42495253682136536, "logps/chosen": -145.9888458251953, "logps/ref_chosen": -59.738033294677734, "logps/ref_rejected": -93.60757446289062, "logps/rejected": -223.30072021484375, "loss": 1.0981, "margin_dpo/margin_mean": 43.44232940673828, "margin_dpo/margin_std": 70.86783599853516, "step": 311 }, { "KL/chosen_KL_mean": -88.2499771118164, "KL/mean": -113.75946044921875, "KL/rejected_KL_mean": -139.26893615722656, "KL/std": 60.36577606201172, "epoch": 0.47165532879818595, "fcm_dpo/beta": 0.01085340604186058, "fcm_dpo/delta": -0.10966580361127853, "fcm_dpo/margin": 51.01895523071289, "fcm_dpo/q_t": 0.37506595253944397, "grad_norm": 12.034233093261719, "learning_rate": 3.1916350007663176e-07, "logits/chosen": 0.6571813821792603, "logits/rejected": 0.5639383792877197, "logps/chosen": -142.06640625, "logps/ref_chosen": -53.816436767578125, "logps/ref_rejected": -68.6575698852539, "logps/rejected": -207.926513671875, "loss": 0.9963, "margin_dpo/margin_mean": 51.018951416015625, "margin_dpo/margin_std": 57.567359924316406, "step": 312 }, { "KL/chosen_KL_mean": -91.30401611328125, "KL/mean": -104.92782592773438, "KL/rejected_KL_mean": -118.5516357421875, "KL/std": 58.888465881347656, "epoch": 0.47316704459561604, "fcm_dpo/beta": 0.011028505861759186, "fcm_dpo/delta": 0.15395116806030273, "fcm_dpo/margin": 27.247615814208984, "fcm_dpo/q_t": 0.4350872337818146, "grad_norm": 14.497456550598145, "learning_rate": 3.178919262911314e-07, "logits/chosen": 0.6942657232284546, "logits/rejected": 0.6728841066360474, "logps/chosen": -151.26138305664062, "logps/ref_chosen": -59.957359313964844, "logps/ref_rejected": -69.31729888916016, "logps/rejected": -187.86892700195312, "loss": 1.2203, "margin_dpo/margin_mean": 27.247615814208984, "margin_dpo/margin_std": 63.275230407714844, "step": 313 }, { "KL/chosen_KL_mean": -83.92698669433594, "KL/mean": -109.30597686767578, "KL/rejected_KL_mean": -134.68496704101562, "KL/std": 57.439422607421875, "epoch": 0.47467876039304613, "fcm_dpo/beta": 0.010885808616876602, "fcm_dpo/delta": -0.10918959230184555, "fcm_dpo/margin": 50.75796890258789, "fcm_dpo/q_t": 0.3786463439464569, "grad_norm": 13.35073184967041, "learning_rate": 3.166184534225087e-07, "logits/chosen": 0.6736915707588196, "logits/rejected": 0.70624178647995, "logps/chosen": -154.1951446533203, "logps/ref_chosen": -70.26815795898438, "logps/ref_rejected": -69.23971557617188, "logps/rejected": -203.9246826171875, "loss": 1.0157, "margin_dpo/margin_mean": 50.757972717285156, "margin_dpo/margin_std": 62.80244445800781, "step": 314 }, { "KL/chosen_KL_mean": -87.91320037841797, "KL/mean": -107.46711730957031, "KL/rejected_KL_mean": -127.02104187011719, "KL/std": 57.641319274902344, "epoch": 0.47619047619047616, "fcm_dpo/beta": 0.010960625484585762, "fcm_dpo/delta": 0.02152669057250023, "fcm_dpo/margin": 39.10783767700195, "fcm_dpo/q_t": 0.4034097194671631, "grad_norm": 13.392065048217773, "learning_rate": 3.1534311709253723e-07, "logits/chosen": 0.6077339053153992, "logits/rejected": 0.5718281269073486, "logps/chosen": -155.7078857421875, "logps/ref_chosen": -67.79469299316406, "logps/ref_rejected": -74.55148315429688, "logps/rejected": -201.572509765625, "loss": 1.1049, "margin_dpo/margin_mean": 39.10784149169922, "margin_dpo/margin_std": 59.44211196899414, "step": 315 }, { "KL/chosen_KL_mean": -83.18527221679688, "KL/mean": -110.18215942382812, "KL/rejected_KL_mean": -137.1790313720703, "KL/std": 60.67741012573242, "epoch": 0.47770219198790626, "fcm_dpo/beta": 0.010814497247338295, "fcm_dpo/delta": -0.14357182383537292, "fcm_dpo/margin": 53.99375915527344, "fcm_dpo/q_t": 0.37073686718940735, "grad_norm": 14.253772735595703, "learning_rate": 3.1406595297511564e-07, "logits/chosen": 0.5644083023071289, "logits/rejected": 0.4203554391860962, "logps/chosen": -138.4737548828125, "logps/ref_chosen": -55.288482666015625, "logps/ref_rejected": -96.15723419189453, "logps/rejected": -233.33627319335938, "loss": 0.9945, "margin_dpo/margin_mean": 53.99375915527344, "margin_dpo/margin_std": 59.711795806884766, "step": 316 }, { "KL/chosen_KL_mean": -75.41068267822266, "KL/mean": -101.35579681396484, "KL/rejected_KL_mean": -127.30091857910156, "KL/std": 56.30128860473633, "epoch": 0.47921390778533635, "fcm_dpo/beta": 0.010403081774711609, "fcm_dpo/delta": -0.09583519399166107, "fcm_dpo/margin": 51.890228271484375, "fcm_dpo/q_t": 0.37759479880332947, "grad_norm": 15.830406188964844, "learning_rate": 3.1278699679526975e-07, "logits/chosen": 0.6743822693824768, "logits/rejected": 0.6287072896957397, "logps/chosen": -129.9920654296875, "logps/ref_chosen": -54.58137512207031, "logps/ref_rejected": -72.77232360839844, "logps/rejected": -200.0732421875, "loss": 1.0127, "margin_dpo/margin_mean": 51.890228271484375, "margin_dpo/margin_std": 61.410675048828125, "step": 317 }, { "KL/chosen_KL_mean": -83.8477783203125, "KL/mean": -102.66434478759766, "KL/rejected_KL_mean": -121.48091125488281, "KL/std": 60.022735595703125, "epoch": 0.48072562358276644, "fcm_dpo/beta": 0.010466702282428741, "fcm_dpo/delta": 0.05843011662364006, "fcm_dpo/margin": 37.633140563964844, "fcm_dpo/q_t": 0.4175593852996826, "grad_norm": 13.778341293334961, "learning_rate": 3.1150628432815336e-07, "logits/chosen": 0.6658343076705933, "logits/rejected": 0.5933891534805298, "logps/chosen": -136.7360076904297, "logps/ref_chosen": -52.88822937011719, "logps/ref_rejected": -80.63988494873047, "logps/rejected": -202.1208038330078, "loss": 1.1904, "margin_dpo/margin_mean": 37.633140563964844, "margin_dpo/margin_std": 80.30189514160156, "step": 318 }, { "KL/chosen_KL_mean": -81.49282836914062, "KL/mean": -108.32252502441406, "KL/rejected_KL_mean": -135.1522216796875, "KL/std": 63.14406204223633, "epoch": 0.48223733938019653, "fcm_dpo/beta": 0.010345933958888054, "fcm_dpo/delta": -0.11138413846492767, "fcm_dpo/margin": 53.65939712524414, "fcm_dpo/q_t": 0.37948736548423767, "grad_norm": 13.567173957824707, "learning_rate": 3.1022385139804707e-07, "logits/chosen": 0.6083593964576721, "logits/rejected": 0.5914223194122314, "logps/chosen": -145.8561553955078, "logps/ref_chosen": -64.36333465576172, "logps/ref_rejected": -79.47296142578125, "logps/rejected": -214.62518310546875, "loss": 1.0291, "margin_dpo/margin_mean": 53.65939712524414, "margin_dpo/margin_std": 72.19100952148438, "step": 319 }, { "KL/chosen_KL_mean": -76.05279541015625, "KL/mean": -97.53375244140625, "KL/rejected_KL_mean": -119.01470947265625, "KL/std": 64.29141235351562, "epoch": 0.4837490551776266, "fcm_dpo/beta": 0.01016431674361229, "fcm_dpo/delta": -0.12435748428106308, "fcm_dpo/margin": 42.96190643310547, "fcm_dpo/q_t": 0.40559324622154236, "grad_norm": 14.700331687927246, "learning_rate": 3.0893973387735683e-07, "logits/chosen": 0.5677880048751831, "logits/rejected": 0.5255244970321655, "logps/chosen": -125.61154174804688, "logps/ref_chosen": -49.558746337890625, "logps/ref_rejected": -71.23444366455078, "logps/rejected": -190.2491455078125, "loss": 1.1179, "margin_dpo/margin_mean": 42.96190643310547, "margin_dpo/margin_std": 68.222412109375, "step": 320 }, { "KL/chosen_KL_mean": -80.76130676269531, "KL/mean": -103.32257080078125, "KL/rejected_KL_mean": -125.88383483886719, "KL/std": 56.74087905883789, "epoch": 0.4852607709750567, "fcm_dpo/beta": 0.009960871189832687, "fcm_dpo/delta": -0.0006339475512504578, "fcm_dpo/margin": 45.12252426147461, "fcm_dpo/q_t": 0.39828598499298096, "grad_norm": 18.722091674804688, "learning_rate": 3.0765396768561004e-07, "logits/chosen": 0.650843620300293, "logits/rejected": 0.6300288438796997, "logps/chosen": -132.84657287597656, "logps/ref_chosen": -52.08526611328125, "logps/ref_rejected": -55.58674621582031, "logps/rejected": -181.4705810546875, "loss": 1.0956, "margin_dpo/margin_mean": 45.12252426147461, "margin_dpo/margin_std": 66.43511962890625, "step": 321 }, { "KL/chosen_KL_mean": -90.73258209228516, "KL/mean": -119.62736511230469, "KL/rejected_KL_mean": -148.5221405029297, "KL/std": 63.579566955566406, "epoch": 0.48677248677248675, "fcm_dpo/beta": 0.009883934631943703, "fcm_dpo/delta": -0.1283361166715622, "fcm_dpo/margin": 57.78955841064453, "fcm_dpo/q_t": 0.370322048664093, "grad_norm": 11.981712341308594, "learning_rate": 3.063665887884511e-07, "logits/chosen": 0.7016029357910156, "logits/rejected": 0.6129163503646851, "logps/chosen": -138.13668823242188, "logps/ref_chosen": -47.404109954833984, "logps/ref_rejected": -73.4260025024414, "logps/rejected": -221.94815063476562, "loss": 0.9847, "margin_dpo/margin_mean": 57.78955841064453, "margin_dpo/margin_std": 62.76979446411133, "step": 322 }, { "KL/chosen_KL_mean": -92.734619140625, "KL/mean": -112.15545654296875, "KL/rejected_KL_mean": -131.57632446289062, "KL/std": 61.908042907714844, "epoch": 0.48828420256991684, "fcm_dpo/beta": 0.009892433881759644, "fcm_dpo/delta": 0.06829862296581268, "fcm_dpo/margin": 38.841705322265625, "fcm_dpo/q_t": 0.4185178875923157, "grad_norm": 14.59123420715332, "learning_rate": 3.0507763319663517e-07, "logits/chosen": 0.5701498985290527, "logits/rejected": 0.49318498373031616, "logps/chosen": -162.74090576171875, "logps/ref_chosen": -70.00630187988281, "logps/ref_rejected": -86.96690368652344, "logps/rejected": -218.543212890625, "loss": 1.176, "margin_dpo/margin_mean": 38.841705322265625, "margin_dpo/margin_std": 78.66917419433594, "step": 323 }, { "KL/chosen_KL_mean": -82.48573303222656, "KL/mean": -107.55412292480469, "KL/rejected_KL_mean": -132.62252807617188, "KL/std": 68.80017852783203, "epoch": 0.4897959183673469, "fcm_dpo/beta": 0.009812276810407639, "fcm_dpo/delta": -0.0446263924241066, "fcm_dpo/margin": 50.13678741455078, "fcm_dpo/q_t": 0.3889816403388977, "grad_norm": 18.301029205322266, "learning_rate": 3.0378713696502097e-07, "logits/chosen": 0.6554989814758301, "logits/rejected": 0.5961704254150391, "logps/chosen": -138.37454223632812, "logps/ref_chosen": -55.88882064819336, "logps/ref_rejected": -75.23088073730469, "logps/rejected": -207.8533935546875, "loss": 1.0441, "margin_dpo/margin_mean": 50.13678741455078, "margin_dpo/margin_std": 63.38947677612305, "step": 324 }, { "KL/chosen_KL_mean": -99.98908996582031, "KL/mean": -125.22124481201172, "KL/rejected_KL_mean": -150.45339965820312, "KL/std": 67.48681640625, "epoch": 0.491307634164777, "fcm_dpo/beta": 0.009717625565826893, "fcm_dpo/delta": -0.04319227859377861, "fcm_dpo/margin": 50.46430587768555, "fcm_dpo/q_t": 0.3927181661128998, "grad_norm": 13.410660743713379, "learning_rate": 3.0249513619156206e-07, "logits/chosen": 0.6126409769058228, "logits/rejected": 0.5459693670272827, "logps/chosen": -164.1361083984375, "logps/ref_chosen": -64.14701843261719, "logps/ref_rejected": -79.91143798828125, "logps/rejected": -230.36483764648438, "loss": 1.0903, "margin_dpo/margin_mean": 50.46430587768555, "margin_dpo/margin_std": 80.43098449707031, "step": 325 }, { "KL/chosen_KL_mean": -118.30567932128906, "KL/mean": -128.3629150390625, "KL/rejected_KL_mean": -138.420166015625, "KL/std": 63.34078598022461, "epoch": 0.4928193499622071, "fcm_dpo/beta": 0.009744199924170971, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 20.11448860168457, "fcm_dpo/q_t": 0.45632901787757874, "grad_norm": 15.810664176940918, "learning_rate": 3.012016670162977e-07, "logits/chosen": 0.5627347230911255, "logits/rejected": 0.5708246827125549, "logps/chosen": -193.8369903564453, "logps/ref_chosen": -75.53131103515625, "logps/ref_rejected": -76.5898666381836, "logps/rejected": -215.01004028320312, "loss": 1.3113, "margin_dpo/margin_mean": 20.114490509033203, "margin_dpo/margin_std": 70.28466796875, "step": 326 }, { "KL/chosen_KL_mean": -105.60182189941406, "KL/mean": -125.68409729003906, "KL/rejected_KL_mean": -145.76638793945312, "KL/std": 67.13113403320312, "epoch": 0.4943310657596372, "fcm_dpo/beta": 0.009856300428509712, "fcm_dpo/delta": 0.055987969040870667, "fcm_dpo/margin": 40.16455078125, "fcm_dpo/q_t": 0.4134991765022278, "grad_norm": 14.7315092086792, "learning_rate": 2.99906765620341e-07, "logits/chosen": 0.5232765674591064, "logits/rejected": 0.4972417950630188, "logps/chosen": -174.93899536132812, "logps/ref_chosen": -69.33717346191406, "logps/ref_rejected": -73.37751770019531, "logps/rejected": -219.14389038085938, "loss": 1.1596, "margin_dpo/margin_mean": 40.16455078125, "margin_dpo/margin_std": 76.08860778808594, "step": 327 }, { "KL/chosen_KL_mean": -92.72146606445312, "KL/mean": -117.269775390625, "KL/rejected_KL_mean": -141.81808471679688, "KL/std": 67.56961059570312, "epoch": 0.4958427815570673, "fcm_dpo/beta": 0.009826852940022945, "fcm_dpo/delta": -0.03414200246334076, "fcm_dpo/margin": 49.09660720825195, "fcm_dpo/q_t": 0.3925701975822449, "grad_norm": 13.953093528747559, "learning_rate": 2.9861046822486766e-07, "logits/chosen": 0.5673099756240845, "logits/rejected": 0.5271446108818054, "logps/chosen": -154.42770385742188, "logps/ref_chosen": -61.70623016357422, "logps/ref_rejected": -83.73808288574219, "logps/rejected": -225.55615234375, "loss": 1.054, "margin_dpo/margin_mean": 49.09661102294922, "margin_dpo/margin_std": 64.48529815673828, "step": 328 }, { "KL/chosen_KL_mean": -104.82115936279297, "KL/mean": -129.3842315673828, "KL/rejected_KL_mean": -153.94729614257812, "KL/std": 70.55194091796875, "epoch": 0.4973544973544973, "fcm_dpo/beta": 0.009733401238918304, "fcm_dpo/delta": -0.029633961617946625, "fcm_dpo/margin": 49.12614440917969, "fcm_dpo/q_t": 0.39622172713279724, "grad_norm": 15.438725471496582, "learning_rate": 2.9731281109010253e-07, "logits/chosen": 0.6905697584152222, "logits/rejected": 0.628017783164978, "logps/chosen": -169.319580078125, "logps/ref_chosen": -64.4984130859375, "logps/ref_rejected": -83.6591796875, "logps/rejected": -237.60647583007812, "loss": 1.077, "margin_dpo/margin_mean": 49.12614440917969, "margin_dpo/margin_std": 72.67239379882812, "step": 329 }, { "KL/chosen_KL_mean": -90.5877685546875, "KL/mean": -115.56993103027344, "KL/rejected_KL_mean": -140.55209350585938, "KL/std": 66.15676879882812, "epoch": 0.4988662131519274, "fcm_dpo/beta": 0.009712828323245049, "fcm_dpo/delta": -0.0372566320002079, "fcm_dpo/margin": 49.964324951171875, "fcm_dpo/q_t": 0.393964946269989, "grad_norm": 13.776653289794922, "learning_rate": 2.9601383051430505e-07, "logits/chosen": 0.6362694501876831, "logits/rejected": 0.5670721530914307, "logps/chosen": -145.3924102783203, "logps/ref_chosen": -54.80464172363281, "logps/ref_rejected": -75.3194351196289, "logps/rejected": -215.87152099609375, "loss": 1.1115, "margin_dpo/margin_mean": 49.96432113647461, "margin_dpo/margin_std": 83.51506042480469, "step": 330 }, { "KL/chosen_KL_mean": -97.71769714355469, "KL/mean": -129.8302001953125, "KL/rejected_KL_mean": -161.9427032470703, "KL/std": 70.34434509277344, "epoch": 0.5003779289493575, "fcm_dpo/beta": 0.00942898727953434, "fcm_dpo/delta": -0.1655063033103943, "fcm_dpo/margin": 64.22500610351562, "fcm_dpo/q_t": 0.36448371410369873, "grad_norm": 12.452539443969727, "learning_rate": 2.947135628327544e-07, "logits/chosen": 0.7674143314361572, "logits/rejected": 0.7401211261749268, "logps/chosen": -156.96026611328125, "logps/ref_chosen": -59.242584228515625, "logps/ref_rejected": -69.87483215332031, "logps/rejected": -231.81753540039062, "loss": 0.9871, "margin_dpo/margin_mean": 64.22500610351562, "margin_dpo/margin_std": 76.57498168945312, "step": 331 }, { "KL/chosen_KL_mean": -100.70648956298828, "KL/mean": -127.30098724365234, "KL/rejected_KL_mean": -153.8954620361328, "KL/std": 67.10773468017578, "epoch": 0.5018896447467877, "fcm_dpo/beta": 0.009348525665700436, "fcm_dpo/delta": -0.050655677914619446, "fcm_dpo/margin": 53.18898010253906, "fcm_dpo/q_t": 0.39014729857444763, "grad_norm": 12.76447868347168, "learning_rate": 2.934120444167326e-07, "logits/chosen": 0.5541119575500488, "logits/rejected": 0.5114009976387024, "logps/chosen": -167.81625366210938, "logps/ref_chosen": -67.10975646972656, "logps/ref_rejected": -77.11839294433594, "logps/rejected": -231.01385498046875, "loss": 1.0501, "margin_dpo/margin_mean": 53.18898010253906, "margin_dpo/margin_std": 69.1185073852539, "step": 332 }, { "KL/chosen_KL_mean": -107.39823150634766, "KL/mean": -136.9010467529297, "KL/rejected_KL_mean": -166.40386962890625, "KL/std": 67.92105102539062, "epoch": 0.5034013605442177, "fcm_dpo/beta": 0.009133165702223778, "fcm_dpo/delta": -0.09388077259063721, "fcm_dpo/margin": 59.0056266784668, "fcm_dpo/q_t": 0.3798203468322754, "grad_norm": 12.019691467285156, "learning_rate": 2.921093116725076e-07, "logits/chosen": 0.6719864010810852, "logits/rejected": 0.5902762413024902, "logps/chosen": -165.77935791015625, "logps/ref_chosen": -58.381134033203125, "logps/ref_rejected": -85.02839660644531, "logps/rejected": -251.4322509765625, "loss": 1.0135, "margin_dpo/margin_mean": 59.0056266784668, "margin_dpo/margin_std": 71.16836547851562, "step": 333 }, { "KL/chosen_KL_mean": -103.21528625488281, "KL/mean": -124.10235595703125, "KL/rejected_KL_mean": -144.98941040039062, "KL/std": 69.16629028320312, "epoch": 0.5049130763416477, "fcm_dpo/beta": 0.009185167029500008, "fcm_dpo/delta": 0.0687691867351532, "fcm_dpo/margin": 41.77412414550781, "fcm_dpo/q_t": 0.41626012325286865, "grad_norm": 13.901957511901855, "learning_rate": 2.9080540104031484e-07, "logits/chosen": 0.6650213599205017, "logits/rejected": 0.6160895824432373, "logps/chosen": -170.1072998046875, "logps/ref_chosen": -66.89199829101562, "logps/ref_rejected": -91.83695220947266, "logps/rejected": -236.8263702392578, "loss": 1.1629, "margin_dpo/margin_mean": 41.77412414550781, "margin_dpo/margin_std": 79.92010498046875, "step": 334 }, { "KL/chosen_KL_mean": -103.90148162841797, "KL/mean": -127.40682983398438, "KL/rejected_KL_mean": -150.9121856689453, "KL/std": 68.83200073242188, "epoch": 0.5064247921390779, "fcm_dpo/beta": 0.009263388812541962, "fcm_dpo/delta": 0.014709331095218658, "fcm_dpo/margin": 47.010711669921875, "fcm_dpo/q_t": 0.4052375257015228, "grad_norm": 18.1312313079834, "learning_rate": 2.895003489933375e-07, "logits/chosen": 0.6490761041641235, "logits/rejected": 0.6102247834205627, "logps/chosen": -165.4159393310547, "logps/ref_chosen": -61.51445770263672, "logps/ref_rejected": -75.68916320800781, "logps/rejected": -226.60134887695312, "loss": 1.1211, "margin_dpo/margin_mean": 47.010711669921875, "margin_dpo/margin_std": 78.88675689697266, "step": 335 }, { "KL/chosen_KL_mean": -116.6889877319336, "KL/mean": -141.385498046875, "KL/rejected_KL_mean": -166.08203125, "KL/std": 72.22708892822266, "epoch": 0.5079365079365079, "fcm_dpo/beta": 0.00915299728512764, "fcm_dpo/delta": -0.003460422158241272, "fcm_dpo/margin": 49.393028259277344, "fcm_dpo/q_t": 0.40451768040657043, "grad_norm": 13.747593879699707, "learning_rate": 2.8819419203668675e-07, "logits/chosen": 0.6120268106460571, "logits/rejected": 0.5861070156097412, "logps/chosen": -185.5390625, "logps/ref_chosen": -68.85006713867188, "logps/ref_rejected": -92.99603271484375, "logps/rejected": -259.07806396484375, "loss": 1.1113, "margin_dpo/margin_mean": 49.393028259277344, "margin_dpo/margin_std": 80.9678726196289, "step": 336 }, { "KL/chosen_KL_mean": -121.39860534667969, "KL/mean": -140.76492309570312, "KL/rejected_KL_mean": -160.13125610351562, "KL/std": 71.30752563476562, "epoch": 0.509448223733938, "fcm_dpo/beta": 0.009357749484479427, "fcm_dpo/delta": 0.09072916209697723, "fcm_dpo/margin": 38.732643127441406, "fcm_dpo/q_t": 0.4197903275489807, "grad_norm": 12.64812183380127, "learning_rate": 2.8688696670638053e-07, "logits/chosen": 0.5565305352210999, "logits/rejected": 0.5242564082145691, "logps/chosen": -194.58644104003906, "logps/ref_chosen": -73.18783569335938, "logps/ref_rejected": -86.89118957519531, "logps/rejected": -247.02244567871094, "loss": 1.169, "margin_dpo/margin_mean": 38.732643127441406, "margin_dpo/margin_std": 74.53764343261719, "step": 337 }, { "KL/chosen_KL_mean": -118.91609954833984, "KL/mean": -139.07220458984375, "KL/rejected_KL_mean": -159.22833251953125, "KL/std": 67.26274108886719, "epoch": 0.5109599395313681, "fcm_dpo/beta": 0.00947779230773449, "fcm_dpo/delta": 0.07068651914596558, "fcm_dpo/margin": 40.312225341796875, "fcm_dpo/q_t": 0.41567832231521606, "grad_norm": 13.105402946472168, "learning_rate": 2.8557870956832133e-07, "logits/chosen": 0.6425968408584595, "logits/rejected": 0.6147984862327576, "logps/chosen": -182.855712890625, "logps/ref_chosen": -63.939613342285156, "logps/ref_rejected": -75.34243774414062, "logps/rejected": -234.57077026367188, "loss": 1.1589, "margin_dpo/margin_mean": 40.312225341796875, "margin_dpo/margin_std": 75.49674224853516, "step": 338 }, { "KL/chosen_KL_mean": -96.85836791992188, "KL/mean": -120.70976257324219, "KL/rejected_KL_mean": -144.56114196777344, "KL/std": 67.41364288330078, "epoch": 0.5124716553287982, "fcm_dpo/beta": 0.009497357532382011, "fcm_dpo/delta": -0.0033246222883462906, "fcm_dpo/margin": 47.70277404785156, "fcm_dpo/q_t": 0.39905327558517456, "grad_norm": 15.153426170349121, "learning_rate": 2.842694572172736e-07, "logits/chosen": 0.7378226518630981, "logits/rejected": 0.6509774923324585, "logps/chosen": -142.40750122070312, "logps/ref_chosen": -45.54913330078125, "logps/ref_rejected": -67.0482177734375, "logps/rejected": -211.609375, "loss": 1.093, "margin_dpo/margin_mean": 47.70277404785156, "margin_dpo/margin_std": 71.71929931640625, "step": 339 }, { "KL/chosen_KL_mean": -109.52645111083984, "KL/mean": -132.99769592285156, "KL/rejected_KL_mean": -156.46893310546875, "KL/std": 73.52427673339844, "epoch": 0.5139833711262283, "fcm_dpo/beta": 0.009554904885590076, "fcm_dpo/delta": 0.0012706257402896881, "fcm_dpo/margin": 46.94248962402344, "fcm_dpo/q_t": 0.4044419229030609, "grad_norm": 13.740220069885254, "learning_rate": 2.8295924627584004e-07, "logits/chosen": 0.6307625770568848, "logits/rejected": 0.6090366840362549, "logps/chosen": -163.5321044921875, "logps/ref_chosen": -54.00564956665039, "logps/ref_rejected": -61.314430236816406, "logps/rejected": -217.7833709716797, "loss": 1.1408, "margin_dpo/margin_mean": 46.94248580932617, "margin_dpo/margin_std": 85.59992218017578, "step": 340 }, { "KL/chosen_KL_mean": -109.53600311279297, "KL/mean": -136.73626708984375, "KL/rejected_KL_mean": -163.9365234375, "KL/std": 72.4736099243164, "epoch": 0.5154950869236583, "fcm_dpo/beta": 0.009304164908826351, "fcm_dpo/delta": -0.06382884830236435, "fcm_dpo/margin": 54.4005241394043, "fcm_dpo/q_t": 0.3888283371925354, "grad_norm": 15.317728996276855, "learning_rate": 2.816481133934373e-07, "logits/chosen": 0.6532335877418518, "logits/rejected": 0.6039453744888306, "logps/chosen": -172.93109130859375, "logps/ref_chosen": -63.39509582519531, "logps/ref_rejected": -76.20973205566406, "logps/rejected": -240.14625549316406, "loss": 1.0768, "margin_dpo/margin_mean": 54.4005241394043, "margin_dpo/margin_std": 76.9643783569336, "step": 341 }, { "KL/chosen_KL_mean": -108.70679473876953, "KL/mean": -135.81114196777344, "KL/rejected_KL_mean": -162.91549682617188, "KL/std": 76.89732360839844, "epoch": 0.5170068027210885, "fcm_dpo/beta": 0.009291356429457664, "fcm_dpo/delta": -0.05691445991396904, "fcm_dpo/margin": 54.20869445800781, "fcm_dpo/q_t": 0.39120563864707947, "grad_norm": 12.805046081542969, "learning_rate": 2.8033609524527046e-07, "logits/chosen": 0.6862562894821167, "logits/rejected": 0.6442948579788208, "logps/chosen": -161.75460815429688, "logps/ref_chosen": -53.047813415527344, "logps/ref_rejected": -68.2854232788086, "logps/rejected": -231.200927734375, "loss": 1.0676, "margin_dpo/margin_mean": 54.20869445800781, "margin_dpo/margin_std": 78.99603271484375, "step": 342 }, { "KL/chosen_KL_mean": -102.84151458740234, "KL/mean": -122.31297302246094, "KL/rejected_KL_mean": -141.784423828125, "KL/std": 71.11018371582031, "epoch": 0.5185185185185185, "fcm_dpo/beta": 0.009219045750796795, "fcm_dpo/delta": -0.040550898760557175, "fcm_dpo/margin": 38.94290542602539, "fcm_dpo/q_t": 0.4192054271697998, "grad_norm": 12.088910102844238, "learning_rate": 2.7902322853130753e-07, "logits/chosen": 0.5472891330718994, "logits/rejected": 0.5373716354370117, "logps/chosen": -173.4200439453125, "logps/ref_chosen": -70.57852935791016, "logps/ref_rejected": -84.73873901367188, "logps/rejected": -226.52316284179688, "loss": 1.167, "margin_dpo/margin_mean": 38.94290542602539, "margin_dpo/margin_std": 72.6668701171875, "step": 343 }, { "KL/chosen_KL_mean": -113.80350494384766, "KL/mean": -141.92852783203125, "KL/rejected_KL_mean": -170.05355834960938, "KL/std": 72.05269622802734, "epoch": 0.5200302343159486, "fcm_dpo/beta": 0.009113233536481857, "fcm_dpo/delta": -0.06617586314678192, "fcm_dpo/margin": 56.250038146972656, "fcm_dpo/q_t": 0.3878791332244873, "grad_norm": 15.124133110046387, "learning_rate": 2.7770954997525274e-07, "logits/chosen": 0.6614542007446289, "logits/rejected": 0.5895565152168274, "logps/chosen": -169.614501953125, "logps/ref_chosen": -55.811004638671875, "logps/ref_rejected": -84.77637481689453, "logps/rejected": -254.82992553710938, "loss": 1.0575, "margin_dpo/margin_mean": 56.250038146972656, "margin_dpo/margin_std": 79.96159362792969, "step": 344 }, { "KL/chosen_KL_mean": -91.85005187988281, "KL/mean": -116.61634826660156, "KL/rejected_KL_mean": -141.3826446533203, "KL/std": 65.72059631347656, "epoch": 0.5215419501133787, "fcm_dpo/beta": 0.009115805849432945, "fcm_dpo/delta": -0.0016993321478366852, "fcm_dpo/margin": 49.532596588134766, "fcm_dpo/q_t": 0.4003925323486328, "grad_norm": 14.183326721191406, "learning_rate": 2.7639509632351927e-07, "logits/chosen": 0.7415436506271362, "logits/rejected": 0.6945260763168335, "logps/chosen": -149.63613891601562, "logps/ref_chosen": -57.78609848022461, "logps/ref_rejected": -78.91847229003906, "logps/rejected": -220.30111694335938, "loss": 1.1019, "margin_dpo/margin_mean": 49.5325927734375, "margin_dpo/margin_std": 78.25393676757812, "step": 345 }, { "KL/chosen_KL_mean": -98.70977783203125, "KL/mean": -128.33563232421875, "KL/rejected_KL_mean": -157.96148681640625, "KL/std": 76.5794677734375, "epoch": 0.5230536659108088, "fcm_dpo/beta": 0.009013254195451736, "fcm_dpo/delta": -0.08877884596586227, "fcm_dpo/margin": 59.251705169677734, "fcm_dpo/q_t": 0.38100147247314453, "grad_norm": 14.048331260681152, "learning_rate": 2.7507990434420123e-07, "logits/chosen": 0.6886954307556152, "logits/rejected": 0.5999356508255005, "logps/chosen": -154.99490356445312, "logps/ref_chosen": -56.285125732421875, "logps/ref_rejected": -91.15303039550781, "logps/rejected": -249.11451721191406, "loss": 1.0448, "margin_dpo/margin_mean": 59.251708984375, "margin_dpo/margin_std": 79.60140991210938, "step": 346 }, { "KL/chosen_KL_mean": -108.50074005126953, "KL/mean": -131.3124542236328, "KL/rejected_KL_mean": -154.1241455078125, "KL/std": 73.10723876953125, "epoch": 0.5245653817082389, "fcm_dpo/beta": 0.008980346843600273, "fcm_dpo/delta": 0.04202822968363762, "fcm_dpo/margin": 45.6234130859375, "fcm_dpo/q_t": 0.40984588861465454, "grad_norm": 15.117777824401855, "learning_rate": 2.737640108260456e-07, "logits/chosen": 0.8087772130966187, "logits/rejected": 0.7555938959121704, "logps/chosen": -162.00027465820312, "logps/ref_chosen": -53.499542236328125, "logps/ref_rejected": -72.52565002441406, "logps/rejected": -226.64981079101562, "loss": 1.1206, "margin_dpo/margin_mean": 45.6234130859375, "margin_dpo/margin_std": 74.35516357421875, "step": 347 }, { "KL/chosen_KL_mean": -97.05572509765625, "KL/mean": -123.86107635498047, "KL/rejected_KL_mean": -150.6664276123047, "KL/std": 69.09683990478516, "epoch": 0.5260770975056689, "fcm_dpo/beta": 0.008932996541261673, "fcm_dpo/delta": -0.030811918899416924, "fcm_dpo/margin": 53.610694885253906, "fcm_dpo/q_t": 0.3972897529602051, "grad_norm": 14.440524101257324, "learning_rate": 2.724474525774229e-07, "logits/chosen": 0.7605525255203247, "logits/rejected": 0.728769063949585, "logps/chosen": -147.8425750732422, "logps/ref_chosen": -50.78684997558594, "logps/ref_rejected": -68.63732147216797, "logps/rejected": -219.30374145507812, "loss": 1.0883, "margin_dpo/margin_mean": 53.61069107055664, "margin_dpo/margin_std": 83.18544006347656, "step": 348 }, { "KL/chosen_KL_mean": -97.84178161621094, "KL/mean": -125.2623062133789, "KL/rejected_KL_mean": -152.68283081054688, "KL/std": 75.20608520507812, "epoch": 0.527588813303099, "fcm_dpo/beta": 0.008946020156145096, "fcm_dpo/delta": -0.04307024925947189, "fcm_dpo/margin": 54.84104919433594, "fcm_dpo/q_t": 0.39336341619491577, "grad_norm": 13.364866256713867, "learning_rate": 2.711302664252973e-07, "logits/chosen": 0.702577531337738, "logits/rejected": 0.6050245761871338, "logps/chosen": -151.16677856445312, "logps/ref_chosen": -53.325008392333984, "logps/ref_rejected": -83.21236419677734, "logps/rejected": -235.8951873779297, "loss": 1.0727, "margin_dpo/margin_mean": 54.84105682373047, "margin_dpo/margin_std": 79.70642852783203, "step": 349 }, { "KL/chosen_KL_mean": -102.39527130126953, "KL/mean": -137.85421752929688, "KL/rejected_KL_mean": -173.31314086914062, "KL/std": 80.3461685180664, "epoch": 0.5291005291005291, "fcm_dpo/beta": 0.008636381477117538, "fcm_dpo/delta": -0.17346930503845215, "fcm_dpo/margin": 70.91788482666016, "fcm_dpo/q_t": 0.3626285195350647, "grad_norm": 15.910598754882812, "learning_rate": 2.698124892141971e-07, "logits/chosen": 0.6693747639656067, "logits/rejected": 0.5895169377326965, "logps/chosen": -164.0210418701172, "logps/ref_chosen": -61.625770568847656, "logps/ref_rejected": -87.63627624511719, "logps/rejected": -260.94940185546875, "loss": 0.9643, "margin_dpo/margin_mean": 70.91788482666016, "margin_dpo/margin_std": 77.03778076171875, "step": 350 }, { "KL/chosen_KL_mean": -97.79383850097656, "KL/mean": -122.77336120605469, "KL/rejected_KL_mean": -147.7528839111328, "KL/std": 68.2166748046875, "epoch": 0.5306122448979592, "fcm_dpo/beta": 0.008579917252063751, "fcm_dpo/delta": 0.02221677638590336, "fcm_dpo/margin": 49.95904541015625, "fcm_dpo/q_t": 0.4026418626308441, "grad_norm": 14.179925918579102, "learning_rate": 2.6849415780518357e-07, "logits/chosen": 0.6331157088279724, "logits/rejected": 0.5555412769317627, "logps/chosen": -154.0501708984375, "logps/ref_chosen": -56.2563362121582, "logps/ref_rejected": -79.11589813232422, "logps/rejected": -226.8687744140625, "loss": 1.1297, "margin_dpo/margin_mean": 49.95904541015625, "margin_dpo/margin_std": 85.19073486328125, "step": 351 }, { "KL/chosen_KL_mean": -99.13749694824219, "KL/mean": -127.44450378417969, "KL/rejected_KL_mean": -155.75152587890625, "KL/std": 76.02853393554688, "epoch": 0.5321239606953893, "fcm_dpo/beta": 0.008539706468582153, "fcm_dpo/delta": -0.03550984710454941, "fcm_dpo/margin": 56.61402893066406, "fcm_dpo/q_t": 0.3938150107860565, "grad_norm": 12.919422149658203, "learning_rate": 2.6717530907482024e-07, "logits/chosen": 0.677651047706604, "logits/rejected": 0.6238071918487549, "logps/chosen": -162.189453125, "logps/ref_chosen": -63.05195236206055, "logps/ref_rejected": -85.52035522460938, "logps/rejected": -241.27188110351562, "loss": 1.0692, "margin_dpo/margin_mean": 56.61402893066406, "margin_dpo/margin_std": 81.5572509765625, "step": 352 }, { "KL/chosen_KL_mean": -96.16493225097656, "KL/mean": -123.23371124267578, "KL/rejected_KL_mean": -150.302490234375, "KL/std": 70.93081665039062, "epoch": 0.5336356764928194, "fcm_dpo/beta": 0.008550824597477913, "fcm_dpo/delta": -0.013591569848358631, "fcm_dpo/margin": 54.137550354003906, "fcm_dpo/q_t": 0.3958391547203064, "grad_norm": 12.857582092285156, "learning_rate": 2.658559799141411e-07, "logits/chosen": 0.6801486015319824, "logits/rejected": 0.6855990886688232, "logps/chosen": -165.1741180419922, "logps/ref_chosen": -69.00918579101562, "logps/ref_rejected": -72.65840148925781, "logps/rejected": -222.96090698242188, "loss": 1.0738, "margin_dpo/margin_mean": 54.137550354003906, "margin_dpo/margin_std": 75.59597778320312, "step": 353 }, { "KL/chosen_KL_mean": -97.02384948730469, "KL/mean": -128.12098693847656, "KL/rejected_KL_mean": -159.21810913085938, "KL/std": 68.00271606445312, "epoch": 0.5351473922902494, "fcm_dpo/beta": 0.008396446704864502, "fcm_dpo/delta": -0.0768899917602539, "fcm_dpo/margin": 62.19426727294922, "fcm_dpo/q_t": 0.3830644488334656, "grad_norm": 14.596508026123047, "learning_rate": 2.6453620722761895e-07, "logits/chosen": 0.7443443536758423, "logits/rejected": 0.6121722459793091, "logps/chosen": -136.81219482421875, "logps/ref_chosen": -39.78833770751953, "logps/ref_rejected": -69.56885528564453, "logps/rejected": -228.7869873046875, "loss": 1.0408, "margin_dpo/margin_mean": 62.19426727294922, "margin_dpo/margin_std": 81.65336608886719, "step": 354 }, { "KL/chosen_KL_mean": -102.66900634765625, "KL/mean": -135.7369842529297, "KL/rejected_KL_mean": -168.80494689941406, "KL/std": 78.304443359375, "epoch": 0.5366591080876795, "fcm_dpo/beta": 0.008291777223348618, "fcm_dpo/delta": -0.10396140813827515, "fcm_dpo/margin": 66.13594055175781, "fcm_dpo/q_t": 0.378852516412735, "grad_norm": 16.355134963989258, "learning_rate": 2.632160279321328e-07, "logits/chosen": 0.6968790292739868, "logits/rejected": 0.5650628805160522, "logps/chosen": -148.9243927001953, "logps/ref_chosen": -46.25537872314453, "logps/ref_rejected": -78.20236206054688, "logps/rejected": -247.00730895996094, "loss": 1.0354, "margin_dpo/margin_mean": 66.13594055175781, "margin_dpo/margin_std": 88.17823791503906, "step": 355 }, { "KL/chosen_KL_mean": -96.81419372558594, "KL/mean": -123.33544921875, "KL/rejected_KL_mean": -149.85671997070312, "KL/std": 73.71067810058594, "epoch": 0.5381708238851096, "fcm_dpo/beta": 0.008203265257179737, "fcm_dpo/delta": 0.015107132494449615, "fcm_dpo/margin": 53.042518615722656, "fcm_dpo/q_t": 0.4048543870449066, "grad_norm": 12.561837196350098, "learning_rate": 2.618954789559356e-07, "logits/chosen": 0.6723008155822754, "logits/rejected": 0.5905227065086365, "logps/chosen": -144.72035217285156, "logps/ref_chosen": -47.906158447265625, "logps/ref_rejected": -74.29397583007812, "logps/rejected": -224.15069580078125, "loss": 1.1379, "margin_dpo/margin_mean": 53.042518615722656, "margin_dpo/margin_std": 94.193359375, "step": 356 }, { "KL/chosen_KL_mean": -115.66548919677734, "KL/mean": -137.5971221923828, "KL/rejected_KL_mean": -159.52877807617188, "KL/std": 73.30964660644531, "epoch": 0.5396825396825397, "fcm_dpo/beta": 0.00813913345336914, "fcm_dpo/delta": -0.06833313405513763, "fcm_dpo/margin": 43.86327362060547, "fcm_dpo/q_t": 0.418012797832489, "grad_norm": 13.875882148742676, "learning_rate": 2.6057459723762076e-07, "logits/chosen": 0.6871232986450195, "logits/rejected": 0.6637519001960754, "logps/chosen": -178.3004913330078, "logps/ref_chosen": -62.63500213623047, "logps/ref_rejected": -65.11399841308594, "logps/rejected": -224.64276123046875, "loss": 1.1691, "margin_dpo/margin_mean": 43.86327362060547, "margin_dpo/margin_std": 79.24894714355469, "step": 357 }, { "KL/chosen_KL_mean": -112.24562072753906, "KL/mean": -142.50857543945312, "KL/rejected_KL_mean": -172.77151489257812, "KL/std": 76.78482055664062, "epoch": 0.5411942554799698, "fcm_dpo/beta": 0.008158707991242409, "fcm_dpo/delta": -0.04745243489742279, "fcm_dpo/margin": 60.52588653564453, "fcm_dpo/q_t": 0.3917842209339142, "grad_norm": 18.900074005126953, "learning_rate": 2.5925341972508954e-07, "logits/chosen": 0.6384870409965515, "logits/rejected": 0.6525646448135376, "logps/chosen": -179.45523071289062, "logps/ref_chosen": -67.20960998535156, "logps/ref_rejected": -69.34715270996094, "logps/rejected": -242.11868286132812, "loss": 1.0702, "margin_dpo/margin_mean": 60.52588653564453, "margin_dpo/margin_std": 85.34688568115234, "step": 358 }, { "KL/chosen_KL_mean": -125.95619201660156, "KL/mean": -141.68975830078125, "KL/rejected_KL_mean": -157.42330932617188, "KL/std": 75.13418579101562, "epoch": 0.5427059712773998, "fcm_dpo/beta": 0.008086791262030602, "fcm_dpo/delta": 0.030342087149620056, "fcm_dpo/margin": 31.467113494873047, "fcm_dpo/q_t": 0.44308972358703613, "grad_norm": 14.887833595275879, "learning_rate": 2.579319833745169e-07, "logits/chosen": 0.6606814861297607, "logits/rejected": 0.6267046928405762, "logps/chosen": -188.4819793701172, "logps/ref_chosen": -62.52578353881836, "logps/ref_rejected": -76.63114929199219, "logps/rejected": -234.05445861816406, "loss": 1.2485, "margin_dpo/margin_mean": 31.467111587524414, "margin_dpo/margin_std": 78.3302993774414, "step": 359 }, { "KL/chosen_KL_mean": -121.25907897949219, "KL/mean": -148.23548889160156, "KL/rejected_KL_mean": -175.21189880371094, "KL/std": 79.970947265625, "epoch": 0.54421768707483, "fcm_dpo/beta": 0.008114282041788101, "fcm_dpo/delta": 0.012742086313664913, "fcm_dpo/margin": 53.95281219482422, "fcm_dpo/q_t": 0.40300747752189636, "grad_norm": 13.132290840148926, "learning_rate": 2.5661032514931834e-07, "logits/chosen": 0.6002212762832642, "logits/rejected": 0.5079668164253235, "logps/chosen": -184.74679565429688, "logps/ref_chosen": -63.48772048950195, "logps/ref_rejected": -90.6891098022461, "logps/rejected": -265.9010009765625, "loss": 1.0905, "margin_dpo/margin_mean": 53.95281219482422, "margin_dpo/margin_std": 78.95816802978516, "step": 360 }, { "KL/chosen_KL_mean": -115.78294372558594, "KL/mean": -146.99960327148438, "KL/rejected_KL_mean": -178.21627807617188, "KL/std": 77.42436218261719, "epoch": 0.54572940287226, "fcm_dpo/beta": 0.008079019375145435, "fcm_dpo/delta": -0.05725649744272232, "fcm_dpo/margin": 62.43333435058594, "fcm_dpo/q_t": 0.38624024391174316, "grad_norm": 12.444530487060547, "learning_rate": 2.552884820191154e-07, "logits/chosen": 0.7191639542579651, "logits/rejected": 0.6731234788894653, "logps/chosen": -173.70008850097656, "logps/ref_chosen": -57.917144775390625, "logps/ref_rejected": -72.39089965820312, "logps/rejected": -250.607177734375, "loss": 1.0394, "margin_dpo/margin_mean": 62.43333053588867, "margin_dpo/margin_std": 79.18809509277344, "step": 361 }, { "KL/chosen_KL_mean": -117.07453918457031, "KL/mean": -151.28367614746094, "KL/rejected_KL_mean": -185.49281311035156, "KL/std": 84.75254821777344, "epoch": 0.54724111866969, "fcm_dpo/beta": 0.007967658340930939, "fcm_dpo/delta": -0.10074599087238312, "fcm_dpo/margin": 68.41827392578125, "fcm_dpo/q_t": 0.3835322856903076, "grad_norm": 13.671100616455078, "learning_rate": 2.53966490958702e-07, "logits/chosen": 0.7531605958938599, "logits/rejected": 0.6426513195037842, "logps/chosen": -180.51800537109375, "logps/ref_chosen": -63.4434700012207, "logps/ref_rejected": -103.45516967773438, "logps/rejected": -288.947998046875, "loss": 1.0478, "margin_dpo/margin_mean": 68.41827392578125, "margin_dpo/margin_std": 98.29641723632812, "step": 362 }, { "KL/chosen_KL_mean": -121.70635986328125, "KL/mean": -153.733642578125, "KL/rejected_KL_mean": -185.76092529296875, "KL/std": 75.45001220703125, "epoch": 0.5487528344671202, "fcm_dpo/beta": 0.007809008937329054, "fcm_dpo/delta": -0.05292538180947304, "fcm_dpo/margin": 64.0545654296875, "fcm_dpo/q_t": 0.38914868235588074, "grad_norm": 14.454474449157715, "learning_rate": 2.526443889470099e-07, "logits/chosen": 0.771051287651062, "logits/rejected": 0.630893349647522, "logps/chosen": -170.35818481445312, "logps/ref_chosen": -48.65182876586914, "logps/ref_rejected": -88.65904235839844, "logps/rejected": -274.41998291015625, "loss": 1.0513, "margin_dpo/margin_mean": 64.0545654296875, "margin_dpo/margin_std": 87.23719787597656, "step": 363 }, { "KL/chosen_KL_mean": -107.86466979980469, "KL/mean": -143.8165740966797, "KL/rejected_KL_mean": -179.76844787597656, "KL/std": 83.8006591796875, "epoch": 0.5502645502645502, "fcm_dpo/beta": 0.007666376419365406, "fcm_dpo/delta": -0.10725465416908264, "fcm_dpo/margin": 71.90379333496094, "fcm_dpo/q_t": 0.3811180293560028, "grad_norm": 11.80333423614502, "learning_rate": 2.513222129660744e-07, "logits/chosen": 0.6106956005096436, "logits/rejected": 0.5245239734649658, "logps/chosen": -165.73574829101562, "logps/ref_chosen": -57.87107467651367, "logps/ref_rejected": -80.95503234863281, "logps/rejected": -260.7234802246094, "loss": 1.0498, "margin_dpo/margin_mean": 71.90379333496094, "margin_dpo/margin_std": 103.74449920654297, "step": 364 }, { "KL/chosen_KL_mean": -100.33560180664062, "KL/mean": -133.744384765625, "KL/rejected_KL_mean": -167.15313720703125, "KL/std": 88.45748138427734, "epoch": 0.5517762660619804, "fcm_dpo/beta": 0.007543657440692186, "fcm_dpo/delta": -0.057265233248472214, "fcm_dpo/margin": 66.81753540039062, "fcm_dpo/q_t": 0.3860504627227783, "grad_norm": 11.33962631225586, "learning_rate": 2.5e-07, "logits/chosen": 0.6893282532691956, "logits/rejected": 0.6836937665939331, "logps/chosen": -165.27777099609375, "logps/ref_chosen": -64.94217681884766, "logps/ref_rejected": -74.8599853515625, "logps/rejected": -242.01312255859375, "loss": 1.0215, "margin_dpo/margin_mean": 66.81754302978516, "margin_dpo/margin_std": 78.03083038330078, "step": 365 }, { "KL/chosen_KL_mean": -99.90927124023438, "KL/mean": -127.51522827148438, "KL/rejected_KL_mean": -155.1211700439453, "KL/std": 76.94725799560547, "epoch": 0.5532879818594104, "fcm_dpo/beta": 0.007609900087118149, "fcm_dpo/delta": 0.0306707676500082, "fcm_dpo/margin": 55.21190643310547, "fcm_dpo/q_t": 0.4082038402557373, "grad_norm": 14.167494773864746, "learning_rate": 2.486777870339255e-07, "logits/chosen": 0.6601539254188538, "logits/rejected": 0.6505333185195923, "logps/chosen": -155.07525634765625, "logps/ref_chosen": -55.16598129272461, "logps/ref_rejected": -65.26121520996094, "logps/rejected": -220.38238525390625, "loss": 1.1391, "margin_dpo/margin_mean": 55.21190643310547, "margin_dpo/margin_std": 98.03353118896484, "step": 366 }, { "KL/chosen_KL_mean": -115.7052001953125, "KL/mean": -143.0133056640625, "KL/rejected_KL_mean": -170.3214111328125, "KL/std": 78.01261901855469, "epoch": 0.5547996976568406, "fcm_dpo/beta": 0.007616519462317228, "fcm_dpo/delta": 0.03548750281333923, "fcm_dpo/margin": 54.61621856689453, "fcm_dpo/q_t": 0.4044143259525299, "grad_norm": 13.201713562011719, "learning_rate": 2.4735561105299014e-07, "logits/chosen": 0.651557207107544, "logits/rejected": 0.5488114953041077, "logps/chosen": -171.71566772460938, "logps/ref_chosen": -56.01046371459961, "logps/ref_rejected": -77.31010437011719, "logps/rejected": -247.6315155029297, "loss": 1.1061, "margin_dpo/margin_mean": 54.61621856689453, "margin_dpo/margin_std": 82.45671844482422, "step": 367 }, { "KL/chosen_KL_mean": -127.14485168457031, "KL/mean": -152.2775421142578, "KL/rejected_KL_mean": -177.41024780273438, "KL/std": 77.05934143066406, "epoch": 0.5563114134542706, "fcm_dpo/beta": 0.007702920585870743, "fcm_dpo/delta": 0.06538652628660202, "fcm_dpo/margin": 50.265403747558594, "fcm_dpo/q_t": 0.41213297843933105, "grad_norm": 15.009796142578125, "learning_rate": 2.46033509041298e-07, "logits/chosen": 0.5128130316734314, "logits/rejected": 0.514595091342926, "logps/chosen": -201.97412109375, "logps/ref_chosen": -74.82927703857422, "logps/ref_rejected": -76.11680603027344, "logps/rejected": -253.5270538330078, "loss": 1.1298, "margin_dpo/margin_mean": 50.265403747558594, "margin_dpo/margin_std": 81.61317443847656, "step": 368 }, { "KL/chosen_KL_mean": -118.11318969726562, "KL/mean": -140.47128295898438, "KL/rejected_KL_mean": -162.8293914794922, "KL/std": 77.84164428710938, "epoch": 0.5578231292517006, "fcm_dpo/beta": 0.007902087643742561, "fcm_dpo/delta": 0.09908389300107956, "fcm_dpo/margin": 44.716209411621094, "fcm_dpo/q_t": 0.42133742570877075, "grad_norm": 12.984330177307129, "learning_rate": 2.447115179808846e-07, "logits/chosen": 0.7052586078643799, "logits/rejected": 0.6519788503646851, "logps/chosen": -176.4394073486328, "logps/ref_chosen": -58.32621765136719, "logps/ref_rejected": -80.92183685302734, "logps/rejected": -243.75123596191406, "loss": 1.165, "margin_dpo/margin_mean": 44.716209411621094, "margin_dpo/margin_std": 82.68061828613281, "step": 369 }, { "KL/chosen_KL_mean": -113.09016418457031, "KL/mean": -145.8719024658203, "KL/rejected_KL_mean": -178.6536407470703, "KL/std": 81.12561798095703, "epoch": 0.5593348450491308, "fcm_dpo/beta": 0.007803306449204683, "fcm_dpo/delta": -0.06524214148521423, "fcm_dpo/margin": 65.5634765625, "fcm_dpo/q_t": 0.3852328062057495, "grad_norm": 13.21735668182373, "learning_rate": 2.4338967485068164e-07, "logits/chosen": 0.7980337142944336, "logits/rejected": 0.7273433208465576, "logps/chosen": -165.973876953125, "logps/ref_chosen": -52.88372039794922, "logps/ref_rejected": -79.43692016601562, "logps/rejected": -258.0905456542969, "loss": 1.0628, "margin_dpo/margin_mean": 65.5634765625, "margin_dpo/margin_std": 94.4046630859375, "step": 370 }, { "KL/chosen_KL_mean": -112.4964599609375, "KL/mean": -140.91160583496094, "KL/rejected_KL_mean": -169.3267364501953, "KL/std": 77.54881286621094, "epoch": 0.5608465608465608, "fcm_dpo/beta": 0.00784794706851244, "fcm_dpo/delta": 0.0034468621015548706, "fcm_dpo/margin": 56.83028030395508, "fcm_dpo/q_t": 0.4004814028739929, "grad_norm": 17.490827560424805, "learning_rate": 2.420680166254831e-07, "logits/chosen": 0.8194184899330139, "logits/rejected": 0.7865846753120422, "logps/chosen": -161.72067260742188, "logps/ref_chosen": -49.224212646484375, "logps/ref_rejected": -63.348472595214844, "logps/rejected": -232.6752166748047, "loss": 1.092, "margin_dpo/margin_mean": 56.83028030395508, "margin_dpo/margin_std": 83.48664855957031, "step": 371 }, { "KL/chosen_KL_mean": -119.92819213867188, "KL/mean": -136.84817504882812, "KL/rejected_KL_mean": -153.76815795898438, "KL/std": 77.0000991821289, "epoch": 0.562358276643991, "fcm_dpo/beta": 0.007822863757610321, "fcm_dpo/delta": 0.02560269646346569, "fcm_dpo/margin": 33.83995819091797, "fcm_dpo/q_t": 0.44220101833343506, "grad_norm": 16.425230026245117, "learning_rate": 2.4074658027491044e-07, "logits/chosen": 0.7374979257583618, "logits/rejected": 0.6381915807723999, "logps/chosen": -172.19775390625, "logps/ref_chosen": -52.269554138183594, "logps/ref_rejected": -72.99522399902344, "logps/rejected": -226.7633819580078, "loss": 1.2855, "margin_dpo/margin_mean": 33.8399543762207, "margin_dpo/margin_std": 99.59162902832031, "step": 372 }, { "KL/chosen_KL_mean": -134.54428100585938, "KL/mean": -158.06080627441406, "KL/rejected_KL_mean": -181.57733154296875, "KL/std": 80.29035186767578, "epoch": 0.563869992441421, "fcm_dpo/beta": 0.007942916825413704, "fcm_dpo/delta": 0.07930424809455872, "fcm_dpo/margin": 47.033042907714844, "fcm_dpo/q_t": 0.41851696372032166, "grad_norm": 13.92640209197998, "learning_rate": 2.394254027623792e-07, "logits/chosen": 0.6737028956413269, "logits/rejected": 0.6056632399559021, "logps/chosen": -195.65728759765625, "logps/ref_chosen": -61.112998962402344, "logps/ref_rejected": -76.24851989746094, "logps/rejected": -257.8258361816406, "loss": 1.1941, "margin_dpo/margin_mean": 47.03303909301758, "margin_dpo/margin_std": 100.17448425292969, "step": 373 }, { "KL/chosen_KL_mean": -114.75213623046875, "KL/mean": -152.3450469970703, "KL/rejected_KL_mean": -189.9379425048828, "KL/std": 82.41384887695312, "epoch": 0.5653817082388511, "fcm_dpo/beta": 0.0077880797907710075, "fcm_dpo/delta": -0.14424122869968414, "fcm_dpo/margin": 75.18580627441406, "fcm_dpo/q_t": 0.37012046575546265, "grad_norm": 15.205571174621582, "learning_rate": 2.381045210440644e-07, "logits/chosen": 0.6250724196434021, "logits/rejected": 0.6265436410903931, "logps/chosen": -187.4213409423828, "logps/ref_chosen": -72.66920471191406, "logps/ref_rejected": -76.83158874511719, "logps/rejected": -266.76953125, "loss": 1.0123, "margin_dpo/margin_mean": 75.1858139038086, "margin_dpo/margin_std": 97.47200012207031, "step": 374 }, { "KL/chosen_KL_mean": -113.03237915039062, "KL/mean": -140.780029296875, "KL/rejected_KL_mean": -168.5276641845703, "KL/std": 88.9169921875, "epoch": 0.5668934240362812, "fcm_dpo/beta": 0.0077394102700054646, "fcm_dpo/delta": 0.02124682255089283, "fcm_dpo/margin": 55.49528121948242, "fcm_dpo/q_t": 0.40597110986709595, "grad_norm": 15.411624908447266, "learning_rate": 2.3678397206786715e-07, "logits/chosen": 0.7049387693405151, "logits/rejected": 0.6460511684417725, "logps/chosen": -170.7156982421875, "logps/ref_chosen": -57.68330383300781, "logps/ref_rejected": -79.34097290039062, "logps/rejected": -247.86863708496094, "loss": 1.1328, "margin_dpo/margin_mean": 55.49528121948242, "margin_dpo/margin_std": 96.86477661132812, "step": 375 }, { "KL/chosen_KL_mean": -120.8951416015625, "KL/mean": -155.48785400390625, "KL/rejected_KL_mean": -190.08056640625, "KL/std": 89.38397216796875, "epoch": 0.5684051398337112, "fcm_dpo/beta": 0.007656695321202278, "fcm_dpo/delta": -0.08457353711128235, "fcm_dpo/margin": 69.18541717529297, "fcm_dpo/q_t": 0.3872775137424469, "grad_norm": 12.870294570922852, "learning_rate": 2.3546379277238103e-07, "logits/chosen": 0.735195517539978, "logits/rejected": 0.6646612882614136, "logps/chosen": -172.5692138671875, "logps/ref_chosen": -51.674072265625, "logps/ref_rejected": -75.69713592529297, "logps/rejected": -265.7777099609375, "loss": 1.067, "margin_dpo/margin_mean": 69.18541717529297, "margin_dpo/margin_std": 104.7180404663086, "step": 376 }, { "KL/chosen_KL_mean": -122.86614990234375, "KL/mean": -146.974853515625, "KL/rejected_KL_mean": -171.0835418701172, "KL/std": 75.72106170654297, "epoch": 0.5699168556311414, "fcm_dpo/beta": 0.007768765091896057, "fcm_dpo/delta": 0.07755792140960693, "fcm_dpo/margin": 48.21738052368164, "fcm_dpo/q_t": 0.41530439257621765, "grad_norm": 13.496039390563965, "learning_rate": 2.3414402008585886e-07, "logits/chosen": 0.7239181995391846, "logits/rejected": 0.6990246772766113, "logps/chosen": -169.044677734375, "logps/ref_chosen": -46.17853546142578, "logps/ref_rejected": -57.756500244140625, "logps/rejected": -228.8400421142578, "loss": 1.1513, "margin_dpo/margin_mean": 48.217384338378906, "margin_dpo/margin_std": 85.44389343261719, "step": 377 }, { "KL/chosen_KL_mean": -121.64041137695312, "KL/mean": -144.5426025390625, "KL/rejected_KL_mean": -167.44479370117188, "KL/std": 83.25043487548828, "epoch": 0.5714285714285714, "fcm_dpo/beta": 0.007907286286354065, "fcm_dpo/delta": 0.09018823504447937, "fcm_dpo/margin": 45.80438232421875, "fcm_dpo/q_t": 0.4190833270549774, "grad_norm": 13.746359825134277, "learning_rate": 2.3282469092517977e-07, "logits/chosen": 0.7368068695068359, "logits/rejected": 0.6892712712287903, "logps/chosen": -180.85928344726562, "logps/ref_chosen": -59.21887969970703, "logps/ref_rejected": -71.24818420410156, "logps/rejected": -238.69297790527344, "loss": 1.1613, "margin_dpo/margin_mean": 45.80437469482422, "margin_dpo/margin_std": 82.95941162109375, "step": 378 }, { "KL/chosen_KL_mean": -119.11299133300781, "KL/mean": -152.1615753173828, "KL/rejected_KL_mean": -185.21017456054688, "KL/std": 85.97073364257812, "epoch": 0.5729402872260015, "fcm_dpo/beta": 0.007825326174497604, "fcm_dpo/delta": -0.07087454199790955, "fcm_dpo/margin": 66.09716796875, "fcm_dpo/q_t": 0.3876013159751892, "grad_norm": 13.917744636535645, "learning_rate": 2.3150584219481643e-07, "logits/chosen": 0.6688319444656372, "logits/rejected": 0.5949773788452148, "logps/chosen": -195.42958068847656, "logps/ref_chosen": -76.31658935546875, "logps/ref_rejected": -104.26200103759766, "logps/rejected": -289.47216796875, "loss": 1.0541, "margin_dpo/margin_mean": 66.09716796875, "margin_dpo/margin_std": 94.03168487548828, "step": 379 }, { "KL/chosen_KL_mean": -103.0672378540039, "KL/mean": -140.85955810546875, "KL/rejected_KL_mean": -178.65188598632812, "KL/std": 78.66189575195312, "epoch": 0.5744520030234316, "fcm_dpo/beta": 0.007645877078175545, "fcm_dpo/delta": -0.13564595580101013, "fcm_dpo/margin": 75.58464813232422, "fcm_dpo/q_t": 0.37022942304611206, "grad_norm": 11.984086990356445, "learning_rate": 2.3018751078580283e-07, "logits/chosen": 0.7207067012786865, "logits/rejected": 0.6817103624343872, "logps/chosen": -164.35040283203125, "logps/ref_chosen": -61.283164978027344, "logps/ref_rejected": -72.38892364501953, "logps/rejected": -251.04080200195312, "loss": 1.0073, "margin_dpo/margin_mean": 75.58465576171875, "margin_dpo/margin_std": 93.482421875, "step": 380 }, { "KL/chosen_KL_mean": -126.96179962158203, "KL/mean": -141.54598999023438, "KL/rejected_KL_mean": -156.13018798828125, "KL/std": 81.3851547241211, "epoch": 0.5759637188208617, "fcm_dpo/beta": 0.0076386271975934505, "fcm_dpo/delta": 0.06993352621793747, "fcm_dpo/margin": 29.168384552001953, "fcm_dpo/q_t": 0.45122307538986206, "grad_norm": 13.51224422454834, "learning_rate": 2.288697335747027e-07, "logits/chosen": 0.6346052885055542, "logits/rejected": 0.6169425249099731, "logps/chosen": -185.17578125, "logps/ref_chosen": -58.2139892578125, "logps/ref_rejected": -60.78669357299805, "logps/rejected": -216.9168701171875, "loss": 1.3022, "margin_dpo/margin_mean": 29.168384552001953, "margin_dpo/margin_std": 96.48456573486328, "step": 381 }, { "KL/chosen_KL_mean": -127.53329467773438, "KL/mean": -153.88670349121094, "KL/rejected_KL_mean": -180.24008178710938, "KL/std": 81.65504455566406, "epoch": 0.5774754346182918, "fcm_dpo/beta": 0.0077784061431884766, "fcm_dpo/delta": 0.04100748896598816, "fcm_dpo/margin": 52.70681381225586, "fcm_dpo/q_t": 0.4081585109233856, "grad_norm": 14.638057708740234, "learning_rate": 2.2755254742257706e-07, "logits/chosen": 0.7190566062927246, "logits/rejected": 0.6629340052604675, "logps/chosen": -189.35861206054688, "logps/ref_chosen": -61.82532501220703, "logps/ref_rejected": -83.0452880859375, "logps/rejected": -263.2853698730469, "loss": 1.1092, "margin_dpo/margin_mean": 52.706817626953125, "margin_dpo/margin_std": 79.73265075683594, "step": 382 }, { "KL/chosen_KL_mean": -127.44752502441406, "KL/mean": -151.69912719726562, "KL/rejected_KL_mean": -175.9507293701172, "KL/std": 81.36878204345703, "epoch": 0.5789871504157218, "fcm_dpo/beta": 0.007807151880115271, "fcm_dpo/delta": 0.07406416535377502, "fcm_dpo/margin": 48.50321960449219, "fcm_dpo/q_t": 0.41831278800964355, "grad_norm": 15.773255348205566, "learning_rate": 2.2623598917395436e-07, "logits/chosen": 0.5699061751365662, "logits/rejected": 0.6028285026550293, "logps/chosen": -208.01077270507812, "logps/ref_chosen": -80.56326293945312, "logps/ref_rejected": -74.62922668457031, "logps/rejected": -250.5799560546875, "loss": 1.1889, "margin_dpo/margin_mean": 48.50322341918945, "margin_dpo/margin_std": 101.3597640991211, "step": 383 }, { "KL/chosen_KL_mean": -120.73493957519531, "KL/mean": -148.515869140625, "KL/rejected_KL_mean": -176.29678344726562, "KL/std": 80.29554748535156, "epoch": 0.5804988662131519, "fcm_dpo/beta": 0.007890386506915092, "fcm_dpo/delta": 0.012110977433621883, "fcm_dpo/margin": 55.561851501464844, "fcm_dpo/q_t": 0.39945393800735474, "grad_norm": 16.28131675720215, "learning_rate": 2.2492009565579875e-07, "logits/chosen": 0.7039550542831421, "logits/rejected": 0.6640291810035706, "logps/chosen": -186.2100830078125, "logps/ref_chosen": -65.47514343261719, "logps/ref_rejected": -79.67378234863281, "logps/rejected": -255.9705810546875, "loss": 1.0962, "margin_dpo/margin_mean": 55.561851501464844, "margin_dpo/margin_std": 83.95750427246094, "step": 384 }, { "KL/chosen_KL_mean": -118.50875854492188, "KL/mean": -154.27658081054688, "KL/rejected_KL_mean": -190.04440307617188, "KL/std": 81.92251586914062, "epoch": 0.582010582010582, "fcm_dpo/beta": 0.007788371294736862, "fcm_dpo/delta": -0.11333450675010681, "fcm_dpo/margin": 71.53565216064453, "fcm_dpo/q_t": 0.3762691617012024, "grad_norm": 13.468798637390137, "learning_rate": 2.2360490367648084e-07, "logits/chosen": 0.6351601481437683, "logits/rejected": 0.5921720266342163, "logps/chosen": -184.56527709960938, "logps/ref_chosen": -66.0565185546875, "logps/ref_rejected": -86.68023681640625, "logps/rejected": -276.7246398925781, "loss": 1.0081, "margin_dpo/margin_mean": 71.53565216064453, "margin_dpo/margin_std": 85.75909423828125, "step": 385 }, { "KL/chosen_KL_mean": -138.46185302734375, "KL/mean": -161.70205688476562, "KL/rejected_KL_mean": -184.94227600097656, "KL/std": 86.34553527832031, "epoch": 0.5835222978080121, "fcm_dpo/beta": 0.0078004514798521996, "fcm_dpo/delta": 0.09082190692424774, "fcm_dpo/margin": 46.48042297363281, "fcm_dpo/q_t": 0.41729068756103516, "grad_norm": 13.531447410583496, "learning_rate": 2.2229045002474724e-07, "logits/chosen": 0.635225772857666, "logits/rejected": 0.5782843828201294, "logps/chosen": -214.08551025390625, "logps/ref_chosen": -75.6236572265625, "logps/ref_rejected": -92.62330627441406, "logps/rejected": -277.5655822753906, "loss": 1.1624, "margin_dpo/margin_mean": 46.48042297363281, "margin_dpo/margin_std": 85.95610046386719, "step": 386 }, { "KL/chosen_KL_mean": -121.05953979492188, "KL/mean": -154.82073974609375, "KL/rejected_KL_mean": -188.58192443847656, "KL/std": 78.66697692871094, "epoch": 0.5850340136054422, "fcm_dpo/beta": 0.007759010884910822, "fcm_dpo/delta": -0.07805424928665161, "fcm_dpo/margin": 67.52236938476562, "fcm_dpo/q_t": 0.38322460651397705, "grad_norm": 15.326830863952637, "learning_rate": 2.209767714686924e-07, "logits/chosen": 0.6912319660186768, "logits/rejected": 0.5814871788024902, "logps/chosen": -168.28125, "logps/ref_chosen": -47.22170639038086, "logps/ref_rejected": -87.338134765625, "logps/rejected": -275.9200439453125, "loss": 1.0211, "margin_dpo/margin_mean": 67.52236938476562, "margin_dpo/margin_std": 82.26397705078125, "step": 387 }, { "KL/chosen_KL_mean": -123.56349182128906, "KL/mean": -145.304443359375, "KL/rejected_KL_mean": -167.04537963867188, "KL/std": 83.71977233886719, "epoch": 0.5865457294028723, "fcm_dpo/beta": 0.007735843770205975, "fcm_dpo/delta": 0.0017733016284182668, "fcm_dpo/margin": 43.481903076171875, "fcm_dpo/q_t": 0.42785027623176575, "grad_norm": 14.318989753723145, "learning_rate": 2.1966390475472954e-07, "logits/chosen": 0.6570320129394531, "logits/rejected": 0.6532206535339355, "logps/chosen": -198.14295959472656, "logps/ref_chosen": -74.5794677734375, "logps/ref_rejected": -79.92558288574219, "logps/rejected": -246.97097778320312, "loss": 1.2134, "margin_dpo/margin_mean": 43.481903076171875, "margin_dpo/margin_std": 98.93060302734375, "step": 388 }, { "KL/chosen_KL_mean": -118.34381103515625, "KL/mean": -151.6417694091797, "KL/rejected_KL_mean": -184.9397430419922, "KL/std": 81.11581420898438, "epoch": 0.5880574452003023, "fcm_dpo/beta": 0.007675846107304096, "fcm_dpo/delta": -0.06442630290985107, "fcm_dpo/margin": 66.59591674804688, "fcm_dpo/q_t": 0.3868892788887024, "grad_norm": 26.09174346923828, "learning_rate": 2.1835188660656265e-07, "logits/chosen": 0.6963808536529541, "logits/rejected": 0.6595287322998047, "logps/chosen": -179.96817016601562, "logps/ref_chosen": -61.624366760253906, "logps/ref_rejected": -76.50978088378906, "logps/rejected": -261.44952392578125, "loss": 1.0465, "margin_dpo/margin_mean": 66.59591674804688, "margin_dpo/margin_std": 89.59396362304688, "step": 389 }, { "KL/chosen_KL_mean": -105.8058090209961, "KL/mean": -132.39251708984375, "KL/rejected_KL_mean": -158.979248046875, "KL/std": 78.03878784179688, "epoch": 0.5895691609977324, "fcm_dpo/beta": 0.0076671745628118515, "fcm_dpo/delta": 0.04408419877290726, "fcm_dpo/margin": 53.173431396484375, "fcm_dpo/q_t": 0.40860432386398315, "grad_norm": 12.027225494384766, "learning_rate": 2.170407537241599e-07, "logits/chosen": 0.7897613048553467, "logits/rejected": 0.7145026922225952, "logps/chosen": -151.67767333984375, "logps/ref_chosen": -45.871864318847656, "logps/ref_rejected": -61.305999755859375, "logps/rejected": -220.28524780273438, "loss": 1.1127, "margin_dpo/margin_mean": 53.17342758178711, "margin_dpo/margin_std": 82.32190704345703, "step": 390 }, { "KL/chosen_KL_mean": -117.52503967285156, "KL/mean": -149.52325439453125, "KL/rejected_KL_mean": -181.52145385742188, "KL/std": 79.88783264160156, "epoch": 0.5910808767951625, "fcm_dpo/beta": 0.00763491727411747, "fcm_dpo/delta": -0.040894802659749985, "fcm_dpo/margin": 63.99640655517578, "fcm_dpo/q_t": 0.3913359045982361, "grad_norm": 12.352397918701172, "learning_rate": 2.1573054278272636e-07, "logits/chosen": 0.6845928430557251, "logits/rejected": 0.616992712020874, "logps/chosen": -175.71205139160156, "logps/ref_chosen": -58.18701171875, "logps/ref_rejected": -83.63442993164062, "logps/rejected": -265.1558837890625, "loss": 1.0819, "margin_dpo/margin_mean": 63.99640655517578, "margin_dpo/margin_std": 96.66595458984375, "step": 391 }, { "KL/chosen_KL_mean": -106.76398468017578, "KL/mean": -140.35391235351562, "KL/rejected_KL_mean": -173.94381713867188, "KL/std": 85.97457122802734, "epoch": 0.5925925925925926, "fcm_dpo/beta": 0.007620956748723984, "fcm_dpo/delta": -0.06586946547031403, "fcm_dpo/margin": 67.17984008789062, "fcm_dpo/q_t": 0.38879668712615967, "grad_norm": 13.259527206420898, "learning_rate": 2.1442129043167873e-07, "logits/chosen": 0.7438907027244568, "logits/rejected": 0.6849305629730225, "logps/chosen": -176.50851440429688, "logps/ref_chosen": -69.7445297241211, "logps/ref_rejected": -94.05877685546875, "logps/rejected": -268.0025939941406, "loss": 1.0735, "margin_dpo/margin_mean": 67.17984008789062, "margin_dpo/margin_std": 100.96359252929688, "step": 392 }, { "KL/chosen_KL_mean": -120.26972961425781, "KL/mean": -155.8102264404297, "KL/rejected_KL_mean": -191.35073852539062, "KL/std": 83.73542785644531, "epoch": 0.5941043083900227, "fcm_dpo/beta": 0.007404985371977091, "fcm_dpo/delta": -0.08186736702919006, "fcm_dpo/margin": 71.08099365234375, "fcm_dpo/q_t": 0.38141682744026184, "grad_norm": 11.25800609588623, "learning_rate": 2.131130332936195e-07, "logits/chosen": 0.7393004894256592, "logits/rejected": 0.6986830830574036, "logps/chosen": -172.60462951660156, "logps/ref_chosen": -52.33489990234375, "logps/ref_rejected": -74.33809661865234, "logps/rejected": -265.6888427734375, "loss": 1.0188, "margin_dpo/margin_mean": 71.08099365234375, "margin_dpo/margin_std": 83.78794860839844, "step": 393 }, { "KL/chosen_KL_mean": -115.41416931152344, "KL/mean": -145.79640197753906, "KL/rejected_KL_mean": -176.17861938476562, "KL/std": 76.976318359375, "epoch": 0.5956160241874527, "fcm_dpo/beta": 0.0074429926462471485, "fcm_dpo/delta": -0.0026309899985790253, "fcm_dpo/margin": 60.76445770263672, "fcm_dpo/q_t": 0.3960072994232178, "grad_norm": 12.980253219604492, "learning_rate": 2.1180580796331323e-07, "logits/chosen": 0.7144701480865479, "logits/rejected": 0.6844040155410767, "logps/chosen": -176.09030151367188, "logps/ref_chosen": -60.6761360168457, "logps/ref_rejected": -71.36074829101562, "logps/rejected": -247.53936767578125, "loss": 1.0635, "margin_dpo/margin_mean": 60.76445770263672, "margin_dpo/margin_std": 76.67507934570312, "step": 394 }, { "KL/chosen_KL_mean": -118.93069458007812, "KL/mean": -146.92593383789062, "KL/rejected_KL_mean": -174.92117309570312, "KL/std": 80.11039733886719, "epoch": 0.5971277399848829, "fcm_dpo/beta": 0.00747601967304945, "fcm_dpo/delta": 0.0324532687664032, "fcm_dpo/margin": 55.990455627441406, "fcm_dpo/q_t": 0.4076528251171112, "grad_norm": 14.716011047363281, "learning_rate": 2.104996510066625e-07, "logits/chosen": 0.7400883436203003, "logits/rejected": 0.6406676173210144, "logps/chosen": -169.53501892089844, "logps/ref_chosen": -50.60432434082031, "logps/ref_rejected": -77.08731079101562, "logps/rejected": -252.00848388671875, "loss": 1.1032, "margin_dpo/margin_mean": 55.99046325683594, "margin_dpo/margin_std": 84.89546203613281, "step": 395 }, { "KL/chosen_KL_mean": -113.71238708496094, "KL/mean": -143.36878967285156, "KL/rejected_KL_mean": -173.02520751953125, "KL/std": 88.9266128540039, "epoch": 0.5986394557823129, "fcm_dpo/beta": 0.007402296643704176, "fcm_dpo/delta": 0.009609527885913849, "fcm_dpo/margin": 59.31281280517578, "fcm_dpo/q_t": 0.4014135003089905, "grad_norm": 12.139056205749512, "learning_rate": 2.0919459895968517e-07, "logits/chosen": 0.7061029672622681, "logits/rejected": 0.6125322580337524, "logps/chosen": -165.07199096679688, "logps/ref_chosen": -51.35961151123047, "logps/ref_rejected": -79.89360046386719, "logps/rejected": -252.91879272460938, "loss": 1.0793, "margin_dpo/margin_mean": 59.31280517578125, "margin_dpo/margin_std": 77.40509033203125, "step": 396 }, { "KL/chosen_KL_mean": -130.69134521484375, "KL/mean": -147.45037841796875, "KL/rejected_KL_mean": -164.2094268798828, "KL/std": 81.297607421875, "epoch": 0.600151171579743, "fcm_dpo/beta": 0.007710670121014118, "fcm_dpo/delta": 0.19634832441806793, "fcm_dpo/margin": 33.518104553222656, "fcm_dpo/q_t": 0.4427967071533203, "grad_norm": 13.76626968383789, "learning_rate": 2.078906883274924e-07, "logits/chosen": 0.6195926070213318, "logits/rejected": 0.5693718791007996, "logps/chosen": -197.14755249023438, "logps/ref_chosen": -66.45622253417969, "logps/ref_rejected": -85.74736785888672, "logps/rejected": -249.95680236816406, "loss": 1.2701, "margin_dpo/margin_mean": 33.518104553222656, "margin_dpo/margin_std": 94.12562561035156, "step": 397 }, { "KL/chosen_KL_mean": -111.14080810546875, "KL/mean": -148.89974975585938, "KL/rejected_KL_mean": -186.65869140625, "KL/std": 85.36833190917969, "epoch": 0.6016628873771731, "fcm_dpo/beta": 0.007586984895169735, "fcm_dpo/delta": -0.1316794604063034, "fcm_dpo/margin": 75.51786804199219, "fcm_dpo/q_t": 0.37503597140312195, "grad_norm": 11.452016830444336, "learning_rate": 2.065879555832674e-07, "logits/chosen": 0.6672321557998657, "logits/rejected": 0.6032127141952515, "logps/chosen": -160.38504028320312, "logps/ref_chosen": -49.244239807128906, "logps/ref_rejected": -75.18949127197266, "logps/rejected": -261.8481750488281, "loss": 0.9958, "margin_dpo/margin_mean": 75.51786804199219, "margin_dpo/margin_std": 90.60173797607422, "step": 398 }, { "KL/chosen_KL_mean": -130.374267578125, "KL/mean": -173.41238403320312, "KL/rejected_KL_mean": -216.45053100585938, "KL/std": 94.34305572509766, "epoch": 0.6031746031746031, "fcm_dpo/beta": 0.0073226685635745525, "fcm_dpo/delta": -0.19398213922977448, "fcm_dpo/margin": 86.07627868652344, "fcm_dpo/q_t": 0.3651480972766876, "grad_norm": 12.927462577819824, "learning_rate": 2.052864371672457e-07, "logits/chosen": 0.6225359439849854, "logits/rejected": 0.48039311170578003, "logps/chosen": -198.68104553222656, "logps/ref_chosen": -68.30679321289062, "logps/ref_rejected": -113.2708511352539, "logps/rejected": -329.72137451171875, "loss": 0.9782, "margin_dpo/margin_mean": 86.07627868652344, "margin_dpo/margin_std": 106.19702911376953, "step": 399 }, { "KL/chosen_KL_mean": -144.94996643066406, "KL/mean": -169.4925537109375, "KL/rejected_KL_mean": -194.03512573242188, "KL/std": 85.74641418457031, "epoch": 0.6046863189720333, "fcm_dpo/beta": 0.007226614747196436, "fcm_dpo/delta": -0.04161422699689865, "fcm_dpo/margin": 49.08516311645508, "fcm_dpo/q_t": 0.41834330558776855, "grad_norm": 17.31791114807129, "learning_rate": 2.0398616948569493e-07, "logits/chosen": 0.6973024606704712, "logits/rejected": 0.6369521617889404, "logps/chosen": -216.5764617919922, "logps/ref_chosen": -71.62649536132812, "logps/ref_rejected": -90.98765563964844, "logps/rejected": -285.02276611328125, "loss": 1.1537, "margin_dpo/margin_mean": 49.08516311645508, "margin_dpo/margin_std": 82.55448150634766, "step": 400 }, { "KL/chosen_KL_mean": -110.20097351074219, "KL/mean": -145.64926147460938, "KL/rejected_KL_mean": -181.0975799560547, "KL/std": 92.50545501708984, "epoch": 0.6061980347694633, "fcm_dpo/beta": 0.0071340445429086685, "fcm_dpo/delta": -0.059332482516765594, "fcm_dpo/margin": 70.89659118652344, "fcm_dpo/q_t": 0.3873620331287384, "grad_norm": 9.548161506652832, "learning_rate": 2.0268718890989752e-07, "logits/chosen": 0.7134464383125305, "logits/rejected": 0.6242314577102661, "logps/chosen": -163.92593383789062, "logps/ref_chosen": -53.72495651245117, "logps/ref_rejected": -75.06304931640625, "logps/rejected": -256.1606140136719, "loss": 1.0338, "margin_dpo/margin_mean": 70.89659118652344, "margin_dpo/margin_std": 88.12057495117188, "step": 401 }, { "KL/chosen_KL_mean": -120.5625, "KL/mean": -148.1356201171875, "KL/rejected_KL_mean": -175.708740234375, "KL/std": 78.65602111816406, "epoch": 0.6077097505668935, "fcm_dpo/beta": 0.007157139014452696, "fcm_dpo/delta": 0.05712353438138962, "fcm_dpo/margin": 55.146236419677734, "fcm_dpo/q_t": 0.4119277596473694, "grad_norm": 14.758689880371094, "learning_rate": 2.013895317751323e-07, "logits/chosen": 0.6909421682357788, "logits/rejected": 0.6727443933486938, "logps/chosen": -182.43643188476562, "logps/ref_chosen": -61.873931884765625, "logps/ref_rejected": -66.15198516845703, "logps/rejected": -241.8607177734375, "loss": 1.1408, "margin_dpo/margin_mean": 55.146236419677734, "margin_dpo/margin_std": 93.9888916015625, "step": 402 }, { "KL/chosen_KL_mean": -127.60636901855469, "KL/mean": -163.28607177734375, "KL/rejected_KL_mean": -198.9657745361328, "KL/std": 91.14511108398438, "epoch": 0.6092214663643235, "fcm_dpo/beta": 0.007140805944800377, "fcm_dpo/delta": -0.06299025565385818, "fcm_dpo/margin": 71.35940551757812, "fcm_dpo/q_t": 0.3879600167274475, "grad_norm": 10.904901504516602, "learning_rate": 2.0009323437965898e-07, "logits/chosen": 0.8188765645027161, "logits/rejected": 0.730493426322937, "logps/chosen": -178.92787170410156, "logps/ref_chosen": -51.321502685546875, "logps/ref_rejected": -86.54010772705078, "logps/rejected": -285.5058898925781, "loss": 1.0464, "margin_dpo/margin_mean": 71.35940551757812, "margin_dpo/margin_std": 95.07559204101562, "step": 403 }, { "KL/chosen_KL_mean": -119.84092712402344, "KL/mean": -156.6473388671875, "KL/rejected_KL_mean": -193.4537353515625, "KL/std": 92.27593231201172, "epoch": 0.6107331821617535, "fcm_dpo/beta": 0.007005881518125534, "fcm_dpo/delta": -0.07096034288406372, "fcm_dpo/margin": 73.61279296875, "fcm_dpo/q_t": 0.38724684715270996, "grad_norm": 14.632216453552246, "learning_rate": 1.9879833298370237e-07, "logits/chosen": 0.6507140398025513, "logits/rejected": 0.5575799345970154, "logps/chosen": -182.10382080078125, "logps/ref_chosen": -62.26288604736328, "logps/ref_rejected": -95.19029998779297, "logps/rejected": -288.64404296875, "loss": 1.0525, "margin_dpo/margin_mean": 73.61280059814453, "margin_dpo/margin_std": 100.62483215332031, "step": 404 }, { "KL/chosen_KL_mean": -123.88162231445312, "KL/mean": -152.46707153320312, "KL/rejected_KL_mean": -181.05255126953125, "KL/std": 82.95892333984375, "epoch": 0.6122448979591837, "fcm_dpo/beta": 0.007027004845440388, "fcm_dpo/delta": 0.04931598901748657, "fcm_dpo/margin": 57.17091751098633, "fcm_dpo/q_t": 0.4105098247528076, "grad_norm": 12.742623329162598, "learning_rate": 1.975048638084379e-07, "logits/chosen": 0.7530736923217773, "logits/rejected": 0.7074278593063354, "logps/chosen": -174.46597290039062, "logps/ref_chosen": -50.5843391418457, "logps/ref_rejected": -65.43156433105469, "logps/rejected": -246.48411560058594, "loss": 1.1115, "margin_dpo/margin_mean": 57.17091369628906, "margin_dpo/margin_std": 83.23585510253906, "step": 405 }, { "KL/chosen_KL_mean": -117.66008758544922, "KL/mean": -155.90249633789062, "KL/rejected_KL_mean": -194.14488220214844, "KL/std": 89.51800537109375, "epoch": 0.6137566137566137, "fcm_dpo/beta": 0.0070230658166110516, "fcm_dpo/delta": -0.09200313687324524, "fcm_dpo/margin": 76.48478698730469, "fcm_dpo/q_t": 0.3798549175262451, "grad_norm": 12.587457656860352, "learning_rate": 1.9621286303497914e-07, "logits/chosen": 0.7463070750236511, "logits/rejected": 0.5869977474212646, "logps/chosen": -166.65570068359375, "logps/ref_chosen": -48.99560546875, "logps/ref_rejected": -92.47774505615234, "logps/rejected": -286.62261962890625, "loss": 1.0331, "margin_dpo/margin_mean": 76.48478698730469, "margin_dpo/margin_std": 99.57765197753906, "step": 406 }, { "KL/chosen_KL_mean": -150.7066650390625, "KL/mean": -180.73129272460938, "KL/rejected_KL_mean": -210.7559356689453, "KL/std": 102.36092376708984, "epoch": 0.6152683295540439, "fcm_dpo/beta": 0.007019747979938984, "fcm_dpo/delta": 0.029542850330471992, "fcm_dpo/margin": 60.04927062988281, "fcm_dpo/q_t": 0.4063884913921356, "grad_norm": 14.38724136352539, "learning_rate": 1.9492236680336483e-07, "logits/chosen": 0.5551047325134277, "logits/rejected": 0.4944635033607483, "logps/chosen": -240.1072235107422, "logps/ref_chosen": -89.40056610107422, "logps/ref_rejected": -99.28775024414062, "logps/rejected": -310.043701171875, "loss": 1.1289, "margin_dpo/margin_mean": 60.04927062988281, "margin_dpo/margin_std": 102.39197540283203, "step": 407 }, { "KL/chosen_KL_mean": -111.80641174316406, "KL/mean": -154.49441528320312, "KL/rejected_KL_mean": -197.18243408203125, "KL/std": 84.27525329589844, "epoch": 0.6167800453514739, "fcm_dpo/beta": 0.006904586683958769, "fcm_dpo/delta": -0.14809463918209076, "fcm_dpo/margin": 85.37601470947266, "fcm_dpo/q_t": 0.3662289083003998, "grad_norm": 11.406484603881836, "learning_rate": 1.9363341121154895e-07, "logits/chosen": 0.7426500916481018, "logits/rejected": 0.6647111177444458, "logps/chosen": -166.5103302001953, "logps/ref_chosen": -54.70391845703125, "logps/ref_rejected": -73.98648834228516, "logps/rejected": -271.1689147949219, "loss": 0.971, "margin_dpo/margin_mean": 85.37600708007812, "margin_dpo/margin_std": 88.67381286621094, "step": 408 }, { "KL/chosen_KL_mean": -139.26519775390625, "KL/mean": -159.29217529296875, "KL/rejected_KL_mean": -179.31918334960938, "KL/std": 72.91958618164062, "epoch": 0.618291761148904, "fcm_dpo/beta": 0.006983055733144283, "fcm_dpo/delta": 0.17536059021949768, "fcm_dpo/margin": 40.053977966308594, "fcm_dpo/q_t": 0.43714118003845215, "grad_norm": 13.305119514465332, "learning_rate": 1.9234603231438994e-07, "logits/chosen": 0.7231974601745605, "logits/rejected": 0.7324143052101135, "logps/chosen": -201.3834228515625, "logps/ref_chosen": -62.11822509765625, "logps/ref_rejected": -61.933509826660156, "logps/rejected": -241.252685546875, "loss": 1.2059, "margin_dpo/margin_mean": 40.053977966308594, "margin_dpo/margin_std": 82.67475891113281, "step": 409 }, { "KL/chosen_KL_mean": -130.318359375, "KL/mean": -164.8314208984375, "KL/rejected_KL_mean": -199.34449768066406, "KL/std": 82.56102752685547, "epoch": 0.6198034769463341, "fcm_dpo/beta": 0.0069589754566550255, "fcm_dpo/delta": -0.03359462320804596, "fcm_dpo/margin": 69.02615356445312, "fcm_dpo/q_t": 0.39046138525009155, "grad_norm": 11.623514175415039, "learning_rate": 1.9106026612264315e-07, "logits/chosen": 0.695282518863678, "logits/rejected": 0.6719921827316284, "logps/chosen": -192.1210174560547, "logps/ref_chosen": -61.80266189575195, "logps/ref_rejected": -76.60002136230469, "logps/rejected": -275.94451904296875, "loss": 1.0414, "margin_dpo/margin_mean": 69.02615356445312, "margin_dpo/margin_std": 80.98117065429688, "step": 410 }, { "KL/chosen_KL_mean": -136.67645263671875, "KL/mean": -169.91497802734375, "KL/rejected_KL_mean": -203.15347290039062, "KL/std": 89.70034790039062, "epoch": 0.6213151927437641, "fcm_dpo/beta": 0.006991643458604813, "fcm_dpo/delta": -0.0155013557523489, "fcm_dpo/margin": 66.47702026367188, "fcm_dpo/q_t": 0.39620882272720337, "grad_norm": 10.967887878417969, "learning_rate": 1.8977614860195296e-07, "logits/chosen": 0.7672784328460693, "logits/rejected": 0.7097710371017456, "logps/chosen": -191.12184143066406, "logps/ref_chosen": -54.44539260864258, "logps/ref_rejected": -74.5650863647461, "logps/rejected": -277.71856689453125, "loss": 1.0737, "margin_dpo/margin_mean": 66.47702026367188, "margin_dpo/margin_std": 94.61786651611328, "step": 411 }, { "KL/chosen_KL_mean": -139.16372680664062, "KL/mean": -169.79412841796875, "KL/rejected_KL_mean": -200.42453002929688, "KL/std": 76.77566528320312, "epoch": 0.6228269085411943, "fcm_dpo/beta": 0.006982295773923397, "fcm_dpo/delta": 0.02307012677192688, "fcm_dpo/margin": 61.26079559326172, "fcm_dpo/q_t": 0.4027141332626343, "grad_norm": 12.258587837219238, "learning_rate": 1.8849371567184662e-07, "logits/chosen": 0.7537434101104736, "logits/rejected": 0.6913366317749023, "logps/chosen": -194.41180419921875, "logps/ref_chosen": -55.248085021972656, "logps/ref_rejected": -68.96623229980469, "logps/rejected": -269.3907470703125, "loss": 1.0801, "margin_dpo/margin_mean": 61.26079177856445, "margin_dpo/margin_std": 81.34800720214844, "step": 412 }, { "KL/chosen_KL_mean": -154.973388671875, "KL/mean": -183.91961669921875, "KL/rejected_KL_mean": -212.8658447265625, "KL/std": 88.90245819091797, "epoch": 0.6243386243386243, "fcm_dpo/beta": 0.0070727551355957985, "fcm_dpo/delta": 0.04212770238518715, "fcm_dpo/margin": 57.892452239990234, "fcm_dpo/q_t": 0.41068193316459656, "grad_norm": 15.375361442565918, "learning_rate": 1.872130032047302e-07, "logits/chosen": 0.557374119758606, "logits/rejected": 0.5279108285903931, "logps/chosen": -223.69415283203125, "logps/ref_chosen": -68.72074890136719, "logps/ref_rejected": -78.76539611816406, "logps/rejected": -291.6312255859375, "loss": 1.1519, "margin_dpo/margin_mean": 57.892452239990234, "margin_dpo/margin_std": 107.5550537109375, "step": 413 }, { "KL/chosen_KL_mean": -137.41403198242188, "KL/mean": -171.55357360839844, "KL/rejected_KL_mean": -205.693115234375, "KL/std": 96.39564514160156, "epoch": 0.6258503401360545, "fcm_dpo/beta": 0.0070444131270051, "fcm_dpo/delta": -0.032540664076805115, "fcm_dpo/margin": 68.27906799316406, "fcm_dpo/q_t": 0.39073142409324646, "grad_norm": 11.638055801391602, "learning_rate": 1.8593404702488436e-07, "logits/chosen": 0.7413473129272461, "logits/rejected": 0.682194709777832, "logps/chosen": -191.55224609375, "logps/ref_chosen": -54.138214111328125, "logps/ref_rejected": -74.65741729736328, "logps/rejected": -280.35052490234375, "loss": 1.0561, "margin_dpo/margin_mean": 68.2790756225586, "margin_dpo/margin_std": 89.36869812011719, "step": 414 }, { "KL/chosen_KL_mean": -135.49588012695312, "KL/mean": -163.8934326171875, "KL/rejected_KL_mean": -192.29098510742188, "KL/std": 89.41297912597656, "epoch": 0.6273620559334845, "fcm_dpo/beta": 0.00707012414932251, "fcm_dpo/delta": 0.050504814833402634, "fcm_dpo/margin": 56.79509735107422, "fcm_dpo/q_t": 0.41016697883605957, "grad_norm": 12.561532974243164, "learning_rate": 1.846568829074628e-07, "logits/chosen": 0.749638557434082, "logits/rejected": 0.7332205772399902, "logps/chosen": -191.4144287109375, "logps/ref_chosen": -55.91856002807617, "logps/ref_rejected": -61.747703552246094, "logps/rejected": -254.0386962890625, "loss": 1.1275, "margin_dpo/margin_mean": 56.79510498046875, "margin_dpo/margin_std": 94.32406616210938, "step": 415 }, { "KL/chosen_KL_mean": -143.9766387939453, "KL/mean": -168.79827880859375, "KL/rejected_KL_mean": -193.61993408203125, "KL/std": 93.01155853271484, "epoch": 0.6288737717309146, "fcm_dpo/beta": 0.007032511755824089, "fcm_dpo/delta": -0.09679044038057327, "fcm_dpo/margin": 49.64328384399414, "fcm_dpo/q_t": 0.42265474796295166, "grad_norm": 14.138065338134766, "learning_rate": 1.8338154657749128e-07, "logits/chosen": 0.7095975875854492, "logits/rejected": 0.6624330282211304, "logps/chosen": -198.69972229003906, "logps/ref_chosen": -54.72308349609375, "logps/ref_rejected": -69.17388916015625, "logps/rejected": -262.7938232421875, "loss": 1.179, "margin_dpo/margin_mean": 49.64328384399414, "margin_dpo/margin_std": 89.73538208007812, "step": 416 }, { "KL/chosen_KL_mean": -149.44570922851562, "KL/mean": -181.24932861328125, "KL/rejected_KL_mean": -213.0529327392578, "KL/std": 86.96218872070312, "epoch": 0.6303854875283447, "fcm_dpo/beta": 0.007011952809989452, "fcm_dpo/delta": 0.003346942365169525, "fcm_dpo/margin": 63.607215881347656, "fcm_dpo/q_t": 0.39805179834365845, "grad_norm": 13.41283893585205, "learning_rate": 1.8210807370886849e-07, "logits/chosen": 0.8221265077590942, "logits/rejected": 0.7597347497940063, "logps/chosen": -206.23696899414062, "logps/ref_chosen": -56.791259765625, "logps/ref_rejected": -68.7791748046875, "logps/rejected": -281.83209228515625, "loss": 1.1164, "margin_dpo/margin_mean": 63.607215881347656, "margin_dpo/margin_std": 105.42054748535156, "step": 417 }, { "KL/chosen_KL_mean": -159.71566772460938, "KL/mean": -186.35943603515625, "KL/rejected_KL_mean": -213.00320434570312, "KL/std": 97.96890258789062, "epoch": 0.6318972033257747, "fcm_dpo/beta": 0.006870288867503405, "fcm_dpo/delta": -0.07140669971704483, "fcm_dpo/margin": 53.28753662109375, "fcm_dpo/q_t": 0.4186713695526123, "grad_norm": 14.105439186096191, "learning_rate": 1.8083649992336825e-07, "logits/chosen": 0.7837098240852356, "logits/rejected": 0.7870790958404541, "logps/chosen": -228.8236541748047, "logps/ref_chosen": -69.10798645019531, "logps/ref_rejected": -75.09132385253906, "logps/rejected": -288.09454345703125, "loss": 1.1693, "margin_dpo/margin_mean": 53.28753662109375, "margin_dpo/margin_std": 99.01031494140625, "step": 418 }, { "KL/chosen_KL_mean": -126.68461608886719, "KL/mean": -165.0189971923828, "KL/rejected_KL_mean": -203.35337829589844, "KL/std": 88.75230407714844, "epoch": 0.6334089191232048, "fcm_dpo/beta": 0.006787517108023167, "fcm_dpo/delta": -0.07435769587755203, "fcm_dpo/margin": 76.66874694824219, "fcm_dpo/q_t": 0.38583898544311523, "grad_norm": 11.909916877746582, "learning_rate": 1.7956686078964255e-07, "logits/chosen": 0.6351292729377747, "logits/rejected": 0.5898942947387695, "logps/chosen": -184.85638427734375, "logps/ref_chosen": -58.1717643737793, "logps/ref_rejected": -71.67066955566406, "logps/rejected": -275.0240478515625, "loss": 1.0373, "margin_dpo/margin_mean": 76.66874694824219, "margin_dpo/margin_std": 101.29539489746094, "step": 419 }, { "KL/chosen_KL_mean": -158.27415466308594, "KL/mean": -177.7630615234375, "KL/rejected_KL_mean": -197.251953125, "KL/std": 92.9329833984375, "epoch": 0.6349206349206349, "fcm_dpo/beta": 0.00685450155287981, "fcm_dpo/delta": 0.06285488605499268, "fcm_dpo/margin": 38.977783203125, "fcm_dpo/q_t": 0.4422045350074768, "grad_norm": 13.392801284790039, "learning_rate": 1.782991918222275e-07, "logits/chosen": 0.7376530170440674, "logits/rejected": 0.694350004196167, "logps/chosen": -215.32766723632812, "logps/ref_chosen": -57.05351257324219, "logps/ref_rejected": -62.670982360839844, "logps/rejected": -259.9229431152344, "loss": 1.2604, "margin_dpo/margin_mean": 38.977783203125, "margin_dpo/margin_std": 105.80848693847656, "step": 420 }, { "KL/chosen_KL_mean": -145.82496643066406, "KL/mean": -174.51287841796875, "KL/rejected_KL_mean": -203.20077514648438, "KL/std": 91.58183288574219, "epoch": 0.636432350718065, "fcm_dpo/beta": 0.006877418141812086, "fcm_dpo/delta": 0.057417359203100204, "fcm_dpo/margin": 57.37579345703125, "fcm_dpo/q_t": 0.41433507204055786, "grad_norm": 14.985981941223145, "learning_rate": 1.7703352848054887e-07, "logits/chosen": 0.6757022738456726, "logits/rejected": 0.6184811592102051, "logps/chosen": -203.14822387695312, "logps/ref_chosen": -57.32324981689453, "logps/ref_rejected": -75.33782958984375, "logps/rejected": -278.5386047363281, "loss": 1.1868, "margin_dpo/margin_mean": 57.375797271728516, "margin_dpo/margin_std": 119.31109619140625, "step": 421 }, { "KL/chosen_KL_mean": -128.066162109375, "KL/mean": -165.34466552734375, "KL/rejected_KL_mean": -202.62313842773438, "KL/std": 90.69136047363281, "epoch": 0.6379440665154951, "fcm_dpo/beta": 0.006889001466333866, "fcm_dpo/delta": -0.0670836940407753, "fcm_dpo/margin": 74.55697631835938, "fcm_dpo/q_t": 0.38474151492118835, "grad_norm": 13.526140213012695, "learning_rate": 1.7576990616793137e-07, "logits/chosen": 0.6843360662460327, "logits/rejected": 0.6799574494361877, "logps/chosen": -195.12374877929688, "logps/ref_chosen": -67.05757141113281, "logps/ref_rejected": -72.12803649902344, "logps/rejected": -274.75115966796875, "loss": 1.0293, "margin_dpo/margin_mean": 74.55697631835938, "margin_dpo/margin_std": 91.39720153808594, "step": 422 }, { "KL/chosen_KL_mean": -130.3348388671875, "KL/mean": -169.43580627441406, "KL/rejected_KL_mean": -208.5367431640625, "KL/std": 92.26570129394531, "epoch": 0.6394557823129252, "fcm_dpo/beta": 0.0067449817433953285, "fcm_dpo/delta": -0.08202774077653885, "fcm_dpo/margin": 78.201904296875, "fcm_dpo/q_t": 0.38311636447906494, "grad_norm": 12.166535377502441, "learning_rate": 1.745083602306071e-07, "logits/chosen": 0.7933779954910278, "logits/rejected": 0.7255642414093018, "logps/chosen": -184.39651489257812, "logps/ref_chosen": -54.06167221069336, "logps/ref_rejected": -76.64092254638672, "logps/rejected": -285.17767333984375, "loss": 1.025, "margin_dpo/margin_mean": 78.201904296875, "margin_dpo/margin_std": 97.99603271484375, "step": 423 }, { "KL/chosen_KL_mean": -138.32078552246094, "KL/mean": -177.15737915039062, "KL/rejected_KL_mean": -215.99398803710938, "KL/std": 86.63241577148438, "epoch": 0.6409674981103552, "fcm_dpo/beta": 0.006620635278522968, "fcm_dpo/delta": -0.0689234584569931, "fcm_dpo/margin": 77.6731948852539, "fcm_dpo/q_t": 0.3853021264076233, "grad_norm": 15.852065086364746, "learning_rate": 1.7324892595672804e-07, "logits/chosen": 0.6060911417007446, "logits/rejected": 0.5659924745559692, "logps/chosen": -191.92965698242188, "logps/ref_chosen": -53.60887145996094, "logps/ref_rejected": -79.2139892578125, "logps/rejected": -295.2079772949219, "loss": 1.0347, "margin_dpo/margin_mean": 77.6731948852539, "margin_dpo/margin_std": 98.35594940185547, "step": 424 }, { "KL/chosen_KL_mean": -139.3746337890625, "KL/mean": -168.45822143554688, "KL/rejected_KL_mean": -197.5418243408203, "KL/std": 84.505615234375, "epoch": 0.6424792139077853, "fcm_dpo/beta": 0.006698750890791416, "fcm_dpo/delta": 0.06281259655952454, "fcm_dpo/margin": 58.16719055175781, "fcm_dpo/q_t": 0.41317880153656006, "grad_norm": 13.068650245666504, "learning_rate": 1.7199163857537824e-07, "logits/chosen": 0.7463515996932983, "logits/rejected": 0.7196699380874634, "logps/chosen": -197.789306640625, "logps/ref_chosen": -58.41468048095703, "logps/ref_rejected": -66.59054565429688, "logps/rejected": -264.13238525390625, "loss": 1.1314, "margin_dpo/margin_mean": 58.16719055175781, "margin_dpo/margin_std": 96.10154724121094, "step": 425 }, { "KL/chosen_KL_mean": -166.42657470703125, "KL/mean": -184.3075714111328, "KL/rejected_KL_mean": -202.18856811523438, "KL/std": 90.15042114257812, "epoch": 0.6439909297052154, "fcm_dpo/beta": 0.006936525925993919, "fcm_dpo/delta": 0.2069658637046814, "fcm_dpo/margin": 35.762001037597656, "fcm_dpo/q_t": 0.4449042081832886, "grad_norm": 18.208776473999023, "learning_rate": 1.7073653325558828e-07, "logits/chosen": 0.7006687521934509, "logits/rejected": 0.7039576768875122, "logps/chosen": -238.13479614257812, "logps/ref_chosen": -71.70822143554688, "logps/ref_rejected": -73.57725524902344, "logps/rejected": -275.76580810546875, "loss": 1.2917, "margin_dpo/margin_mean": 35.762001037597656, "margin_dpo/margin_std": 112.21145629882812, "step": 426 }, { "KL/chosen_KL_mean": -157.49887084960938, "KL/mean": -188.9153289794922, "KL/rejected_KL_mean": -220.331787109375, "KL/std": 99.3335952758789, "epoch": 0.6455026455026455, "fcm_dpo/beta": 0.007007491309195757, "fcm_dpo/delta": 0.010125400498509407, "fcm_dpo/margin": 62.83293914794922, "fcm_dpo/q_t": 0.40540921688079834, "grad_norm": 14.336414337158203, "learning_rate": 1.6948364510535218e-07, "logits/chosen": 0.6997349262237549, "logits/rejected": 0.6372700929641724, "logps/chosen": -216.14163208007812, "logps/ref_chosen": -58.64276885986328, "logps/ref_rejected": -86.25437927246094, "logps/rejected": -306.586181640625, "loss": 1.1242, "margin_dpo/margin_mean": 62.83293914794922, "margin_dpo/margin_std": 109.12852478027344, "step": 427 }, { "KL/chosen_KL_mean": -151.69796752929688, "KL/mean": -185.02224731445312, "KL/rejected_KL_mean": -218.3465118408203, "KL/std": 100.86468505859375, "epoch": 0.6470143613000756, "fcm_dpo/beta": 0.007032737135887146, "fcm_dpo/delta": -0.020013775676488876, "fcm_dpo/margin": 66.64852905273438, "fcm_dpo/q_t": 0.3979109227657318, "grad_norm": 12.575758934020996, "learning_rate": 1.6823300917064458e-07, "logits/chosen": 0.6532288789749146, "logits/rejected": 0.6143096685409546, "logps/chosen": -218.2940216064453, "logps/ref_chosen": -66.5960464477539, "logps/ref_rejected": -82.3941650390625, "logps/rejected": -300.74066162109375, "loss": 1.1049, "margin_dpo/margin_mean": 66.64852905273438, "margin_dpo/margin_std": 107.984130859375, "step": 428 }, { "KL/chosen_KL_mean": -154.6398468017578, "KL/mean": -180.8323974609375, "KL/rejected_KL_mean": -207.02491760253906, "KL/std": 84.92485046386719, "epoch": 0.6485260770975056, "fcm_dpo/beta": 0.007110248319804668, "fcm_dpo/delta": 0.07964736223220825, "fcm_dpo/margin": 52.385066986083984, "fcm_dpo/q_t": 0.41526907682418823, "grad_norm": 14.710054397583008, "learning_rate": 1.669846604344412e-07, "logits/chosen": 0.6934762001037598, "logits/rejected": 0.7128642797470093, "logps/chosen": -211.64955139160156, "logps/ref_chosen": -57.00970458984375, "logps/ref_rejected": -59.86549377441406, "logps/rejected": -266.8904113769531, "loss": 1.1572, "margin_dpo/margin_mean": 52.38506317138672, "margin_dpo/margin_std": 94.00373077392578, "step": 429 }, { "KL/chosen_KL_mean": -137.94509887695312, "KL/mean": -179.18109130859375, "KL/rejected_KL_mean": -220.41705322265625, "KL/std": 86.64157104492188, "epoch": 0.6500377928949358, "fcm_dpo/beta": 0.006960110738873482, "fcm_dpo/delta": -0.131709486246109, "fcm_dpo/margin": 82.47195434570312, "fcm_dpo/q_t": 0.36921268701553345, "grad_norm": 13.911884307861328, "learning_rate": 1.6573863381573954e-07, "logits/chosen": 0.6297199726104736, "logits/rejected": 0.6272151470184326, "logps/chosen": -197.50830078125, "logps/ref_chosen": -59.563194274902344, "logps/ref_rejected": -70.52289581298828, "logps/rejected": -290.93994140625, "loss": 1.0029, "margin_dpo/margin_mean": 82.47195434570312, "margin_dpo/margin_std": 98.84146118164062, "step": 430 }, { "KL/chosen_KL_mean": -136.78176879882812, "KL/mean": -167.47360229492188, "KL/rejected_KL_mean": -198.16543579101562, "KL/std": 88.53604125976562, "epoch": 0.6515495086923658, "fcm_dpo/beta": 0.006935927085578442, "fcm_dpo/delta": 0.02532074972987175, "fcm_dpo/margin": 61.3836669921875, "fcm_dpo/q_t": 0.405121386051178, "grad_norm": 13.702001571655273, "learning_rate": 1.6449496416858282e-07, "logits/chosen": 0.7203608751296997, "logits/rejected": 0.6643437147140503, "logps/chosen": -186.9821014404297, "logps/ref_chosen": -50.20032501220703, "logps/ref_rejected": -77.81680297851562, "logps/rejected": -275.98223876953125, "loss": 1.1097, "margin_dpo/margin_mean": 61.3836669921875, "margin_dpo/margin_std": 96.4728775024414, "step": 431 }, { "KL/chosen_KL_mean": -142.304443359375, "KL/mean": -174.9478759765625, "KL/rejected_KL_mean": -207.59133911132812, "KL/std": 86.27505493164062, "epoch": 0.6530612244897959, "fcm_dpo/beta": 0.006943239830434322, "fcm_dpo/delta": -0.003489219583570957, "fcm_dpo/margin": 65.28689575195312, "fcm_dpo/q_t": 0.3982744514942169, "grad_norm": 14.40606689453125, "learning_rate": 1.632536862810844e-07, "logits/chosen": 0.7262982130050659, "logits/rejected": 0.6743370294570923, "logps/chosen": -203.96719360351562, "logps/ref_chosen": -61.662757873535156, "logps/ref_rejected": -83.94496154785156, "logps/rejected": -291.53631591796875, "loss": 1.0941, "margin_dpo/margin_mean": 65.28689575195312, "margin_dpo/margin_std": 99.4925765991211, "step": 432 }, { "KL/chosen_KL_mean": -141.8938751220703, "KL/mean": -179.70875549316406, "KL/rejected_KL_mean": -217.52365112304688, "KL/std": 83.667236328125, "epoch": 0.654572940287226, "fcm_dpo/beta": 0.006893531884998083, "fcm_dpo/delta": -0.07524176687002182, "fcm_dpo/margin": 75.62977600097656, "fcm_dpo/q_t": 0.3817235231399536, "grad_norm": 13.342097282409668, "learning_rate": 1.6201483487445515e-07, "logits/chosen": 0.8095422983169556, "logits/rejected": 0.8106013536453247, "logps/chosen": -205.623046875, "logps/ref_chosen": -63.72917938232422, "logps/ref_rejected": -65.8391342163086, "logps/rejected": -283.36279296875, "loss": 1.0331, "margin_dpo/margin_mean": 75.6297607421875, "margin_dpo/margin_std": 95.17214965820312, "step": 433 }, { "KL/chosen_KL_mean": -120.78085327148438, "KL/mean": -161.9495849609375, "KL/rejected_KL_mean": -203.11831665039062, "KL/std": 91.25202941894531, "epoch": 0.656084656084656, "fcm_dpo/beta": 0.006684788502752781, "fcm_dpo/delta": -0.1083269864320755, "fcm_dpo/margin": 82.33747100830078, "fcm_dpo/q_t": 0.375964879989624, "grad_norm": 12.028138160705566, "learning_rate": 1.6077844460203204e-07, "logits/chosen": 0.8332556486129761, "logits/rejected": 0.7700821161270142, "logps/chosen": -168.75416564941406, "logps/ref_chosen": -47.97331619262695, "logps/ref_rejected": -72.51132202148438, "logps/rejected": -275.629638671875, "loss": 1.0374, "margin_dpo/margin_mean": 82.33747100830078, "margin_dpo/margin_std": 109.64669799804688, "step": 434 }, { "KL/chosen_KL_mean": -145.86680603027344, "KL/mean": -177.239501953125, "KL/rejected_KL_mean": -208.6121826171875, "KL/std": 87.94329833984375, "epoch": 0.6575963718820862, "fcm_dpo/beta": 0.006757765542715788, "fcm_dpo/delta": 0.02654809132218361, "fcm_dpo/margin": 62.74540710449219, "fcm_dpo/q_t": 0.4058331251144409, "grad_norm": 15.042875289916992, "learning_rate": 1.5954455004830878e-07, "logits/chosen": 0.8290047645568848, "logits/rejected": 0.7902365922927856, "logps/chosen": -202.9270477294922, "logps/ref_chosen": -57.06024932861328, "logps/ref_rejected": -71.69146728515625, "logps/rejected": -280.30364990234375, "loss": 1.1169, "margin_dpo/margin_mean": 62.745399475097656, "margin_dpo/margin_std": 101.01614379882812, "step": 435 }, { "KL/chosen_KL_mean": -145.9172821044922, "KL/mean": -172.13966369628906, "KL/rejected_KL_mean": -198.362060546875, "KL/std": 86.71946716308594, "epoch": 0.6591080876795162, "fcm_dpo/beta": 0.006820957642048597, "fcm_dpo/delta": 0.09580140560865402, "fcm_dpo/margin": 52.444766998291016, "fcm_dpo/q_t": 0.4212391674518585, "grad_norm": 15.2418794631958, "learning_rate": 1.5831318572796847e-07, "logits/chosen": 0.7261216640472412, "logits/rejected": 0.6715189218521118, "logps/chosen": -202.07533264160156, "logps/ref_chosen": -56.158050537109375, "logps/ref_rejected": -67.63787841796875, "logps/rejected": -265.99993896484375, "loss": 1.1837, "margin_dpo/margin_mean": 52.444766998291016, "margin_dpo/margin_std": 106.6795654296875, "step": 436 }, { "KL/chosen_KL_mean": -152.7239990234375, "KL/mean": -183.65553283691406, "KL/rejected_KL_mean": -214.5870819091797, "KL/std": 94.8659896850586, "epoch": 0.6606198034769464, "fcm_dpo/beta": 0.00675150565803051, "fcm_dpo/delta": -0.08352308720350266, "fcm_dpo/margin": 61.86308288574219, "fcm_dpo/q_t": 0.4087408185005188, "grad_norm": 16.527193069458008, "learning_rate": 1.5708438608491815e-07, "logits/chosen": 0.7057990431785583, "logits/rejected": 0.5852597951889038, "logps/chosen": -209.70977783203125, "logps/ref_chosen": -56.98578643798828, "logps/ref_rejected": -85.61524963378906, "logps/rejected": -300.20233154296875, "loss": 1.1707, "margin_dpo/margin_mean": 61.86308288574219, "margin_dpo/margin_std": 120.68586730957031, "step": 437 }, { "KL/chosen_KL_mean": -131.84364318847656, "KL/mean": -173.12020874023438, "KL/rejected_KL_mean": -214.39675903320312, "KL/std": 98.55461120605469, "epoch": 0.6621315192743764, "fcm_dpo/beta": 0.006669066846370697, "fcm_dpo/delta": -0.1062895730137825, "fcm_dpo/margin": 82.55313110351562, "fcm_dpo/q_t": 0.38156580924987793, "grad_norm": 13.007272720336914, "learning_rate": 1.558581854913253e-07, "logits/chosen": 0.805670440196991, "logits/rejected": 0.7402873039245605, "logps/chosen": -173.12142944335938, "logps/ref_chosen": -41.27777862548828, "logps/ref_rejected": -65.33840942382812, "logps/rejected": -279.73516845703125, "loss": 1.0229, "margin_dpo/margin_mean": 82.55313110351562, "margin_dpo/margin_std": 109.02909851074219, "step": 438 }, { "KL/chosen_KL_mean": -149.43386840820312, "KL/mean": -186.30145263671875, "KL/rejected_KL_mean": -223.1690216064453, "KL/std": 99.49622344970703, "epoch": 0.6636432350718064, "fcm_dpo/beta": 0.006624848581850529, "fcm_dpo/delta": -0.04157322645187378, "fcm_dpo/margin": 73.73515319824219, "fcm_dpo/q_t": 0.39052993059158325, "grad_norm": 13.092687606811523, "learning_rate": 1.5463461824665658e-07, "logits/chosen": 0.6147295832633972, "logits/rejected": 0.5832754373550415, "logps/chosen": -230.8515167236328, "logps/ref_chosen": -81.41764831542969, "logps/ref_rejected": -94.72309875488281, "logps/rejected": -317.8921203613281, "loss": 1.0622, "margin_dpo/margin_mean": 73.73515319824219, "margin_dpo/margin_std": 99.93533325195312, "step": 439 }, { "KL/chosen_KL_mean": -129.64244079589844, "KL/mean": -165.80941772460938, "KL/rejected_KL_mean": -201.9763946533203, "KL/std": 88.39790344238281, "epoch": 0.6651549508692366, "fcm_dpo/beta": 0.006558232009410858, "fcm_dpo/delta": -0.025874171406030655, "fcm_dpo/margin": 72.33395385742188, "fcm_dpo/q_t": 0.39480775594711304, "grad_norm": 24.257293701171875, "learning_rate": 1.534137185767178e-07, "logits/chosen": 0.7508381605148315, "logits/rejected": 0.6510604619979858, "logps/chosen": -172.18063354492188, "logps/ref_chosen": -42.538185119628906, "logps/ref_rejected": -69.78813934326172, "logps/rejected": -271.7645263671875, "loss": 1.0767, "margin_dpo/margin_mean": 72.33395385742188, "margin_dpo/margin_std": 103.9335708618164, "step": 440 }, { "KL/chosen_KL_mean": -131.30650329589844, "KL/mean": -170.871826171875, "KL/rejected_KL_mean": -210.43714904785156, "KL/std": 95.84239196777344, "epoch": 0.6666666666666666, "fcm_dpo/beta": 0.006433665752410889, "fcm_dpo/delta": -0.06287907809019089, "fcm_dpo/margin": 79.13067626953125, "fcm_dpo/q_t": 0.3833358883857727, "grad_norm": 15.967854499816895, "learning_rate": 1.521955206326976e-07, "logits/chosen": 0.6979551315307617, "logits/rejected": 0.6069918870925903, "logps/chosen": -188.89971923828125, "logps/ref_chosen": -57.593223571777344, "logps/ref_rejected": -84.82878875732422, "logps/rejected": -295.26593017578125, "loss": 1.0171, "margin_dpo/margin_mean": 79.13067626953125, "margin_dpo/margin_std": 87.90113830566406, "step": 441 }, { "KL/chosen_KL_mean": -157.26657104492188, "KL/mean": -197.35610961914062, "KL/rejected_KL_mean": -237.4456787109375, "KL/std": 95.72440338134766, "epoch": 0.6681783824640968, "fcm_dpo/beta": 0.006357879377901554, "fcm_dpo/delta": -0.06341610103845596, "fcm_dpo/margin": 80.17911529541016, "fcm_dpo/q_t": 0.38497215509414673, "grad_norm": 14.785154342651367, "learning_rate": 1.5098005849021078e-07, "logits/chosen": 0.6711052656173706, "logits/rejected": 0.628170907497406, "logps/chosen": -224.727783203125, "logps/ref_chosen": -67.46121978759766, "logps/ref_rejected": -89.0693588256836, "logps/rejected": -326.5150451660156, "loss": 1.0311, "margin_dpo/margin_mean": 80.17912292480469, "margin_dpo/margin_std": 98.379638671875, "step": 442 }, { "KL/chosen_KL_mean": -129.59542846679688, "KL/mean": -178.85983276367188, "KL/rejected_KL_mean": -228.12423706054688, "KL/std": 101.99020385742188, "epoch": 0.6696900982615268, "fcm_dpo/beta": 0.006190933287143707, "fcm_dpo/delta": -0.17065072059631348, "fcm_dpo/margin": 98.52880859375, "fcm_dpo/q_t": 0.3641754388809204, "grad_norm": 12.864545822143555, "learning_rate": 1.4976736614834662e-07, "logits/chosen": 0.744697093963623, "logits/rejected": 0.6813009977340698, "logps/chosen": -184.3915252685547, "logps/ref_chosen": -54.79610061645508, "logps/ref_rejected": -77.80781555175781, "logps/rejected": -305.93206787109375, "loss": 0.9731, "margin_dpo/margin_mean": 98.52880859375, "margin_dpo/margin_std": 113.32872772216797, "step": 443 }, { "KL/chosen_KL_mean": -160.04959106445312, "KL/mean": -181.8095703125, "KL/rejected_KL_mean": -203.569580078125, "KL/std": 100.75285339355469, "epoch": 0.671201814058957, "fcm_dpo/beta": 0.006196248810738325, "fcm_dpo/delta": 0.04157605022192001, "fcm_dpo/margin": 43.519989013671875, "fcm_dpo/q_t": 0.4417763352394104, "grad_norm": 16.792774200439453, "learning_rate": 1.4855747752871654e-07, "logits/chosen": 0.7347674369812012, "logits/rejected": 0.6471496224403381, "logps/chosen": -218.79864501953125, "logps/ref_chosen": -58.749061584472656, "logps/ref_rejected": -86.87396240234375, "logps/rejected": -290.44354248046875, "loss": 1.247, "margin_dpo/margin_mean": 43.519989013671875, "margin_dpo/margin_std": 110.95816040039062, "step": 444 }, { "KL/chosen_KL_mean": -150.85191345214844, "KL/mean": -192.16619873046875, "KL/rejected_KL_mean": -233.48049926757812, "KL/std": 93.49421691894531, "epoch": 0.672713529856387, "fcm_dpo/beta": 0.006143460050225258, "fcm_dpo/delta": -0.060683369636535645, "fcm_dpo/margin": 82.62857818603516, "fcm_dpo/q_t": 0.38386598229408264, "grad_norm": 12.43869400024414, "learning_rate": 1.473504264745062e-07, "logits/chosen": 0.675220251083374, "logits/rejected": 0.6696294546127319, "logps/chosen": -211.76934814453125, "logps/ref_chosen": -60.91743850708008, "logps/ref_rejected": -71.5637435913086, "logps/rejected": -305.04425048828125, "loss": 1.0258, "margin_dpo/margin_mean": 82.62857818603516, "margin_dpo/margin_std": 97.68711853027344, "step": 445 }, { "KL/chosen_KL_mean": -136.88546752929688, "KL/mean": -178.89035034179688, "KL/rejected_KL_mean": -220.89523315429688, "KL/std": 95.0992431640625, "epoch": 0.674225245653817, "fcm_dpo/beta": 0.006117424461990595, "fcm_dpo/delta": -0.06857110559940338, "fcm_dpo/margin": 84.00977325439453, "fcm_dpo/q_t": 0.38333478569984436, "grad_norm": 12.378578186035156, "learning_rate": 1.461462467495284e-07, "logits/chosen": 0.7436650395393372, "logits/rejected": 0.6679918766021729, "logps/chosen": -185.68470764160156, "logps/ref_chosen": -48.79924774169922, "logps/ref_rejected": -71.8719482421875, "logps/rejected": -292.7671813964844, "loss": 1.0182, "margin_dpo/margin_mean": 84.00977325439453, "margin_dpo/margin_std": 91.514404296875, "step": 446 }, { "KL/chosen_KL_mean": -132.49256896972656, "KL/mean": -180.810791015625, "KL/rejected_KL_mean": -229.1290283203125, "KL/std": 94.01569366455078, "epoch": 0.6757369614512472, "fcm_dpo/beta": 0.005898823030292988, "fcm_dpo/delta": -0.12822897732257843, "fcm_dpo/margin": 96.63645935058594, "fcm_dpo/q_t": 0.36920368671417236, "grad_norm": 13.93958854675293, "learning_rate": 1.4494497203727843e-07, "logits/chosen": 0.6841769814491272, "logits/rejected": 0.5845425128936768, "logps/chosen": -186.17529296875, "logps/ref_chosen": -53.682716369628906, "logps/ref_rejected": -88.17315673828125, "logps/rejected": -317.30218505859375, "loss": 0.9901, "margin_dpo/margin_mean": 96.63645935058594, "margin_dpo/margin_std": 106.86634826660156, "step": 447 }, { "KL/chosen_KL_mean": -145.4189453125, "KL/mean": -183.78155517578125, "KL/rejected_KL_mean": -222.14418029785156, "KL/std": 96.9927749633789, "epoch": 0.6772486772486772, "fcm_dpo/beta": 0.005885988939553499, "fcm_dpo/delta": -0.0016819881275296211, "fcm_dpo/margin": 76.72523498535156, "fcm_dpo/q_t": 0.3971629738807678, "grad_norm": 9.772438049316406, "learning_rate": 1.4374663593999256e-07, "logits/chosen": 0.7255531549453735, "logits/rejected": 0.6777403950691223, "logps/chosen": -199.17019653320312, "logps/ref_chosen": -53.75125503540039, "logps/ref_rejected": -77.17623901367188, "logps/rejected": -299.3204345703125, "loss": 1.0711, "margin_dpo/margin_mean": 76.72523498535156, "margin_dpo/margin_std": 103.92894744873047, "step": 448 }, { "KL/chosen_KL_mean": -172.9747772216797, "KL/mean": -192.73361206054688, "KL/rejected_KL_mean": -212.49246215820312, "KL/std": 100.64846801757812, "epoch": 0.6787603930461074, "fcm_dpo/beta": 0.005969383753836155, "fcm_dpo/delta": 0.07266371697187424, "fcm_dpo/margin": 39.5177116394043, "fcm_dpo/q_t": 0.44623565673828125, "grad_norm": 19.279781341552734, "learning_rate": 1.4255127197770707e-07, "logits/chosen": 0.5781837701797485, "logits/rejected": 0.5781469345092773, "logps/chosen": -248.80215454101562, "logps/ref_chosen": -75.82737731933594, "logps/ref_rejected": -82.20687866210938, "logps/rejected": -294.6993408203125, "loss": 1.2458, "margin_dpo/margin_mean": 39.5177116394043, "margin_dpo/margin_std": 97.13715362548828, "step": 449 }, { "KL/chosen_KL_mean": -142.44183349609375, "KL/mean": -173.92239379882812, "KL/rejected_KL_mean": -205.40293884277344, "KL/std": 99.94340515136719, "epoch": 0.6802721088435374, "fcm_dpo/beta": 0.006016598083078861, "fcm_dpo/delta": 0.07402040809392929, "fcm_dpo/margin": 62.96111297607422, "fcm_dpo/q_t": 0.41852670907974243, "grad_norm": 12.642474174499512, "learning_rate": 1.4135891358732205e-07, "logits/chosen": 0.7799849510192871, "logits/rejected": 0.6756715774536133, "logps/chosen": -189.55755615234375, "logps/ref_chosen": -47.11572265625, "logps/ref_rejected": -78.7546615600586, "logps/rejected": -284.1575927734375, "loss": 1.153, "margin_dpo/margin_mean": 62.96112060546875, "margin_dpo/margin_std": 114.33680725097656, "step": 450 }, { "KL/chosen_KL_mean": -143.79136657714844, "KL/mean": -172.44857788085938, "KL/rejected_KL_mean": -201.1057891845703, "KL/std": 95.12226104736328, "epoch": 0.6817838246409675, "fcm_dpo/beta": 0.006161220371723175, "fcm_dpo/delta": 0.10015745460987091, "fcm_dpo/margin": 57.314430236816406, "fcm_dpo/q_t": 0.42018577456474304, "grad_norm": 13.891311645507812, "learning_rate": 1.4016959412166437e-07, "logits/chosen": 0.6785339117050171, "logits/rejected": 0.63237464427948, "logps/chosen": -207.14181518554688, "logps/ref_chosen": -63.350440979003906, "logps/ref_rejected": -76.28530883789062, "logps/rejected": -277.39111328125, "loss": 1.149, "margin_dpo/margin_mean": 57.314430236816406, "margin_dpo/margin_std": 97.8006362915039, "step": 451 }, { "KL/chosen_KL_mean": -144.1630859375, "KL/mean": -176.96450805664062, "KL/rejected_KL_mean": -209.76593017578125, "KL/std": 91.98998260498047, "epoch": 0.6832955404383976, "fcm_dpo/beta": 0.006219558417797089, "fcm_dpo/delta": 0.04377629607915878, "fcm_dpo/margin": 65.60285949707031, "fcm_dpo/q_t": 0.4090433716773987, "grad_norm": 14.884453773498535, "learning_rate": 1.3898334684855645e-07, "logits/chosen": 0.6643181443214417, "logits/rejected": 0.5869356393814087, "logps/chosen": -199.74891662597656, "logps/ref_chosen": -55.58583450317383, "logps/ref_rejected": -77.68738555908203, "logps/rejected": -287.45330810546875, "loss": 1.132, "margin_dpo/margin_mean": 65.60285949707031, "margin_dpo/margin_std": 111.69452667236328, "step": 452 }, { "KL/chosen_KL_mean": -144.0391387939453, "KL/mean": -176.14346313476562, "KL/rejected_KL_mean": -208.247802734375, "KL/std": 94.88691711425781, "epoch": 0.6848072562358276, "fcm_dpo/beta": 0.006271724589169025, "fcm_dpo/delta": 0.049313317984342575, "fcm_dpo/margin": 64.20864868164062, "fcm_dpo/q_t": 0.4105488657951355, "grad_norm": 15.54833698272705, "learning_rate": 1.3780020494988445e-07, "logits/chosen": 0.680355429649353, "logits/rejected": 0.6594034433364868, "logps/chosen": -205.81735229492188, "logps/ref_chosen": -61.778202056884766, "logps/ref_rejected": -71.51403045654297, "logps/rejected": -279.7618408203125, "loss": 1.1303, "margin_dpo/margin_mean": 64.20864868164062, "margin_dpo/margin_std": 107.24003601074219, "step": 453 }, { "KL/chosen_KL_mean": -132.9048614501953, "KL/mean": -169.541748046875, "KL/rejected_KL_mean": -206.17861938476562, "KL/std": 95.81768035888672, "epoch": 0.6863189720332578, "fcm_dpo/beta": 0.0062804133631289005, "fcm_dpo/delta": -0.01071581058204174, "fcm_dpo/margin": 73.27376556396484, "fcm_dpo/q_t": 0.3961000144481659, "grad_norm": 13.032012939453125, "learning_rate": 1.366202015206706e-07, "logits/chosen": 0.723847508430481, "logits/rejected": 0.6875928044319153, "logps/chosen": -184.5, "logps/ref_chosen": -51.59515380859375, "logps/ref_rejected": -63.96732711791992, "logps/rejected": -270.14593505859375, "loss": 1.0833, "margin_dpo/margin_mean": 73.27375793457031, "margin_dpo/margin_std": 108.64644622802734, "step": 454 }, { "KL/chosen_KL_mean": -154.15054321289062, "KL/mean": -188.80511474609375, "KL/rejected_KL_mean": -223.4596710205078, "KL/std": 95.76663208007812, "epoch": 0.6878306878306878, "fcm_dpo/beta": 0.006290521007031202, "fcm_dpo/delta": 0.01463963370770216, "fcm_dpo/margin": 69.30914306640625, "fcm_dpo/q_t": 0.40427112579345703, "grad_norm": 13.61314868927002, "learning_rate": 1.354433695681474e-07, "logits/chosen": 0.5870033502578735, "logits/rejected": 0.5615803003311157, "logps/chosen": -224.80224609375, "logps/ref_chosen": -70.65170288085938, "logps/ref_rejected": -77.44276428222656, "logps/rejected": -300.9024353027344, "loss": 1.0927, "margin_dpo/margin_mean": 69.30914306640625, "margin_dpo/margin_std": 103.27223205566406, "step": 455 }, { "KL/chosen_KL_mean": -149.3287353515625, "KL/mean": -182.22406005859375, "KL/rejected_KL_mean": -215.119384765625, "KL/std": 94.61996459960938, "epoch": 0.6893424036281179, "fcm_dpo/beta": 0.006323617417365313, "fcm_dpo/delta": 0.035434018820524216, "fcm_dpo/margin": 65.79066467285156, "fcm_dpo/q_t": 0.40913882851600647, "grad_norm": 17.2503719329834, "learning_rate": 1.3426974201083439e-07, "logits/chosen": 0.631847620010376, "logits/rejected": 0.5720112323760986, "logps/chosen": -205.72702026367188, "logps/ref_chosen": -56.398284912109375, "logps/ref_rejected": -82.61642456054688, "logps/rejected": -297.7358093261719, "loss": 1.1165, "margin_dpo/margin_mean": 65.79066467285156, "margin_dpo/margin_std": 107.55733489990234, "step": 456 }, { "KL/chosen_KL_mean": -147.87754821777344, "KL/mean": -182.2469482421875, "KL/rejected_KL_mean": -216.61634826660156, "KL/std": 97.08610534667969, "epoch": 0.690854119425548, "fcm_dpo/beta": 0.0063408189453184605, "fcm_dpo/delta": 0.01462765596807003, "fcm_dpo/margin": 68.7387924194336, "fcm_dpo/q_t": 0.40054911375045776, "grad_norm": 13.827205657958984, "learning_rate": 1.3309935167761717e-07, "logits/chosen": 0.8234766721725464, "logits/rejected": 0.7447128295898438, "logps/chosen": -192.59811401367188, "logps/ref_chosen": -44.72057342529297, "logps/ref_rejected": -68.1158676147461, "logps/rejected": -284.7322082519531, "loss": 1.0766, "margin_dpo/margin_mean": 68.7387924194336, "margin_dpo/margin_std": 91.6793212890625, "step": 457 }, { "KL/chosen_KL_mean": -142.45022583007812, "KL/mean": -179.52110290527344, "KL/rejected_KL_mean": -216.5919952392578, "KL/std": 99.73497009277344, "epoch": 0.6923658352229781, "fcm_dpo/beta": 0.006352492608129978, "fcm_dpo/delta": -0.022026075050234795, "fcm_dpo/margin": 74.14176940917969, "fcm_dpo/q_t": 0.3945360779762268, "grad_norm": 13.504679679870605, "learning_rate": 1.3193223130682936e-07, "logits/chosen": 0.7190458178520203, "logits/rejected": 0.6072895526885986, "logps/chosen": -192.4559326171875, "logps/ref_chosen": -50.00569152832031, "logps/ref_rejected": -87.50015258789062, "logps/rejected": -304.0921630859375, "loss": 1.0825, "margin_dpo/margin_mean": 74.14176940917969, "margin_dpo/margin_std": 110.75813293457031, "step": 458 }, { "KL/chosen_KL_mean": -136.78713989257812, "KL/mean": -181.85948181152344, "KL/rejected_KL_mean": -226.93182373046875, "KL/std": 114.09321594238281, "epoch": 0.6938775510204082, "fcm_dpo/beta": 0.006281477864831686, "fcm_dpo/delta": -0.12387188524007797, "fcm_dpo/margin": 90.14466857910156, "fcm_dpo/q_t": 0.3718501925468445, "grad_norm": 12.169325828552246, "learning_rate": 1.3076841354533658e-07, "logits/chosen": 0.7203759551048279, "logits/rejected": 0.690513014793396, "logps/chosen": -202.16510009765625, "logps/ref_chosen": -65.37794494628906, "logps/ref_rejected": -88.19244384765625, "logps/rejected": -315.124267578125, "loss": 1.0117, "margin_dpo/margin_mean": 90.14466857910156, "margin_dpo/margin_std": 107.88766479492188, "step": 459 }, { "KL/chosen_KL_mean": -149.89292907714844, "KL/mean": -193.3321075439453, "KL/rejected_KL_mean": -236.77130126953125, "KL/std": 100.56085205078125, "epoch": 0.6953892668178382, "fcm_dpo/beta": 0.006080328952521086, "fcm_dpo/delta": -0.08334297686815262, "fcm_dpo/margin": 86.87835693359375, "fcm_dpo/q_t": 0.3830181360244751, "grad_norm": 12.787505149841309, "learning_rate": 1.2960793094762345e-07, "logits/chosen": 0.7253769636154175, "logits/rejected": 0.6118832230567932, "logps/chosen": -214.45458984375, "logps/ref_chosen": -64.5616683959961, "logps/ref_rejected": -88.67890167236328, "logps/rejected": -325.4501953125, "loss": 1.0242, "margin_dpo/margin_mean": 86.87835693359375, "margin_dpo/margin_std": 108.14886474609375, "step": 460 }, { "KL/chosen_KL_mean": -124.567138671875, "KL/mean": -168.6573028564453, "KL/rejected_KL_mean": -212.74746704101562, "KL/std": 93.67964172363281, "epoch": 0.6969009826152683, "fcm_dpo/beta": 0.00594598101451993, "fcm_dpo/delta": -0.08142174780368805, "fcm_dpo/margin": 88.1803207397461, "fcm_dpo/q_t": 0.3814476430416107, "grad_norm": 11.927419662475586, "learning_rate": 1.2845081597488286e-07, "logits/chosen": 0.834221363067627, "logits/rejected": 0.754862904548645, "logps/chosen": -174.04505920410156, "logps/ref_chosen": -49.4779167175293, "logps/ref_rejected": -72.65262603759766, "logps/rejected": -285.40008544921875, "loss": 1.0205, "margin_dpo/margin_mean": 88.18031311035156, "margin_dpo/margin_std": 100.82464599609375, "step": 461 }, { "KL/chosen_KL_mean": -137.95655822753906, "KL/mean": -181.53204345703125, "KL/rejected_KL_mean": -225.10751342773438, "KL/std": 91.580078125, "epoch": 0.6984126984126984, "fcm_dpo/beta": 0.005903769284486771, "fcm_dpo/delta": -0.06840892881155014, "fcm_dpo/margin": 87.15096282958984, "fcm_dpo/q_t": 0.38177424669265747, "grad_norm": 12.905652046203613, "learning_rate": 1.27297100994108e-07, "logits/chosen": 0.7005641460418701, "logits/rejected": 0.6541897058486938, "logps/chosen": -198.45167541503906, "logps/ref_chosen": -60.4951171875, "logps/ref_rejected": -74.82136535644531, "logps/rejected": -299.92889404296875, "loss": 1.0221, "margin_dpo/margin_mean": 87.15095520019531, "margin_dpo/margin_std": 102.30754852294922, "step": 462 }, { "KL/chosen_KL_mean": -165.4473876953125, "KL/mean": -191.84483337402344, "KL/rejected_KL_mean": -218.24227905273438, "KL/std": 91.26084899902344, "epoch": 0.6999244142101285, "fcm_dpo/beta": 0.0059958938509225845, "fcm_dpo/delta": 0.138011172413826, "fcm_dpo/margin": 52.794891357421875, "fcm_dpo/q_t": 0.4275769591331482, "grad_norm": 18.58067512512207, "learning_rate": 1.2614681827718695e-07, "logits/chosen": 0.7361761331558228, "logits/rejected": 0.7334294319152832, "logps/chosen": -233.13250732421875, "logps/ref_chosen": -67.68511962890625, "logps/ref_rejected": -71.32196044921875, "logps/rejected": -289.564208984375, "loss": 1.1823, "margin_dpo/margin_mean": 52.79489517211914, "margin_dpo/margin_std": 99.66201782226562, "step": 463 }, { "KL/chosen_KL_mean": -150.11727905273438, "KL/mean": -191.15919494628906, "KL/rejected_KL_mean": -232.2011260986328, "KL/std": 103.76582336425781, "epoch": 0.7014361300075586, "fcm_dpo/beta": 0.006046008318662643, "fcm_dpo/delta": -0.04882443696260452, "fcm_dpo/margin": 82.08385467529297, "fcm_dpo/q_t": 0.3890731930732727, "grad_norm": 12.270607948303223, "learning_rate": 1.2500000000000005e-07, "logits/chosen": 0.7439402937889099, "logits/rejected": 0.7154823541641235, "logps/chosen": -209.2829132080078, "logps/ref_chosen": -59.16564178466797, "logps/ref_rejected": -69.56146240234375, "logps/rejected": -301.7625732421875, "loss": 1.0684, "margin_dpo/margin_mean": 82.08385467529297, "margin_dpo/margin_std": 115.9161376953125, "step": 464 }, { "KL/chosen_KL_mean": -157.9586181640625, "KL/mean": -192.47720336914062, "KL/rejected_KL_mean": -226.9957733154297, "KL/std": 97.54608154296875, "epoch": 0.7029478458049887, "fcm_dpo/beta": 0.006019483786076307, "fcm_dpo/delta": 0.0358240120112896, "fcm_dpo/margin": 69.03716278076172, "fcm_dpo/q_t": 0.4071248471736908, "grad_norm": 13.349013328552246, "learning_rate": 1.238566782415197e-07, "logits/chosen": 0.7654883861541748, "logits/rejected": 0.7077640295028687, "logps/chosen": -216.4722900390625, "logps/ref_chosen": -58.513671875, "logps/ref_rejected": -84.31745910644531, "logps/rejected": -311.313232421875, "loss": 1.1183, "margin_dpo/margin_mean": 69.03716278076172, "margin_dpo/margin_std": 110.43283081054688, "step": 465 }, { "KL/chosen_KL_mean": -170.02719116210938, "KL/mean": -190.02499389648438, "KL/rejected_KL_mean": -210.02281188964844, "KL/std": 105.02227783203125, "epoch": 0.7044595616024187, "fcm_dpo/beta": 0.006132124457508326, "fcm_dpo/delta": 0.06729494035243988, "fcm_dpo/margin": 39.99561309814453, "fcm_dpo/q_t": 0.4432896673679352, "grad_norm": 20.618934631347656, "learning_rate": 1.2271688498291334e-07, "logits/chosen": 0.710273265838623, "logits/rejected": 0.7162467241287231, "logps/chosen": -243.2930145263672, "logps/ref_chosen": -73.26580810546875, "logps/ref_rejected": -74.83621215820312, "logps/rejected": -284.8590087890625, "loss": 1.2579, "margin_dpo/margin_mean": 39.99561309814453, "margin_dpo/margin_std": 106.95750427246094, "step": 466 }, { "KL/chosen_KL_mean": -150.3333740234375, "KL/mean": -183.27291870117188, "KL/rejected_KL_mean": -216.21246337890625, "KL/std": 102.4985580444336, "epoch": 0.7059712773998488, "fcm_dpo/beta": 0.006167709827423096, "fcm_dpo/delta": 0.04555200785398483, "fcm_dpo/margin": 65.87907409667969, "fcm_dpo/q_t": 0.4112234115600586, "grad_norm": 11.9649658203125, "learning_rate": 1.2158065210664848e-07, "logits/chosen": 0.784168004989624, "logits/rejected": 0.6448608636856079, "logps/chosen": -197.912841796875, "logps/ref_chosen": -47.57947540283203, "logps/ref_rejected": -78.68522644042969, "logps/rejected": -294.897705078125, "loss": 1.117, "margin_dpo/margin_mean": 65.87907409667969, "margin_dpo/margin_std": 107.44978332519531, "step": 467 }, { "KL/chosen_KL_mean": -141.78823852539062, "KL/mean": -185.31454467773438, "KL/rejected_KL_mean": -228.84085083007812, "KL/std": 100.20425415039062, "epoch": 0.7074829931972789, "fcm_dpo/beta": 0.006109039299190044, "fcm_dpo/delta": -0.08640697598457336, "fcm_dpo/margin": 87.0526123046875, "fcm_dpo/q_t": 0.37998446822166443, "grad_norm": 18.61052131652832, "learning_rate": 1.204480113956011e-07, "logits/chosen": 0.6854424476623535, "logits/rejected": 0.6727529168128967, "logps/chosen": -205.71603393554688, "logps/ref_chosen": -63.92778778076172, "logps/ref_rejected": -76.51626586914062, "logps/rejected": -305.35711669921875, "loss": 1.0337, "margin_dpo/margin_mean": 87.0526123046875, "margin_dpo/margin_std": 113.15974426269531, "step": 468 }, { "KL/chosen_KL_mean": -144.42633056640625, "KL/mean": -183.31387329101562, "KL/rejected_KL_mean": -222.201416015625, "KL/std": 97.5907974243164, "epoch": 0.708994708994709, "fcm_dpo/beta": 0.006026652175933123, "fcm_dpo/delta": -0.020541474223136902, "fcm_dpo/margin": 77.77510070800781, "fcm_dpo/q_t": 0.39321011304855347, "grad_norm": 13.9988431930542, "learning_rate": 1.1931899453216697e-07, "logits/chosen": 0.7652057409286499, "logits/rejected": 0.7501240968704224, "logps/chosen": -203.48451232910156, "logps/ref_chosen": -59.05818176269531, "logps/ref_rejected": -75.67672729492188, "logps/rejected": -297.8781433105469, "loss": 1.0453, "margin_dpo/margin_mean": 77.77510070800781, "margin_dpo/margin_std": 91.79092407226562, "step": 469 }, { "KL/chosen_KL_mean": -139.2176055908203, "KL/mean": -177.15162658691406, "KL/rejected_KL_mean": -215.08563232421875, "KL/std": 95.38059997558594, "epoch": 0.7105064247921391, "fcm_dpo/beta": 0.006067805923521519, "fcm_dpo/delta": -0.011047637090086937, "fcm_dpo/margin": 75.86803436279297, "fcm_dpo/q_t": 0.396224707365036, "grad_norm": 12.56069278717041, "learning_rate": 1.1819363309737438e-07, "logits/chosen": 0.7496016025543213, "logits/rejected": 0.6867067217826843, "logps/chosen": -187.08505249023438, "logps/ref_chosen": -47.86743927001953, "logps/ref_rejected": -65.96859741210938, "logps/rejected": -281.0542297363281, "loss": 1.0764, "margin_dpo/margin_mean": 75.86803436279297, "margin_dpo/margin_std": 106.75106811523438, "step": 470 }, { "KL/chosen_KL_mean": -134.6875762939453, "KL/mean": -176.7728729248047, "KL/rejected_KL_mean": -218.85816955566406, "KL/std": 94.34480285644531, "epoch": 0.7120181405895691, "fcm_dpo/beta": 0.005986287258565426, "fcm_dpo/delta": -0.05681845545768738, "fcm_dpo/margin": 84.17059326171875, "fcm_dpo/q_t": 0.384713351726532, "grad_norm": 13.130870819091797, "learning_rate": 1.1707195857000215e-07, "logits/chosen": 0.7122259140014648, "logits/rejected": 0.6619666814804077, "logps/chosen": -192.46542358398438, "logps/ref_chosen": -57.777854919433594, "logps/ref_rejected": -73.81172180175781, "logps/rejected": -292.6698913574219, "loss": 1.0365, "margin_dpo/margin_mean": 84.17059326171875, "margin_dpo/margin_std": 104.3567123413086, "step": 471 }, { "KL/chosen_KL_mean": -134.97317504882812, "KL/mean": -168.95550537109375, "KL/rejected_KL_mean": -202.93785095214844, "KL/std": 98.41152954101562, "epoch": 0.7135298563869993, "fcm_dpo/beta": 0.0060343313962221146, "fcm_dpo/delta": 0.04116275534033775, "fcm_dpo/margin": 67.96469116210938, "fcm_dpo/q_t": 0.41018474102020264, "grad_norm": 13.530449867248535, "learning_rate": 1.1595400232569768e-07, "logits/chosen": 0.7044432163238525, "logits/rejected": 0.6615398526191711, "logps/chosen": -190.8818359375, "logps/ref_chosen": -55.908668518066406, "logps/ref_rejected": -74.70294189453125, "logps/rejected": -277.64080810546875, "loss": 1.1442, "margin_dpo/margin_mean": 67.96469116210938, "margin_dpo/margin_std": 123.55488586425781, "step": 472 }, { "KL/chosen_KL_mean": -138.88629150390625, "KL/mean": -180.80670166015625, "KL/rejected_KL_mean": -222.72711181640625, "KL/std": 114.38705444335938, "epoch": 0.7150415721844293, "fcm_dpo/beta": 0.005974816158413887, "fcm_dpo/delta": -0.05364374816417694, "fcm_dpo/margin": 83.84082794189453, "fcm_dpo/q_t": 0.3932644724845886, "grad_norm": 15.695032119750977, "learning_rate": 1.1483979563610069e-07, "logits/chosen": 0.7801591157913208, "logits/rejected": 0.6740267276763916, "logps/chosen": -193.04718017578125, "logps/ref_chosen": -54.16088104248047, "logps/ref_rejected": -92.76789855957031, "logps/rejected": -315.4949951171875, "loss": 1.0901, "margin_dpo/margin_mean": 83.8408203125, "margin_dpo/margin_std": 134.4425048828125, "step": 473 }, { "KL/chosen_KL_mean": -142.9180908203125, "KL/mean": -177.65631103515625, "KL/rejected_KL_mean": -212.39453125, "KL/std": 101.67129516601562, "epoch": 0.7165532879818595, "fcm_dpo/beta": 0.005997130647301674, "fcm_dpo/delta": 0.034741394221782684, "fcm_dpo/margin": 69.47645568847656, "fcm_dpo/q_t": 0.40932339429855347, "grad_norm": 18.85576629638672, "learning_rate": 1.1372936966796709e-07, "logits/chosen": 0.7585621476173401, "logits/rejected": 0.6906800270080566, "logps/chosen": -189.60379028320312, "logps/ref_chosen": -46.685707092285156, "logps/ref_rejected": -71.44731903076172, "logps/rejected": -283.84185791015625, "loss": 1.1316, "margin_dpo/margin_mean": 69.47645568847656, "margin_dpo/margin_std": 120.51841735839844, "step": 474 }, { "KL/chosen_KL_mean": -136.62750244140625, "KL/mean": -186.64437866210938, "KL/rejected_KL_mean": -236.6612548828125, "KL/std": 102.17213439941406, "epoch": 0.7180650037792895, "fcm_dpo/beta": 0.005852770060300827, "fcm_dpo/delta": -0.14472949504852295, "fcm_dpo/margin": 100.03376770019531, "fcm_dpo/q_t": 0.36991116404533386, "grad_norm": 10.036908149719238, "learning_rate": 1.126227554822985e-07, "logits/chosen": 0.7174779176712036, "logits/rejected": 0.6689811944961548, "logps/chosen": -195.11480712890625, "logps/ref_chosen": -58.4873046875, "logps/ref_rejected": -87.00187683105469, "logps/rejected": -323.66314697265625, "loss": 0.9834, "margin_dpo/margin_mean": 100.03376770019531, "margin_dpo/margin_std": 113.25033569335938, "step": 475 }, { "KL/chosen_KL_mean": -166.2928924560547, "KL/mean": -197.72381591796875, "KL/rejected_KL_mean": -229.15475463867188, "KL/std": 107.59396362304688, "epoch": 0.7195767195767195, "fcm_dpo/beta": 0.005927635356783867, "fcm_dpo/delta": 0.07958348095417023, "fcm_dpo/margin": 62.861881256103516, "fcm_dpo/q_t": 0.418168306350708, "grad_norm": 14.75139045715332, "learning_rate": 1.1151998403347243e-07, "logits/chosen": 0.6256033778190613, "logits/rejected": 0.6278666257858276, "logps/chosen": -241.6745147705078, "logps/ref_chosen": -75.38162231445312, "logps/ref_rejected": -76.99822235107422, "logps/rejected": -306.1529846191406, "loss": 1.1569, "margin_dpo/margin_mean": 62.861881256103516, "margin_dpo/margin_std": 114.92433166503906, "step": 476 }, { "KL/chosen_KL_mean": -164.40463256835938, "KL/mean": -199.77825927734375, "KL/rejected_KL_mean": -235.1519012451172, "KL/std": 107.18777465820312, "epoch": 0.7210884353741497, "fcm_dpo/beta": 0.0059497784823179245, "fcm_dpo/delta": 0.03035357967019081, "fcm_dpo/margin": 70.74728393554688, "fcm_dpo/q_t": 0.4081571102142334, "grad_norm": 15.135383605957031, "learning_rate": 1.1042108616837692e-07, "logits/chosen": 0.7753346562385559, "logits/rejected": 0.726446270942688, "logps/chosen": -225.47802734375, "logps/ref_chosen": -61.073387145996094, "logps/ref_rejected": -81.34375, "logps/rejected": -316.49566650390625, "loss": 1.1603, "margin_dpo/margin_mean": 70.74728393554688, "margin_dpo/margin_std": 136.30105590820312, "step": 477 }, { "KL/chosen_KL_mean": -145.6103973388672, "KL/mean": -173.1075439453125, "KL/rejected_KL_mean": -200.6046905517578, "KL/std": 91.68518829345703, "epoch": 0.7226001511715797, "fcm_dpo/beta": 0.006051356438547373, "fcm_dpo/delta": 0.1214258000254631, "fcm_dpo/margin": 54.99429702758789, "fcm_dpo/q_t": 0.4251078963279724, "grad_norm": 16.03772735595703, "learning_rate": 1.0932609262554746e-07, "logits/chosen": 0.6870132088661194, "logits/rejected": 0.6989500522613525, "logps/chosen": -202.7777099609375, "logps/ref_chosen": -57.16731643676758, "logps/ref_rejected": -53.30917739868164, "logps/rejected": -253.9138641357422, "loss": 1.1908, "margin_dpo/margin_mean": 54.99428939819336, "margin_dpo/margin_std": 111.90939331054688, "step": 478 }, { "KL/chosen_KL_mean": -150.79190063476562, "KL/mean": -175.4144287109375, "KL/rejected_KL_mean": -200.03695678710938, "KL/std": 92.3194580078125, "epoch": 0.7241118669690099, "fcm_dpo/beta": 0.006136768963187933, "fcm_dpo/delta": 0.024153757840394974, "fcm_dpo/margin": 49.24507522583008, "fcm_dpo/q_t": 0.4326751232147217, "grad_norm": 15.933855056762695, "learning_rate": 1.0823503403430734e-07, "logits/chosen": 0.7064397931098938, "logits/rejected": 0.6684491634368896, "logps/chosen": -209.70521545410156, "logps/ref_chosen": -58.91331481933594, "logps/ref_rejected": -63.7403450012207, "logps/rejected": -263.7773132324219, "loss": 1.2137, "margin_dpo/margin_mean": 49.24507141113281, "margin_dpo/margin_std": 109.21455383300781, "step": 479 }, { "KL/chosen_KL_mean": -154.2449951171875, "KL/mean": -192.55990600585938, "KL/rejected_KL_mean": -230.87478637695312, "KL/std": 95.96563720703125, "epoch": 0.7256235827664399, "fcm_dpo/beta": 0.0061669060960412025, "fcm_dpo/delta": -0.02497832477092743, "fcm_dpo/margin": 76.62979125976562, "fcm_dpo/q_t": 0.39488065242767334, "grad_norm": 17.156362533569336, "learning_rate": 1.0714794091391072e-07, "logits/chosen": 0.6846721768379211, "logits/rejected": 0.6770238280296326, "logps/chosen": -217.04562377929688, "logps/ref_chosen": -62.80061340332031, "logps/ref_rejected": -67.58859252929688, "logps/rejected": -298.46337890625, "loss": 1.0847, "margin_dpo/margin_mean": 76.62979125976562, "margin_dpo/margin_std": 112.43879699707031, "step": 480 }, { "KL/chosen_KL_mean": -152.42149353027344, "KL/mean": -187.66525268554688, "KL/rejected_KL_mean": -222.90902709960938, "KL/std": 100.48785400390625, "epoch": 0.72713529856387, "fcm_dpo/beta": 0.006108821835368872, "fcm_dpo/delta": 0.020193161442875862, "fcm_dpo/margin": 70.48753356933594, "fcm_dpo/q_t": 0.40493303537368774, "grad_norm": 14.735018730163574, "learning_rate": 1.0606484367268906e-07, "logits/chosen": 0.6829352974891663, "logits/rejected": 0.6849657297134399, "logps/chosen": -217.70797729492188, "logps/ref_chosen": -65.28649139404297, "logps/ref_rejected": -70.78668212890625, "logps/rejected": -293.6957092285156, "loss": 1.1072, "margin_dpo/margin_mean": 70.48753356933594, "margin_dpo/margin_std": 112.09474182128906, "step": 481 }, { "KL/chosen_KL_mean": -170.580810546875, "KL/mean": -207.0380401611328, "KL/rejected_KL_mean": -243.49526977539062, "KL/std": 107.98014831542969, "epoch": 0.7286470143613001, "fcm_dpo/beta": 0.00615047849714756, "fcm_dpo/delta": 0.0014284886419773102, "fcm_dpo/margin": 72.91446685791016, "fcm_dpo/q_t": 0.40593189001083374, "grad_norm": 16.25693130493164, "learning_rate": 1.0498577260720048e-07, "logits/chosen": 0.6703163385391235, "logits/rejected": 0.521394670009613, "logps/chosen": -231.48699951171875, "logps/ref_chosen": -60.906185150146484, "logps/ref_rejected": -103.44656372070312, "logps/rejected": -346.94183349609375, "loss": 1.1419, "margin_dpo/margin_mean": 72.91446685791016, "margin_dpo/margin_std": 136.18093872070312, "step": 482 }, { "KL/chosen_KL_mean": -142.95286560058594, "KL/mean": -186.32806396484375, "KL/rejected_KL_mean": -229.70327758789062, "KL/std": 97.5377426147461, "epoch": 0.7301587301587301, "fcm_dpo/beta": 0.0060723647475242615, "fcm_dpo/delta": -0.0809704065322876, "fcm_dpo/margin": 86.75039672851562, "fcm_dpo/q_t": 0.38196927309036255, "grad_norm": 13.32763957977295, "learning_rate": 1.0391075790138232e-07, "logits/chosen": 0.7873438000679016, "logits/rejected": 0.6818989515304565, "logps/chosen": -196.14486694335938, "logps/ref_chosen": -53.192012786865234, "logps/ref_rejected": -81.83927154541016, "logps/rejected": -311.54254150390625, "loss": 1.0308, "margin_dpo/margin_mean": 86.75039672851562, "margin_dpo/margin_std": 110.80722045898438, "step": 483 }, { "KL/chosen_KL_mean": -149.74163818359375, "KL/mean": -176.8128662109375, "KL/rejected_KL_mean": -203.8840789794922, "KL/std": 95.46682739257812, "epoch": 0.7316704459561603, "fcm_dpo/beta": 0.006167872808873653, "fcm_dpo/delta": 0.11922429502010345, "fcm_dpo/margin": 54.142452239990234, "fcm_dpo/q_t": 0.4233402609825134, "grad_norm": 17.306833267211914, "learning_rate": 1.0283982962570681e-07, "logits/chosen": 0.7923409342765808, "logits/rejected": 0.7621510028839111, "logps/chosen": -207.5111083984375, "logps/ref_chosen": -57.76945877075195, "logps/ref_rejected": -71.6829833984375, "logps/rejected": -275.56707763671875, "loss": 1.1463, "margin_dpo/margin_mean": 54.142452239990234, "margin_dpo/margin_std": 85.11724853515625, "step": 484 }, { "KL/chosen_KL_mean": -152.07223510742188, "KL/mean": -183.80691528320312, "KL/rejected_KL_mean": -215.54156494140625, "KL/std": 97.00895690917969, "epoch": 0.7331821617535903, "fcm_dpo/beta": 0.006109254900366068, "fcm_dpo/delta": -0.057364415377378464, "fcm_dpo/margin": 63.46931457519531, "fcm_dpo/q_t": 0.4122996926307678, "grad_norm": 14.168609619140625, "learning_rate": 1.0177301773633992e-07, "logits/chosen": 0.7602566480636597, "logits/rejected": 0.7402700185775757, "logps/chosen": -208.70809936523438, "logps/ref_chosen": -56.63584899902344, "logps/ref_rejected": -70.85614013671875, "logps/rejected": -286.397705078125, "loss": 1.1199, "margin_dpo/margin_mean": 63.46931457519531, "margin_dpo/margin_std": 95.38653564453125, "step": 485 }, { "KL/chosen_KL_mean": -176.1962127685547, "KL/mean": -206.28851318359375, "KL/rejected_KL_mean": -236.38079833984375, "KL/std": 112.07026672363281, "epoch": 0.7346938775510204, "fcm_dpo/beta": 0.006182005628943443, "fcm_dpo/delta": 0.0809686928987503, "fcm_dpo/margin": 60.1845817565918, "fcm_dpo/q_t": 0.4215923547744751, "grad_norm": 12.775336265563965, "learning_rate": 1.007103520743035e-07, "logits/chosen": 0.776502251625061, "logits/rejected": 0.6627446413040161, "logps/chosen": -232.54324340820312, "logps/ref_chosen": -56.347023010253906, "logps/ref_rejected": -85.97221374511719, "logps/rejected": -322.35302734375, "loss": 1.1926, "margin_dpo/margin_mean": 60.18458557128906, "margin_dpo/margin_std": 129.93482971191406, "step": 486 }, { "KL/chosen_KL_mean": -157.27584838867188, "KL/mean": -193.54388427734375, "KL/rejected_KL_mean": -229.81195068359375, "KL/std": 102.25735473632812, "epoch": 0.7362055933484505, "fcm_dpo/beta": 0.0062069520354270935, "fcm_dpo/delta": -0.0002405631821602583, "fcm_dpo/margin": 72.53608703613281, "fcm_dpo/q_t": 0.40433698892593384, "grad_norm": 14.756111145019531, "learning_rate": 9.965186236464046e-08, "logits/chosen": 0.823411762714386, "logits/rejected": 0.7664943337440491, "logps/chosen": -217.89306640625, "logps/ref_chosen": -60.617218017578125, "logps/ref_rejected": -82.50975036621094, "logps/rejected": -312.3216857910156, "loss": 1.1053, "margin_dpo/margin_mean": 72.53608703613281, "margin_dpo/margin_std": 120.83760070800781, "step": 487 }, { "KL/chosen_KL_mean": -145.82931518554688, "KL/mean": -185.6131591796875, "KL/rejected_KL_mean": -225.39697265625, "KL/std": 97.22247314453125, "epoch": 0.7377173091458806, "fcm_dpo/beta": 0.006201518699526787, "fcm_dpo/delta": -0.046092621982097626, "fcm_dpo/margin": 79.56764221191406, "fcm_dpo/q_t": 0.39139896631240845, "grad_norm": 17.251914978027344, "learning_rate": 9.859757821558337e-08, "logits/chosen": 0.7474915981292725, "logits/rejected": 0.6937694549560547, "logps/chosen": -208.93836975097656, "logps/ref_chosen": -63.10905075073242, "logps/ref_rejected": -82.49348449707031, "logps/rejected": -307.89044189453125, "loss": 1.0545, "margin_dpo/margin_mean": 79.56764221191406, "margin_dpo/margin_std": 107.11445617675781, "step": 488 }, { "KL/chosen_KL_mean": -171.64735412597656, "KL/mean": -195.64874267578125, "KL/rejected_KL_mean": -219.65011596679688, "KL/std": 107.1324462890625, "epoch": 0.7392290249433107, "fcm_dpo/beta": 0.006290414370596409, "fcm_dpo/delta": 0.15272760391235352, "fcm_dpo/margin": 48.002750396728516, "fcm_dpo/q_t": 0.43335413932800293, "grad_norm": 14.13779067993164, "learning_rate": 9.754752911772615e-08, "logits/chosen": 0.7004495859146118, "logits/rejected": 0.656787633895874, "logps/chosen": -236.63632202148438, "logps/ref_chosen": -64.98896026611328, "logps/ref_rejected": -84.39607238769531, "logps/rejected": -304.04620361328125, "loss": 1.2294, "margin_dpo/margin_mean": 48.002750396728516, "margin_dpo/margin_std": 114.78465270996094, "step": 489 }, { "KL/chosen_KL_mean": -154.73526000976562, "KL/mean": -183.689453125, "KL/rejected_KL_mean": -212.64366149902344, "KL/std": 103.60529327392578, "epoch": 0.7407407407407407, "fcm_dpo/beta": 0.006394874304533005, "fcm_dpo/delta": 0.08273956179618835, "fcm_dpo/margin": 57.908416748046875, "fcm_dpo/q_t": 0.4194272756576538, "grad_norm": 13.123128890991211, "learning_rate": 9.650174444319956e-08, "logits/chosen": 0.7914144396781921, "logits/rejected": 0.7736309170722961, "logps/chosen": -216.6439971923828, "logps/ref_chosen": -61.90874481201172, "logps/ref_rejected": -70.58566284179688, "logps/rejected": -283.22930908203125, "loss": 1.2074, "margin_dpo/margin_mean": 57.908416748046875, "margin_dpo/margin_std": 129.22569274902344, "step": 490 }, { "KL/chosen_KL_mean": -150.71292114257812, "KL/mean": -181.09390258789062, "KL/rejected_KL_mean": -211.47488403320312, "KL/std": 94.73994445800781, "epoch": 0.7422524565381708, "fcm_dpo/beta": 0.0064573828130960464, "fcm_dpo/delta": 0.05928955599665642, "fcm_dpo/margin": 60.761940002441406, "fcm_dpo/q_t": 0.4131777584552765, "grad_norm": 13.748710632324219, "learning_rate": 9.546025344484868e-08, "logits/chosen": 0.675628125667572, "logits/rejected": 0.6172371506690979, "logps/chosen": -206.18862915039062, "logps/ref_chosen": -55.47570037841797, "logps/ref_rejected": -78.70318603515625, "logps/rejected": -290.17803955078125, "loss": 1.1328, "margin_dpo/margin_mean": 60.761940002441406, "margin_dpo/margin_std": 100.07881164550781, "step": 491 }, { "KL/chosen_KL_mean": -177.33346557617188, "KL/mean": -204.11441040039062, "KL/rejected_KL_mean": -230.89535522460938, "KL/std": 107.49058532714844, "epoch": 0.7437641723356009, "fcm_dpo/beta": 0.006651863921433687, "fcm_dpo/delta": 0.09622493386268616, "fcm_dpo/margin": 53.56187438964844, "fcm_dpo/q_t": 0.42310231924057007, "grad_norm": 16.768766403198242, "learning_rate": 9.442308525541589e-08, "logits/chosen": 0.7349872589111328, "logits/rejected": 0.6678378582000732, "logps/chosen": -244.61985778808594, "logps/ref_chosen": -67.28638458251953, "logps/ref_rejected": -82.78628540039062, "logps/rejected": -313.681640625, "loss": 1.2139, "margin_dpo/margin_mean": 53.5618782043457, "margin_dpo/margin_std": 119.93057250976562, "step": 492 }, { "KL/chosen_KL_mean": -146.36495971679688, "KL/mean": -186.2227783203125, "KL/rejected_KL_mean": -226.0806121826172, "KL/std": 105.06057739257812, "epoch": 0.745275888133031, "fcm_dpo/beta": 0.006630874238908291, "fcm_dpo/delta": -0.08393573015928268, "fcm_dpo/margin": 79.71566772460938, "fcm_dpo/q_t": 0.385233610868454, "grad_norm": 15.71538257598877, "learning_rate": 9.339026888672468e-08, "logits/chosen": 0.692692220211029, "logits/rejected": 0.621835470199585, "logps/chosen": -202.29244995117188, "logps/ref_chosen": -55.92750549316406, "logps/ref_rejected": -79.12149810791016, "logps/rejected": -305.20208740234375, "loss": 1.0568, "margin_dpo/margin_mean": 79.71566772460938, "margin_dpo/margin_std": 111.91082763671875, "step": 493 }, { "KL/chosen_KL_mean": -150.67694091796875, "KL/mean": -185.75869750976562, "KL/rejected_KL_mean": -220.8404998779297, "KL/std": 104.7548828125, "epoch": 0.7467876039304611, "fcm_dpo/beta": 0.0065204668790102005, "fcm_dpo/delta": -0.007957713678479195, "fcm_dpo/margin": 70.16357421875, "fcm_dpo/q_t": 0.401015967130661, "grad_norm": 15.422093391418457, "learning_rate": 9.236183322886945e-08, "logits/chosen": 0.6373249888420105, "logits/rejected": 0.5916510224342346, "logps/chosen": -218.63104248046875, "logps/ref_chosen": -67.95410919189453, "logps/ref_rejected": -90.50865173339844, "logps/rejected": -311.3491516113281, "loss": 1.1375, "margin_dpo/margin_mean": 70.16357421875, "margin_dpo/margin_std": 129.2499542236328, "step": 494 }, { "KL/chosen_KL_mean": -147.87957763671875, "KL/mean": -175.8821563720703, "KL/rejected_KL_mean": -203.88473510742188, "KL/std": 102.84738159179688, "epoch": 0.7482993197278912, "fcm_dpo/beta": 0.00664489483460784, "fcm_dpo/delta": 0.07983549684286118, "fcm_dpo/margin": 56.00517272949219, "fcm_dpo/q_t": 0.42484697699546814, "grad_norm": 19.291988372802734, "learning_rate": 9.133780704940594e-08, "logits/chosen": 0.7651863098144531, "logits/rejected": 0.7069584131240845, "logps/chosen": -200.50503540039062, "logps/ref_chosen": -52.62546157836914, "logps/ref_rejected": -72.06781005859375, "logps/rejected": -275.9525451660156, "loss": 1.1991, "margin_dpo/margin_mean": 56.005165100097656, "margin_dpo/margin_std": 125.55036163330078, "step": 495 }, { "KL/chosen_KL_mean": -161.83767700195312, "KL/mean": -200.32595825195312, "KL/rejected_KL_mean": -238.8142547607422, "KL/std": 111.56478881835938, "epoch": 0.7498110355253212, "fcm_dpo/beta": 0.006527569144964218, "fcm_dpo/delta": -0.05702162906527519, "fcm_dpo/margin": 76.9765853881836, "fcm_dpo/q_t": 0.4012041985988617, "grad_norm": 13.641803741455078, "learning_rate": 9.031821899254797e-08, "logits/chosen": 0.7423851490020752, "logits/rejected": 0.6301846504211426, "logps/chosen": -219.43499755859375, "logps/ref_chosen": -57.597320556640625, "logps/ref_rejected": -94.36127471923828, "logps/rejected": -333.175537109375, "loss": 1.1167, "margin_dpo/margin_mean": 76.9765853881836, "margin_dpo/margin_std": 141.50401306152344, "step": 496 }, { "KL/chosen_KL_mean": -163.78262329101562, "KL/mean": -201.80593872070312, "KL/rejected_KL_mean": -239.82925415039062, "KL/std": 103.0841064453125, "epoch": 0.7513227513227513, "fcm_dpo/beta": 0.006504581309854984, "fcm_dpo/delta": -0.04710300266742706, "fcm_dpo/margin": 76.04662322998047, "fcm_dpo/q_t": 0.38855764269828796, "grad_norm": 14.237540245056152, "learning_rate": 8.930309757836516e-08, "logits/chosen": 0.7395977973937988, "logits/rejected": 0.7060419321060181, "logps/chosen": -236.57257080078125, "logps/ref_chosen": -72.78994750976562, "logps/ref_rejected": -89.48483276367188, "logps/rejected": -329.3140869140625, "loss": 1.0734, "margin_dpo/margin_mean": 76.04662322998047, "margin_dpo/margin_std": 111.859619140625, "step": 497 }, { "KL/chosen_KL_mean": -146.14315795898438, "KL/mean": -186.0857696533203, "KL/rejected_KL_mean": -226.02841186523438, "KL/std": 97.7952880859375, "epoch": 0.7528344671201814, "fcm_dpo/beta": 0.00642948504537344, "fcm_dpo/delta": -0.06713449209928513, "fcm_dpo/margin": 79.88525390625, "fcm_dpo/q_t": 0.38355961441993713, "grad_norm": 16.2513484954834, "learning_rate": 8.829247120198563e-08, "logits/chosen": 0.7451342344284058, "logits/rejected": 0.7235029935836792, "logps/chosen": -214.5088653564453, "logps/ref_chosen": -68.36572265625, "logps/ref_rejected": -71.28846740722656, "logps/rejected": -297.31689453125, "loss": 1.0396, "margin_dpo/margin_mean": 79.88526153564453, "margin_dpo/margin_std": 104.2471694946289, "step": 498 }, { "KL/chosen_KL_mean": -148.076904296875, "KL/mean": -188.3824462890625, "KL/rejected_KL_mean": -228.68801879882812, "KL/std": 107.20639038085938, "epoch": 0.7543461829176115, "fcm_dpo/beta": 0.006349165923893452, "fcm_dpo/delta": -0.06516575068235397, "fcm_dpo/margin": 80.61111450195312, "fcm_dpo/q_t": 0.39157482981681824, "grad_norm": 18.03789710998535, "learning_rate": 8.728636813280163e-08, "logits/chosen": 0.717039942741394, "logits/rejected": 0.6537841558456421, "logps/chosen": -209.98573303222656, "logps/ref_chosen": -61.90882873535156, "logps/ref_rejected": -91.9411392211914, "logps/rejected": -320.629150390625, "loss": 1.1156, "margin_dpo/margin_mean": 80.6111068725586, "margin_dpo/margin_std": 139.66329956054688, "step": 499 }, { "KL/chosen_KL_mean": -154.04995727539062, "KL/mean": -187.4798583984375, "KL/rejected_KL_mean": -220.9097442626953, "KL/std": 95.2972640991211, "epoch": 0.7558578987150416, "fcm_dpo/beta": 0.0063400594517588615, "fcm_dpo/delta": 0.027207905426621437, "fcm_dpo/margin": 66.85978698730469, "fcm_dpo/q_t": 0.40306317806243896, "grad_norm": 16.55160140991211, "learning_rate": 8.628481651367875e-08, "logits/chosen": 0.6679513454437256, "logits/rejected": 0.6745506525039673, "logps/chosen": -224.27578735351562, "logps/ref_chosen": -70.225830078125, "logps/ref_rejected": -71.72203063964844, "logps/rejected": -292.63177490234375, "loss": 1.1462, "margin_dpo/margin_mean": 66.85978698730469, "margin_dpo/margin_std": 121.90635681152344, "step": 500 }, { "KL/chosen_KL_mean": -154.33560180664062, "KL/mean": -184.90542602539062, "KL/rejected_KL_mean": -215.47528076171875, "KL/std": 102.1161880493164, "epoch": 0.7573696145124716, "fcm_dpo/beta": 0.006436236202716827, "fcm_dpo/delta": 0.05852247402071953, "fcm_dpo/margin": 61.139686584472656, "fcm_dpo/q_t": 0.4098885953426361, "grad_norm": 12.187454223632812, "learning_rate": 8.528784436016878e-08, "logits/chosen": 0.725420355796814, "logits/rejected": 0.7295132279396057, "logps/chosen": -218.9344024658203, "logps/ref_chosen": -64.59880828857422, "logps/ref_rejected": -70.59329223632812, "logps/rejected": -286.0685729980469, "loss": 1.0981, "margin_dpo/margin_mean": 61.139678955078125, "margin_dpo/margin_std": 82.2115707397461, "step": 501 }, { "KL/chosen_KL_mean": -153.7998046875, "KL/mean": -186.72488403320312, "KL/rejected_KL_mean": -219.6499481201172, "KL/std": 107.63894653320312, "epoch": 0.7588813303099018, "fcm_dpo/beta": 0.006486368365585804, "fcm_dpo/delta": 0.023503903299570084, "fcm_dpo/margin": 65.85012817382812, "fcm_dpo/q_t": 0.40453580021858215, "grad_norm": 17.191099166870117, "learning_rate": 8.4295479559726e-08, "logits/chosen": 0.7299618721008301, "logits/rejected": 0.6835644841194153, "logps/chosen": -219.26644897460938, "logps/ref_chosen": -65.46662902832031, "logps/ref_rejected": -90.22233581542969, "logps/rejected": -309.8722839355469, "loss": 1.1001, "margin_dpo/margin_mean": 65.85012817382812, "margin_dpo/margin_std": 98.904296875, "step": 502 }, { "KL/chosen_KL_mean": -139.2997283935547, "KL/mean": -173.0640869140625, "KL/rejected_KL_mean": -206.82846069335938, "KL/std": 94.63929748535156, "epoch": 0.7603930461073318, "fcm_dpo/beta": 0.00647312868386507, "fcm_dpo/delta": 0.013444061391055584, "fcm_dpo/margin": 67.52874755859375, "fcm_dpo/q_t": 0.401027113199234, "grad_norm": 12.830598831176758, "learning_rate": 8.330774987092712e-08, "logits/chosen": 0.717086672782898, "logits/rejected": 0.7222782373428345, "logps/chosen": -191.13449096679688, "logps/ref_chosen": -51.83476257324219, "logps/ref_rejected": -57.62522506713867, "logps/rejected": -264.4537048339844, "loss": 1.1121, "margin_dpo/margin_mean": 67.52874755859375, "margin_dpo/margin_std": 108.20802307128906, "step": 503 }, { "KL/chosen_KL_mean": -143.835693359375, "KL/mean": -188.02395629882812, "KL/rejected_KL_mean": -232.21218872070312, "KL/std": 94.23897552490234, "epoch": 0.7619047619047619, "fcm_dpo/beta": 0.006416743155568838, "fcm_dpo/delta": -0.12435752153396606, "fcm_dpo/margin": 88.37649536132812, "fcm_dpo/q_t": 0.371462345123291, "grad_norm": 14.499225616455078, "learning_rate": 8.232468292269479e-08, "logits/chosen": 0.7206200361251831, "logits/rejected": 0.7034634351730347, "logps/chosen": -212.48690795898438, "logps/ref_chosen": -68.65119934082031, "logps/ref_rejected": -77.91394805908203, "logps/rejected": -310.12615966796875, "loss": 0.991, "margin_dpo/margin_mean": 88.3764877319336, "margin_dpo/margin_std": 96.57669067382812, "step": 504 }, { "KL/chosen_KL_mean": -151.95733642578125, "KL/mean": -181.8119659423828, "KL/rejected_KL_mean": -211.66659545898438, "KL/std": 109.19900512695312, "epoch": 0.763416477702192, "fcm_dpo/beta": 0.006348426453769207, "fcm_dpo/delta": 0.07310114800930023, "fcm_dpo/margin": 59.709259033203125, "fcm_dpo/q_t": 0.41927751898765564, "grad_norm": 15.147652626037598, "learning_rate": 8.134630621352483e-08, "logits/chosen": 0.7179839611053467, "logits/rejected": 0.6813881397247314, "logps/chosen": -211.9561767578125, "logps/ref_chosen": -59.99884796142578, "logps/ref_rejected": -76.88048553466797, "logps/rejected": -288.5470886230469, "loss": 1.1806, "margin_dpo/margin_mean": 59.709259033203125, "margin_dpo/margin_std": 120.05146789550781, "step": 505 }, { "KL/chosen_KL_mean": -151.50389099121094, "KL/mean": -183.47604370117188, "KL/rejected_KL_mean": -215.44818115234375, "KL/std": 96.9446792602539, "epoch": 0.764928193499622, "fcm_dpo/beta": 0.006442956626415253, "fcm_dpo/delta": 0.03961796313524246, "fcm_dpo/margin": 63.944297790527344, "fcm_dpo/q_t": 0.40785694122314453, "grad_norm": 15.846113204956055, "learning_rate": 8.037264711071698e-08, "logits/chosen": 0.701289176940918, "logits/rejected": 0.687470555305481, "logps/chosen": -221.5751953125, "logps/ref_chosen": -70.07130432128906, "logps/ref_rejected": -82.03775024414062, "logps/rejected": -297.4859313964844, "loss": 1.1567, "margin_dpo/margin_mean": 63.944297790527344, "margin_dpo/margin_std": 120.85015869140625, "step": 506 }, { "KL/chosen_KL_mean": -162.0356903076172, "KL/mean": -198.49484252929688, "KL/rejected_KL_mean": -234.95399475097656, "KL/std": 108.42218017578125, "epoch": 0.7664399092970522, "fcm_dpo/beta": 0.006417691707611084, "fcm_dpo/delta": -0.019571736454963684, "fcm_dpo/margin": 72.91832733154297, "fcm_dpo/q_t": 0.40374889969825745, "grad_norm": 15.331804275512695, "learning_rate": 7.940373284960933e-08, "logits/chosen": 0.7045880556106567, "logits/rejected": 0.6585378646850586, "logps/chosen": -234.042724609375, "logps/ref_chosen": -72.00703430175781, "logps/ref_rejected": -93.94987487792969, "logps/rejected": -328.90386962890625, "loss": 1.1235, "margin_dpo/margin_mean": 72.91831970214844, "margin_dpo/margin_std": 127.28765869140625, "step": 507 }, { "KL/chosen_KL_mean": -148.79429626464844, "KL/mean": -191.38870239257812, "KL/rejected_KL_mean": -233.9831085205078, "KL/std": 114.09550476074219, "epoch": 0.7679516250944822, "fcm_dpo/beta": 0.006400927901268005, "fcm_dpo/delta": -0.1012621819972992, "fcm_dpo/margin": 85.18881225585938, "fcm_dpo/q_t": 0.3824944794178009, "grad_norm": 17.070287704467773, "learning_rate": 7.843959053281663e-08, "logits/chosen": 0.6818482875823975, "logits/rejected": 0.5656284093856812, "logps/chosen": -209.01422119140625, "logps/ref_chosen": -60.21992492675781, "logps/ref_rejected": -95.9200668334961, "logps/rejected": -329.9031982421875, "loss": 1.0558, "margin_dpo/margin_mean": 85.18881225585938, "margin_dpo/margin_std": 121.49737548828125, "step": 508 }, { "KL/chosen_KL_mean": -159.0069580078125, "KL/mean": -190.4010009765625, "KL/rejected_KL_mean": -221.79501342773438, "KL/std": 101.37274169921875, "epoch": 0.7694633408919124, "fcm_dpo/beta": 0.006350069306790829, "fcm_dpo/delta": 0.05341218784451485, "fcm_dpo/margin": 62.78805160522461, "fcm_dpo/q_t": 0.41115373373031616, "grad_norm": 17.830284118652344, "learning_rate": 7.748024712947204e-08, "logits/chosen": 0.6710245013237, "logits/rejected": 0.649438738822937, "logps/chosen": -225.27713012695312, "logps/ref_chosen": -66.27017211914062, "logps/ref_rejected": -71.73065185546875, "logps/rejected": -293.5256652832031, "loss": 1.1576, "margin_dpo/margin_mean": 62.78805160522461, "margin_dpo/margin_std": 117.54412841796875, "step": 509 }, { "KL/chosen_KL_mean": -157.57489013671875, "KL/mean": -197.96450805664062, "KL/rejected_KL_mean": -238.3541259765625, "KL/std": 110.64065551757812, "epoch": 0.7709750566893424, "fcm_dpo/beta": 0.006295748520642519, "fcm_dpo/delta": -0.06247474253177643, "fcm_dpo/margin": 80.77923583984375, "fcm_dpo/q_t": 0.3951059579849243, "grad_norm": 14.694042205810547, "learning_rate": 7.652572947447272e-08, "logits/chosen": 0.8205969929695129, "logits/rejected": 0.727057933807373, "logps/chosen": -211.11976623535156, "logps/ref_chosen": -53.54487609863281, "logps/ref_rejected": -91.36648559570312, "logps/rejected": -329.7206115722656, "loss": 1.1097, "margin_dpo/margin_mean": 80.77923583984375, "margin_dpo/margin_std": 140.24911499023438, "step": 510 }, { "KL/chosen_KL_mean": -148.6263427734375, "KL/mean": -194.43792724609375, "KL/rejected_KL_mean": -240.24951171875, "KL/std": 100.10411071777344, "epoch": 0.7724867724867724, "fcm_dpo/beta": 0.006196873262524605, "fcm_dpo/delta": -0.12475550174713135, "fcm_dpo/margin": 91.62318420410156, "fcm_dpo/q_t": 0.37250590324401855, "grad_norm": 16.96703338623047, "learning_rate": 7.557606426772961e-08, "logits/chosen": 0.7572273015975952, "logits/rejected": 0.7002270817756653, "logps/chosen": -204.47071838378906, "logps/ref_chosen": -55.844383239746094, "logps/ref_rejected": -86.49819946289062, "logps/rejected": -326.7477111816406, "loss": 1.0031, "margin_dpo/margin_mean": 91.62318420410156, "margin_dpo/margin_std": 108.27359008789062, "step": 511 }, { "KL/chosen_KL_mean": -153.93524169921875, "KL/mean": -182.5100555419922, "KL/rejected_KL_mean": -211.08486938476562, "KL/std": 93.58463287353516, "epoch": 0.7739984882842026, "fcm_dpo/beta": 0.006222460884600878, "fcm_dpo/delta": 0.09797990322113037, "fcm_dpo/margin": 57.149635314941406, "fcm_dpo/q_t": 0.4194071888923645, "grad_norm": 21.45284080505371, "learning_rate": 7.463127807341966e-08, "logits/chosen": 0.6186809539794922, "logits/rejected": 0.6180380582809448, "logps/chosen": -215.58827209472656, "logps/ref_chosen": -61.653038024902344, "logps/ref_rejected": -72.83148193359375, "logps/rejected": -283.9163513183594, "loss": 1.1794, "margin_dpo/margin_mean": 57.14964294433594, "margin_dpo/margin_std": 112.34751892089844, "step": 512 }, { "KL/chosen_KL_mean": -137.3463592529297, "KL/mean": -174.01168823242188, "KL/rejected_KL_mean": -210.67701721191406, "KL/std": 95.78628540039062, "epoch": 0.7755102040816326, "fcm_dpo/beta": 0.006251271814107895, "fcm_dpo/delta": -0.008913135156035423, "fcm_dpo/margin": 73.33065032958984, "fcm_dpo/q_t": 0.39796391129493713, "grad_norm": 11.89295482635498, "learning_rate": 7.369139731924401e-08, "logits/chosen": 0.816609263420105, "logits/rejected": 0.7631587386131287, "logps/chosen": -188.1989288330078, "logps/ref_chosen": -50.85256576538086, "logps/ref_rejected": -69.21754455566406, "logps/rejected": -279.8945617675781, "loss": 1.0705, "margin_dpo/margin_mean": 73.33065032958984, "margin_dpo/margin_std": 101.00604248046875, "step": 513 }, { "KL/chosen_KL_mean": -153.13296508789062, "KL/mean": -195.20777893066406, "KL/rejected_KL_mean": -237.2825927734375, "KL/std": 104.29148864746094, "epoch": 0.7770219198790628, "fcm_dpo/beta": 0.0062185414135456085, "fcm_dpo/delta": -0.07742288708686829, "fcm_dpo/margin": 84.14960479736328, "fcm_dpo/q_t": 0.3840315639972687, "grad_norm": 15.968329429626465, "learning_rate": 7.275644829568747e-08, "logits/chosen": 0.7511647343635559, "logits/rejected": 0.7200503349304199, "logps/chosen": -222.51791381835938, "logps/ref_chosen": -69.38493347167969, "logps/ref_rejected": -83.32447814941406, "logps/rejected": -320.6070556640625, "loss": 1.052, "margin_dpo/margin_mean": 84.14961242675781, "margin_dpo/margin_std": 117.60154724121094, "step": 514 }, { "KL/chosen_KL_mean": -162.0972442626953, "KL/mean": -195.05587768554688, "KL/rejected_KL_mean": -228.0145263671875, "KL/std": 97.84542846679688, "epoch": 0.7785336356764928, "fcm_dpo/beta": 0.006194347515702248, "fcm_dpo/delta": 0.043475374579429626, "fcm_dpo/margin": 65.9172592163086, "fcm_dpo/q_t": 0.4080887734889984, "grad_norm": 15.475692749023438, "learning_rate": 7.182645715528435e-08, "logits/chosen": 0.7449712753295898, "logits/rejected": 0.6657108068466187, "logps/chosen": -215.78427124023438, "logps/ref_chosen": -53.687034606933594, "logps/ref_rejected": -83.59614562988281, "logps/rejected": -311.61065673828125, "loss": 1.1306, "margin_dpo/margin_mean": 65.9172592163086, "margin_dpo/margin_std": 111.5773696899414, "step": 515 }, { "KL/chosen_KL_mean": -140.24081420898438, "KL/mean": -171.61367797851562, "KL/rejected_KL_mean": -202.98654174804688, "KL/std": 97.13780212402344, "epoch": 0.780045351473923, "fcm_dpo/beta": 0.0062939864583313465, "fcm_dpo/delta": 0.056877270340919495, "fcm_dpo/margin": 62.74571228027344, "fcm_dpo/q_t": 0.4098943769931793, "grad_norm": 17.302248001098633, "learning_rate": 7.090144991188568e-08, "logits/chosen": 0.7184008359909058, "logits/rejected": 0.6824341416358948, "logps/chosen": -197.14254760742188, "logps/ref_chosen": -56.9017219543457, "logps/ref_rejected": -67.83477783203125, "logps/rejected": -270.8213195800781, "loss": 1.1527, "margin_dpo/margin_mean": 62.7457160949707, "margin_dpo/margin_std": 114.64778900146484, "step": 516 }, { "KL/chosen_KL_mean": -167.3245849609375, "KL/mean": -192.08224487304688, "KL/rejected_KL_mean": -216.83987426757812, "KL/std": 102.61844635009766, "epoch": 0.781557067271353, "fcm_dpo/beta": 0.006441779434680939, "fcm_dpo/delta": 0.1345398724079132, "fcm_dpo/margin": 49.51531219482422, "fcm_dpo/q_t": 0.43090230226516724, "grad_norm": 16.201496124267578, "learning_rate": 6.998145243993284e-08, "logits/chosen": 0.7385038137435913, "logits/rejected": 0.7405730485916138, "logps/chosen": -229.0997314453125, "logps/ref_chosen": -61.775142669677734, "logps/ref_rejected": -62.88270950317383, "logps/rejected": -279.72259521484375, "loss": 1.202, "margin_dpo/margin_mean": 49.51531219482422, "margin_dpo/margin_std": 106.55221557617188, "step": 517 }, { "KL/chosen_KL_mean": -143.09214782714844, "KL/mean": -177.44760131835938, "KL/rejected_KL_mean": -211.80307006835938, "KL/std": 100.13240051269531, "epoch": 0.783068783068783, "fcm_dpo/beta": 0.006467289291322231, "fcm_dpo/delta": 0.005887898616492748, "fcm_dpo/margin": 68.71090698242188, "fcm_dpo/q_t": 0.40420591831207275, "grad_norm": 13.878226280212402, "learning_rate": 6.906649047373245e-08, "logits/chosen": 0.7254435420036316, "logits/rejected": 0.6841272115707397, "logps/chosen": -205.1173858642578, "logps/ref_chosen": -62.02523422241211, "logps/ref_rejected": -79.06085205078125, "logps/rejected": -290.8639221191406, "loss": 1.109, "margin_dpo/margin_mean": 68.71090698242188, "margin_dpo/margin_std": 112.37191009521484, "step": 518 }, { "KL/chosen_KL_mean": -171.6400604248047, "KL/mean": -191.7677764892578, "KL/rejected_KL_mean": -211.89549255371094, "KL/std": 102.14537811279297, "epoch": 0.7845804988662132, "fcm_dpo/beta": 0.006517443805932999, "fcm_dpo/delta": 0.0746041014790535, "fcm_dpo/margin": 40.25545883178711, "fcm_dpo/q_t": 0.4420524835586548, "grad_norm": 21.90707778930664, "learning_rate": 6.815658960673781e-08, "logits/chosen": 0.745517373085022, "logits/rejected": 0.6990114450454712, "logps/chosen": -233.24642944335938, "logps/ref_chosen": -61.60636901855469, "logps/ref_rejected": -74.50727844238281, "logps/rejected": -286.40277099609375, "loss": 1.3203, "margin_dpo/margin_mean": 40.25545883178711, "margin_dpo/margin_std": 135.0486602783203, "step": 519 }, { "KL/chosen_KL_mean": -157.34315490722656, "KL/mean": -187.73007202148438, "KL/rejected_KL_mean": -218.11697387695312, "KL/std": 101.43348693847656, "epoch": 0.7860922146636432, "fcm_dpo/beta": 0.006574805825948715, "fcm_dpo/delta": 0.05200670659542084, "fcm_dpo/margin": 60.773807525634766, "fcm_dpo/q_t": 0.412746787071228, "grad_norm": 16.04285430908203, "learning_rate": 6.725177529083209e-08, "logits/chosen": 0.7855746746063232, "logits/rejected": 0.7317807078361511, "logps/chosen": -220.21658325195312, "logps/ref_chosen": -62.87343215942383, "logps/ref_rejected": -76.505615234375, "logps/rejected": -294.62255859375, "loss": 1.1375, "margin_dpo/margin_mean": 60.773807525634766, "margin_dpo/margin_std": 104.73448181152344, "step": 520 }, { "KL/chosen_KL_mean": -154.99295043945312, "KL/mean": -199.1806640625, "KL/rejected_KL_mean": -243.36837768554688, "KL/std": 101.28709411621094, "epoch": 0.7876039304610734, "fcm_dpo/beta": 0.00651928223669529, "fcm_dpo/delta": -0.13365957140922546, "fcm_dpo/margin": 88.37542724609375, "fcm_dpo/q_t": 0.3741111755371094, "grad_norm": 13.415428161621094, "learning_rate": 6.63520728356167e-08, "logits/chosen": 0.6261991262435913, "logits/rejected": 0.5538595914840698, "logps/chosen": -219.1996307373047, "logps/ref_chosen": -64.20668029785156, "logps/ref_rejected": -92.28083038330078, "logps/rejected": -335.64923095703125, "loss": 1.0104, "margin_dpo/margin_mean": 88.37542724609375, "margin_dpo/margin_std": 113.64947509765625, "step": 521 }, { "KL/chosen_KL_mean": -156.89218139648438, "KL/mean": -183.0986328125, "KL/rejected_KL_mean": -209.30506896972656, "KL/std": 101.82064056396484, "epoch": 0.7891156462585034, "fcm_dpo/beta": 0.006550894118845463, "fcm_dpo/delta": 0.11058982461690903, "fcm_dpo/margin": 52.412906646728516, "fcm_dpo/q_t": 0.42578643560409546, "grad_norm": 16.121353149414062, "learning_rate": 6.545750740770336e-08, "logits/chosen": 0.7278043627738953, "logits/rejected": 0.7202616333961487, "logps/chosen": -215.26190185546875, "logps/ref_chosen": -58.369720458984375, "logps/ref_rejected": -68.79248046875, "logps/rejected": -278.0975341796875, "loss": 1.2466, "margin_dpo/margin_mean": 52.41290283203125, "margin_dpo/margin_std": 133.0802764892578, "step": 522 }, { "KL/chosen_KL_mean": -160.9173583984375, "KL/mean": -192.372314453125, "KL/rejected_KL_mean": -223.8272705078125, "KL/std": 102.92727661132812, "epoch": 0.7906273620559335, "fcm_dpo/beta": 0.006632317788898945, "fcm_dpo/delta": 0.034202300012111664, "fcm_dpo/margin": 62.9099006652832, "fcm_dpo/q_t": 0.40550702810287476, "grad_norm": 17.54810333251953, "learning_rate": 6.456810403001012e-08, "logits/chosen": 0.7431254386901855, "logits/rejected": 0.6213644742965698, "logps/chosen": -226.63058471679688, "logps/ref_chosen": -65.71324157714844, "logps/ref_rejected": -91.98896789550781, "logps/rejected": -315.81622314453125, "loss": 1.1495, "margin_dpo/margin_mean": 62.9099006652832, "margin_dpo/margin_std": 116.45848083496094, "step": 523 }, { "KL/chosen_KL_mean": -139.40798950195312, "KL/mean": -172.82437133789062, "KL/rejected_KL_mean": -206.24075317382812, "KL/std": 98.07855987548828, "epoch": 0.7921390778533636, "fcm_dpo/beta": 0.006677767261862755, "fcm_dpo/delta": 0.003585953265428543, "fcm_dpo/margin": 66.83275604248047, "fcm_dpo/q_t": 0.3996120095252991, "grad_norm": 15.2020845413208, "learning_rate": 6.368388758106134e-08, "logits/chosen": 0.6536479592323303, "logits/rejected": 0.6307451725006104, "logps/chosen": -215.75924682617188, "logps/ref_chosen": -76.35124969482422, "logps/ref_rejected": -89.96072387695312, "logps/rejected": -296.20147705078125, "loss": 1.0911, "margin_dpo/margin_mean": 66.83275604248047, "margin_dpo/margin_std": 99.37472534179688, "step": 524 }, { "KL/chosen_KL_mean": -158.2174530029297, "KL/mean": -185.13754272460938, "KL/rejected_KL_mean": -212.05763244628906, "KL/std": 96.78825378417969, "epoch": 0.7936507936507936, "fcm_dpo/beta": 0.006735939532518387, "fcm_dpo/delta": 0.09070742875337601, "fcm_dpo/margin": 53.840171813964844, "fcm_dpo/q_t": 0.4220663607120514, "grad_norm": 21.1476993560791, "learning_rate": 6.280488279429185e-08, "logits/chosen": 0.5505712032318115, "logits/rejected": 0.5474892854690552, "logps/chosen": -233.71324157714844, "logps/ref_chosen": -75.49578857421875, "logps/ref_rejected": -84.04852294921875, "logps/rejected": -296.10614013671875, "loss": 1.1904, "margin_dpo/margin_mean": 53.840171813964844, "margin_dpo/margin_std": 114.42605590820312, "step": 525 }, { "KL/chosen_KL_mean": -165.60940551757812, "KL/mean": -190.29290771484375, "KL/rejected_KL_mean": -214.9763946533203, "KL/std": 99.83271789550781, "epoch": 0.7951625094482238, "fcm_dpo/beta": 0.00672984030097723, "fcm_dpo/delta": -0.03424973785877228, "fcm_dpo/margin": 49.36698913574219, "fcm_dpo/q_t": 0.42518430948257446, "grad_norm": 15.3506498336792, "learning_rate": 6.193111425735515e-08, "logits/chosen": 0.766860842704773, "logits/rejected": 0.6968935132026672, "logps/chosen": -226.90182495117188, "logps/ref_chosen": -61.29241943359375, "logps/ref_rejected": -82.47763061523438, "logps/rejected": -297.45404052734375, "loss": 1.1969, "margin_dpo/margin_mean": 49.36698913574219, "margin_dpo/margin_std": 101.69459533691406, "step": 526 }, { "KL/chosen_KL_mean": -173.23233032226562, "KL/mean": -194.60577392578125, "KL/rejected_KL_mean": -215.97921752929688, "KL/std": 98.76846313476562, "epoch": 0.7966742252456538, "fcm_dpo/beta": 0.006750140339136124, "fcm_dpo/delta": 0.030073152855038643, "fcm_dpo/margin": 42.74687957763672, "fcm_dpo/q_t": 0.4377078413963318, "grad_norm": 18.568347930908203, "learning_rate": 6.106260641143546e-08, "logits/chosen": 0.8117989301681519, "logits/rejected": 0.728463888168335, "logps/chosen": -234.7049560546875, "logps/ref_chosen": -61.472625732421875, "logps/ref_rejected": -90.52831268310547, "logps/rejected": -306.50750732421875, "loss": 1.2515, "margin_dpo/margin_mean": 42.74687957763672, "margin_dpo/margin_std": 111.45413208007812, "step": 527 }, { "KL/chosen_KL_mean": -153.25848388671875, "KL/mean": -178.26425170898438, "KL/rejected_KL_mean": -203.27000427246094, "KL/std": 95.63511657714844, "epoch": 0.7981859410430839, "fcm_dpo/beta": 0.006859553977847099, "fcm_dpo/delta": 0.11088088154792786, "fcm_dpo/margin": 50.01153564453125, "fcm_dpo/q_t": 0.42585062980651855, "grad_norm": 18.767499923706055, "learning_rate": 6.019938355056422e-08, "logits/chosen": 0.679128885269165, "logits/rejected": 0.6063876748085022, "logps/chosen": -212.0504913330078, "logps/ref_chosen": -58.792015075683594, "logps/ref_rejected": -71.82516479492188, "logps/rejected": -275.09515380859375, "loss": 1.2378, "margin_dpo/margin_mean": 50.011531829833984, "margin_dpo/margin_std": 122.185546875, "step": 528 }, { "KL/chosen_KL_mean": -146.73153686523438, "KL/mean": -195.88995361328125, "KL/rejected_KL_mean": -245.04833984375, "KL/std": 95.25759887695312, "epoch": 0.799697656840514, "fcm_dpo/beta": 0.00668944651260972, "fcm_dpo/delta": -0.22261568903923035, "fcm_dpo/margin": 98.31681823730469, "fcm_dpo/q_t": 0.3527362644672394, "grad_norm": 15.684530258178711, "learning_rate": 5.934146982094049e-08, "logits/chosen": 0.6630944013595581, "logits/rejected": 0.6121984720230103, "logps/chosen": -201.802490234375, "logps/ref_chosen": -55.070960998535156, "logps/ref_rejected": -75.44007873535156, "logps/rejected": -320.4884033203125, "loss": 0.9411, "margin_dpo/margin_mean": 98.31681823730469, "margin_dpo/margin_std": 103.48916625976562, "step": 529 }, { "KL/chosen_KL_mean": -148.173828125, "KL/mean": -179.78138732910156, "KL/rejected_KL_mean": -211.3889617919922, "KL/std": 100.672607421875, "epoch": 0.8012093726379441, "fcm_dpo/beta": 0.00664612278342247, "fcm_dpo/delta": 0.031186042353510857, "fcm_dpo/margin": 63.215126037597656, "fcm_dpo/q_t": 0.4094647169113159, "grad_norm": 19.30486297607422, "learning_rate": 5.848888922025552e-08, "logits/chosen": 0.7495627403259277, "logits/rejected": 0.7060422897338867, "logps/chosen": -204.91763305664062, "logps/ref_chosen": -56.743812561035156, "logps/ref_rejected": -76.6692123413086, "logps/rejected": -288.05816650390625, "loss": 1.1221, "margin_dpo/margin_mean": 63.215126037597656, "margin_dpo/margin_std": 104.92845153808594, "step": 530 }, { "KL/chosen_KL_mean": -147.54202270507812, "KL/mean": -180.4340057373047, "KL/rejected_KL_mean": -213.32598876953125, "KL/std": 99.5765380859375, "epoch": 0.8027210884353742, "fcm_dpo/beta": 0.006687507964670658, "fcm_dpo/delta": 0.01040719449520111, "fcm_dpo/margin": 65.78396606445312, "fcm_dpo/q_t": 0.40361833572387695, "grad_norm": 15.21304988861084, "learning_rate": 5.7641665597021435e-08, "logits/chosen": 0.7009578347206116, "logits/rejected": 0.6236515045166016, "logps/chosen": -198.65847778320312, "logps/ref_chosen": -51.116455078125, "logps/ref_rejected": -79.52884674072266, "logps/rejected": -292.8548278808594, "loss": 1.113, "margin_dpo/margin_mean": 65.78396606445312, "margin_dpo/margin_std": 107.75035095214844, "step": 531 }, { "KL/chosen_KL_mean": -171.36239624023438, "KL/mean": -206.0482635498047, "KL/rejected_KL_mean": -240.734130859375, "KL/std": 98.20024871826172, "epoch": 0.8042328042328042, "fcm_dpo/beta": 0.006664278917014599, "fcm_dpo/delta": -0.012907939031720161, "fcm_dpo/margin": 69.37173461914062, "fcm_dpo/q_t": 0.3980584144592285, "grad_norm": 15.120401382446289, "learning_rate": 5.679982264990424e-08, "logits/chosen": 0.6691586971282959, "logits/rejected": 0.6198612451553345, "logps/chosen": -229.64234924316406, "logps/ref_chosen": -58.279945373535156, "logps/ref_rejected": -78.05426788330078, "logps/rejected": -318.78839111328125, "loss": 1.1031, "margin_dpo/margin_mean": 69.37173461914062, "margin_dpo/margin_std": 111.1090087890625, "step": 532 }, { "KL/chosen_KL_mean": -123.17243957519531, "KL/mean": -161.1618194580078, "KL/rejected_KL_mean": -199.15118408203125, "KL/std": 104.36534118652344, "epoch": 0.8057445200302343, "fcm_dpo/beta": 0.006617809645831585, "fcm_dpo/delta": -0.05558159202337265, "fcm_dpo/margin": 75.97874450683594, "fcm_dpo/q_t": 0.3897656202316284, "grad_norm": 14.80583381652832, "learning_rate": 5.596338392706076e-08, "logits/chosen": 0.8223803639411926, "logits/rejected": 0.7595813274383545, "logps/chosen": -179.5904541015625, "logps/ref_chosen": -56.41801071166992, "logps/ref_rejected": -73.89324951171875, "logps/rejected": -273.04443359375, "loss": 1.0669, "margin_dpo/margin_mean": 75.97874450683594, "margin_dpo/margin_std": 110.16397094726562, "step": 533 }, { "KL/chosen_KL_mean": -152.75064086914062, "KL/mean": -184.16172790527344, "KL/rejected_KL_mean": -215.57281494140625, "KL/std": 96.92642974853516, "epoch": 0.8072562358276644, "fcm_dpo/beta": 0.006597267463803291, "fcm_dpo/delta": 0.036938875913619995, "fcm_dpo/margin": 62.82218551635742, "fcm_dpo/q_t": 0.410610169172287, "grad_norm": 14.558494567871094, "learning_rate": 5.513237282548033e-08, "logits/chosen": 0.7041269540786743, "logits/rejected": 0.6697344183921814, "logps/chosen": -213.49932861328125, "logps/ref_chosen": -60.748687744140625, "logps/ref_rejected": -73.8623046875, "logps/rejected": -289.43511962890625, "loss": 1.1546, "margin_dpo/margin_mean": 62.82218551635742, "margin_dpo/margin_std": 118.70053100585938, "step": 534 }, { "KL/chosen_KL_mean": -163.18838500976562, "KL/mean": -192.66116333007812, "KL/rejected_KL_mean": -222.1339111328125, "KL/std": 103.63077545166016, "epoch": 0.8087679516250945, "fcm_dpo/beta": 0.006703161634504795, "fcm_dpo/delta": 0.0569588765501976, "fcm_dpo/margin": 58.94554138183594, "fcm_dpo/q_t": 0.41441118717193604, "grad_norm": 17.145980834960938, "learning_rate": 5.430681259032957e-08, "logits/chosen": 0.6563804149627686, "logits/rejected": 0.5968525409698486, "logps/chosen": -224.8258056640625, "logps/ref_chosen": -61.637413024902344, "logps/ref_rejected": -80.93138885498047, "logps/rejected": -303.0653076171875, "loss": 1.1646, "margin_dpo/margin_mean": 58.94554138183594, "margin_dpo/margin_std": 113.44522857666016, "step": 535 }, { "KL/chosen_KL_mean": -142.1412353515625, "KL/mean": -186.98748779296875, "KL/rejected_KL_mean": -231.83370971679688, "KL/std": 103.05687713623047, "epoch": 0.8102796674225246, "fcm_dpo/beta": 0.0065511735156178474, "fcm_dpo/delta": -0.14660747349262238, "fcm_dpo/margin": 89.69249725341797, "fcm_dpo/q_t": 0.3693081736564636, "grad_norm": 11.731171607971191, "learning_rate": 5.3486726314303175e-08, "logits/chosen": 0.7670760154724121, "logits/rejected": 0.6842254996299744, "logps/chosen": -194.03021240234375, "logps/ref_chosen": -51.88897705078125, "logps/ref_rejected": -73.34864044189453, "logps/rejected": -305.182373046875, "loss": 0.9762, "margin_dpo/margin_mean": 89.69249725341797, "margin_dpo/margin_std": 99.07676696777344, "step": 536 }, { "KL/chosen_KL_mean": -161.25852966308594, "KL/mean": -199.32022094726562, "KL/rejected_KL_mean": -237.38189697265625, "KL/std": 110.19611358642578, "epoch": 0.8117913832199547, "fcm_dpo/beta": 0.006451202556490898, "fcm_dpo/delta": -0.043621353805065155, "fcm_dpo/margin": 76.12335205078125, "fcm_dpo/q_t": 0.39416471123695374, "grad_norm": 14.4218168258667, "learning_rate": 5.267213693697695e-08, "logits/chosen": 0.8012008666992188, "logits/rejected": 0.7109647393226624, "logps/chosen": -215.50714111328125, "logps/ref_chosen": -54.248619079589844, "logps/ref_rejected": -94.94343566894531, "logps/rejected": -332.3253173828125, "loss": 1.0885, "margin_dpo/margin_mean": 76.12335205078125, "margin_dpo/margin_std": 119.11538696289062, "step": 537 }, { "KL/chosen_KL_mean": -160.41598510742188, "KL/mean": -198.71536254882812, "KL/rejected_KL_mean": -237.01473999023438, "KL/std": 106.37571716308594, "epoch": 0.8133030990173847, "fcm_dpo/beta": 0.006427483633160591, "fcm_dpo/delta": -0.044511329382658005, "fcm_dpo/margin": 76.59874725341797, "fcm_dpo/q_t": 0.3906959891319275, "grad_norm": 15.044438362121582, "learning_rate": 5.1863067244167144e-08, "logits/chosen": 0.7274729013442993, "logits/rejected": 0.7041901350021362, "logps/chosen": -230.509521484375, "logps/ref_chosen": -70.09353637695312, "logps/ref_rejected": -79.49833679199219, "logps/rejected": -316.5130615234375, "loss": 1.0591, "margin_dpo/margin_mean": 76.5987548828125, "margin_dpo/margin_std": 106.91598510742188, "step": 538 }, { "KL/chosen_KL_mean": -166.6041259765625, "KL/mean": -195.6445770263672, "KL/rejected_KL_mean": -224.68502807617188, "KL/std": 99.87994384765625, "epoch": 0.8148148148148148, "fcm_dpo/beta": 0.0064649423584342, "fcm_dpo/delta": 0.07749071717262268, "fcm_dpo/margin": 58.080909729003906, "fcm_dpo/q_t": 0.41956430673599243, "grad_norm": 16.343154907226562, "learning_rate": 5.105953986729195e-08, "logits/chosen": 0.6749646663665771, "logits/rejected": 0.594658374786377, "logps/chosen": -228.5358123779297, "logps/ref_chosen": -61.93169403076172, "logps/ref_rejected": -84.08946228027344, "logps/rejected": -308.77447509765625, "loss": 1.1496, "margin_dpo/margin_mean": 58.080909729003906, "margin_dpo/margin_std": 105.43345642089844, "step": 539 }, { "KL/chosen_KL_mean": -153.3197021484375, "KL/mean": -197.894287109375, "KL/rejected_KL_mean": -242.4688720703125, "KL/std": 114.05096435546875, "epoch": 0.8163265306122449, "fcm_dpo/beta": 0.006410318426787853, "fcm_dpo/delta": -0.1286935806274414, "fcm_dpo/margin": 89.14921569824219, "fcm_dpo/q_t": 0.3729906976222992, "grad_norm": 13.589789390563965, "learning_rate": 5.026157728273966e-08, "logits/chosen": 0.7904075980186462, "logits/rejected": 0.6958364248275757, "logps/chosen": -216.02394104003906, "logps/ref_chosen": -62.704254150390625, "logps/ref_rejected": -95.63597106933594, "logps/rejected": -338.1048583984375, "loss": 1.0036, "margin_dpo/margin_mean": 89.14920806884766, "margin_dpo/margin_std": 107.2660903930664, "step": 540 }, { "KL/chosen_KL_mean": -148.68814086914062, "KL/mean": -186.67027282714844, "KL/rejected_KL_mean": -224.65240478515625, "KL/std": 100.93985748291016, "epoch": 0.817838246409675, "fcm_dpo/beta": 0.00628033559769392, "fcm_dpo/delta": -0.029025848954916, "fcm_dpo/margin": 75.96427917480469, "fcm_dpo/q_t": 0.3912792205810547, "grad_norm": 13.795503616333008, "learning_rate": 4.9469201811239035e-08, "logits/chosen": 0.7655197381973267, "logits/rejected": 0.7934525012969971, "logps/chosen": -211.16897583007812, "logps/ref_chosen": -62.48084259033203, "logps/ref_rejected": -57.55541229248047, "logps/rejected": -282.20782470703125, "loss": 1.0564, "margin_dpo/margin_mean": 75.96427154541016, "margin_dpo/margin_std": 98.35800170898438, "step": 541 }, { "KL/chosen_KL_mean": -129.79156494140625, "KL/mean": -171.60101318359375, "KL/rejected_KL_mean": -213.4104461669922, "KL/std": 96.41845703125, "epoch": 0.8193499622071051, "fcm_dpo/beta": 0.0062217069789767265, "fcm_dpo/delta": -0.07427109777927399, "fcm_dpo/margin": 83.61888122558594, "fcm_dpo/q_t": 0.38352102041244507, "grad_norm": 14.153130531311035, "learning_rate": 4.868243561723534e-08, "logits/chosen": 0.7965934872627258, "logits/rejected": 0.7499839067459106, "logps/chosen": -179.2464599609375, "logps/ref_chosen": -49.454891204833984, "logps/ref_rejected": -65.33275604248047, "logps/rejected": -278.74322509765625, "loss": 1.0525, "margin_dpo/margin_mean": 83.61888122558594, "margin_dpo/margin_std": 116.61778259277344, "step": 542 }, { "KL/chosen_KL_mean": -141.03268432617188, "KL/mean": -181.77450561523438, "KL/rejected_KL_mean": -222.51632690429688, "KL/std": 98.75093841552734, "epoch": 0.8208616780045351, "fcm_dpo/beta": 0.006175430491566658, "fcm_dpo/delta": -0.05604030564427376, "fcm_dpo/margin": 81.48362731933594, "fcm_dpo/q_t": 0.38659557700157166, "grad_norm": 11.417715072631836, "learning_rate": 4.790130070827028e-08, "logits/chosen": 0.7406554222106934, "logits/rejected": 0.6567230224609375, "logps/chosen": -192.133544921875, "logps/ref_chosen": -51.100860595703125, "logps/ref_rejected": -76.06130981445312, "logps/rejected": -298.57763671875, "loss": 1.0504, "margin_dpo/margin_mean": 81.48362731933594, "margin_dpo/margin_std": 107.75604248046875, "step": 543 }, { "KL/chosen_KL_mean": -151.23812866210938, "KL/mean": -197.67628479003906, "KL/rejected_KL_mean": -244.11444091796875, "KL/std": 108.5754165649414, "epoch": 0.8223733938019653, "fcm_dpo/beta": 0.006031567230820656, "fcm_dpo/delta": -0.11675632745027542, "fcm_dpo/margin": 92.87631225585938, "fcm_dpo/q_t": 0.3768364191055298, "grad_norm": 15.823902130126953, "learning_rate": 4.7125818934366454e-08, "logits/chosen": 0.7370050549507141, "logits/rejected": 0.6611154675483704, "logps/chosen": -211.51535034179688, "logps/ref_chosen": -60.2772331237793, "logps/ref_rejected": -88.40553283691406, "logps/rejected": -332.51995849609375, "loss": 1.0305, "margin_dpo/margin_mean": 92.87631225585938, "margin_dpo/margin_std": 125.10038757324219, "step": 544 }, { "KL/chosen_KL_mean": -164.6317901611328, "KL/mean": -191.05908203125, "KL/rejected_KL_mean": -217.48635864257812, "KL/std": 101.46080017089844, "epoch": 0.8238851095993953, "fcm_dpo/beta": 0.006120487116277218, "fcm_dpo/delta": 0.1305941343307495, "fcm_dpo/margin": 52.85456848144531, "fcm_dpo/q_t": 0.4277143180370331, "grad_norm": 15.269746780395508, "learning_rate": 4.635601198741607e-08, "logits/chosen": 0.7444198131561279, "logits/rejected": 0.6830329895019531, "logps/chosen": -226.24703979492188, "logps/ref_chosen": -61.61524963378906, "logps/ref_rejected": -78.71266174316406, "logps/rejected": -296.19903564453125, "loss": 1.1929, "margin_dpo/margin_mean": 52.85456848144531, "margin_dpo/margin_std": 108.56221771240234, "step": 545 }, { "KL/chosen_KL_mean": -155.8973388671875, "KL/mean": -185.46197509765625, "KL/rejected_KL_mean": -215.02658081054688, "KL/std": 95.04618835449219, "epoch": 0.8253968253968254, "fcm_dpo/beta": 0.0062283361330628395, "fcm_dpo/delta": 0.08490745723247528, "fcm_dpo/margin": 59.12923812866211, "fcm_dpo/q_t": 0.4165213108062744, "grad_norm": 18.64433479309082, "learning_rate": 4.559190140057428e-08, "logits/chosen": 0.8147549629211426, "logits/rejected": 0.8096719980239868, "logps/chosen": -215.21060180664062, "logps/ref_chosen": -59.313262939453125, "logps/ref_rejected": -64.73631286621094, "logps/rejected": -279.76287841796875, "loss": 1.1728, "margin_dpo/margin_mean": 59.129241943359375, "margin_dpo/margin_std": 115.27635192871094, "step": 546 }, { "KL/chosen_KL_mean": -137.16912841796875, "KL/mean": -179.7300567626953, "KL/rejected_KL_mean": -222.29100036621094, "KL/std": 101.09002685546875, "epoch": 0.8269085411942555, "fcm_dpo/beta": 0.006165103986859322, "fcm_dpo/delta": -0.07947796583175659, "fcm_dpo/margin": 85.12187194824219, "fcm_dpo/q_t": 0.38214462995529175, "grad_norm": 14.160149574279785, "learning_rate": 4.483350854765672e-08, "logits/chosen": 0.6544277667999268, "logits/rejected": 0.5950570106506348, "logps/chosen": -192.1458740234375, "logps/ref_chosen": -54.97674560546875, "logps/ref_rejected": -75.35922241210938, "logps/rejected": -297.65020751953125, "loss": 1.0434, "margin_dpo/margin_mean": 85.12186431884766, "margin_dpo/margin_std": 114.97239685058594, "step": 547 }, { "KL/chosen_KL_mean": -162.11642456054688, "KL/mean": -187.776611328125, "KL/rejected_KL_mean": -213.43678283691406, "KL/std": 100.71829223632812, "epoch": 0.8284202569916855, "fcm_dpo/beta": 0.006303782574832439, "fcm_dpo/delta": 0.13005885481834412, "fcm_dpo/margin": 51.32037353515625, "fcm_dpo/q_t": 0.4288024604320526, "grad_norm": 17.33949851989746, "learning_rate": 4.4080854642541826e-08, "logits/chosen": 0.6553746461868286, "logits/rejected": 0.5998683571815491, "logps/chosen": -225.32708740234375, "logps/ref_chosen": -63.21067428588867, "logps/ref_rejected": -81.23347473144531, "logps/rejected": -294.6702575683594, "loss": 1.2002, "margin_dpo/margin_mean": 51.32037353515625, "margin_dpo/margin_std": 109.20315551757812, "step": 548 }, { "KL/chosen_KL_mean": -158.01779174804688, "KL/mean": -190.70941162109375, "KL/rejected_KL_mean": -223.4010467529297, "KL/std": 108.44243621826172, "epoch": 0.8299319727891157, "fcm_dpo/beta": 0.006333451718091965, "fcm_dpo/delta": 0.03725190833210945, "fcm_dpo/margin": 65.38323211669922, "fcm_dpo/q_t": 0.411038339138031, "grad_norm": 18.26466941833496, "learning_rate": 4.333396073857723e-08, "logits/chosen": 0.7768402099609375, "logits/rejected": 0.7113950252532959, "logps/chosen": -222.2913055419922, "logps/ref_chosen": -64.27351379394531, "logps/ref_rejected": -92.31663513183594, "logps/rejected": -315.7176818847656, "loss": 1.1632, "margin_dpo/margin_mean": 65.38322448730469, "margin_dpo/margin_std": 127.63583374023438, "step": 549 }, { "KL/chosen_KL_mean": -163.54788208007812, "KL/mean": -186.1416015625, "KL/rejected_KL_mean": -208.73532104492188, "KL/std": 97.8836669921875, "epoch": 0.8314436885865457, "fcm_dpo/beta": 0.006398816127330065, "fcm_dpo/delta": 0.02116929367184639, "fcm_dpo/margin": 45.18742370605469, "fcm_dpo/q_t": 0.435605525970459, "grad_norm": 16.312294006347656, "learning_rate": 4.259284772799099e-08, "logits/chosen": 0.7508084774017334, "logits/rejected": 0.7250527143478394, "logps/chosen": -219.7783203125, "logps/ref_chosen": -56.230438232421875, "logps/ref_rejected": -62.59788513183594, "logps/rejected": -271.33319091796875, "loss": 1.228, "margin_dpo/margin_mean": 45.18741989135742, "margin_dpo/margin_std": 105.23531341552734, "step": 550 }, { "KL/chosen_KL_mean": -165.9162139892578, "KL/mean": -195.33651733398438, "KL/rejected_KL_mean": -224.75680541992188, "KL/std": 105.70631408691406, "epoch": 0.8329554043839759, "fcm_dpo/beta": 0.006481049582362175, "fcm_dpo/delta": 0.07112284004688263, "fcm_dpo/margin": 58.84058380126953, "fcm_dpo/q_t": 0.4186520576477051, "grad_norm": 15.41645622253418, "learning_rate": 4.1857536341307176e-08, "logits/chosen": 0.7407591342926025, "logits/rejected": 0.7093254923820496, "logps/chosen": -233.66342163085938, "logps/ref_chosen": -67.74720764160156, "logps/ref_rejected": -87.04285430908203, "logps/rejected": -311.7996826171875, "loss": 1.1393, "margin_dpo/margin_mean": 58.840579986572266, "margin_dpo/margin_std": 104.4417724609375, "step": 551 }, { "KL/chosen_KL_mean": -154.389892578125, "KL/mean": -186.97584533691406, "KL/rejected_KL_mean": -219.56179809570312, "KL/std": 104.70506286621094, "epoch": 0.8344671201814059, "fcm_dpo/beta": 0.006548226810991764, "fcm_dpo/delta": 0.023566871881484985, "fcm_dpo/margin": 65.17192077636719, "fcm_dpo/q_t": 0.40180903673171997, "grad_norm": 18.85803985595703, "learning_rate": 4.112804714676593e-08, "logits/chosen": 0.7013387680053711, "logits/rejected": 0.6492581367492676, "logps/chosen": -217.31614685058594, "logps/ref_chosen": -62.92625427246094, "logps/ref_rejected": -82.98365783691406, "logps/rejected": -302.5454406738281, "loss": 1.1028, "margin_dpo/margin_mean": 65.17192077636719, "margin_dpo/margin_std": 98.27408599853516, "step": 552 }, { "KL/chosen_KL_mean": -167.21725463867188, "KL/mean": -199.59527587890625, "KL/rejected_KL_mean": -231.97332763671875, "KL/std": 102.98626708984375, "epoch": 0.8359788359788359, "fcm_dpo/beta": 0.0065464479848742485, "fcm_dpo/delta": 0.027236608788371086, "fcm_dpo/margin": 64.75605773925781, "fcm_dpo/q_t": 0.41023218631744385, "grad_norm": 15.29988956451416, "learning_rate": 4.0404400549748144e-08, "logits/chosen": 0.7155384421348572, "logits/rejected": 0.61528480052948, "logps/chosen": -223.2557373046875, "logps/ref_chosen": -56.038490295410156, "logps/ref_rejected": -84.48454284667969, "logps/rejected": -316.4578552246094, "loss": 1.1541, "margin_dpo/margin_mean": 64.75605773925781, "margin_dpo/margin_std": 124.07582092285156, "step": 553 }, { "KL/chosen_KL_mean": -151.11111450195312, "KL/mean": -190.245361328125, "KL/rejected_KL_mean": -229.37960815429688, "KL/std": 99.22693634033203, "epoch": 0.8374905517762661, "fcm_dpo/beta": 0.006496269255876541, "fcm_dpo/delta": -0.061591919511556625, "fcm_dpo/margin": 78.26849365234375, "fcm_dpo/q_t": 0.384976327419281, "grad_norm": 14.839645385742188, "learning_rate": 3.968661679220467e-08, "logits/chosen": 0.6921082735061646, "logits/rejected": 0.6812867522239685, "logps/chosen": -215.64170837402344, "logps/ref_chosen": -64.53059387207031, "logps/ref_rejected": -71.2155990600586, "logps/rejected": -300.59521484375, "loss": 1.0557, "margin_dpo/margin_mean": 78.26849365234375, "margin_dpo/margin_std": 108.02368927001953, "step": 554 }, { "KL/chosen_KL_mean": -164.1957244873047, "KL/mean": -194.96737670898438, "KL/rejected_KL_mean": -225.73904418945312, "KL/std": 99.13853454589844, "epoch": 0.8390022675736961, "fcm_dpo/beta": 0.006575894542038441, "fcm_dpo/delta": 0.045365020632743835, "fcm_dpo/margin": 61.54331970214844, "fcm_dpo/q_t": 0.4076317250728607, "grad_norm": 15.397841453552246, "learning_rate": 3.89747159520904e-08, "logits/chosen": 0.7093448638916016, "logits/rejected": 0.689207911491394, "logps/chosen": -230.84764099121094, "logps/ref_chosen": -66.65191650390625, "logps/ref_rejected": -68.6667251586914, "logps/rejected": -294.40576171875, "loss": 1.1715, "margin_dpo/margin_mean": 61.54331970214844, "margin_dpo/margin_std": 118.01376342773438, "step": 555 }, { "KL/chosen_KL_mean": -163.47509765625, "KL/mean": -192.88812255859375, "KL/rejected_KL_mean": -222.30117797851562, "KL/std": 104.06493377685547, "epoch": 0.8405139833711263, "fcm_dpo/beta": 0.006583062466233969, "fcm_dpo/delta": 0.06530771404504776, "fcm_dpo/margin": 58.82606506347656, "fcm_dpo/q_t": 0.416615754365921, "grad_norm": 15.477357864379883, "learning_rate": 3.826871794280192e-08, "logits/chosen": 0.771374523639679, "logits/rejected": 0.7252951860427856, "logps/chosen": -216.30746459960938, "logps/ref_chosen": -52.832366943359375, "logps/ref_rejected": -64.49044036865234, "logps/rejected": -286.7916259765625, "loss": 1.1748, "margin_dpo/margin_mean": 58.82606887817383, "margin_dpo/margin_std": 116.84832763671875, "step": 556 }, { "KL/chosen_KL_mean": -161.71389770507812, "KL/mean": -203.29481506347656, "KL/rejected_KL_mean": -244.87571716308594, "KL/std": 100.98245239257812, "epoch": 0.8420256991685563, "fcm_dpo/beta": 0.006499768700450659, "fcm_dpo/delta": -0.09647659212350845, "fcm_dpo/margin": 83.16181945800781, "fcm_dpo/q_t": 0.3794446587562561, "grad_norm": 12.793874740600586, "learning_rate": 3.756864251262143e-08, "logits/chosen": 0.8473179340362549, "logits/rejected": 0.7723125219345093, "logps/chosen": -216.74989318847656, "logps/ref_chosen": -55.03598403930664, "logps/ref_rejected": -75.80644989013672, "logps/rejected": -320.68218994140625, "loss": 1.0115, "margin_dpo/margin_mean": 83.16181945800781, "margin_dpo/margin_std": 97.54216003417969, "step": 557 }, { "KL/chosen_KL_mean": -148.48394775390625, "KL/mean": -194.65689086914062, "KL/rejected_KL_mean": -240.8298797607422, "KL/std": 108.04835510253906, "epoch": 0.8435374149659864, "fcm_dpo/beta": 0.006330681033432484, "fcm_dpo/delta": -0.1439516693353653, "fcm_dpo/margin": 92.34593200683594, "fcm_dpo/q_t": 0.3717380166053772, "grad_norm": 12.148514747619629, "learning_rate": 3.687450924416341e-08, "logits/chosen": 0.7686564922332764, "logits/rejected": 0.7162346243858337, "logps/chosen": -211.71029663085938, "logps/ref_chosen": -63.226348876953125, "logps/ref_rejected": -91.46881866455078, "logps/rejected": -332.2987060546875, "loss": 0.9954, "margin_dpo/margin_mean": 92.3459243774414, "margin_dpo/margin_std": 111.07440185546875, "step": 558 }, { "KL/chosen_KL_mean": -153.1084442138672, "KL/mean": -193.3884735107422, "KL/rejected_KL_mean": -233.66848754882812, "KL/std": 106.25447845458984, "epoch": 0.8450491307634165, "fcm_dpo/beta": 0.006216675043106079, "fcm_dpo/delta": -0.05483238771557808, "fcm_dpo/margin": 80.56004333496094, "fcm_dpo/q_t": 0.39433592557907104, "grad_norm": 11.37843132019043, "learning_rate": 3.6186337553827743e-08, "logits/chosen": 0.7013551592826843, "logits/rejected": 0.6412140727043152, "logps/chosen": -214.63009643554688, "logps/ref_chosen": -61.521644592285156, "logps/ref_rejected": -82.83859252929688, "logps/rejected": -316.507080078125, "loss": 1.0877, "margin_dpo/margin_mean": 80.56004333496094, "margin_dpo/margin_std": 127.60150146484375, "step": 559 }, { "KL/chosen_KL_mean": -166.9397430419922, "KL/mean": -204.06809997558594, "KL/rejected_KL_mean": -241.1964569091797, "KL/std": 102.28352355957031, "epoch": 0.8465608465608465, "fcm_dpo/beta": 0.006258917972445488, "fcm_dpo/delta": -0.01593797653913498, "fcm_dpo/margin": 74.25672912597656, "fcm_dpo/q_t": 0.39539068937301636, "grad_norm": 17.991178512573242, "learning_rate": 3.550414669125573e-08, "logits/chosen": 0.7269060611724854, "logits/rejected": 0.6896910667419434, "logps/chosen": -227.58096313476562, "logps/ref_chosen": -60.64122009277344, "logps/ref_rejected": -78.75474548339844, "logps/rejected": -319.9512023925781, "loss": 1.064, "margin_dpo/margin_mean": 74.25672912597656, "margin_dpo/margin_std": 97.35572052001953, "step": 560 }, { "KL/chosen_KL_mean": -152.1344757080078, "KL/mean": -186.82179260253906, "KL/rejected_KL_mean": -221.50912475585938, "KL/std": 100.64840698242188, "epoch": 0.8480725623582767, "fcm_dpo/beta": 0.006227419711649418, "fcm_dpo/delta": 0.01874781772494316, "fcm_dpo/margin": 69.3746566772461, "fcm_dpo/q_t": 0.4056755304336548, "grad_norm": 14.884553909301758, "learning_rate": 3.482795573879241e-08, "logits/chosen": 0.6993428468704224, "logits/rejected": 0.6668655872344971, "logps/chosen": -214.63307189941406, "logps/ref_chosen": -62.49859619140625, "logps/ref_rejected": -78.72064208984375, "logps/rejected": -300.2297668457031, "loss": 1.1103, "margin_dpo/margin_mean": 69.37466430664062, "margin_dpo/margin_std": 110.47499084472656, "step": 561 }, { "KL/chosen_KL_mean": -162.52401733398438, "KL/mean": -207.02621459960938, "KL/rejected_KL_mean": -251.52841186523438, "KL/std": 116.5802001953125, "epoch": 0.8495842781557067, "fcm_dpo/beta": 0.006109355017542839, "fcm_dpo/delta": -0.10087047517299652, "fcm_dpo/margin": 89.00437927246094, "fcm_dpo/q_t": 0.3810436725616455, "grad_norm": 16.537874221801758, "learning_rate": 3.415778361095226e-08, "logits/chosen": 0.7176022529602051, "logits/rejected": 0.6851276159286499, "logps/chosen": -237.30575561523438, "logps/ref_chosen": -74.78173828125, "logps/ref_rejected": -92.63499450683594, "logps/rejected": -344.16339111328125, "loss": 1.0226, "margin_dpo/margin_mean": 89.00437927246094, "margin_dpo/margin_std": 112.19898986816406, "step": 562 }, { "KL/chosen_KL_mean": -135.99513244628906, "KL/mean": -173.78982543945312, "KL/rejected_KL_mean": -211.58447265625, "KL/std": 90.8531494140625, "epoch": 0.8510959939531368, "fcm_dpo/beta": 0.00610891031101346, "fcm_dpo/delta": -0.012339213863015175, "fcm_dpo/margin": 75.58935546875, "fcm_dpo/q_t": 0.395784854888916, "grad_norm": 20.83038330078125, "learning_rate": 3.349364905389032e-08, "logits/chosen": 0.8373708128929138, "logits/rejected": 0.7844498157501221, "logps/chosen": -186.19363403320312, "logps/ref_chosen": -50.19850158691406, "logps/ref_rejected": -66.76687622070312, "logps/rejected": -278.35137939453125, "loss": 1.0922, "margin_dpo/margin_mean": 75.58935546875, "margin_dpo/margin_std": 115.21420288085938, "step": 563 }, { "KL/chosen_KL_mean": -137.42124938964844, "KL/mean": -185.23602294921875, "KL/rejected_KL_mean": -233.05078125, "KL/std": 98.76502990722656, "epoch": 0.8526077097505669, "fcm_dpo/beta": 0.005984083749353886, "fcm_dpo/delta": -0.12979558110237122, "fcm_dpo/margin": 95.62950897216797, "fcm_dpo/q_t": 0.37068864703178406, "grad_norm": 12.641019821166992, "learning_rate": 3.283557064487785e-08, "logits/chosen": 0.7246955633163452, "logits/rejected": 0.6956311464309692, "logps/chosen": -193.162109375, "logps/ref_chosen": -55.7408447265625, "logps/ref_rejected": -74.82323455810547, "logps/rejected": -307.8740234375, "loss": 1.0071, "margin_dpo/margin_mean": 95.6295166015625, "margin_dpo/margin_std": 117.49134826660156, "step": 564 }, { "KL/chosen_KL_mean": -166.86520385742188, "KL/mean": -201.83352661132812, "KL/rejected_KL_mean": -236.80184936523438, "KL/std": 102.82844543457031, "epoch": 0.854119425547997, "fcm_dpo/beta": 0.00600344967097044, "fcm_dpo/delta": 0.030905822291970253, "fcm_dpo/margin": 69.93665313720703, "fcm_dpo/q_t": 0.40452465415000916, "grad_norm": 14.577558517456055, "learning_rate": 3.218356679178252e-08, "logits/chosen": 0.7704143524169922, "logits/rejected": 0.7189521193504333, "logps/chosen": -225.20257568359375, "logps/ref_chosen": -58.33738327026367, "logps/ref_rejected": -78.31776428222656, "logps/rejected": -315.11962890625, "loss": 1.0977, "margin_dpo/margin_mean": 69.93665313720703, "margin_dpo/margin_std": 101.45330810546875, "step": 565 }, { "KL/chosen_KL_mean": -157.60694885253906, "KL/mean": -189.88328552246094, "KL/rejected_KL_mean": -222.15963745117188, "KL/std": 105.84686279296875, "epoch": 0.8556311413454271, "fcm_dpo/beta": 0.006068835034966469, "fcm_dpo/delta": 0.05984632670879364, "fcm_dpo/margin": 64.55268859863281, "fcm_dpo/q_t": 0.41507333517074585, "grad_norm": 14.73648452758789, "learning_rate": 3.1537655732553764e-08, "logits/chosen": 0.7199764847755432, "logits/rejected": 0.7104334831237793, "logps/chosen": -228.8306884765625, "logps/ref_chosen": -71.22373962402344, "logps/ref_rejected": -71.11601257324219, "logps/rejected": -293.275634765625, "loss": 1.1818, "margin_dpo/margin_mean": 64.55268096923828, "margin_dpo/margin_std": 132.97817993164062, "step": 566 }, { "KL/chosen_KL_mean": -150.5260009765625, "KL/mean": -190.51409912109375, "KL/rejected_KL_mean": -230.50216674804688, "KL/std": 99.58441925048828, "epoch": 0.8571428571428571, "fcm_dpo/beta": 0.005988474003970623, "fcm_dpo/delta": -0.03157001733779907, "fcm_dpo/margin": 79.97616577148438, "fcm_dpo/q_t": 0.39104723930358887, "grad_norm": 12.800858497619629, "learning_rate": 3.089785553471233e-08, "logits/chosen": 0.7463260889053345, "logits/rejected": 0.6602100133895874, "logps/chosen": -203.19528198242188, "logps/ref_chosen": -52.669273376464844, "logps/ref_rejected": -74.34785461425781, "logps/rejected": -304.85003662109375, "loss": 1.0607, "margin_dpo/margin_mean": 79.97615814208984, "margin_dpo/margin_std": 105.27865600585938, "step": 567 }, { "KL/chosen_KL_mean": -138.55270385742188, "KL/mean": -187.60003662109375, "KL/rejected_KL_mean": -236.64739990234375, "KL/std": 106.38749694824219, "epoch": 0.8586545729402872, "fcm_dpo/beta": 0.005939141847193241, "fcm_dpo/delta": -0.14087416231632233, "fcm_dpo/margin": 98.09469604492188, "fcm_dpo/q_t": 0.37110963463783264, "grad_norm": 15.389481544494629, "learning_rate": 3.026418409484513e-08, "logits/chosen": 0.7434817552566528, "logits/rejected": 0.66554856300354, "logps/chosen": -190.73069763183594, "logps/ref_chosen": -52.178001403808594, "logps/ref_rejected": -85.8277587890625, "logps/rejected": -322.47515869140625, "loss": 0.9842, "margin_dpo/margin_mean": 98.09469604492188, "margin_dpo/margin_std": 107.54985046386719, "step": 568 }, { "KL/chosen_KL_mean": -163.41534423828125, "KL/mean": -187.918212890625, "KL/rejected_KL_mean": -212.42108154296875, "KL/std": 106.85481262207031, "epoch": 0.8601662887377173, "fcm_dpo/beta": 0.005847351625561714, "fcm_dpo/delta": -0.004478311166167259, "fcm_dpo/margin": 49.0057258605957, "fcm_dpo/q_t": 0.43568170070648193, "grad_norm": 16.01313591003418, "learning_rate": 2.963665913810451e-08, "logits/chosen": 0.6617103219032288, "logits/rejected": 0.6303982734680176, "logps/chosen": -226.06460571289062, "logps/ref_chosen": -62.649261474609375, "logps/ref_rejected": -75.4298324584961, "logps/rejected": -287.85089111328125, "loss": 1.2172, "margin_dpo/margin_mean": 49.0057258605957, "margin_dpo/margin_std": 106.43392181396484, "step": 569 }, { "KL/chosen_KL_mean": -142.8104248046875, "KL/mean": -189.96177673339844, "KL/rejected_KL_mean": -237.1131134033203, "KL/std": 98.74018096923828, "epoch": 0.8616780045351474, "fcm_dpo/beta": 0.005793171934783459, "fcm_dpo/delta": -0.1019757091999054, "fcm_dpo/margin": 94.30270385742188, "fcm_dpo/q_t": 0.3777024745941162, "grad_norm": 13.21369457244873, "learning_rate": 2.9015298217712453e-08, "logits/chosen": 0.7050719261169434, "logits/rejected": 0.6235053539276123, "logps/chosen": -192.85223388671875, "logps/ref_chosen": -50.04179382324219, "logps/ref_rejected": -78.27146911621094, "logps/rejected": -315.38458251953125, "loss": 1.0105, "margin_dpo/margin_mean": 94.30270385742188, "margin_dpo/margin_std": 112.3070068359375, "step": 570 }, { "KL/chosen_KL_mean": -155.06976318359375, "KL/mean": -181.69549560546875, "KL/rejected_KL_mean": -208.3212432861328, "KL/std": 95.75420379638672, "epoch": 0.8631897203325775, "fcm_dpo/beta": 0.005875091999769211, "fcm_dpo/delta": 0.14064154028892517, "fcm_dpo/margin": 53.25146484375, "fcm_dpo/q_t": 0.4286976754665375, "grad_norm": 14.657941818237305, "learning_rate": 2.840011871446962e-08, "logits/chosen": 0.7359094023704529, "logits/rejected": 0.7131412625312805, "logps/chosen": -208.72657775878906, "logps/ref_chosen": -53.65681457519531, "logps/ref_rejected": -66.13298034667969, "logps/rejected": -274.4542236328125, "loss": 1.1985, "margin_dpo/margin_mean": 53.25146484375, "margin_dpo/margin_std": 108.87671661376953, "step": 571 }, { "KL/chosen_KL_mean": -156.80117797851562, "KL/mean": -187.74636840820312, "KL/rejected_KL_mean": -218.69155883789062, "KL/std": 101.79773712158203, "epoch": 0.8647014361300076, "fcm_dpo/beta": 0.005991543643176556, "fcm_dpo/delta": 0.08145475387573242, "fcm_dpo/margin": 61.89037322998047, "fcm_dpo/q_t": 0.4158562421798706, "grad_norm": 14.93289852142334, "learning_rate": 2.7791137836269158e-08, "logits/chosen": 0.7083245515823364, "logits/rejected": 0.7496261596679688, "logps/chosen": -231.61911010742188, "logps/ref_chosen": -74.81792449951172, "logps/ref_rejected": -65.88681030273438, "logps/rejected": -284.578369140625, "loss": 1.1395, "margin_dpo/margin_mean": 61.89037322998047, "margin_dpo/margin_std": 102.08415222167969, "step": 572 }, { "KL/chosen_KL_mean": -169.649169921875, "KL/mean": -205.01141357421875, "KL/rejected_KL_mean": -240.37362670898438, "KL/std": 111.67261505126953, "epoch": 0.8662131519274376, "fcm_dpo/beta": 0.006012643687427044, "fcm_dpo/delta": 0.025858554989099503, "fcm_dpo/margin": 70.72447204589844, "fcm_dpo/q_t": 0.4091688096523285, "grad_norm": 14.856910705566406, "learning_rate": 2.718837261761528e-08, "logits/chosen": 0.7295148968696594, "logits/rejected": 0.6869486570358276, "logps/chosen": -238.37481689453125, "logps/ref_chosen": -68.72564697265625, "logps/ref_rejected": -88.16201782226562, "logps/rejected": -328.53564453125, "loss": 1.1504, "margin_dpo/margin_mean": 70.72447204589844, "margin_dpo/margin_std": 133.98379516601562, "step": 573 }, { "KL/chosen_KL_mean": -151.06890869140625, "KL/mean": -195.67068481445312, "KL/rejected_KL_mean": -240.2724609375, "KL/std": 102.99846649169922, "epoch": 0.8677248677248677, "fcm_dpo/beta": 0.005980664864182472, "fcm_dpo/delta": -0.08848445862531662, "fcm_dpo/margin": 89.20353698730469, "fcm_dpo/q_t": 0.3796440064907074, "grad_norm": 11.192665100097656, "learning_rate": 2.659183991914696e-08, "logits/chosen": 0.8058528900146484, "logits/rejected": 0.7434405088424683, "logps/chosen": -207.38229370117188, "logps/ref_chosen": -56.31340026855469, "logps/ref_rejected": -83.91553497314453, "logps/rejected": -324.18798828125, "loss": 1.0011, "margin_dpo/margin_mean": 89.20353698730469, "margin_dpo/margin_std": 96.93377685546875, "step": 574 }, { "KL/chosen_KL_mean": -156.55935668945312, "KL/mean": -188.05662536621094, "KL/rejected_KL_mean": -219.55389404296875, "KL/std": 103.39370727539062, "epoch": 0.8692365835222978, "fcm_dpo/beta": 0.005861110985279083, "fcm_dpo/delta": -0.04574337974190712, "fcm_dpo/margin": 62.99452590942383, "fcm_dpo/q_t": 0.42001163959503174, "grad_norm": 13.875605583190918, "learning_rate": 2.600155642716606e-08, "logits/chosen": 0.7368036508560181, "logits/rejected": 0.661503791809082, "logps/chosen": -221.14349365234375, "logps/ref_chosen": -64.5841293334961, "logps/ref_rejected": -93.47034454345703, "logps/rejected": -313.02423095703125, "loss": 1.1786, "margin_dpo/margin_mean": 62.99452590942383, "margin_dpo/margin_std": 123.26811981201172, "step": 575 }, { "KL/chosen_KL_mean": -138.5884246826172, "KL/mean": -185.07730102539062, "KL/rejected_KL_mean": -231.56613159179688, "KL/std": 105.1303939819336, "epoch": 0.8707482993197279, "fcm_dpo/beta": 0.005738668143749237, "fcm_dpo/delta": -0.0901963859796524, "fcm_dpo/margin": 92.97773742675781, "fcm_dpo/q_t": 0.3803936243057251, "grad_norm": 13.09487533569336, "learning_rate": 2.5417538653170754e-08, "logits/chosen": 0.7440035343170166, "logits/rejected": 0.6456351280212402, "logps/chosen": -191.86895751953125, "logps/ref_chosen": -53.28052520751953, "logps/ref_rejected": -84.2000503540039, "logps/rejected": -315.76617431640625, "loss": 1.0266, "margin_dpo/margin_mean": 92.97773742675781, "margin_dpo/margin_std": 115.10040283203125, "step": 576 }, { "KL/chosen_KL_mean": -154.52947998046875, "KL/mean": -185.0371856689453, "KL/rejected_KL_mean": -215.54489135742188, "KL/std": 100.77213287353516, "epoch": 0.872260015117158, "fcm_dpo/beta": 0.005833870265632868, "fcm_dpo/delta": 0.09760239720344543, "fcm_dpo/margin": 61.01540756225586, "fcm_dpo/q_t": 0.4187750220298767, "grad_norm": 13.304019927978516, "learning_rate": 2.4839802933393607e-08, "logits/chosen": 0.7207918763160706, "logits/rejected": 0.7140774130821228, "logps/chosen": -216.85415649414062, "logps/ref_chosen": -62.32468795776367, "logps/ref_rejected": -67.300537109375, "logps/rejected": -282.8454284667969, "loss": 1.1637, "margin_dpo/margin_mean": 61.01540756225586, "margin_dpo/margin_std": 109.60237884521484, "step": 577 }, { "KL/chosen_KL_mean": -146.52560424804688, "KL/mean": -176.1517333984375, "KL/rejected_KL_mean": -205.7778778076172, "KL/std": 105.66199493408203, "epoch": 0.873771730914588, "fcm_dpo/beta": 0.005955866537988186, "fcm_dpo/delta": 0.10069636255502701, "fcm_dpo/margin": 59.252296447753906, "fcm_dpo/q_t": 0.4234742522239685, "grad_norm": 13.605378150939941, "learning_rate": 2.4268365428344733e-08, "logits/chosen": 0.7855771780014038, "logits/rejected": 0.7710686922073364, "logps/chosen": -203.18118286132812, "logps/ref_chosen": -56.65557861328125, "logps/ref_rejected": -68.21835327148438, "logps/rejected": -273.9962158203125, "loss": 1.1829, "margin_dpo/margin_mean": 59.25229263305664, "margin_dpo/margin_std": 120.24205017089844, "step": 578 }, { "KL/chosen_KL_mean": -152.49461364746094, "KL/mean": -196.47821044921875, "KL/rejected_KL_mean": -240.4617919921875, "KL/std": 98.71546936035156, "epoch": 0.8752834467120182, "fcm_dpo/beta": 0.005913109518587589, "fcm_dpo/delta": -0.0742032453417778, "fcm_dpo/margin": 87.96717834472656, "fcm_dpo/q_t": 0.3802490234375, "grad_norm": 12.734554290771484, "learning_rate": 2.3703242122359357e-08, "logits/chosen": 0.6896715760231018, "logits/rejected": 0.6705402731895447, "logps/chosen": -209.3042755126953, "logps/ref_chosen": -56.809661865234375, "logps/ref_rejected": -68.09613037109375, "logps/rejected": -308.55792236328125, "loss": 1.0138, "margin_dpo/margin_mean": 87.96717834472656, "margin_dpo/margin_std": 100.14832305908203, "step": 579 }, { "KL/chosen_KL_mean": -154.41006469726562, "KL/mean": -191.92547607421875, "KL/rejected_KL_mean": -229.44088745117188, "KL/std": 115.38453674316406, "epoch": 0.8767951625094482, "fcm_dpo/beta": 0.005934194196015596, "fcm_dpo/delta": 0.004452776163816452, "fcm_dpo/margin": 75.03083801269531, "fcm_dpo/q_t": 0.4045197367668152, "grad_norm": 14.501690864562988, "learning_rate": 2.3144448823151392e-08, "logits/chosen": 0.7188555002212524, "logits/rejected": 0.6699697971343994, "logps/chosen": -212.11016845703125, "logps/ref_chosen": -57.70011520385742, "logps/ref_rejected": -77.90664672851562, "logps/rejected": -307.3475341796875, "loss": 1.121, "margin_dpo/margin_mean": 75.03083801269531, "margin_dpo/margin_std": 128.20059204101562, "step": 580 }, { "KL/chosen_KL_mean": -169.00424194335938, "KL/mean": -203.2427978515625, "KL/rejected_KL_mean": -237.48133850097656, "KL/std": 101.91188049316406, "epoch": 0.8783068783068783, "fcm_dpo/beta": 0.00592381926253438, "fcm_dpo/delta": 0.046096865087747574, "fcm_dpo/margin": 68.47708129882812, "fcm_dpo/q_t": 0.4113912880420685, "grad_norm": 14.175301551818848, "learning_rate": 2.259200116137039e-08, "logits/chosen": 0.7493467330932617, "logits/rejected": 0.6887227296829224, "logps/chosen": -228.33660888671875, "logps/ref_chosen": -59.332359313964844, "logps/ref_rejected": -83.64482116699219, "logps/rejected": -321.12615966796875, "loss": 1.1411, "margin_dpo/margin_mean": 68.47708129882812, "margin_dpo/margin_std": 121.07502746582031, "step": 581 }, { "KL/chosen_KL_mean": -154.54629516601562, "KL/mean": -187.54974365234375, "KL/rejected_KL_mean": -220.55316162109375, "KL/std": 97.96818542480469, "epoch": 0.8798185941043084, "fcm_dpo/beta": 0.006023784168064594, "fcm_dpo/delta": 0.05431080237030983, "fcm_dpo/margin": 66.00688934326172, "fcm_dpo/q_t": 0.41208571195602417, "grad_norm": 11.859220504760742, "learning_rate": 2.204591459016525e-08, "logits/chosen": 0.7633649706840515, "logits/rejected": 0.7924540042877197, "logps/chosen": -218.7091522216797, "logps/ref_chosen": -64.16285705566406, "logps/ref_rejected": -58.632896423339844, "logps/rejected": -279.1860656738281, "loss": 1.1323, "margin_dpo/margin_mean": 66.00688934326172, "margin_dpo/margin_std": 110.81221771240234, "step": 582 }, { "KL/chosen_KL_mean": -153.4376220703125, "KL/mean": -195.10647583007812, "KL/rejected_KL_mean": -236.77532958984375, "KL/std": 110.63685607910156, "epoch": 0.8813303099017384, "fcm_dpo/beta": 0.005971607752144337, "fcm_dpo/delta": -0.05028872191905975, "fcm_dpo/margin": 83.33770751953125, "fcm_dpo/q_t": 0.3901503384113312, "grad_norm": 15.806873321533203, "learning_rate": 2.1506204384751064e-08, "logits/chosen": 0.8192166090011597, "logits/rejected": 0.7225203514099121, "logps/chosen": -205.30999755859375, "logps/ref_chosen": -51.87239456176758, "logps/ref_rejected": -83.86331176757812, "logps/rejected": -320.6386413574219, "loss": 1.08, "margin_dpo/margin_mean": 83.33770751953125, "margin_dpo/margin_std": 126.09477233886719, "step": 583 }, { "KL/chosen_KL_mean": -140.74923706054688, "KL/mean": -176.01983642578125, "KL/rejected_KL_mean": -211.29043579101562, "KL/std": 99.2724609375, "epoch": 0.8828420256991686, "fcm_dpo/beta": 0.005971814971417189, "fcm_dpo/delta": 0.02986559458076954, "fcm_dpo/margin": 70.54119873046875, "fcm_dpo/q_t": 0.4095681309700012, "grad_norm": 15.029725074768066, "learning_rate": 2.09728856419826e-08, "logits/chosen": 0.889620304107666, "logits/rejected": 0.784419059753418, "logps/chosen": -187.32061767578125, "logps/ref_chosen": -46.571388244628906, "logps/ref_rejected": -80.67969512939453, "logps/rejected": -291.9701232910156, "loss": 1.1435, "margin_dpo/margin_mean": 70.54119873046875, "margin_dpo/margin_std": 128.3501434326172, "step": 584 }, { "KL/chosen_KL_mean": -163.51336669921875, "KL/mean": -190.11109924316406, "KL/rejected_KL_mean": -216.7088623046875, "KL/std": 110.62606811523438, "epoch": 0.8843537414965986, "fcm_dpo/beta": 0.006146551109850407, "fcm_dpo/delta": 0.12602292001247406, "fcm_dpo/margin": 53.19551086425781, "fcm_dpo/q_t": 0.42664623260498047, "grad_norm": 13.625960350036621, "learning_rate": 2.044597327993153e-08, "logits/chosen": 0.7172322869300842, "logits/rejected": 0.6654783487319946, "logps/chosen": -221.6378936767578, "logps/ref_chosen": -58.124534606933594, "logps/ref_rejected": -79.00538635253906, "logps/rejected": -295.7142333984375, "loss": 1.1917, "margin_dpo/margin_mean": 53.19550323486328, "margin_dpo/margin_std": 108.43853759765625, "step": 585 }, { "KL/chosen_KL_mean": -150.9927978515625, "KL/mean": -185.2848663330078, "KL/rejected_KL_mean": -219.57691955566406, "KL/std": 92.50978088378906, "epoch": 0.8858654572940288, "fcm_dpo/beta": 0.006165110971778631, "fcm_dpo/delta": 0.02831181511282921, "fcm_dpo/margin": 68.5841064453125, "fcm_dpo/q_t": 0.40376946330070496, "grad_norm": 13.801816940307617, "learning_rate": 1.9925482037469187e-08, "logits/chosen": 0.7588971853256226, "logits/rejected": 0.7175912857055664, "logps/chosen": -205.09445190429688, "logps/ref_chosen": -54.10163879394531, "logps/ref_rejected": -63.72113037109375, "logps/rejected": -283.29803466796875, "loss": 1.0728, "margin_dpo/margin_mean": 68.58411407470703, "margin_dpo/margin_std": 85.91049194335938, "step": 586 }, { "KL/chosen_KL_mean": -158.9512481689453, "KL/mean": -192.81903076171875, "KL/rejected_KL_mean": -226.68679809570312, "KL/std": 108.18991088867188, "epoch": 0.8873771730914588, "fcm_dpo/beta": 0.006208654493093491, "fcm_dpo/delta": 0.030746201053261757, "fcm_dpo/margin": 67.73554229736328, "fcm_dpo/q_t": 0.4009004235267639, "grad_norm": 16.25068473815918, "learning_rate": 1.9411426473854687e-08, "logits/chosen": 0.7532941102981567, "logits/rejected": 0.7503918409347534, "logps/chosen": -222.36843872070312, "logps/ref_chosen": -63.41719436645508, "logps/ref_rejected": -63.47003936767578, "logps/rejected": -290.1568603515625, "loss": 1.1813, "margin_dpo/margin_mean": 67.73554229736328, "margin_dpo/margin_std": 144.77902221679688, "step": 587 }, { "KL/chosen_KL_mean": -161.81015014648438, "KL/mean": -199.49249267578125, "KL/rejected_KL_mean": -237.1748504638672, "KL/std": 107.10841369628906, "epoch": 0.8888888888888888, "fcm_dpo/beta": 0.006185667589306831, "fcm_dpo/delta": -0.017407085746526718, "fcm_dpo/margin": 75.36467742919922, "fcm_dpo/q_t": 0.3991840183734894, "grad_norm": 17.871604919433594, "learning_rate": 1.890382096832699e-08, "logits/chosen": 0.8103761672973633, "logits/rejected": 0.7680736780166626, "logps/chosen": -224.01119995117188, "logps/ref_chosen": -62.20103454589844, "logps/ref_rejected": -82.10249328613281, "logps/rejected": -319.27734375, "loss": 1.1208, "margin_dpo/margin_mean": 75.36467742919922, "margin_dpo/margin_std": 130.9927978515625, "step": 588 }, { "KL/chosen_KL_mean": -151.50381469726562, "KL/mean": -190.66334533691406, "KL/rejected_KL_mean": -229.8228759765625, "KL/std": 103.41250610351562, "epoch": 0.890400604686319, "fcm_dpo/beta": 0.006190561689436436, "fcm_dpo/delta": -0.03668287768959999, "fcm_dpo/margin": 78.31907653808594, "fcm_dpo/q_t": 0.3893091678619385, "grad_norm": 12.957806587219238, "learning_rate": 1.840267971970344e-08, "logits/chosen": 0.7153310775756836, "logits/rejected": 0.686347246170044, "logps/chosen": -208.21742248535156, "logps/ref_chosen": -56.71361541748047, "logps/ref_rejected": -76.7366943359375, "logps/rejected": -306.5595703125, "loss": 1.0425, "margin_dpo/margin_mean": 78.31907653808594, "margin_dpo/margin_std": 96.69180297851562, "step": 589 }, { "KL/chosen_KL_mean": -165.386474609375, "KL/mean": -201.8364715576172, "KL/rejected_KL_mean": -238.28648376464844, "KL/std": 94.99177551269531, "epoch": 0.891912320483749, "fcm_dpo/beta": 0.006181138567626476, "fcm_dpo/delta": -0.0008702166378498077, "fcm_dpo/margin": 72.9000015258789, "fcm_dpo/q_t": 0.3985363841056824, "grad_norm": 16.859054565429688, "learning_rate": 1.7908016745981856e-08, "logits/chosen": 0.6825237274169922, "logits/rejected": 0.6479649543762207, "logps/chosen": -231.90029907226562, "logps/ref_chosen": -66.5138168334961, "logps/ref_rejected": -85.70820617675781, "logps/rejected": -323.99468994140625, "loss": 1.0834, "margin_dpo/margin_mean": 72.9000015258789, "margin_dpo/margin_std": 103.98199462890625, "step": 590 }, { "KL/chosen_KL_mean": -139.999755859375, "KL/mean": -186.31524658203125, "KL/rejected_KL_mean": -232.63076782226562, "KL/std": 111.80885314941406, "epoch": 0.8934240362811792, "fcm_dpo/beta": 0.006001078523695469, "fcm_dpo/delta": -0.11483590304851532, "fcm_dpo/margin": 92.63101196289062, "fcm_dpo/q_t": 0.3807540535926819, "grad_norm": 14.555268287658691, "learning_rate": 1.7419845883949098e-08, "logits/chosen": 0.8046550750732422, "logits/rejected": 0.7481690645217896, "logps/chosen": -200.69692993164062, "logps/ref_chosen": -60.697181701660156, "logps/ref_rejected": -86.12278747558594, "logps/rejected": -318.7535400390625, "loss": 1.0569, "margin_dpo/margin_mean": 92.63101196289062, "margin_dpo/margin_std": 135.70516967773438, "step": 591 }, { "KL/chosen_KL_mean": -153.17800903320312, "KL/mean": -188.54193115234375, "KL/rejected_KL_mean": -223.90585327148438, "KL/std": 96.40587615966797, "epoch": 0.8949357520786092, "fcm_dpo/beta": 0.006005411501973867, "fcm_dpo/delta": 0.02572956308722496, "fcm_dpo/margin": 70.72784423828125, "fcm_dpo/q_t": 0.4077094793319702, "grad_norm": 14.989432334899902, "learning_rate": 1.6938180788793556e-08, "logits/chosen": 0.8055961728096008, "logits/rejected": 0.6956392526626587, "logps/chosen": -204.4153289794922, "logps/ref_chosen": -51.237327575683594, "logps/ref_rejected": -81.60242462158203, "logps/rejected": -305.50830078125, "loss": 1.1064, "margin_dpo/margin_mean": 70.72784423828125, "margin_dpo/margin_std": 108.61327362060547, "step": 592 }, { "KL/chosen_KL_mean": -127.48636627197266, "KL/mean": -166.19288635253906, "KL/rejected_KL_mean": -204.8994140625, "KL/std": 98.92471313476562, "epoch": 0.8964474678760394, "fcm_dpo/beta": 0.006023376248776913, "fcm_dpo/delta": -0.017201118171215057, "fcm_dpo/margin": 77.41304016113281, "fcm_dpo/q_t": 0.3943125307559967, "grad_norm": 16.600597381591797, "learning_rate": 1.6463034933723336e-08, "logits/chosen": 0.7461457848548889, "logits/rejected": 0.6567406058311462, "logps/chosen": -169.56637573242188, "logps/ref_chosen": -42.08000183105469, "logps/ref_rejected": -68.47499084472656, "logps/rejected": -273.3743896484375, "loss": 1.0807, "margin_dpo/margin_mean": 77.41304016113281, "margin_dpo/margin_std": 112.75965881347656, "step": 593 }, { "KL/chosen_KL_mean": -161.6719970703125, "KL/mean": -194.70840454101562, "KL/rejected_KL_mean": -227.74481201171875, "KL/std": 98.35874938964844, "epoch": 0.8979591836734694, "fcm_dpo/beta": 0.006076568737626076, "fcm_dpo/delta": 0.05052534490823746, "fcm_dpo/margin": 66.07281494140625, "fcm_dpo/q_t": 0.40820130705833435, "grad_norm": 14.272443771362305, "learning_rate": 1.5994421609589385e-08, "logits/chosen": 0.6612966060638428, "logits/rejected": 0.6492611169815063, "logps/chosen": -225.33065795898438, "logps/ref_chosen": -63.658668518066406, "logps/ref_rejected": -70.35597229003906, "logps/rejected": -298.10076904296875, "loss": 1.1112, "margin_dpo/margin_mean": 66.07281494140625, "margin_dpo/margin_std": 99.67398071289062, "step": 594 }, { "KL/chosen_KL_mean": -149.42576599121094, "KL/mean": -192.97630310058594, "KL/rejected_KL_mean": -236.52682495117188, "KL/std": 105.02578735351562, "epoch": 0.8994708994708994, "fcm_dpo/beta": 0.006041559390723705, "fcm_dpo/delta": -0.08044502139091492, "fcm_dpo/margin": 87.10105895996094, "fcm_dpo/q_t": 0.38504302501678467, "grad_norm": 11.805750846862793, "learning_rate": 1.553235392451377e-08, "logits/chosen": 0.8043855428695679, "logits/rejected": 0.717462956905365, "logps/chosen": -205.64453125, "logps/ref_chosen": -56.21875762939453, "logps/ref_rejected": -83.95773315429688, "logps/rejected": -320.48455810546875, "loss": 1.0636, "margin_dpo/margin_mean": 87.10105895996094, "margin_dpo/margin_std": 128.87863159179688, "step": 595 }, { "KL/chosen_KL_mean": -173.06805419921875, "KL/mean": -190.773193359375, "KL/rejected_KL_mean": -208.47833251953125, "KL/std": 95.94255065917969, "epoch": 0.9009826152683296, "fcm_dpo/beta": 0.0060940892435610294, "fcm_dpo/delta": 0.08267121016979218, "fcm_dpo/margin": 35.410255432128906, "fcm_dpo/q_t": 0.45276933908462524, "grad_norm": 16.265701293945312, "learning_rate": 1.507684480352292e-08, "logits/chosen": 0.6956692934036255, "logits/rejected": 0.7181705236434937, "logps/chosen": -241.54891967773438, "logps/ref_chosen": -68.48088073730469, "logps/ref_rejected": -61.732967376708984, "logps/rejected": -270.2113037109375, "loss": 1.274, "margin_dpo/margin_mean": 35.410255432128906, "margin_dpo/margin_std": 101.649169921875, "step": 596 }, { "KL/chosen_KL_mean": -132.4046173095703, "KL/mean": -166.00335693359375, "KL/rejected_KL_mean": -199.6020965576172, "KL/std": 85.7354736328125, "epoch": 0.9024943310657596, "fcm_dpo/beta": 0.006142480298876762, "fcm_dpo/delta": 0.0386665053665638, "fcm_dpo/margin": 67.1974868774414, "fcm_dpo/q_t": 0.40726274251937866, "grad_norm": 11.388983726501465, "learning_rate": 1.4627906988186111e-08, "logits/chosen": 0.7288523316383362, "logits/rejected": 0.7139878273010254, "logps/chosen": -181.26211547851562, "logps/ref_chosen": -48.85750961303711, "logps/ref_rejected": -55.068084716796875, "logps/rejected": -254.67018127441406, "loss": 1.1035, "margin_dpo/margin_mean": 67.19747924804688, "margin_dpo/margin_std": 99.90011596679688, "step": 597 }, { "KL/chosen_KL_mean": -173.84255981445312, "KL/mean": -195.55624389648438, "KL/rejected_KL_mean": -217.26992797851562, "KL/std": 101.8652572631836, "epoch": 0.9040060468631897, "fcm_dpo/beta": 0.00618097186088562, "fcm_dpo/delta": 0.06402421742677689, "fcm_dpo/margin": 43.42737579345703, "fcm_dpo/q_t": 0.4409489929676056, "grad_norm": 14.127278327941895, "learning_rate": 1.4185553036259095e-08, "logits/chosen": 0.740046501159668, "logits/rejected": 0.6648821234703064, "logps/chosen": -232.72970581054688, "logps/ref_chosen": -58.88715362548828, "logps/ref_rejected": -81.43145751953125, "logps/rejected": -298.7013854980469, "loss": 1.2401, "margin_dpo/margin_mean": 43.42737579345703, "margin_dpo/margin_std": 107.77723693847656, "step": 598 }, { "KL/chosen_KL_mean": -173.7288055419922, "KL/mean": -199.68463134765625, "KL/rejected_KL_mean": -225.64044189453125, "KL/std": 97.53303527832031, "epoch": 0.9055177626606198, "fcm_dpo/beta": 0.006358982063829899, "fcm_dpo/delta": 0.1232747808098793, "fcm_dpo/margin": 51.91165542602539, "fcm_dpo/q_t": 0.42809587717056274, "grad_norm": 16.50663185119629, "learning_rate": 1.3749795321332885e-08, "logits/chosen": 0.79831862449646, "logits/rejected": 0.7588490843772888, "logps/chosen": -231.33599853515625, "logps/ref_chosen": -57.60719299316406, "logps/ref_rejected": -71.80469512939453, "logps/rejected": -297.44512939453125, "loss": 1.2006, "margin_dpo/margin_mean": 51.911651611328125, "margin_dpo/margin_std": 111.52732849121094, "step": 599 }, { "KL/chosen_KL_mean": -158.45321655273438, "KL/mean": -189.308837890625, "KL/rejected_KL_mean": -220.16444396972656, "KL/std": 106.662109375, "epoch": 0.9070294784580499, "fcm_dpo/beta": 0.00634386483579874, "fcm_dpo/delta": -0.05036870762705803, "fcm_dpo/margin": 61.711246490478516, "fcm_dpo/q_t": 0.41562899947166443, "grad_norm": 18.35607147216797, "learning_rate": 1.3320646032487393e-08, "logits/chosen": 0.7958853244781494, "logits/rejected": 0.7406545877456665, "logps/chosen": -216.89552307128906, "logps/ref_chosen": -58.44231414794922, "logps/ref_rejected": -83.64639282226562, "logps/rejected": -303.81085205078125, "loss": 1.1548, "margin_dpo/margin_mean": 61.71124267578125, "margin_dpo/margin_std": 113.1817398071289, "step": 600 }, { "KL/chosen_KL_mean": -142.54322814941406, "KL/mean": -184.44918823242188, "KL/rejected_KL_mean": -226.35516357421875, "KL/std": 111.73482513427734, "epoch": 0.90854119425548, "fcm_dpo/beta": 0.006213693879544735, "fcm_dpo/delta": -0.07541825622320175, "fcm_dpo/margin": 83.81192016601562, "fcm_dpo/q_t": 0.3867299258708954, "grad_norm": 12.669417381286621, "learning_rate": 1.2898117173950868e-08, "logits/chosen": 0.7443298101425171, "logits/rejected": 0.669115424156189, "logps/chosen": -198.13755798339844, "logps/ref_chosen": -55.59432601928711, "logps/ref_rejected": -83.68630981445312, "logps/rejected": -310.04150390625, "loss": 1.072, "margin_dpo/margin_mean": 83.81192779541016, "margin_dpo/margin_std": 126.72483825683594, "step": 601 }, { "KL/chosen_KL_mean": -132.24453735351562, "KL/mean": -170.80950927734375, "KL/rejected_KL_mean": -209.37448120117188, "KL/std": 96.84636688232422, "epoch": 0.91005291005291, "fcm_dpo/beta": 0.0062125981785357, "fcm_dpo/delta": -0.030824126675724983, "fcm_dpo/margin": 77.12992858886719, "fcm_dpo/q_t": 0.39198559522628784, "grad_norm": 14.065460205078125, "learning_rate": 1.2482220564763667e-08, "logits/chosen": 0.780281126499176, "logits/rejected": 0.7512023448944092, "logps/chosen": -188.59371948242188, "logps/ref_chosen": -56.349185943603516, "logps/ref_rejected": -71.9959716796875, "logps/rejected": -281.37042236328125, "loss": 1.0584, "margin_dpo/margin_mean": 77.12992858886719, "margin_dpo/margin_std": 103.31945037841797, "step": 602 }, { "KL/chosen_KL_mean": -146.39669799804688, "KL/mean": -184.16403198242188, "KL/rejected_KL_mean": -221.93136596679688, "KL/std": 96.0953140258789, "epoch": 0.9115646258503401, "fcm_dpo/beta": 0.006151379551738501, "fcm_dpo/delta": -0.015490264631807804, "fcm_dpo/margin": 75.53468322753906, "fcm_dpo/q_t": 0.3961607813835144, "grad_norm": 13.6494779586792, "learning_rate": 1.2072967838448051e-08, "logits/chosen": 0.7146183252334595, "logits/rejected": 0.6660758256912231, "logps/chosen": -199.56509399414062, "logps/ref_chosen": -53.16838836669922, "logps/ref_rejected": -73.8604736328125, "logps/rejected": -295.7918701171875, "loss": 1.0812, "margin_dpo/margin_mean": 75.53468322753906, "margin_dpo/margin_std": 110.68234252929688, "step": 603 }, { "KL/chosen_KL_mean": -142.59632873535156, "KL/mean": -174.91241455078125, "KL/rejected_KL_mean": -207.22850036621094, "KL/std": 92.63813781738281, "epoch": 0.9130763416477702, "fcm_dpo/beta": 0.00621896144002676, "fcm_dpo/delta": 0.04987533390522003, "fcm_dpo/margin": 64.63217163085938, "fcm_dpo/q_t": 0.41217827796936035, "grad_norm": 16.16542625427246, "learning_rate": 1.1670370442682459e-08, "logits/chosen": 0.7201324701309204, "logits/rejected": 0.7276899814605713, "logps/chosen": -215.24575805664062, "logps/ref_chosen": -72.64942169189453, "logps/ref_rejected": -69.8792724609375, "logps/rejected": -277.1077880859375, "loss": 1.153, "margin_dpo/margin_mean": 64.63217163085938, "margin_dpo/margin_std": 120.90769958496094, "step": 604 }, { "KL/chosen_KL_mean": -160.7579345703125, "KL/mean": -195.69827270507812, "KL/rejected_KL_mean": -230.63864135742188, "KL/std": 96.85035705566406, "epoch": 0.9145880574452003, "fcm_dpo/beta": 0.006217213813215494, "fcm_dpo/delta": 0.016090279445052147, "fcm_dpo/margin": 69.88069915771484, "fcm_dpo/q_t": 0.40307921171188354, "grad_norm": 15.524698257446289, "learning_rate": 1.1274439638981532e-08, "logits/chosen": 0.7722749710083008, "logits/rejected": 0.7222751975059509, "logps/chosen": -222.3707733154297, "logps/ref_chosen": -61.61284637451172, "logps/ref_rejected": -79.34398651123047, "logps/rejected": -309.98260498046875, "loss": 1.1192, "margin_dpo/margin_mean": 69.88069915771484, "margin_dpo/margin_std": 115.388671875, "step": 605 }, { "KL/chosen_KL_mean": -142.37429809570312, "KL/mean": -184.49478149414062, "KL/rejected_KL_mean": -226.61526489257812, "KL/std": 100.42843627929688, "epoch": 0.9160997732426304, "fcm_dpo/beta": 0.006196511909365654, "fcm_dpo/delta": -0.0759873166680336, "fcm_dpo/margin": 84.24095916748047, "fcm_dpo/q_t": 0.38542690873146057, "grad_norm": 15.91579818725586, "learning_rate": 1.0885186502381016e-08, "logits/chosen": 0.710547685623169, "logits/rejected": 0.648972749710083, "logps/chosen": -196.83853149414062, "logps/ref_chosen": -54.46424102783203, "logps/ref_rejected": -79.62708282470703, "logps/rejected": -306.2423400878906, "loss": 1.0475, "margin_dpo/margin_mean": 84.240966796875, "margin_dpo/margin_std": 115.52052307128906, "step": 606 }, { "KL/chosen_KL_mean": -156.99851989746094, "KL/mean": -193.15847778320312, "KL/rejected_KL_mean": -229.3184051513672, "KL/std": 97.73643493652344, "epoch": 0.9176114890400605, "fcm_dpo/beta": 0.006090829148888588, "fcm_dpo/delta": 0.007767722010612488, "fcm_dpo/margin": 72.31989288330078, "fcm_dpo/q_t": 0.40162938833236694, "grad_norm": 13.030501365661621, "learning_rate": 1.0502621921127774e-08, "logits/chosen": 0.7018817663192749, "logits/rejected": 0.6800510287284851, "logps/chosen": -219.85939025878906, "logps/ref_chosen": -62.86086654663086, "logps/ref_rejected": -72.5501937866211, "logps/rejected": -301.86859130859375, "loss": 1.1058, "margin_dpo/margin_mean": 72.31988525390625, "margin_dpo/margin_std": 109.04088592529297, "step": 607 }, { "KL/chosen_KL_mean": -160.4464111328125, "KL/mean": -199.41481018066406, "KL/rejected_KL_mean": -238.38323974609375, "KL/std": 107.04619598388672, "epoch": 0.9191232048374905, "fcm_dpo/beta": 0.006139816716313362, "fcm_dpo/delta": -0.029955772683024406, "fcm_dpo/margin": 77.93682098388672, "fcm_dpo/q_t": 0.39407122135162354, "grad_norm": 13.91819953918457, "learning_rate": 1.0126756596375685e-08, "logits/chosen": 0.7119251489639282, "logits/rejected": 0.6361026167869568, "logps/chosen": -223.62713623046875, "logps/ref_chosen": -63.18071746826172, "logps/ref_rejected": -99.15888214111328, "logps/rejected": -337.5421142578125, "loss": 1.0655, "margin_dpo/margin_mean": 77.93682098388672, "margin_dpo/margin_std": 108.65492248535156, "step": 608 }, { "KL/chosen_KL_mean": -142.321533203125, "KL/mean": -183.87535095214844, "KL/rejected_KL_mean": -225.42916870117188, "KL/std": 96.29520416259766, "epoch": 0.9206349206349206, "fcm_dpo/beta": 0.0060390448197722435, "fcm_dpo/delta": -0.05546504259109497, "fcm_dpo/margin": 83.10763549804688, "fcm_dpo/q_t": 0.3844478726387024, "grad_norm": 12.680366516113281, "learning_rate": 9.757601041885694e-09, "logits/chosen": 0.8203383684158325, "logits/rejected": 0.7830733060836792, "logps/chosen": -190.9447479248047, "logps/ref_chosen": -48.62322235107422, "logps/ref_rejected": -68.28271484375, "logps/rejected": -293.7118835449219, "loss": 1.0301, "margin_dpo/margin_mean": 83.10763549804688, "margin_dpo/margin_std": 95.72496795654297, "step": 609 }, { "KL/chosen_KL_mean": -155.60647583007812, "KL/mean": -194.73910522460938, "KL/rejected_KL_mean": -233.8717498779297, "KL/std": 105.99610900878906, "epoch": 0.9221466364323507, "fcm_dpo/beta": 0.006009879522025585, "fcm_dpo/delta": -0.021769745275378227, "fcm_dpo/margin": 78.26527404785156, "fcm_dpo/q_t": 0.39677220582962036, "grad_norm": 14.142727851867676, "learning_rate": 9.395165583732379e-09, "logits/chosen": 0.7283831238746643, "logits/rejected": 0.7220611572265625, "logps/chosen": -228.27159118652344, "logps/ref_chosen": -72.66513061523438, "logps/ref_rejected": -87.15310668945312, "logps/rejected": -321.02484130859375, "loss": 1.0927, "margin_dpo/margin_mean": 78.26527404785156, "margin_dpo/margin_std": 121.27104187011719, "step": 610 }, { "KL/chosen_KL_mean": -143.68881225585938, "KL/mean": -174.88534545898438, "KL/rejected_KL_mean": -206.0819091796875, "KL/std": 93.48977661132812, "epoch": 0.9236583522297808, "fcm_dpo/beta": 0.0060781436040997505, "fcm_dpo/delta": 0.07361207157373428, "fcm_dpo/margin": 62.39308166503906, "fcm_dpo/q_t": 0.4132322669029236, "grad_norm": 14.4437894821167, "learning_rate": 9.03946036001449e-09, "logits/chosen": 0.7655025124549866, "logits/rejected": 0.7187894582748413, "logps/chosen": -191.99737548828125, "logps/ref_chosen": -48.30857849121094, "logps/ref_rejected": -70.6141128540039, "logps/rejected": -276.6960144042969, "loss": 1.1224, "margin_dpo/margin_mean": 62.3930778503418, "margin_dpo/margin_std": 95.25655364990234, "step": 611 }, { "KL/chosen_KL_mean": -158.34677124023438, "KL/mean": -204.57286071777344, "KL/rejected_KL_mean": -250.7989501953125, "KL/std": 100.17593383789062, "epoch": 0.9251700680272109, "fcm_dpo/beta": 0.006016192026436329, "fcm_dpo/delta": -0.11244214326143265, "fcm_dpo/margin": 92.45218658447266, "fcm_dpo/q_t": 0.37426358461380005, "grad_norm": 12.516098022460938, "learning_rate": 8.690495320571839e-09, "logits/chosen": 0.6408207416534424, "logits/rejected": 0.5755380988121033, "logps/chosen": -219.5783233642578, "logps/ref_chosen": -61.23155975341797, "logps/ref_rejected": -94.37979888916016, "logps/rejected": -345.1787414550781, "loss": 1.0041, "margin_dpo/margin_mean": 92.45218658447266, "margin_dpo/margin_std": 109.68734741210938, "step": 612 }, { "KL/chosen_KL_mean": -136.57814025878906, "KL/mean": -178.27291870117188, "KL/rejected_KL_mean": -219.96771240234375, "KL/std": 107.65634155273438, "epoch": 0.926681783824641, "fcm_dpo/beta": 0.0059585003182291985, "fcm_dpo/delta": -0.04942707717418671, "fcm_dpo/margin": 83.38955688476562, "fcm_dpo/q_t": 0.38701191544532776, "grad_norm": 12.486554145812988, "learning_rate": 8.348280226706722e-09, "logits/chosen": 0.6936590075492859, "logits/rejected": 0.6893400549888611, "logps/chosen": -190.56124877929688, "logps/ref_chosen": -53.98310852050781, "logps/ref_rejected": -58.32208251953125, "logps/rejected": -278.289794921875, "loss": 1.0483, "margin_dpo/margin_mean": 83.38955688476562, "margin_dpo/margin_std": 108.76756286621094, "step": 613 }, { "KL/chosen_KL_mean": -156.5811004638672, "KL/mean": -194.799560546875, "KL/rejected_KL_mean": -233.0180206298828, "KL/std": 92.09423828125, "epoch": 0.9281934996220711, "fcm_dpo/beta": 0.0059523265808820724, "fcm_dpo/delta": -0.005938872694969177, "fcm_dpo/margin": 76.43690490722656, "fcm_dpo/q_t": 0.39591896533966064, "grad_norm": 15.995917320251465, "learning_rate": 8.012824650910937e-09, "logits/chosen": 0.7563154101371765, "logits/rejected": 0.7482579946517944, "logps/chosen": -216.8241424560547, "logps/ref_chosen": -60.24303436279297, "logps/ref_rejected": -72.26258850097656, "logps/rejected": -305.2806091308594, "loss": 1.0741, "margin_dpo/margin_mean": 76.43690490722656, "margin_dpo/margin_std": 103.75457763671875, "step": 614 }, { "KL/chosen_KL_mean": -163.4049072265625, "KL/mean": -206.7666473388672, "KL/rejected_KL_mean": -250.12840270996094, "KL/std": 120.01435852050781, "epoch": 0.9297052154195011, "fcm_dpo/beta": 0.005815756507217884, "fcm_dpo/delta": -0.05904611572623253, "fcm_dpo/margin": 86.7235107421875, "fcm_dpo/q_t": 0.38891440629959106, "grad_norm": 14.163253784179688, "learning_rate": 7.684137976598088e-09, "logits/chosen": 0.6970840692520142, "logits/rejected": 0.6417770385742188, "logps/chosen": -235.49957275390625, "logps/ref_chosen": -72.09467315673828, "logps/ref_rejected": -104.02980041503906, "logps/rejected": -354.158203125, "loss": 1.0777, "margin_dpo/margin_mean": 86.7235107421875, "margin_dpo/margin_std": 130.05892944335938, "step": 615 }, { "KL/chosen_KL_mean": -156.443603515625, "KL/mean": -189.8626251220703, "KL/rejected_KL_mean": -223.28164672851562, "KL/std": 110.390869140625, "epoch": 0.9312169312169312, "fcm_dpo/beta": 0.005887066014111042, "fcm_dpo/delta": 0.058877281844615936, "fcm_dpo/margin": 66.8380126953125, "fcm_dpo/q_t": 0.41178950667381287, "grad_norm": 13.02852725982666, "learning_rate": 7.36222939784098e-09, "logits/chosen": 0.760650634765625, "logits/rejected": 0.6907193660736084, "logps/chosen": -214.97433471679688, "logps/ref_chosen": -58.530723571777344, "logps/ref_rejected": -75.48025512695312, "logps/rejected": -298.76190185546875, "loss": 1.1227, "margin_dpo/margin_mean": 66.83802032470703, "margin_dpo/margin_std": 106.22430419921875, "step": 616 }, { "KL/chosen_KL_mean": -164.33099365234375, "KL/mean": -196.11001586914062, "KL/rejected_KL_mean": -227.88905334472656, "KL/std": 100.60113525390625, "epoch": 0.9327286470143613, "fcm_dpo/beta": 0.005934374872595072, "fcm_dpo/delta": 0.07501719892024994, "fcm_dpo/margin": 63.55805969238281, "fcm_dpo/q_t": 0.4166525602340698, "grad_norm": 16.678241729736328, "learning_rate": 7.047107919114586e-09, "logits/chosen": 0.7422701120376587, "logits/rejected": 0.6935194730758667, "logps/chosen": -221.93966674804688, "logps/ref_chosen": -57.608673095703125, "logps/ref_rejected": -81.22109985351562, "logps/rejected": -309.11016845703125, "loss": 1.1452, "margin_dpo/margin_mean": 63.55805969238281, "margin_dpo/margin_std": 107.30416870117188, "step": 617 }, { "KL/chosen_KL_mean": -151.17364501953125, "KL/mean": -188.66285705566406, "KL/rejected_KL_mean": -226.15208435058594, "KL/std": 111.13240051269531, "epoch": 0.9342403628117913, "fcm_dpo/beta": 0.005985327996313572, "fcm_dpo/delta": 0.0010824277997016907, "fcm_dpo/margin": 74.97844696044922, "fcm_dpo/q_t": 0.4020419120788574, "grad_norm": 15.610330581665039, "learning_rate": 6.738782355044048e-09, "logits/chosen": 0.7219952940940857, "logits/rejected": 0.6295669674873352, "logps/chosen": -207.8695831298828, "logps/ref_chosen": -56.69594192504883, "logps/ref_rejected": -85.92362976074219, "logps/rejected": -312.0757141113281, "loss": 1.085, "margin_dpo/margin_mean": 74.97845458984375, "margin_dpo/margin_std": 110.57818603515625, "step": 618 }, { "KL/chosen_KL_mean": -145.59751892089844, "KL/mean": -185.35507202148438, "KL/rejected_KL_mean": -225.11264038085938, "KL/std": 103.94949340820312, "epoch": 0.9357520786092215, "fcm_dpo/beta": 0.005967982113361359, "fcm_dpo/delta": -0.025933532044291496, "fcm_dpo/margin": 79.51509094238281, "fcm_dpo/q_t": 0.39350175857543945, "grad_norm": 13.841119766235352, "learning_rate": 6.437261330158206e-09, "logits/chosen": 0.8068792223930359, "logits/rejected": 0.7342312335968018, "logps/chosen": -199.65594482421875, "logps/ref_chosen": -54.05841827392578, "logps/ref_rejected": -83.55493927001953, "logps/rejected": -308.6675720214844, "loss": 1.0728, "margin_dpo/margin_mean": 79.51509857177734, "margin_dpo/margin_std": 114.1060562133789, "step": 619 }, { "KL/chosen_KL_mean": -148.67767333984375, "KL/mean": -179.8364715576172, "KL/rejected_KL_mean": -210.99526977539062, "KL/std": 95.24564361572266, "epoch": 0.9372637944066515, "fcm_dpo/beta": 0.005929121747612953, "fcm_dpo/delta": -0.07515688985586166, "fcm_dpo/margin": 62.317569732666016, "fcm_dpo/q_t": 0.41601672768592834, "grad_norm": 14.274309158325195, "learning_rate": 6.142553278648238e-09, "logits/chosen": 0.7464447021484375, "logits/rejected": 0.7477549314498901, "logps/chosen": -212.04739379882812, "logps/ref_chosen": -63.36971664428711, "logps/ref_rejected": -65.68269348144531, "logps/rejected": -276.6779479980469, "loss": 1.1448, "margin_dpo/margin_mean": 62.31756591796875, "margin_dpo/margin_std": 99.23009490966797, "step": 620 }, { "KL/chosen_KL_mean": -147.39321899414062, "KL/mean": -183.0356903076172, "KL/rejected_KL_mean": -218.67816162109375, "KL/std": 103.13090515136719, "epoch": 0.9387755102040817, "fcm_dpo/beta": 0.005933044478297234, "fcm_dpo/delta": 0.027808479964733124, "fcm_dpo/margin": 71.2849349975586, "fcm_dpo/q_t": 0.4077685475349426, "grad_norm": 13.713377952575684, "learning_rate": 5.854666444131934e-09, "logits/chosen": 0.764801025390625, "logits/rejected": 0.6597084999084473, "logps/chosen": -199.71444702148438, "logps/ref_chosen": -52.321224212646484, "logps/ref_rejected": -88.09001159667969, "logps/rejected": -306.7681884765625, "loss": 1.1263, "margin_dpo/margin_mean": 71.2849349975586, "margin_dpo/margin_std": 120.7091293334961, "step": 621 }, { "KL/chosen_KL_mean": -162.29052734375, "KL/mean": -197.90365600585938, "KL/rejected_KL_mean": -233.51678466796875, "KL/std": 108.23118591308594, "epoch": 0.9402872260015117, "fcm_dpo/beta": 0.005962342023849487, "fcm_dpo/delta": 0.026074469089508057, "fcm_dpo/margin": 71.22624206542969, "fcm_dpo/q_t": 0.4056706428527832, "grad_norm": 16.656646728515625, "learning_rate": 5.573608879422875e-09, "logits/chosen": 0.6871777772903442, "logits/rejected": 0.6430974006652832, "logps/chosen": -222.15599060058594, "logps/ref_chosen": -59.86545944213867, "logps/ref_rejected": -81.86668395996094, "logps/rejected": -315.38348388671875, "loss": 1.1095, "margin_dpo/margin_mean": 71.22624969482422, "margin_dpo/margin_std": 111.18820190429688, "step": 622 }, { "KL/chosen_KL_mean": -171.44595336914062, "KL/mean": -209.40542602539062, "KL/rejected_KL_mean": -247.36492919921875, "KL/std": 113.62255096435547, "epoch": 0.9417989417989417, "fcm_dpo/beta": 0.005917124915868044, "fcm_dpo/delta": 0.00023915618658065796, "fcm_dpo/margin": 75.9189682006836, "fcm_dpo/q_t": 0.39986979961395264, "grad_norm": 14.932882308959961, "learning_rate": 5.299388446305342e-09, "logits/chosen": 0.7615326046943665, "logits/rejected": 0.7113196849822998, "logps/chosen": -238.8144073486328, "logps/ref_chosen": -67.36846160888672, "logps/ref_rejected": -82.02733612060547, "logps/rejected": -329.39227294921875, "loss": 1.0966, "margin_dpo/margin_mean": 75.9189682006836, "margin_dpo/margin_std": 115.97571563720703, "step": 623 }, { "KL/chosen_KL_mean": -144.95953369140625, "KL/mean": -184.875, "KL/rejected_KL_mean": -224.79046630859375, "KL/std": 106.45075988769531, "epoch": 0.9433106575963719, "fcm_dpo/beta": 0.00591125525534153, "fcm_dpo/delta": -0.023217620328068733, "fcm_dpo/margin": 79.8309326171875, "fcm_dpo/q_t": 0.3964461088180542, "grad_norm": 14.505139350891113, "learning_rate": 5.03201281531429e-09, "logits/chosen": 0.7227747440338135, "logits/rejected": 0.6352113485336304, "logps/chosen": -195.986083984375, "logps/ref_chosen": -51.02655029296875, "logps/ref_rejected": -76.49203491210938, "logps/rejected": -301.282470703125, "loss": 1.0784, "margin_dpo/margin_mean": 79.8309326171875, "margin_dpo/margin_std": 116.97268676757812, "step": 624 }, { "KL/chosen_KL_mean": -147.682861328125, "KL/mean": -178.62127685546875, "KL/rejected_KL_mean": -209.5596923828125, "KL/std": 101.56578063964844, "epoch": 0.9448223733938019, "fcm_dpo/beta": 0.005996185354888439, "fcm_dpo/delta": 0.08198390901088715, "fcm_dpo/margin": 61.876800537109375, "fcm_dpo/q_t": 0.42064088582992554, "grad_norm": 13.062463760375977, "learning_rate": 4.7714894655209174e-09, "logits/chosen": 0.8105379343032837, "logits/rejected": 0.7237575054168701, "logps/chosen": -201.89048767089844, "logps/ref_chosen": -54.20761489868164, "logps/ref_rejected": -84.93669128417969, "logps/rejected": -294.49639892578125, "loss": 1.1721, "margin_dpo/margin_mean": 61.87679672241211, "margin_dpo/margin_std": 122.6982421875, "step": 625 }, { "KL/chosen_KL_mean": -138.61166381835938, "KL/mean": -181.34934997558594, "KL/rejected_KL_mean": -224.0870361328125, "KL/std": 108.27032470703125, "epoch": 0.9463340891912321, "fcm_dpo/beta": 0.005954947788268328, "fcm_dpo/delta": -0.06229614093899727, "fcm_dpo/margin": 85.47537231445312, "fcm_dpo/q_t": 0.391404390335083, "grad_norm": 14.3599271774292, "learning_rate": 4.517825684323323e-09, "logits/chosen": 0.8098399639129639, "logits/rejected": 0.6851488351821899, "logps/chosen": -183.67367553710938, "logps/ref_chosen": -45.06201934814453, "logps/ref_rejected": -89.66368103027344, "logps/rejected": -313.750732421875, "loss": 1.0736, "margin_dpo/margin_mean": 85.47537231445312, "margin_dpo/margin_std": 131.77513122558594, "step": 626 }, { "KL/chosen_KL_mean": -158.8922119140625, "KL/mean": -203.9259033203125, "KL/rejected_KL_mean": -248.9595947265625, "KL/std": 111.73031616210938, "epoch": 0.9478458049886621, "fcm_dpo/beta": 0.0058575589209795, "fcm_dpo/delta": -0.08222895115613937, "fcm_dpo/margin": 90.06739044189453, "fcm_dpo/q_t": 0.3802756369113922, "grad_norm": 14.234342575073242, "learning_rate": 4.271028567242818e-09, "logits/chosen": 0.6816971898078918, "logits/rejected": 0.5688222646713257, "logps/chosen": -217.68325805664062, "logps/ref_chosen": -58.791053771972656, "logps/ref_rejected": -94.90802001953125, "logps/rejected": -343.86761474609375, "loss": 1.0285, "margin_dpo/margin_mean": 90.06739044189453, "margin_dpo/margin_std": 113.25086975097656, "step": 627 }, { "KL/chosen_KL_mean": -146.272216796875, "KL/mean": -189.48312377929688, "KL/rejected_KL_mean": -232.6940460205078, "KL/std": 106.63853454589844, "epoch": 0.9493575207860923, "fcm_dpo/beta": 0.005843169055879116, "fcm_dpo/delta": -0.0587296336889267, "fcm_dpo/margin": 86.42182159423828, "fcm_dpo/q_t": 0.38549649715423584, "grad_norm": 14.816391944885254, "learning_rate": 4.0311050177251895e-09, "logits/chosen": 0.7511180639266968, "logits/rejected": 0.7128807306289673, "logps/chosen": -199.0758056640625, "logps/ref_chosen": -52.80357360839844, "logps/ref_rejected": -76.49468994140625, "logps/rejected": -309.188720703125, "loss": 1.0765, "margin_dpo/margin_mean": 86.42181396484375, "margin_dpo/margin_std": 121.38540649414062, "step": 628 }, { "KL/chosen_KL_mean": -155.6796112060547, "KL/mean": -187.944091796875, "KL/rejected_KL_mean": -220.2086181640625, "KL/std": 97.90316772460938, "epoch": 0.9508692365835223, "fcm_dpo/beta": 0.00585499033331871, "fcm_dpo/delta": 0.07480396330356598, "fcm_dpo/margin": 64.52902221679688, "fcm_dpo/q_t": 0.4162479639053345, "grad_norm": 13.809697151184082, "learning_rate": 3.798061746947995e-09, "logits/chosen": 0.7253426313400269, "logits/rejected": 0.7244319915771484, "logps/chosen": -226.3970947265625, "logps/ref_chosen": -70.71749877929688, "logps/ref_rejected": -78.96273803710938, "logps/rejected": -299.1713562011719, "loss": 1.1289, "margin_dpo/margin_mean": 64.52902221679688, "margin_dpo/margin_std": 103.2763671875, "step": 629 }, { "KL/chosen_KL_mean": -145.9853515625, "KL/mean": -188.92074584960938, "KL/rejected_KL_mean": -231.8561553955078, "KL/std": 106.13909149169922, "epoch": 0.9523809523809523, "fcm_dpo/beta": 0.005835742689669132, "fcm_dpo/delta": -0.053825244307518005, "fcm_dpo/margin": 85.87080383300781, "fcm_dpo/q_t": 0.38744089007377625, "grad_norm": 11.524528503417969, "learning_rate": 3.5719052736323806e-09, "logits/chosen": 0.6896286010742188, "logits/rejected": 0.6513473391532898, "logps/chosen": -202.186767578125, "logps/ref_chosen": -56.201412200927734, "logps/ref_rejected": -74.69807434082031, "logps/rejected": -306.5542297363281, "loss": 1.048, "margin_dpo/margin_mean": 85.87081146240234, "margin_dpo/margin_std": 113.66392517089844, "step": 630 }, { "KL/chosen_KL_mean": -139.0296630859375, "KL/mean": -189.2365264892578, "KL/rejected_KL_mean": -239.44338989257812, "KL/std": 112.87970733642578, "epoch": 0.9538926681783825, "fcm_dpo/beta": 0.005641968920826912, "fcm_dpo/delta": -0.12655234336853027, "fcm_dpo/margin": 100.41372680664062, "fcm_dpo/q_t": 0.37543022632598877, "grad_norm": 13.914621353149414, "learning_rate": 3.352641923861144e-09, "logits/chosen": 0.8196430206298828, "logits/rejected": 0.7138614654541016, "logps/chosen": -197.85025024414062, "logps/ref_chosen": -58.82059860229492, "logps/ref_rejected": -96.51437377929688, "logps/rejected": -335.957763671875, "loss": 1.0156, "margin_dpo/margin_mean": 100.41372680664062, "margin_dpo/margin_std": 124.60462951660156, "step": 631 }, { "KL/chosen_KL_mean": -142.07379150390625, "KL/mean": -187.84475708007812, "KL/rejected_KL_mean": -233.61572265625, "KL/std": 99.48883056640625, "epoch": 0.9554043839758125, "fcm_dpo/beta": 0.005615268833935261, "fcm_dpo/delta": -0.06745753437280655, "fcm_dpo/margin": 91.54193878173828, "fcm_dpo/q_t": 0.3823985457420349, "grad_norm": 13.333562850952148, "learning_rate": 3.140277830901428e-09, "logits/chosen": 0.7539972066879272, "logits/rejected": 0.7377300262451172, "logps/chosen": -200.85983276367188, "logps/ref_chosen": -58.786048889160156, "logps/ref_rejected": -67.21923828125, "logps/rejected": -300.8349609375, "loss": 1.0209, "margin_dpo/margin_mean": 91.54194641113281, "margin_dpo/margin_std": 106.66963195800781, "step": 632 }, { "KL/chosen_KL_mean": -137.31719970703125, "KL/mean": -173.36279296875, "KL/rejected_KL_mean": -209.40838623046875, "KL/std": 96.00003051757812, "epoch": 0.9569160997732427, "fcm_dpo/beta": 0.005642236210405827, "fcm_dpo/delta": 0.04457578808069229, "fcm_dpo/margin": 72.09117889404297, "fcm_dpo/q_t": 0.4096265137195587, "grad_norm": 13.390053749084473, "learning_rate": 2.9348189350335007e-09, "logits/chosen": 0.7287853956222534, "logits/rejected": 0.6764748096466064, "logps/chosen": -189.4473876953125, "logps/ref_chosen": -52.13019561767578, "logps/ref_rejected": -67.23016357421875, "logps/rejected": -276.6385498046875, "loss": 1.1101, "margin_dpo/margin_mean": 72.09117889404297, "margin_dpo/margin_std": 109.14178466796875, "step": 633 }, { "KL/chosen_KL_mean": -165.64334106445312, "KL/mean": -179.42112731933594, "KL/rejected_KL_mean": -193.19894409179688, "KL/std": 99.49234008789062, "epoch": 0.9584278155706727, "fcm_dpo/beta": 0.005632580257952213, "fcm_dpo/delta": 0.0, "fcm_dpo/margin": 27.555599212646484, "fcm_dpo/q_t": 0.4662253260612488, "grad_norm": 16.822216033935547, "learning_rate": 2.736270983384276e-09, "logits/chosen": 0.8007980585098267, "logits/rejected": 0.8156324625015259, "logps/chosen": -226.62313842773438, "logps/ref_chosen": -60.97979736328125, "logps/ref_rejected": -58.50825119018555, "logps/rejected": -251.70718383789062, "loss": 1.3355, "margin_dpo/margin_mean": 27.555599212646484, "margin_dpo/margin_std": 114.27915954589844, "step": 634 }, { "KL/chosen_KL_mean": -180.67120361328125, "KL/mean": -209.85873413085938, "KL/rejected_KL_mean": -239.0462646484375, "KL/std": 98.8995361328125, "epoch": 0.9599395313681028, "fcm_dpo/beta": 0.0057508512400090694, "fcm_dpo/delta": 0.11767241358757019, "fcm_dpo/margin": 58.37506866455078, "fcm_dpo/q_t": 0.4253365993499756, "grad_norm": 12.797456741333008, "learning_rate": 2.5446395297668287e-09, "logits/chosen": 0.6555688381195068, "logits/rejected": 0.6032085418701172, "logps/chosen": -246.64422607421875, "logps/ref_chosen": -65.9730224609375, "logps/ref_rejected": -85.61317443847656, "logps/rejected": -324.659423828125, "loss": 1.1911, "margin_dpo/margin_mean": 58.37506866455078, "margin_dpo/margin_std": 119.31085205078125, "step": 635 }, { "KL/chosen_KL_mean": -141.2942657470703, "KL/mean": -181.30242919921875, "KL/rejected_KL_mean": -221.31060791015625, "KL/std": 100.77838134765625, "epoch": 0.9614512471655329, "fcm_dpo/beta": 0.005744201131165028, "fcm_dpo/delta": -0.010235387831926346, "fcm_dpo/margin": 80.01634979248047, "fcm_dpo/q_t": 0.39512956142425537, "grad_norm": 10.96211051940918, "learning_rate": 2.359929934524829e-09, "logits/chosen": 0.7572035789489746, "logits/rejected": 0.661196231842041, "logps/chosen": -190.43443298339844, "logps/ref_chosen": -49.140167236328125, "logps/ref_rejected": -81.26971435546875, "logps/rejected": -302.580322265625, "loss": 1.0661, "margin_dpo/margin_mean": 80.01634979248047, "margin_dpo/margin_std": 106.34614562988281, "step": 636 }, { "KL/chosen_KL_mean": -188.81138610839844, "KL/mean": -218.619873046875, "KL/rejected_KL_mean": -248.4283447265625, "KL/std": 99.71638488769531, "epoch": 0.9629629629629629, "fcm_dpo/beta": 0.005864589475095272, "fcm_dpo/delta": 0.10340671241283417, "fcm_dpo/margin": 59.616939544677734, "fcm_dpo/q_t": 0.4231225848197937, "grad_norm": 14.910876274108887, "learning_rate": 2.1821473643827137e-09, "logits/chosen": 0.7136276960372925, "logits/rejected": 0.6566417217254639, "logps/chosen": -262.5079650878906, "logps/ref_chosen": -73.69658660888672, "logps/ref_rejected": -83.01487731933594, "logps/rejected": -331.4432373046875, "loss": 1.1901, "margin_dpo/margin_mean": 59.616943359375, "margin_dpo/margin_std": 124.95484924316406, "step": 637 }, { "KL/chosen_KL_mean": -163.7374267578125, "KL/mean": -199.96839904785156, "KL/rejected_KL_mean": -236.19937133789062, "KL/std": 100.8640365600586, "epoch": 0.9644746787603931, "fcm_dpo/beta": 0.005893657449632883, "fcm_dpo/delta": 0.023963581770658493, "fcm_dpo/margin": 72.46197509765625, "fcm_dpo/q_t": 0.40526115894317627, "grad_norm": 13.63633918762207, "learning_rate": 2.0112967923011646e-09, "logits/chosen": 0.7319703698158264, "logits/rejected": 0.6906919479370117, "logps/chosen": -226.5189971923828, "logps/ref_chosen": -62.78158187866211, "logps/ref_rejected": -85.40478515625, "logps/rejected": -321.6041564941406, "loss": 1.1007, "margin_dpo/margin_mean": 72.46197509765625, "margin_dpo/margin_std": 110.03543090820312, "step": 638 }, { "KL/chosen_KL_mean": -146.77230834960938, "KL/mean": -188.85699462890625, "KL/rejected_KL_mean": -230.94168090820312, "KL/std": 107.85531616210938, "epoch": 0.9659863945578231, "fcm_dpo/beta": 0.005871212109923363, "fcm_dpo/delta": -0.04644831269979477, "fcm_dpo/margin": 84.16936492919922, "fcm_dpo/q_t": 0.39135512709617615, "grad_norm": 17.854429244995117, "learning_rate": 1.847382997337943e-09, "logits/chosen": 0.7332069873809814, "logits/rejected": 0.6473530530929565, "logps/chosen": -200.53887939453125, "logps/ref_chosen": -53.76658630371094, "logps/ref_rejected": -72.30009460449219, "logps/rejected": -303.24176025390625, "loss": 1.0569, "margin_dpo/margin_mean": 84.16937255859375, "margin_dpo/margin_std": 116.0023422241211, "step": 639 }, { "KL/chosen_KL_mean": -153.05740356445312, "KL/mean": -189.9259796142578, "KL/rejected_KL_mean": -226.7945556640625, "KL/std": 101.85699462890625, "epoch": 0.9674981103552532, "fcm_dpo/beta": 0.005848909728229046, "fcm_dpo/delta": 0.01944734714925289, "fcm_dpo/margin": 73.73715209960938, "fcm_dpo/q_t": 0.40298590064048767, "grad_norm": 13.762663841247559, "learning_rate": 1.690410564514244e-09, "logits/chosen": 0.7831774950027466, "logits/rejected": 0.7236679792404175, "logps/chosen": -204.47518920898438, "logps/ref_chosen": -51.41777801513672, "logps/ref_rejected": -77.27879333496094, "logps/rejected": -304.0733642578125, "loss": 1.1012, "margin_dpo/margin_mean": 73.73715209960938, "margin_dpo/margin_std": 112.12767028808594, "step": 640 }, { "KL/chosen_KL_mean": -159.89456176757812, "KL/mean": -193.94393920898438, "KL/rejected_KL_mean": -227.99334716796875, "KL/std": 94.81710815429688, "epoch": 0.9690098261526833, "fcm_dpo/beta": 0.005902102682739496, "fcm_dpo/delta": 0.05009445920586586, "fcm_dpo/margin": 68.09879302978516, "fcm_dpo/q_t": 0.40735727548599243, "grad_norm": 14.137347221374512, "learning_rate": 1.5403838846864692e-09, "logits/chosen": 0.6990131139755249, "logits/rejected": 0.6868042945861816, "logps/chosen": -230.94921875, "logps/ref_chosen": -71.0546646118164, "logps/ref_rejected": -82.2440185546875, "logps/rejected": -310.23736572265625, "loss": 1.0898, "margin_dpo/margin_mean": 68.09879302978516, "margin_dpo/margin_std": 89.69477081298828, "step": 641 }, { "KL/chosen_KL_mean": -167.1101837158203, "KL/mean": -193.14244079589844, "KL/rejected_KL_mean": -219.17469787597656, "KL/std": 93.20387268066406, "epoch": 0.9705215419501134, "fcm_dpo/beta": 0.00595252588391304, "fcm_dpo/delta": 0.018339350819587708, "fcm_dpo/margin": 52.064510345458984, "fcm_dpo/q_t": 0.43100064992904663, "grad_norm": 14.998382568359375, "learning_rate": 1.3973071544233218e-09, "logits/chosen": 0.6767123937606812, "logits/rejected": 0.6978293657302856, "logps/chosen": -236.03945922851562, "logps/ref_chosen": -68.92927551269531, "logps/ref_rejected": -70.85682678222656, "logps/rejected": -290.0315246582031, "loss": 1.2075, "margin_dpo/margin_mean": 52.064510345458984, "margin_dpo/margin_std": 112.48196411132812, "step": 642 }, { "KL/chosen_KL_mean": -152.78616333007812, "KL/mean": -192.1639404296875, "KL/rejected_KL_mean": -231.5417022705078, "KL/std": 100.897216796875, "epoch": 0.9720332577475435, "fcm_dpo/beta": 0.005934578832238913, "fcm_dpo/delta": -0.018240533769130707, "fcm_dpo/margin": 78.75552368164062, "fcm_dpo/q_t": 0.3963284492492676, "grad_norm": 19.15582847595215, "learning_rate": 1.261184375888541e-09, "logits/chosen": 0.7067471742630005, "logits/rejected": 0.627659261226654, "logps/chosen": -218.09521484375, "logps/ref_chosen": -65.30903625488281, "logps/ref_rejected": -83.61613464355469, "logps/rejected": -315.1578369140625, "loss": 1.0852, "margin_dpo/margin_mean": 78.75552368164062, "margin_dpo/margin_std": 117.13465881347656, "step": 643 }, { "KL/chosen_KL_mean": -144.2310333251953, "KL/mean": -170.76400756835938, "KL/rejected_KL_mean": -197.2969970703125, "KL/std": 90.6378402709961, "epoch": 0.9735449735449735, "fcm_dpo/beta": 0.005933812819421291, "fcm_dpo/delta": 0.004993738606572151, "fcm_dpo/margin": 53.065940856933594, "fcm_dpo/q_t": 0.42780280113220215, "grad_norm": 11.877394676208496, "learning_rate": 1.1320193567288527e-09, "logits/chosen": 0.8293318748474121, "logits/rejected": 0.7982121706008911, "logps/chosen": -195.233642578125, "logps/ref_chosen": -51.002601623535156, "logps/ref_rejected": -64.46372985839844, "logps/rejected": -261.7607421875, "loss": 1.2014, "margin_dpo/margin_mean": 53.06593704223633, "margin_dpo/margin_std": 110.60516357421875, "step": 644 }, { "KL/chosen_KL_mean": -157.09957885742188, "KL/mean": -190.3580780029297, "KL/rejected_KL_mean": -223.61656188964844, "KL/std": 98.42587280273438, "epoch": 0.9750566893424036, "fcm_dpo/beta": 0.005985003896057606, "fcm_dpo/delta": 0.05404530465602875, "fcm_dpo/margin": 66.51698303222656, "fcm_dpo/q_t": 0.409078985452652, "grad_norm": 15.517080307006836, "learning_rate": 1.0098157099674987e-09, "logits/chosen": 0.6929702758789062, "logits/rejected": 0.6790943145751953, "logps/chosen": -218.06298828125, "logps/ref_chosen": -60.963409423828125, "logps/ref_rejected": -69.73353576660156, "logps/rejected": -293.35009765625, "loss": 1.111, "margin_dpo/margin_mean": 66.51698303222656, "margin_dpo/margin_std": 100.13095092773438, "step": 645 }, { "KL/chosen_KL_mean": -173.18130493164062, "KL/mean": -204.287841796875, "KL/rejected_KL_mean": -235.39434814453125, "KL/std": 114.00927734375, "epoch": 0.9765684051398337, "fcm_dpo/beta": 0.006071906536817551, "fcm_dpo/delta": 0.07504133880138397, "fcm_dpo/margin": 62.21302032470703, "fcm_dpo/q_t": 0.41674911975860596, "grad_norm": 13.308674812316895, "learning_rate": 8.945768539031783e-10, "logits/chosen": 0.7898342609405518, "logits/rejected": 0.732674241065979, "logps/chosen": -235.47137451171875, "logps/ref_chosen": -62.290069580078125, "logps/ref_rejected": -85.54812622070312, "logps/rejected": -320.9424743652344, "loss": 1.1574, "margin_dpo/margin_mean": 62.21302795410156, "margin_dpo/margin_std": 115.69244384765625, "step": 646 }, { "KL/chosen_KL_mean": -169.27145385742188, "KL/mean": -218.55142211914062, "KL/rejected_KL_mean": -267.83135986328125, "KL/std": 112.50706481933594, "epoch": 0.9780801209372638, "fcm_dpo/beta": 0.005969959311187267, "fcm_dpo/delta": -0.14687490463256836, "fcm_dpo/margin": 98.55989837646484, "fcm_dpo/q_t": 0.36799299716949463, "grad_norm": 15.381766319274902, "learning_rate": 7.863060120144316e-10, "logits/chosen": 0.7591278553009033, "logits/rejected": 0.6683089733123779, "logps/chosen": -236.78732299804688, "logps/ref_chosen": -67.515869140625, "logps/ref_rejected": -101.50871276855469, "logps/rejected": -369.340087890625, "loss": 0.982, "margin_dpo/margin_mean": 98.55990600585938, "margin_dpo/margin_std": 111.36669921875, "step": 647 }, { "KL/chosen_KL_mean": -169.76194763183594, "KL/mean": -204.1524658203125, "KL/rejected_KL_mean": -238.54298400878906, "KL/std": 98.05025482177734, "epoch": 0.9795918367346939, "fcm_dpo/beta": 0.005927952937781811, "fcm_dpo/delta": 0.04385180026292801, "fcm_dpo/margin": 68.78103637695312, "fcm_dpo/q_t": 0.41050416231155396, "grad_norm": 13.526182174682617, "learning_rate": 6.850062128694045e-10, "logits/chosen": 0.6800918579101562, "logits/rejected": 0.6235531568527222, "logps/chosen": -234.35787963867188, "logps/ref_chosen": -64.59593963623047, "logps/ref_rejected": -83.384033203125, "logps/rejected": -321.9270324707031, "loss": 1.1538, "margin_dpo/margin_mean": 68.78103637695312, "margin_dpo/margin_std": 125.53382873535156, "step": 648 }, { "KL/chosen_KL_mean": -153.29359436035156, "KL/mean": -189.05458068847656, "KL/rejected_KL_mean": -224.81556701660156, "KL/std": 97.22737121582031, "epoch": 0.981103552532124, "fcm_dpo/beta": 0.005967048928141594, "fcm_dpo/delta": 0.024066461250185966, "fcm_dpo/margin": 71.52197265625, "fcm_dpo/q_t": 0.4042567014694214, "grad_norm": 17.365737915039062, "learning_rate": 5.906802900412788e-10, "logits/chosen": 0.762288510799408, "logits/rejected": 0.7033209800720215, "logps/chosen": -202.60324096679688, "logps/ref_chosen": -49.30964660644531, "logps/ref_rejected": -73.73710632324219, "logps/rejected": -298.55267333984375, "loss": 1.118, "margin_dpo/margin_mean": 71.52197265625, "margin_dpo/margin_std": 116.39321899414062, "step": 649 }, { "KL/chosen_KL_mean": -149.021728515625, "KL/mean": -186.00106811523438, "KL/rejected_KL_mean": -222.98040771484375, "KL/std": 99.01878356933594, "epoch": 0.982615268329554, "fcm_dpo/beta": 0.005991187412291765, "fcm_dpo/delta": 0.007154183462262154, "fcm_dpo/margin": 73.95868682861328, "fcm_dpo/q_t": 0.4018860459327698, "grad_norm": 13.914726257324219, "learning_rate": 5.033308820289184e-10, "logits/chosen": 0.7910748720169067, "logits/rejected": 0.732347309589386, "logps/chosen": -204.08499145507812, "logps/ref_chosen": -55.06325912475586, "logps/ref_rejected": -77.39610290527344, "logps/rejected": -300.37652587890625, "loss": 1.1102, "margin_dpo/margin_mean": 73.95867919921875, "margin_dpo/margin_std": 119.37321472167969, "step": 650 }, { "KL/chosen_KL_mean": -147.79086303710938, "KL/mean": -179.89984130859375, "KL/rejected_KL_mean": -212.00881958007812, "KL/std": 98.75411987304688, "epoch": 0.9841269841269841, "fcm_dpo/beta": 0.006077755242586136, "fcm_dpo/delta": 0.061789728701114655, "fcm_dpo/margin": 64.21796417236328, "fcm_dpo/q_t": 0.4144824147224426, "grad_norm": 13.73145580291748, "learning_rate": 4.2296043218295606e-10, "logits/chosen": 0.783795952796936, "logits/rejected": 0.7152203321456909, "logps/chosen": -201.85601806640625, "logps/ref_chosen": -54.065162658691406, "logps/ref_rejected": -77.79080200195312, "logps/rejected": -289.79962158203125, "loss": 1.1266, "margin_dpo/margin_mean": 64.21795654296875, "margin_dpo/margin_std": 103.98927307128906, "step": 651 }, { "KL/chosen_KL_mean": -174.13955688476562, "KL/mean": -204.1993408203125, "KL/rejected_KL_mean": -234.25912475585938, "KL/std": 107.51026153564453, "epoch": 0.9856386999244142, "fcm_dpo/beta": 0.006125830113887787, "fcm_dpo/delta": 0.0847388356924057, "fcm_dpo/margin": 60.11957931518555, "fcm_dpo/q_t": 0.42180266976356506, "grad_norm": 15.607612609863281, "learning_rate": 3.4957118863768176e-10, "logits/chosen": 0.7536121606826782, "logits/rejected": 0.7091095447540283, "logps/chosen": -237.77984619140625, "logps/ref_chosen": -63.64030456542969, "logps/ref_rejected": -78.86882019042969, "logps/rejected": -313.1279602050781, "loss": 1.192, "margin_dpo/margin_mean": 60.11957550048828, "margin_dpo/margin_std": 127.72785949707031, "step": 652 }, { "KL/chosen_KL_mean": -154.40162658691406, "KL/mean": -190.7852783203125, "KL/rejected_KL_mean": -227.1689453125, "KL/std": 96.48133850097656, "epoch": 0.9871504157218443, "fcm_dpo/beta": 0.0062039922922849655, "fcm_dpo/delta": -0.001763913780450821, "fcm_dpo/margin": 72.76734161376953, "fcm_dpo/q_t": 0.3998035192489624, "grad_norm": 15.49831771850586, "learning_rate": 2.831652042480093e-10, "logits/chosen": 0.7264477014541626, "logits/rejected": 0.6846983432769775, "logps/chosen": -216.07000732421875, "logps/ref_chosen": -61.668373107910156, "logps/ref_rejected": -73.83012390136719, "logps/rejected": -300.99908447265625, "loss": 1.0966, "margin_dpo/margin_mean": 72.76734161376953, "margin_dpo/margin_std": 111.39743041992188, "step": 653 }, { "KL/chosen_KL_mean": -161.70181274414062, "KL/mean": -195.36509704589844, "KL/rejected_KL_mean": -229.02838134765625, "KL/std": 114.16594696044922, "epoch": 0.9886621315192744, "fcm_dpo/beta": 0.006077038589864969, "fcm_dpo/delta": -0.08529359847307205, "fcm_dpo/margin": 67.32656860351562, "fcm_dpo/q_t": 0.40859442949295044, "grad_norm": 14.157605171203613, "learning_rate": 2.2374433653205016e-10, "logits/chosen": 0.7062954306602478, "logits/rejected": 0.6156010627746582, "logps/chosen": -219.27008056640625, "logps/ref_chosen": -57.568267822265625, "logps/ref_rejected": -87.74789428710938, "logps/rejected": -316.7762756347656, "loss": 1.1233, "margin_dpo/margin_mean": 67.32656860351562, "margin_dpo/margin_std": 105.35501098632812, "step": 654 }, { "KL/chosen_KL_mean": -127.7425308227539, "KL/mean": -176.96739196777344, "KL/rejected_KL_mean": -226.1922607421875, "KL/std": 99.56523895263672, "epoch": 0.9901738473167044, "fcm_dpo/beta": 0.0058916169218719006, "fcm_dpo/delta": -0.14149729907512665, "fcm_dpo/margin": 98.44974517822266, "fcm_dpo/q_t": 0.3675011992454529, "grad_norm": 12.291767120361328, "learning_rate": 1.7131024761923852e-10, "logits/chosen": 0.6882774829864502, "logits/rejected": 0.6106013059616089, "logps/chosen": -179.88967895507812, "logps/ref_chosen": -52.14714813232422, "logps/ref_rejected": -80.85014343261719, "logps/rejected": -307.04241943359375, "loss": 0.9693, "margin_dpo/margin_mean": 98.44974517822266, "margin_dpo/margin_std": 96.58932495117188, "step": 655 }, { "KL/chosen_KL_mean": -167.70416259765625, "KL/mean": -204.19580078125, "KL/rejected_KL_mean": -240.68743896484375, "KL/std": 101.8294677734375, "epoch": 0.9916855631141346, "fcm_dpo/beta": 0.005923710763454437, "fcm_dpo/delta": 0.018466537818312645, "fcm_dpo/margin": 72.98329162597656, "fcm_dpo/q_t": 0.4031839966773987, "grad_norm": 11.903160095214844, "learning_rate": 1.2586440420372934e-10, "logits/chosen": 0.656846821308136, "logits/rejected": 0.617106556892395, "logps/chosen": -240.96087646484375, "logps/ref_chosen": -73.25672912597656, "logps/ref_rejected": -85.35127258300781, "logps/rejected": -326.0386962890625, "loss": 1.1019, "margin_dpo/margin_mean": 72.98329162597656, "margin_dpo/margin_std": 112.23123168945312, "step": 656 }, { "KL/chosen_KL_mean": -148.2847900390625, "KL/mean": -192.8415069580078, "KL/rejected_KL_mean": -237.39822387695312, "KL/std": 112.21992492675781, "epoch": 0.9931972789115646, "fcm_dpo/beta": 0.005868775770068169, "fcm_dpo/delta": -0.07695280015468597, "fcm_dpo/margin": 89.11343383789062, "fcm_dpo/q_t": 0.3840072751045227, "grad_norm": 12.307347297668457, "learning_rate": 8.740807750345913e-11, "logits/chosen": 0.8304808735847473, "logits/rejected": 0.7517132759094238, "logps/chosen": -198.0081787109375, "logps/ref_chosen": -49.72339630126953, "logps/ref_rejected": -75.1568603515625, "logps/rejected": -312.5550842285156, "loss": 1.0473, "margin_dpo/margin_mean": 89.11343383789062, "margin_dpo/margin_std": 122.4747314453125, "step": 657 }, { "KL/chosen_KL_mean": -158.35809326171875, "KL/mean": -194.7518768310547, "KL/rejected_KL_mean": -231.14566040039062, "KL/std": 115.96528625488281, "epoch": 0.9947089947089947, "fcm_dpo/beta": 0.005895557813346386, "fcm_dpo/delta": 0.020794283598661423, "fcm_dpo/margin": 72.78756713867188, "fcm_dpo/q_t": 0.40627312660217285, "grad_norm": 12.473671913146973, "learning_rate": 5.594234322453539e-11, "logits/chosen": 0.7611560225486755, "logits/rejected": 0.7182252407073975, "logps/chosen": -221.40443420410156, "logps/ref_chosen": -63.04634094238281, "logps/ref_rejected": -83.44963073730469, "logps/rejected": -314.59527587890625, "loss": 1.1497, "margin_dpo/margin_mean": 72.78756713867188, "margin_dpo/margin_std": 135.44158935546875, "step": 658 }, { "KL/chosen_KL_mean": -161.2940673828125, "KL/mean": -189.77117919921875, "KL/rejected_KL_mean": -218.24826049804688, "KL/std": 100.78056335449219, "epoch": 0.9962207105064248, "fcm_dpo/beta": 0.00593742448836565, "fcm_dpo/delta": 0.11584506928920746, "fcm_dpo/margin": 56.954185485839844, "fcm_dpo/q_t": 0.4244096875190735, "grad_norm": 17.622331619262695, "learning_rate": 3.146808153123293e-11, "logits/chosen": 0.7817635536193848, "logits/rejected": 0.7224591970443726, "logps/chosen": -216.374267578125, "logps/ref_chosen": -55.0802001953125, "logps/ref_rejected": -71.91049194335938, "logps/rejected": -290.15875244140625, "loss": 1.1874, "margin_dpo/margin_mean": 56.95417785644531, "margin_dpo/margin_std": 114.34848022460938, "step": 659 }, { "KL/chosen_KL_mean": -152.11578369140625, "KL/mean": -198.0364990234375, "KL/rejected_KL_mean": -243.95724487304688, "KL/std": 109.25594329833984, "epoch": 0.9977324263038548, "fcm_dpo/beta": 0.005916485097259283, "fcm_dpo/delta": -0.09869644045829773, "fcm_dpo/margin": 91.84146118164062, "fcm_dpo/q_t": 0.37992629408836365, "grad_norm": 14.751081466674805, "learning_rate": 1.3985977021235829e-11, "logits/chosen": 0.8116575479507446, "logits/rejected": 0.7513267993927002, "logps/chosen": -206.64169311523438, "logps/ref_chosen": -54.525917053222656, "logps/ref_rejected": -81.23604583740234, "logps/rejected": -325.19329833984375, "loss": 1.0144, "margin_dpo/margin_mean": 91.84146118164062, "margin_dpo/margin_std": 115.3442611694336, "step": 660 }, { "KL/chosen_KL_mean": -175.55447387695312, "KL/mean": -204.49343872070312, "KL/rejected_KL_mean": -233.43243408203125, "KL/std": 109.31103515625, "epoch": 0.999244142101285, "fcm_dpo/beta": 0.005994308274239302, "fcm_dpo/delta": 0.1062980443239212, "fcm_dpo/margin": 57.877967834472656, "fcm_dpo/q_t": 0.42280131578445435, "grad_norm": 12.12573528289795, "learning_rate": 3.4965187065971735e-12, "logits/chosen": 0.693326473236084, "logits/rejected": 0.6238353252410889, "logps/chosen": -235.92709350585938, "logps/ref_chosen": -60.37263870239258, "logps/ref_rejected": -77.42874145507812, "logps/rejected": -310.8612060546875, "loss": 1.1979, "margin_dpo/margin_mean": 57.877967834472656, "margin_dpo/margin_std": 124.64874267578125, "step": 661 }, { "epoch": 0.999244142101285, "step": 661, "total_flos": 0.0, "train_loss": 1.143015539862565, "train_runtime": 1644.2601, "train_samples_per_second": 25.748, "train_steps_per_second": 0.402 } ], "logging_steps": 1, "max_steps": 661, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 8, "trial_name": null, "trial_params": null }