6543 lines
214 KiB
JSON
6543 lines
214 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5714285714285714,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"completion_length": 2571.2083587646484,
|
|
"epoch": 0.001142857142857143,
|
|
"grad_norm": 0.19510559737682343,
|
|
"kl": 0.0,
|
|
"learning_rate": 2e-08,
|
|
"loss": 0.0,
|
|
"reward": 0.4897647276520729,
|
|
"reward_std": 0.8290339335799217,
|
|
"rewards/cosine_scaled_reward": -0.015534311532974243,
|
|
"rewards/format_reward": 0.5208333488553762,
|
|
"step": 1
|
|
},
|
|
{
|
|
"completion_length": 2804.395881652832,
|
|
"epoch": 0.002285714285714286,
|
|
"grad_norm": 0.18415163457393646,
|
|
"kl": 0.0,
|
|
"learning_rate": 4e-08,
|
|
"loss": 0.0,
|
|
"reward": 0.27539755403995514,
|
|
"reward_std": 0.42092563211917877,
|
|
"rewards/cosine_scaled_reward": -0.04980122856795788,
|
|
"rewards/format_reward": 0.37500000558793545,
|
|
"step": 2
|
|
},
|
|
{
|
|
"completion_length": 3361.1458435058594,
|
|
"epoch": 0.0034285714285714284,
|
|
"grad_norm": 0.16567473113536835,
|
|
"kl": 4.006922245025635e-05,
|
|
"learning_rate": 6e-08,
|
|
"loss": 0.0,
|
|
"reward": -0.23245980869978666,
|
|
"reward_std": 0.5928730629384518,
|
|
"rewards/cosine_scaled_reward": -0.17872990405885503,
|
|
"rewards/format_reward": 0.1250000037252903,
|
|
"step": 3
|
|
},
|
|
{
|
|
"completion_length": 2153.729202270508,
|
|
"epoch": 0.004571428571428572,
|
|
"grad_norm": 0.25097447633743286,
|
|
"kl": 4.071742296218872e-05,
|
|
"learning_rate": 8e-08,
|
|
"loss": 0.0,
|
|
"reward": 0.3372869056183845,
|
|
"reward_std": 0.6931154392659664,
|
|
"rewards/cosine_scaled_reward": -0.14385656313970685,
|
|
"rewards/format_reward": 0.6250000037252903,
|
|
"step": 4
|
|
},
|
|
{
|
|
"completion_length": 3400.375030517578,
|
|
"epoch": 0.005714285714285714,
|
|
"grad_norm": 0.224583700299263,
|
|
"kl": 4.339590668678284e-05,
|
|
"learning_rate": 1e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.30452448688447475,
|
|
"reward_std": 0.5627153031527996,
|
|
"rewards/cosine_scaled_reward": -0.24601224437355995,
|
|
"rewards/format_reward": 0.18750000558793545,
|
|
"step": 5
|
|
},
|
|
{
|
|
"completion_length": 3246.2291717529297,
|
|
"epoch": 0.006857142857142857,
|
|
"grad_norm": 0.17905186116695404,
|
|
"kl": 4.048645496368408e-05,
|
|
"learning_rate": 1.2e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.04279324598610401,
|
|
"reward_std": 0.5993511825799942,
|
|
"rewards/cosine_scaled_reward": -0.18806329306971747,
|
|
"rewards/format_reward": 0.3333333469927311,
|
|
"step": 6
|
|
},
|
|
{
|
|
"completion_length": 2938.0209197998047,
|
|
"epoch": 0.008,
|
|
"grad_norm": 0.19757460057735443,
|
|
"kl": 2.4452805519104004e-05,
|
|
"learning_rate": 1.4e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.2690338185057044,
|
|
"reward_std": 0.7015659939497709,
|
|
"rewards/cosine_scaled_reward": -0.11548309866338968,
|
|
"rewards/format_reward": 0.5000000111758709,
|
|
"step": 7
|
|
},
|
|
{
|
|
"completion_length": 2751.770866394043,
|
|
"epoch": 0.009142857142857144,
|
|
"grad_norm": 0.1658889204263687,
|
|
"kl": 1.728162169456482e-05,
|
|
"learning_rate": 1.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.6224091164767742,
|
|
"reward_std": 0.7972168251872063,
|
|
"rewards/cosine_scaled_reward": 0.09245455078780651,
|
|
"rewards/format_reward": 0.43750001303851604,
|
|
"step": 8
|
|
},
|
|
{
|
|
"completion_length": 3031.7084045410156,
|
|
"epoch": 0.010285714285714285,
|
|
"grad_norm": 0.2120562642812729,
|
|
"kl": 3.610551357269287e-05,
|
|
"learning_rate": 1.8e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.18008227972313762,
|
|
"reward_std": 0.8001800198107958,
|
|
"rewards/cosine_scaled_reward": -0.10787552827969193,
|
|
"rewards/format_reward": 0.39583334140479565,
|
|
"step": 9
|
|
},
|
|
{
|
|
"completion_length": 2753.208366394043,
|
|
"epoch": 0.011428571428571429,
|
|
"grad_norm": 0.21144917607307434,
|
|
"kl": 3.1989067792892456e-05,
|
|
"learning_rate": 2e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.08000842481851578,
|
|
"reward_std": 0.7203316055238247,
|
|
"rewards/cosine_scaled_reward": -0.14749579317867756,
|
|
"rewards/format_reward": 0.3750000111758709,
|
|
"step": 10
|
|
},
|
|
{
|
|
"completion_length": 3293.2708435058594,
|
|
"epoch": 0.012571428571428572,
|
|
"grad_norm": 0.1650972217321396,
|
|
"kl": 3.8489699363708496e-05,
|
|
"learning_rate": 2.1999999999999998e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.41089826077222824,
|
|
"reward_std": 0.4038824327290058,
|
|
"rewards/cosine_scaled_reward": -0.2783657982945442,
|
|
"rewards/format_reward": 0.1458333395421505,
|
|
"step": 11
|
|
},
|
|
{
|
|
"completion_length": 2683.5000915527344,
|
|
"epoch": 0.013714285714285714,
|
|
"grad_norm": 0.19341549277305603,
|
|
"kl": 4.114210605621338e-05,
|
|
"learning_rate": 2.4e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.43104756623506546,
|
|
"reward_std": 0.5792003609240055,
|
|
"rewards/cosine_scaled_reward": -0.11780955828726292,
|
|
"rewards/format_reward": 0.6666666828095913,
|
|
"step": 12
|
|
},
|
|
{
|
|
"completion_length": 2886.541702270508,
|
|
"epoch": 0.014857142857142857,
|
|
"grad_norm": 0.22984430193901062,
|
|
"kl": 3.719329833984375e-05,
|
|
"learning_rate": 2.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.23748547211289406,
|
|
"reward_std": 0.7873078212141991,
|
|
"rewards/cosine_scaled_reward": -0.06875726429279894,
|
|
"rewards/format_reward": 0.3750000149011612,
|
|
"step": 13
|
|
},
|
|
{
|
|
"completion_length": 2877.312530517578,
|
|
"epoch": 0.016,
|
|
"grad_norm": 0.2078430950641632,
|
|
"kl": 2.997368574142456e-05,
|
|
"learning_rate": 2.8e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.14176954282447696,
|
|
"reward_std": 0.7100466191768646,
|
|
"rewards/cosine_scaled_reward": -0.11661523208022118,
|
|
"rewards/format_reward": 0.3750000074505806,
|
|
"step": 14
|
|
},
|
|
{
|
|
"completion_length": 2681.437511444092,
|
|
"epoch": 0.017142857142857144,
|
|
"grad_norm": 0.20677024126052856,
|
|
"kl": 2.692500129342079e-05,
|
|
"learning_rate": 3e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.5315095772966743,
|
|
"reward_std": 0.6707141287624836,
|
|
"rewards/cosine_scaled_reward": 0.04700478911399841,
|
|
"rewards/format_reward": 0.43750000558793545,
|
|
"step": 15
|
|
},
|
|
{
|
|
"completion_length": 3505.2708435058594,
|
|
"epoch": 0.018285714285714287,
|
|
"grad_norm": 0.19309639930725098,
|
|
"kl": 3.673136234283447e-05,
|
|
"learning_rate": 3.2e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.2943936800584197,
|
|
"reward_std": 0.6762014180421829,
|
|
"rewards/cosine_scaled_reward": -0.19928016886115074,
|
|
"rewards/format_reward": 0.10416666977107525,
|
|
"step": 16
|
|
},
|
|
{
|
|
"completion_length": 2520.7916984558105,
|
|
"epoch": 0.019428571428571427,
|
|
"grad_norm": 0.2713626027107239,
|
|
"kl": 3.7260353565216064e-05,
|
|
"learning_rate": 3.4000000000000003e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.29364213347435,
|
|
"reward_std": 0.7413498945534229,
|
|
"rewards/cosine_scaled_reward": -0.09276227621012367,
|
|
"rewards/format_reward": 0.4791666828095913,
|
|
"step": 17
|
|
},
|
|
{
|
|
"completion_length": 2989.500030517578,
|
|
"epoch": 0.02057142857142857,
|
|
"grad_norm": 0.14791515469551086,
|
|
"kl": 2.292729914188385e-05,
|
|
"learning_rate": 3.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.22529255971312523,
|
|
"reward_std": 0.5643694922327995,
|
|
"rewards/cosine_scaled_reward": -0.08527039736509323,
|
|
"rewards/format_reward": 0.39583333395421505,
|
|
"step": 18
|
|
},
|
|
{
|
|
"completion_length": 2874.250030517578,
|
|
"epoch": 0.021714285714285714,
|
|
"grad_norm": 0.18149082362651825,
|
|
"kl": 2.5779008865356445e-05,
|
|
"learning_rate": 3.7999999999999996e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.6877359300851822,
|
|
"reward_std": 0.7566238529980183,
|
|
"rewards/cosine_scaled_reward": 0.1355345994234085,
|
|
"rewards/format_reward": 0.416666679084301,
|
|
"step": 19
|
|
},
|
|
{
|
|
"completion_length": 2473.520854949951,
|
|
"epoch": 0.022857142857142857,
|
|
"grad_norm": 0.20354878902435303,
|
|
"kl": 1.685088500380516e-05,
|
|
"learning_rate": 4e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.5272735962644219,
|
|
"reward_std": 0.7334154956042767,
|
|
"rewards/cosine_scaled_reward": -0.059279868844896555,
|
|
"rewards/format_reward": 0.6458333414047956,
|
|
"step": 20
|
|
},
|
|
{
|
|
"completion_length": 2523.520835876465,
|
|
"epoch": 0.024,
|
|
"grad_norm": 0.26596662402153015,
|
|
"kl": 4.191696643829346e-05,
|
|
"learning_rate": 4.1999999999999995e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.5646272003650665,
|
|
"reward_std": 0.5900244954973459,
|
|
"rewards/cosine_scaled_reward": 0.04273026343435049,
|
|
"rewards/format_reward": 0.47916666977107525,
|
|
"step": 21
|
|
},
|
|
{
|
|
"completion_length": 2035.3542251586914,
|
|
"epoch": 0.025142857142857144,
|
|
"grad_norm": 0.2960587441921234,
|
|
"kl": 3.464333713054657e-05,
|
|
"learning_rate": 4.3999999999999997e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.637257520109415,
|
|
"reward_std": 0.8020393662154675,
|
|
"rewards/cosine_scaled_reward": -0.03553791396552697,
|
|
"rewards/format_reward": 0.7083333488553762,
|
|
"step": 22
|
|
},
|
|
{
|
|
"completion_length": 2664.3125610351562,
|
|
"epoch": 0.026285714285714287,
|
|
"grad_norm": 0.21004951000213623,
|
|
"kl": 3.3404678106307983e-05,
|
|
"learning_rate": 4.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.22290698066353798,
|
|
"reward_std": 0.7860734751448035,
|
|
"rewards/cosine_scaled_reward": -0.0968798566609621,
|
|
"rewards/format_reward": 0.4166666753590107,
|
|
"step": 23
|
|
},
|
|
{
|
|
"completion_length": 2712.6875610351562,
|
|
"epoch": 0.027428571428571427,
|
|
"grad_norm": 0.2664627432823181,
|
|
"kl": 2.100318670272827e-05,
|
|
"learning_rate": 4.8e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.7516965055838227,
|
|
"reward_std": 1.1073086112737656,
|
|
"rewards/cosine_scaled_reward": 0.07376489660236984,
|
|
"rewards/format_reward": 0.6041666865348816,
|
|
"step": 24
|
|
},
|
|
{
|
|
"completion_length": 2708.1458740234375,
|
|
"epoch": 0.02857142857142857,
|
|
"grad_norm": 0.22358901798725128,
|
|
"kl": 3.844499588012695e-05,
|
|
"learning_rate": 5e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.1855073875049129,
|
|
"reward_std": 0.6800910420715809,
|
|
"rewards/cosine_scaled_reward": -0.08432964608073235,
|
|
"rewards/format_reward": 0.3541666753590107,
|
|
"step": 25
|
|
},
|
|
{
|
|
"completion_length": 3107.541717529297,
|
|
"epoch": 0.029714285714285714,
|
|
"grad_norm": 0.1617579460144043,
|
|
"kl": 2.820044755935669e-05,
|
|
"learning_rate": 5.2e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.587481252849102,
|
|
"reward_std": 0.693414282053709,
|
|
"rewards/cosine_scaled_reward": 0.07499059912515804,
|
|
"rewards/format_reward": 0.43750001303851604,
|
|
"step": 26
|
|
},
|
|
{
|
|
"completion_length": 2998.666702270508,
|
|
"epoch": 0.030857142857142857,
|
|
"grad_norm": 0.18699905276298523,
|
|
"kl": 2.479786053299904e-05,
|
|
"learning_rate": 5.4e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.12037789449095726,
|
|
"reward_std": 0.8231884613633156,
|
|
"rewards/cosine_scaled_reward": -0.12731105368584394,
|
|
"rewards/format_reward": 0.37500000558793545,
|
|
"step": 27
|
|
},
|
|
{
|
|
"completion_length": 2880.4375534057617,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.1849849373102188,
|
|
"kl": 2.5503337383270264e-05,
|
|
"learning_rate": 5.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.38755445554852486,
|
|
"reward_std": 0.8283951133489609,
|
|
"rewards/cosine_scaled_reward": -0.004139451950322837,
|
|
"rewards/format_reward": 0.39583334140479565,
|
|
"step": 28
|
|
},
|
|
{
|
|
"completion_length": 3422.604217529297,
|
|
"epoch": 0.03314285714285714,
|
|
"grad_norm": 0.17641501128673553,
|
|
"kl": 2.0990148186683655e-05,
|
|
"learning_rate": 5.8e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.2620885446667671,
|
|
"reward_std": 0.563602440059185,
|
|
"rewards/cosine_scaled_reward": -0.20396093931049109,
|
|
"rewards/format_reward": 0.14583333767950535,
|
|
"step": 29
|
|
},
|
|
{
|
|
"completion_length": 2927.229217529297,
|
|
"epoch": 0.03428571428571429,
|
|
"grad_norm": 0.20999222993850708,
|
|
"kl": 3.0543655157089233e-05,
|
|
"learning_rate": 6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.3440352368634194,
|
|
"reward_std": 0.9646327681839466,
|
|
"rewards/cosine_scaled_reward": -0.0467323949560523,
|
|
"rewards/format_reward": 0.43750001676380634,
|
|
"step": 30
|
|
},
|
|
{
|
|
"completion_length": 2953.437530517578,
|
|
"epoch": 0.03542857142857143,
|
|
"grad_norm": 0.18049225211143494,
|
|
"kl": 4.493445158004761e-05,
|
|
"learning_rate": 6.2e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.20754884742200375,
|
|
"reward_std": 0.8056708425283432,
|
|
"rewards/cosine_scaled_reward": -0.08372558187693357,
|
|
"rewards/format_reward": 0.37500000931322575,
|
|
"step": 31
|
|
},
|
|
{
|
|
"completion_length": 3194.937545776367,
|
|
"epoch": 0.036571428571428574,
|
|
"grad_norm": 0.15885497629642487,
|
|
"kl": 2.6959925889968872e-05,
|
|
"learning_rate": 6.4e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.04864098597317934,
|
|
"reward_std": 0.6213513053953648,
|
|
"rewards/cosine_scaled_reward": -0.12151284422725439,
|
|
"rewards/format_reward": 0.2916666716337204,
|
|
"step": 32
|
|
},
|
|
{
|
|
"completion_length": 3312.2291870117188,
|
|
"epoch": 0.037714285714285714,
|
|
"grad_norm": 0.15803882479667664,
|
|
"kl": 4.449114203453064e-05,
|
|
"learning_rate": 6.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.20429637841880322,
|
|
"reward_std": 0.770494450815022,
|
|
"rewards/cosine_scaled_reward": -0.03326847730204463,
|
|
"rewards/format_reward": 0.27083334140479565,
|
|
"step": 33
|
|
},
|
|
{
|
|
"completion_length": 2483.666702270508,
|
|
"epoch": 0.038857142857142854,
|
|
"grad_norm": 0.22648921608924866,
|
|
"kl": 7.438473403453827e-05,
|
|
"learning_rate": 6.800000000000001e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.7040222510695457,
|
|
"reward_std": 0.8860063031315804,
|
|
"rewards/cosine_scaled_reward": 0.07076112227514386,
|
|
"rewards/format_reward": 0.5625000074505806,
|
|
"step": 34
|
|
},
|
|
{
|
|
"completion_length": 2937.750015258789,
|
|
"epoch": 0.04,
|
|
"grad_norm": 0.20356310904026031,
|
|
"kl": 4.690885543823242e-05,
|
|
"learning_rate": 7e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.31086206063628197,
|
|
"reward_std": 0.8583472333848476,
|
|
"rewards/cosine_scaled_reward": -0.032068971544504166,
|
|
"rewards/format_reward": 0.37500000558793545,
|
|
"step": 35
|
|
},
|
|
{
|
|
"completion_length": 3399.6458435058594,
|
|
"epoch": 0.04114285714285714,
|
|
"grad_norm": 0.17643427848815918,
|
|
"kl": 4.9777328968048096e-05,
|
|
"learning_rate": 7.2e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.3302987515926361,
|
|
"reward_std": 0.4607627494260669,
|
|
"rewards/cosine_scaled_reward": -0.22764937952160835,
|
|
"rewards/format_reward": 0.1250000037252903,
|
|
"step": 36
|
|
},
|
|
{
|
|
"completion_length": 3218.916702270508,
|
|
"epoch": 0.04228571428571429,
|
|
"grad_norm": 0.16882504522800446,
|
|
"kl": 3.8273632526397705e-05,
|
|
"learning_rate": 7.4e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.16163601353764534,
|
|
"reward_std": 0.4668765738606453,
|
|
"rewards/cosine_scaled_reward": -0.21623468212783337,
|
|
"rewards/format_reward": 0.2708333395421505,
|
|
"step": 37
|
|
},
|
|
{
|
|
"completion_length": 3279.5416870117188,
|
|
"epoch": 0.04342857142857143,
|
|
"grad_norm": 0.15908610820770264,
|
|
"kl": 4.918128252029419e-05,
|
|
"learning_rate": 7.599999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.20695911906659603,
|
|
"reward_std": 0.5721107684075832,
|
|
"rewards/cosine_scaled_reward": -0.176396232098341,
|
|
"rewards/format_reward": 0.1458333395421505,
|
|
"step": 38
|
|
},
|
|
{
|
|
"completion_length": 2879.375045776367,
|
|
"epoch": 0.044571428571428574,
|
|
"grad_norm": 0.2147342413663864,
|
|
"kl": 7.034093141555786e-05,
|
|
"learning_rate": 7.799999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.29712690226733685,
|
|
"reward_std": 0.6135409390553832,
|
|
"rewards/cosine_scaled_reward": -0.04935322143137455,
|
|
"rewards/format_reward": 0.3958333432674408,
|
|
"step": 39
|
|
},
|
|
{
|
|
"completion_length": 2469.687530517578,
|
|
"epoch": 0.045714285714285714,
|
|
"grad_norm": 0.19995440542697906,
|
|
"kl": 0.00012910738587379456,
|
|
"learning_rate": 8e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.2901697149500251,
|
|
"reward_std": 0.5493389200419188,
|
|
"rewards/cosine_scaled_reward": -0.13616514671593904,
|
|
"rewards/format_reward": 0.5625000074505806,
|
|
"step": 40
|
|
},
|
|
{
|
|
"completion_length": 3059.3333740234375,
|
|
"epoch": 0.046857142857142854,
|
|
"grad_norm": 0.1593063771724701,
|
|
"kl": 5.719810724258423e-05,
|
|
"learning_rate": 8.199999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.08104978175833821,
|
|
"reward_std": 0.537400247529149,
|
|
"rewards/cosine_scaled_reward": -0.21760822273790836,
|
|
"rewards/format_reward": 0.3541666716337204,
|
|
"step": 41
|
|
},
|
|
{
|
|
"completion_length": 2785.979190826416,
|
|
"epoch": 0.048,
|
|
"grad_norm": 0.24719306826591492,
|
|
"kl": 5.231797695159912e-05,
|
|
"learning_rate": 8.399999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.2765159234404564,
|
|
"reward_std": 0.3159199729561806,
|
|
"rewards/cosine_scaled_reward": -0.29450796730816364,
|
|
"rewards/format_reward": 0.3125,
|
|
"step": 42
|
|
},
|
|
{
|
|
"completion_length": 3143.4583435058594,
|
|
"epoch": 0.04914285714285714,
|
|
"grad_norm": 0.17697609961032867,
|
|
"kl": 5.420297384262085e-05,
|
|
"learning_rate": 8.599999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.027741093188524246,
|
|
"reward_std": 0.5976252444088459,
|
|
"rewards/cosine_scaled_reward": -0.14928722102195024,
|
|
"rewards/format_reward": 0.2708333358168602,
|
|
"step": 43
|
|
},
|
|
{
|
|
"completion_length": 2617.750045776367,
|
|
"epoch": 0.05028571428571429,
|
|
"grad_norm": 0.30272603034973145,
|
|
"kl": 0.00017141178250312805,
|
|
"learning_rate": 8.799999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.54763038828969,
|
|
"reward_std": 0.6350081358104944,
|
|
"rewards/cosine_scaled_reward": -0.007434792350977659,
|
|
"rewards/format_reward": 0.5625000055879354,
|
|
"step": 44
|
|
},
|
|
{
|
|
"completion_length": 3381.1458740234375,
|
|
"epoch": 0.05142857142857143,
|
|
"grad_norm": 0.14729151129722595,
|
|
"kl": 5.987286567687988e-05,
|
|
"learning_rate": 9e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.1284105316735804,
|
|
"reward_std": 0.6408776678144932,
|
|
"rewards/cosine_scaled_reward": -0.06079472857527435,
|
|
"rewards/format_reward": 0.25,
|
|
"step": 45
|
|
},
|
|
{
|
|
"completion_length": 3190.1458740234375,
|
|
"epoch": 0.052571428571428575,
|
|
"grad_norm": 0.18984851241111755,
|
|
"kl": 0.0001230388879776001,
|
|
"learning_rate": 9.2e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.2710073352791369,
|
|
"reward_std": 0.37906504422426224,
|
|
"rewards/cosine_scaled_reward": -0.22925367206335068,
|
|
"rewards/format_reward": 0.18750000186264515,
|
|
"step": 46
|
|
},
|
|
{
|
|
"completion_length": 2773.666748046875,
|
|
"epoch": 0.053714285714285714,
|
|
"grad_norm": 0.22614669799804688,
|
|
"kl": 7.295981049537659e-05,
|
|
"learning_rate": 9.399999999999999e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.6267627328634262,
|
|
"reward_std": 1.0455659702420235,
|
|
"rewards/cosine_scaled_reward": 0.0633813701570034,
|
|
"rewards/format_reward": 0.5000000111758709,
|
|
"step": 47
|
|
},
|
|
{
|
|
"completion_length": 2853.6250534057617,
|
|
"epoch": 0.054857142857142854,
|
|
"grad_norm": 0.2049134522676468,
|
|
"kl": 0.00029557570815086365,
|
|
"learning_rate": 9.6e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.1033776430413127,
|
|
"reward_std": 0.891391895711422,
|
|
"rewards/cosine_scaled_reward": -0.11497785244137049,
|
|
"rewards/format_reward": 0.33333334140479565,
|
|
"step": 48
|
|
},
|
|
{
|
|
"completion_length": 2345.6250610351562,
|
|
"epoch": 0.056,
|
|
"grad_norm": 0.21897369623184204,
|
|
"kl": 0.00014978647232055664,
|
|
"learning_rate": 9.8e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.6087969962973148,
|
|
"reward_std": 0.7800232023000717,
|
|
"rewards/cosine_scaled_reward": -0.008101530373096466,
|
|
"rewards/format_reward": 0.6250000037252903,
|
|
"step": 49
|
|
},
|
|
{
|
|
"completion_length": 3047.5000228881836,
|
|
"epoch": 0.05714285714285714,
|
|
"grad_norm": 0.17249740660190582,
|
|
"kl": 0.0002017766237258911,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"reward": 0.43377939984202385,
|
|
"reward_std": 0.7017503455281258,
|
|
"rewards/cosine_scaled_reward": 0.029389701783657074,
|
|
"rewards/format_reward": 0.37500000931322575,
|
|
"step": 50
|
|
},
|
|
{
|
|
"completion_length": 2343.145881652832,
|
|
"epoch": 0.05828571428571429,
|
|
"grad_norm": 0.2304115891456604,
|
|
"kl": 0.0005070865154266357,
|
|
"learning_rate": 9.999890338174275e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.30895326659083366,
|
|
"reward_std": 0.47219153563492,
|
|
"rewards/cosine_scaled_reward": -0.10594002925790846,
|
|
"rewards/format_reward": 0.5208333395421505,
|
|
"step": 51
|
|
},
|
|
{
|
|
"completion_length": 3017.1042098999023,
|
|
"epoch": 0.05942857142857143,
|
|
"grad_norm": 0.22551533579826355,
|
|
"kl": 0.0004254281520843506,
|
|
"learning_rate": 9.999561358041868e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.3134525269269943,
|
|
"reward_std": 0.7696055620908737,
|
|
"rewards/cosine_scaled_reward": -0.020357078406959772,
|
|
"rewards/format_reward": 0.35416667349636555,
|
|
"step": 52
|
|
},
|
|
{
|
|
"completion_length": 2933.2084197998047,
|
|
"epoch": 0.060571428571428575,
|
|
"grad_norm": 0.1710565686225891,
|
|
"kl": 0.00019855797290802002,
|
|
"learning_rate": 9.999013075636804e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.4190669923555106,
|
|
"reward_std": 0.9724426595494151,
|
|
"rewards/cosine_scaled_reward": -0.030049838591367006,
|
|
"rewards/format_reward": 0.4791666753590107,
|
|
"step": 53
|
|
},
|
|
{
|
|
"completion_length": 2728.2083740234375,
|
|
"epoch": 0.061714285714285715,
|
|
"grad_norm": 0.1685902327299118,
|
|
"kl": 8.179806172847748e-05,
|
|
"learning_rate": 9.998245517681593e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.9252486824989319,
|
|
"reward_std": 1.0930374152958393,
|
|
"rewards/cosine_scaled_reward": 0.16054099425673485,
|
|
"rewards/format_reward": 0.6041666772216558,
|
|
"step": 54
|
|
},
|
|
{
|
|
"completion_length": 2987.250030517578,
|
|
"epoch": 0.06285714285714286,
|
|
"grad_norm": 0.1715623438358307,
|
|
"kl": 0.0003556758165359497,
|
|
"learning_rate": 9.997258721585931e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.34154442069120705,
|
|
"reward_std": 0.7232516668736935,
|
|
"rewards/cosine_scaled_reward": -0.03756110556423664,
|
|
"rewards/format_reward": 0.41666667349636555,
|
|
"step": 55
|
|
},
|
|
{
|
|
"completion_length": 3029.250045776367,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.15501734614372253,
|
|
"kl": 0.00012418627738952637,
|
|
"learning_rate": 9.996052735444862e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.25998372526373714,
|
|
"reward_std": 0.6867683958262205,
|
|
"rewards/cosine_scaled_reward": -0.047091471031308174,
|
|
"rewards/format_reward": 0.3541666716337204,
|
|
"step": 56
|
|
},
|
|
{
|
|
"completion_length": 3392.9791870117188,
|
|
"epoch": 0.06514285714285714,
|
|
"grad_norm": 0.1358395218849182,
|
|
"kl": 5.584210157394409e-05,
|
|
"learning_rate": 9.994627618036452e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.13618062436580658,
|
|
"reward_std": 0.6030133794993162,
|
|
"rewards/cosine_scaled_reward": -0.18267363915219903,
|
|
"rewards/format_reward": 0.2291666679084301,
|
|
"step": 57
|
|
},
|
|
{
|
|
"completion_length": 2349.08341217041,
|
|
"epoch": 0.06628571428571428,
|
|
"grad_norm": 0.22305431962013245,
|
|
"kl": 0.0013600699603557587,
|
|
"learning_rate": 9.992983438818915e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5223081162257586,
|
|
"reward_std": 0.9040666744112968,
|
|
"rewards/cosine_scaled_reward": -0.030512610450387,
|
|
"rewards/format_reward": 0.5833333395421505,
|
|
"step": 58
|
|
},
|
|
{
|
|
"completion_length": 2854.125030517578,
|
|
"epoch": 0.06742857142857143,
|
|
"grad_norm": 0.1728586107492447,
|
|
"kl": 7.890164852142334e-05,
|
|
"learning_rate": 9.991120277927223e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.1912415586411953,
|
|
"reward_std": 0.6860835738480091,
|
|
"rewards/cosine_scaled_reward": -0.06062923185527325,
|
|
"rewards/format_reward": 0.31250000558793545,
|
|
"step": 59
|
|
},
|
|
{
|
|
"completion_length": 3058.0208587646484,
|
|
"epoch": 0.06857142857142857,
|
|
"grad_norm": 0.15029877424240112,
|
|
"kl": 0.0001646280288696289,
|
|
"learning_rate": 9.989038226169207e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.1796177178621292,
|
|
"reward_std": 0.5032580140978098,
|
|
"rewards/cosine_scaled_reward": -0.2356421989388764,
|
|
"rewards/format_reward": 0.29166667349636555,
|
|
"step": 60
|
|
},
|
|
{
|
|
"completion_length": 3021.9375915527344,
|
|
"epoch": 0.06971428571428571,
|
|
"grad_norm": 0.18782874941825867,
|
|
"kl": 0.00031859055161476135,
|
|
"learning_rate": 9.98673738502114e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.2405167557299137,
|
|
"reward_std": 0.8873464465141296,
|
|
"rewards/cosine_scaled_reward": -0.11932496633380651,
|
|
"rewards/format_reward": 0.4791666679084301,
|
|
"step": 61
|
|
},
|
|
{
|
|
"completion_length": 2797.437515258789,
|
|
"epoch": 0.07085714285714285,
|
|
"grad_norm": 0.19510804116725922,
|
|
"kl": 0.0028433650732040405,
|
|
"learning_rate": 9.98421786662277e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.6466732956469059,
|
|
"reward_std": 1.1665390692651272,
|
|
"rewards/cosine_scaled_reward": 0.06291996827349067,
|
|
"rewards/format_reward": 0.520833345130086,
|
|
"step": 62
|
|
},
|
|
{
|
|
"completion_length": 2213.6458740234375,
|
|
"epoch": 0.072,
|
|
"grad_norm": 0.21749359369277954,
|
|
"kl": 0.0012581497430801392,
|
|
"learning_rate": 9.981479793771866e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.9055937305092812,
|
|
"reward_std": 0.8001900352537632,
|
|
"rewards/cosine_scaled_reward": 0.08821351453661919,
|
|
"rewards/format_reward": 0.7291666716337204,
|
|
"step": 63
|
|
},
|
|
{
|
|
"completion_length": 2778.3333435058594,
|
|
"epoch": 0.07314285714285715,
|
|
"grad_norm": 0.18261142075061798,
|
|
"kl": 0.0003489851951599121,
|
|
"learning_rate": 9.97852329991824e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.38868435472249985,
|
|
"reward_std": 0.6305454671382904,
|
|
"rewards/cosine_scaled_reward": -0.02440785290673375,
|
|
"rewards/format_reward": 0.4375000074505806,
|
|
"step": 64
|
|
},
|
|
{
|
|
"completion_length": 2673.687530517578,
|
|
"epoch": 0.07428571428571429,
|
|
"grad_norm": 0.1992972046136856,
|
|
"kl": 0.00033433735370635986,
|
|
"learning_rate": 9.975348529157229e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.3928167396225035,
|
|
"reward_std": 0.7381406147032976,
|
|
"rewards/cosine_scaled_reward": -0.07442497462034225,
|
|
"rewards/format_reward": 0.5416666734963655,
|
|
"step": 65
|
|
},
|
|
{
|
|
"completion_length": 2147.3750038146973,
|
|
"epoch": 0.07542857142857143,
|
|
"grad_norm": 0.22494584321975708,
|
|
"kl": 0.0011640042066574097,
|
|
"learning_rate": 9.971955636222684e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.42546659521758556,
|
|
"reward_std": 0.5198768675327301,
|
|
"rewards/cosine_scaled_reward": -0.03726671263575554,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 66
|
|
},
|
|
{
|
|
"completion_length": 3342.7708740234375,
|
|
"epoch": 0.07657142857142857,
|
|
"grad_norm": 0.14088153839111328,
|
|
"kl": 0.00046347081661224365,
|
|
"learning_rate": 9.968344786479415e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.42950472608208656,
|
|
"reward_std": 0.378866882994771,
|
|
"rewards/cosine_scaled_reward": -0.31891903653740883,
|
|
"rewards/format_reward": 0.2083333395421505,
|
|
"step": 67
|
|
},
|
|
{
|
|
"completion_length": 2170.7083854675293,
|
|
"epoch": 0.07771428571428571,
|
|
"grad_norm": 0.29666775465011597,
|
|
"kl": 0.0014674067497253418,
|
|
"learning_rate": 9.964516155915151e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.4676838363520801,
|
|
"reward_std": 0.7767233625054359,
|
|
"rewards/cosine_scaled_reward": -0.03699141927063465,
|
|
"rewards/format_reward": 0.5416666679084301,
|
|
"step": 68
|
|
},
|
|
{
|
|
"completion_length": 2446.1458740234375,
|
|
"epoch": 0.07885714285714286,
|
|
"grad_norm": 0.22960010170936584,
|
|
"kl": 0.0015052556991577148,
|
|
"learning_rate": 9.960469931131936e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.21741511672735214,
|
|
"reward_std": 0.8107622135430574,
|
|
"rewards/cosine_scaled_reward": -0.1412924542091787,
|
|
"rewards/format_reward": 0.500000013038516,
|
|
"step": 69
|
|
},
|
|
{
|
|
"completion_length": 3091.6041717529297,
|
|
"epoch": 0.08,
|
|
"grad_norm": 0.19971789419651031,
|
|
"kl": 0.0011597275733947754,
|
|
"learning_rate": 9.956206309337066e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.12162317335605621,
|
|
"reward_std": 0.5483939237892628,
|
|
"rewards/cosine_scaled_reward": -0.11627176497131586,
|
|
"rewards/format_reward": 0.35416666977107525,
|
|
"step": 70
|
|
},
|
|
{
|
|
"completion_length": 2609.0208587646484,
|
|
"epoch": 0.08114285714285714,
|
|
"grad_norm": 0.23139537870883942,
|
|
"kl": 0.0016418173909187317,
|
|
"learning_rate": 9.951725498333448e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.2501606810837984,
|
|
"reward_std": 0.5798324253410101,
|
|
"rewards/cosine_scaled_reward": -0.0832529878243804,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 71
|
|
},
|
|
{
|
|
"completion_length": 3072.9583740234375,
|
|
"epoch": 0.08228571428571428,
|
|
"grad_norm": 0.22307687997817993,
|
|
"kl": 0.0010285675525665283,
|
|
"learning_rate": 9.947027716509488e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.06439175084233284,
|
|
"reward_std": 0.7740810364484787,
|
|
"rewards/cosine_scaled_reward": -0.16761254286393523,
|
|
"rewards/format_reward": 0.2708333395421505,
|
|
"step": 72
|
|
},
|
|
{
|
|
"completion_length": 3455.875030517578,
|
|
"epoch": 0.08342857142857144,
|
|
"grad_norm": 0.150093212723732,
|
|
"kl": 0.00021230429410934448,
|
|
"learning_rate": 9.942113192828444e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.021114151924848557,
|
|
"reward_std": 0.7219380252063274,
|
|
"rewards/cosine_scaled_reward": -0.10430707037448883,
|
|
"rewards/format_reward": 0.18750000558793545,
|
|
"step": 73
|
|
},
|
|
{
|
|
"completion_length": 3296.250045776367,
|
|
"epoch": 0.08457142857142858,
|
|
"grad_norm": 0.17310041189193726,
|
|
"kl": 0.0007840991020202637,
|
|
"learning_rate": 9.93698216681727e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.0612453930079937,
|
|
"reward_std": 0.5310056991875172,
|
|
"rewards/cosine_scaled_reward": -0.14520604407880455,
|
|
"rewards/format_reward": 0.22916666977107525,
|
|
"step": 74
|
|
},
|
|
{
|
|
"completion_length": 3076.0833740234375,
|
|
"epoch": 0.08571428571428572,
|
|
"grad_norm": 0.14123524725437164,
|
|
"kl": 0.0008448450826108456,
|
|
"learning_rate": 9.931634888554935e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.21019641309976578,
|
|
"reward_std": 0.4503815546631813,
|
|
"rewards/cosine_scaled_reward": -0.05115179833956063,
|
|
"rewards/format_reward": 0.31250000186264515,
|
|
"step": 75
|
|
},
|
|
{
|
|
"completion_length": 2814.2500228881836,
|
|
"epoch": 0.08685714285714285,
|
|
"grad_norm": 0.20728114247322083,
|
|
"kl": 0.0001634061336517334,
|
|
"learning_rate": 9.926071618660237e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.06560860201716423,
|
|
"reward_std": 0.5740169659256935,
|
|
"rewards/cosine_scaled_reward": -0.17552903201431036,
|
|
"rewards/format_reward": 0.41666666977107525,
|
|
"step": 76
|
|
},
|
|
{
|
|
"completion_length": 3305.1041870117188,
|
|
"epoch": 0.088,
|
|
"grad_norm": 0.14447131752967834,
|
|
"kl": 0.0003515356220304966,
|
|
"learning_rate": 9.9202926282791e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.06417501904070377,
|
|
"reward_std": 0.5662869866937399,
|
|
"rewards/cosine_scaled_reward": -0.15708751417696476,
|
|
"rewards/format_reward": 0.25000000186264515,
|
|
"step": 77
|
|
},
|
|
{
|
|
"completion_length": 3341.875,
|
|
"epoch": 0.08914285714285715,
|
|
"grad_norm": 0.14708709716796875,
|
|
"kl": 0.00022222846746444702,
|
|
"learning_rate": 9.91429819907136e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.07691416330635548,
|
|
"reward_std": 0.7588405385613441,
|
|
"rewards/cosine_scaled_reward": -0.09695957787334919,
|
|
"rewards/format_reward": 0.27083334140479565,
|
|
"step": 78
|
|
},
|
|
{
|
|
"completion_length": 2390.979179382324,
|
|
"epoch": 0.09028571428571429,
|
|
"grad_norm": 0.2777462303638458,
|
|
"kl": 0.0015588998794555664,
|
|
"learning_rate": 9.908088623197048e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.19228350277990103,
|
|
"reward_std": 0.7328854473307729,
|
|
"rewards/cosine_scaled_reward": -0.1746915839612484,
|
|
"rewards/format_reward": 0.5416666734963655,
|
|
"step": 79
|
|
},
|
|
{
|
|
"completion_length": 3182.437530517578,
|
|
"epoch": 0.09142857142857143,
|
|
"grad_norm": 0.18599167466163635,
|
|
"kl": 0.0007250010967254639,
|
|
"learning_rate": 9.901664203302124e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.44475090876221657,
|
|
"reward_std": 0.6929970532655716,
|
|
"rewards/cosine_scaled_reward": 0.024458803236484528,
|
|
"rewards/format_reward": 0.3958333358168602,
|
|
"step": 80
|
|
},
|
|
{
|
|
"completion_length": 2998.7500534057617,
|
|
"epoch": 0.09257142857142857,
|
|
"grad_norm": 0.23842793703079224,
|
|
"kl": 0.0027131736278533936,
|
|
"learning_rate": 9.895025252503755e-07,
|
|
"loss": 0.0001,
|
|
"reward": -0.0324825718998909,
|
|
"reward_std": 0.6861623711884022,
|
|
"rewards/cosine_scaled_reward": -0.16207461850717664,
|
|
"rewards/format_reward": 0.29166667349636555,
|
|
"step": 81
|
|
},
|
|
{
|
|
"completion_length": 2781.791717529297,
|
|
"epoch": 0.09371428571428571,
|
|
"grad_norm": 0.19417299330234528,
|
|
"kl": 0.0011616647243499756,
|
|
"learning_rate": 9.888172094375033e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.3426995016634464,
|
|
"reward_std": 0.756939671933651,
|
|
"rewards/cosine_scaled_reward": -0.03698359243571758,
|
|
"rewards/format_reward": 0.41666666977107525,
|
|
"step": 82
|
|
},
|
|
{
|
|
"completion_length": 2735.4791946411133,
|
|
"epoch": 0.09485714285714286,
|
|
"grad_norm": 0.23736146092414856,
|
|
"kl": 0.001489013433456421,
|
|
"learning_rate": 9.881105062929221e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.022416603518649936,
|
|
"reward_std": 0.33409483451396227,
|
|
"rewards/cosine_scaled_reward": -0.14504170510917902,
|
|
"rewards/format_reward": 0.3125,
|
|
"step": 83
|
|
},
|
|
{
|
|
"completion_length": 3006.7708587646484,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.16109317541122437,
|
|
"kl": 0.0031517744064331055,
|
|
"learning_rate": 9.873824502603459e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.2841636799275875,
|
|
"reward_std": 0.9676609244197607,
|
|
"rewards/cosine_scaled_reward": -0.055834827944636345,
|
|
"rewards/format_reward": 0.39583334140479565,
|
|
"step": 84
|
|
},
|
|
{
|
|
"completion_length": 2945.0416870117188,
|
|
"epoch": 0.09714285714285714,
|
|
"grad_norm": 0.14022095501422882,
|
|
"kl": 0.00030300021171569824,
|
|
"learning_rate": 9.866330768241983e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.3169367350637913,
|
|
"reward_std": 0.8734622374176979,
|
|
"rewards/cosine_scaled_reward": -0.08111499063670635,
|
|
"rewards/format_reward": 0.47916667722165585,
|
|
"step": 85
|
|
},
|
|
{
|
|
"completion_length": 2773.3333740234375,
|
|
"epoch": 0.09828571428571428,
|
|
"grad_norm": 0.20840014517307281,
|
|
"kl": 0.0014702677726745605,
|
|
"learning_rate": 9.85862422507884e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.3505563363432884,
|
|
"reward_std": 0.50963762588799,
|
|
"rewards/cosine_scaled_reward": -0.06430518021807075,
|
|
"rewards/format_reward": 0.47916666977107525,
|
|
"step": 86
|
|
},
|
|
{
|
|
"completion_length": 2771.8333740234375,
|
|
"epoch": 0.09942857142857142,
|
|
"grad_norm": 0.2091607004404068,
|
|
"kl": 0.0014937222003936768,
|
|
"learning_rate": 9.850705248720068e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.21895797760225832,
|
|
"reward_std": 0.8198871687054634,
|
|
"rewards/cosine_scaled_reward": -0.11968768760561943,
|
|
"rewards/format_reward": 0.45833334513008595,
|
|
"step": 87
|
|
},
|
|
{
|
|
"completion_length": 2588.979248046875,
|
|
"epoch": 0.10057142857142858,
|
|
"grad_norm": 0.20578493177890778,
|
|
"kl": 0.003347158432006836,
|
|
"learning_rate": 9.8425742251254e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.8585769310593605,
|
|
"reward_std": 1.0248939394950867,
|
|
"rewards/cosine_scaled_reward": 0.11678845560527407,
|
|
"rewards/format_reward": 0.6250000093132257,
|
|
"step": 88
|
|
},
|
|
{
|
|
"completion_length": 3318.6459045410156,
|
|
"epoch": 0.10171428571428572,
|
|
"grad_norm": 0.198954775929451,
|
|
"kl": 0.0015476346015930176,
|
|
"learning_rate": 9.83423155058946e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.03354086447507143,
|
|
"reward_std": 0.9196067973971367,
|
|
"rewards/cosine_scaled_reward": -0.10822956170886755,
|
|
"rewards/format_reward": 0.2500000074505806,
|
|
"step": 89
|
|
},
|
|
{
|
|
"completion_length": 2533.125045776367,
|
|
"epoch": 0.10285714285714286,
|
|
"grad_norm": 0.30491241812705994,
|
|
"kl": 0.0027846097946166992,
|
|
"learning_rate": 9.825677631722435e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.04410627228207886,
|
|
"reward_std": 0.5515282340347767,
|
|
"rewards/cosine_scaled_reward": -0.21753019373863935,
|
|
"rewards/format_reward": 0.47916667722165585,
|
|
"step": 90
|
|
},
|
|
{
|
|
"completion_length": 3007.5000610351562,
|
|
"epoch": 0.104,
|
|
"grad_norm": 0.1628594547510147,
|
|
"kl": 0.0011910051107406616,
|
|
"learning_rate": 9.816912885430258e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.4382549002766609,
|
|
"reward_std": 0.8723863288760185,
|
|
"rewards/cosine_scaled_reward": 0.010794118046760559,
|
|
"rewards/format_reward": 0.416666679084301,
|
|
"step": 91
|
|
},
|
|
{
|
|
"completion_length": 2512.5833587646484,
|
|
"epoch": 0.10514285714285715,
|
|
"grad_norm": 0.19360074400901794,
|
|
"kl": 0.0010070949792861938,
|
|
"learning_rate": 9.807937738894303e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.32041475689038634,
|
|
"reward_std": 0.6855251621454954,
|
|
"rewards/cosine_scaled_reward": -0.11062596645206213,
|
|
"rewards/format_reward": 0.5416666697710752,
|
|
"step": 92
|
|
},
|
|
{
|
|
"completion_length": 3344.4166870117188,
|
|
"epoch": 0.10628571428571429,
|
|
"grad_norm": 0.1997271329164505,
|
|
"kl": 0.0018374919891357422,
|
|
"learning_rate": 9.798752629550546e-07,
|
|
"loss": 0.0001,
|
|
"reward": -0.34033770859241486,
|
|
"reward_std": 0.5492488071322441,
|
|
"rewards/cosine_scaled_reward": -0.2535021901130676,
|
|
"rewards/format_reward": 0.1666666716337204,
|
|
"step": 93
|
|
},
|
|
{
|
|
"completion_length": 3007.375015258789,
|
|
"epoch": 0.10742857142857143,
|
|
"grad_norm": 0.18923112750053406,
|
|
"kl": 0.0023823007941246033,
|
|
"learning_rate": 9.78935800506826e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.13416577805764973,
|
|
"reward_std": 0.4046405693516135,
|
|
"rewards/cosine_scaled_reward": -0.06833377946168184,
|
|
"rewards/format_reward": 0.27083333395421505,
|
|
"step": 94
|
|
},
|
|
{
|
|
"completion_length": 3444.0416870117188,
|
|
"epoch": 0.10857142857142857,
|
|
"grad_norm": 0.14268328249454498,
|
|
"kl": 0.0009976625442504883,
|
|
"learning_rate": 9.779754323328192e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.09058984462171793,
|
|
"reward_std": 0.7453176453709602,
|
|
"rewards/cosine_scaled_reward": -0.15987825905904174,
|
|
"rewards/format_reward": 0.2291666716337204,
|
|
"step": 95
|
|
},
|
|
{
|
|
"completion_length": 2678.1875228881836,
|
|
"epoch": 0.10971428571428571,
|
|
"grad_norm": 0.19071771204471588,
|
|
"kl": 0.0018395986407995224,
|
|
"learning_rate": 9.769942052400235e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.348697304725647,
|
|
"reward_std": 0.6102097555994987,
|
|
"rewards/cosine_scaled_reward": -0.04440134949982166,
|
|
"rewards/format_reward": 0.43750000186264515,
|
|
"step": 96
|
|
},
|
|
{
|
|
"completion_length": 2990.000030517578,
|
|
"epoch": 0.11085714285714286,
|
|
"grad_norm": 0.20323379337787628,
|
|
"kl": 0.001025363802909851,
|
|
"learning_rate": 9.759921670520634e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.41594838351011276,
|
|
"reward_std": 0.7984455898404121,
|
|
"rewards/cosine_scaled_reward": 0.02047417126595974,
|
|
"rewards/format_reward": 0.3750000111758709,
|
|
"step": 97
|
|
},
|
|
{
|
|
"completion_length": 2679.2083587646484,
|
|
"epoch": 0.112,
|
|
"grad_norm": 0.19446168839931488,
|
|
"kl": 0.0004923343658447266,
|
|
"learning_rate": 9.749693666068663e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.38869317155331373,
|
|
"reward_std": 0.564236119389534,
|
|
"rewards/cosine_scaled_reward": -0.07648676075041294,
|
|
"rewards/format_reward": 0.541666679084301,
|
|
"step": 98
|
|
},
|
|
{
|
|
"completion_length": 2801.7708435058594,
|
|
"epoch": 0.11314285714285714,
|
|
"grad_norm": 0.20274780690670013,
|
|
"kl": 0.0009057521820068359,
|
|
"learning_rate": 9.739258537542835e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.04034796729683876,
|
|
"reward_std": 0.43001002445816994,
|
|
"rewards/cosine_scaled_reward": -0.1152426817570813,
|
|
"rewards/format_reward": 0.27083333395421505,
|
|
"step": 99
|
|
},
|
|
{
|
|
"completion_length": 2591.125030517578,
|
|
"epoch": 0.11428571428571428,
|
|
"grad_norm": 0.19084717333316803,
|
|
"kl": 0.001154184341430664,
|
|
"learning_rate": 9.728616793536587e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.6981350090354681,
|
|
"reward_std": 0.9793934002518654,
|
|
"rewards/cosine_scaled_reward": 0.06781747564673424,
|
|
"rewards/format_reward": 0.5625000111758709,
|
|
"step": 100
|
|
},
|
|
{
|
|
"completion_length": 2723.312530517578,
|
|
"epoch": 0.11542857142857142,
|
|
"grad_norm": 0.22346965968608856,
|
|
"kl": 0.001171112060546875,
|
|
"learning_rate": 9.717768952713511e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.2876786937122233,
|
|
"reward_std": 0.4630406089127064,
|
|
"rewards/cosine_scaled_reward": -0.05407731421291828,
|
|
"rewards/format_reward": 0.39583333395421505,
|
|
"step": 101
|
|
},
|
|
{
|
|
"completion_length": 2054.187545776367,
|
|
"epoch": 0.11657142857142858,
|
|
"grad_norm": 0.23816390335559845,
|
|
"kl": 0.002776503562927246,
|
|
"learning_rate": 9.706715543782064e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.6102155670523643,
|
|
"reward_std": 0.7314882390201092,
|
|
"rewards/cosine_scaled_reward": -0.059475560672581196,
|
|
"rewards/format_reward": 0.7291666865348816,
|
|
"step": 102
|
|
},
|
|
{
|
|
"completion_length": 2666.833396911621,
|
|
"epoch": 0.11771428571428572,
|
|
"grad_norm": 0.2530282139778137,
|
|
"kl": 0.0017848014831542969,
|
|
"learning_rate": 9.695457105469804e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.2429911457002163,
|
|
"reward_std": 0.6969506703317165,
|
|
"rewards/cosine_scaled_reward": -0.11808777041733265,
|
|
"rewards/format_reward": 0.479166679084301,
|
|
"step": 103
|
|
},
|
|
{
|
|
"completion_length": 2740.9791870117188,
|
|
"epoch": 0.11885714285714286,
|
|
"grad_norm": 0.20160914957523346,
|
|
"kl": 0.0023827552795410156,
|
|
"learning_rate": 9.683994186497132e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.20196924358606339,
|
|
"reward_std": 0.5999510791152716,
|
|
"rewards/cosine_scaled_reward": -0.0865153931081295,
|
|
"rewards/format_reward": 0.3750000037252903,
|
|
"step": 104
|
|
},
|
|
{
|
|
"completion_length": 2462.1041870117188,
|
|
"epoch": 0.12,
|
|
"grad_norm": 0.24578723311424255,
|
|
"kl": 0.0011974573135375977,
|
|
"learning_rate": 9.672327345550543e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.29713789327070117,
|
|
"reward_std": 0.8025108277797699,
|
|
"rewards/cosine_scaled_reward": -0.080597716383636,
|
|
"rewards/format_reward": 0.45833334140479565,
|
|
"step": 105
|
|
},
|
|
{
|
|
"completion_length": 2199.854179382324,
|
|
"epoch": 0.12114285714285715,
|
|
"grad_norm": 0.19964368641376495,
|
|
"kl": 0.002254962921142578,
|
|
"learning_rate": 9.66045715125541e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.8605472417548299,
|
|
"reward_std": 0.7241987711749971,
|
|
"rewards/cosine_scaled_reward": 0.11777362413704395,
|
|
"rewards/format_reward": 0.6250000037252903,
|
|
"step": 106
|
|
},
|
|
{
|
|
"completion_length": 2900.125030517578,
|
|
"epoch": 0.12228571428571429,
|
|
"grad_norm": 0.1984090954065323,
|
|
"kl": 0.0013356208801269531,
|
|
"learning_rate": 9.648384182148252e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.3114801459014416,
|
|
"reward_std": 0.5947921872138977,
|
|
"rewards/cosine_scaled_reward": -0.08384328025022114,
|
|
"rewards/format_reward": 0.47916666977107525,
|
|
"step": 107
|
|
},
|
|
{
|
|
"completion_length": 2570.6041870117188,
|
|
"epoch": 0.12342857142857143,
|
|
"grad_norm": 0.2010282427072525,
|
|
"kl": 0.0011126399040222168,
|
|
"learning_rate": 9.636109026648554e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.6098845191299915,
|
|
"reward_std": 0.8683119043707848,
|
|
"rewards/cosine_scaled_reward": 0.02369226049631834,
|
|
"rewards/format_reward": 0.5625000074505806,
|
|
"step": 108
|
|
},
|
|
{
|
|
"completion_length": 2960.645896911621,
|
|
"epoch": 0.12457142857142857,
|
|
"grad_norm": 0.19048207998275757,
|
|
"kl": 0.0006784200668334961,
|
|
"learning_rate": 9.623632283030077e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.38045269437134266,
|
|
"reward_std": 0.6336373277008533,
|
|
"rewards/cosine_scaled_reward": 0.002726326696574688,
|
|
"rewards/format_reward": 0.3750000037252903,
|
|
"step": 109
|
|
},
|
|
{
|
|
"completion_length": 2726.1042098999023,
|
|
"epoch": 0.12571428571428572,
|
|
"grad_norm": 0.20842185616493225,
|
|
"kl": 0.0014406442642211914,
|
|
"learning_rate": 9.610954559391704e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.2915392220020294,
|
|
"reward_std": 0.891454428434372,
|
|
"rewards/cosine_scaled_reward": -0.10423038713634014,
|
|
"rewards/format_reward": 0.5000000093132257,
|
|
"step": 110
|
|
},
|
|
{
|
|
"completion_length": 2772.9375534057617,
|
|
"epoch": 0.12685714285714286,
|
|
"grad_norm": 0.20696447789669037,
|
|
"kl": 0.0015645027160644531,
|
|
"learning_rate": 9.598076473627796e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.20833251508884132,
|
|
"reward_std": 0.7267865724861622,
|
|
"rewards/cosine_scaled_reward": -0.12500040791928768,
|
|
"rewards/format_reward": 0.4583333395421505,
|
|
"step": 111
|
|
},
|
|
{
|
|
"completion_length": 2978.0208892822266,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.16811113059520721,
|
|
"kl": 0.000897526741027832,
|
|
"learning_rate": 9.58499865339809e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.3418322764337063,
|
|
"reward_std": 0.65217125415802,
|
|
"rewards/cosine_scaled_reward": -0.05825053807348013,
|
|
"rewards/format_reward": 0.4583333358168602,
|
|
"step": 112
|
|
},
|
|
{
|
|
"completion_length": 2559.7083740234375,
|
|
"epoch": 0.12914285714285714,
|
|
"grad_norm": 0.20631085336208344,
|
|
"kl": 0.0014824867248535156,
|
|
"learning_rate": 9.571721736097088e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.43655770644545555,
|
|
"reward_std": 0.7786016501486301,
|
|
"rewards/cosine_scaled_reward": -0.03172116680070758,
|
|
"rewards/format_reward": 0.5000000093132257,
|
|
"step": 113
|
|
},
|
|
{
|
|
"completion_length": 2495.7917137145996,
|
|
"epoch": 0.13028571428571428,
|
|
"grad_norm": 0.22188131511211395,
|
|
"kl": 0.0031752586364746094,
|
|
"learning_rate": 9.55824636882301e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.20582257118076086,
|
|
"reward_std": 0.7198268510401249,
|
|
"rewards/cosine_scaled_reward": -0.18875539442524314,
|
|
"rewards/format_reward": 0.5833333469927311,
|
|
"step": 114
|
|
},
|
|
{
|
|
"completion_length": 2754.6667098999023,
|
|
"epoch": 0.13142857142857142,
|
|
"grad_norm": 0.19496048986911774,
|
|
"kl": 0.0033437013626098633,
|
|
"learning_rate": 9.54457320834625e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.02730092965066433,
|
|
"reward_std": 0.5944252610206604,
|
|
"rewards/cosine_scaled_reward": -0.2050995440222323,
|
|
"rewards/format_reward": 0.4375000074505806,
|
|
"step": 115
|
|
},
|
|
{
|
|
"completion_length": 3369.375030517578,
|
|
"epoch": 0.13257142857142856,
|
|
"grad_norm": 0.1768704503774643,
|
|
"kl": 0.001748800277709961,
|
|
"learning_rate": 9.530702921077358e-07,
|
|
"loss": 0.0001,
|
|
"reward": -0.17860433738678694,
|
|
"reward_std": 0.7088249325752258,
|
|
"rewards/cosine_scaled_reward": -0.15180217241868377,
|
|
"rewards/format_reward": 0.1250000037252903,
|
|
"step": 116
|
|
},
|
|
{
|
|
"completion_length": 3029.687530517578,
|
|
"epoch": 0.1337142857142857,
|
|
"grad_norm": 0.22546739876270294,
|
|
"kl": 0.0027513504028320312,
|
|
"learning_rate": 9.516636183034564e-07,
|
|
"loss": 0.0001,
|
|
"reward": -0.09248590935021639,
|
|
"reward_std": 0.6045599468052387,
|
|
"rewards/cosine_scaled_reward": -0.19207628909498453,
|
|
"rewards/format_reward": 0.2916666753590107,
|
|
"step": 117
|
|
},
|
|
{
|
|
"completion_length": 2870.8958740234375,
|
|
"epoch": 0.13485714285714287,
|
|
"grad_norm": 0.16772252321243286,
|
|
"kl": 0.0015900135040283203,
|
|
"learning_rate": 9.502373679810839e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.46789007633924484,
|
|
"reward_std": 0.8386504454538226,
|
|
"rewards/cosine_scaled_reward": 0.015195020474493504,
|
|
"rewards/format_reward": 0.4375000074505806,
|
|
"step": 118
|
|
},
|
|
{
|
|
"completion_length": 2562.3125610351562,
|
|
"epoch": 0.136,
|
|
"grad_norm": 0.21363668143749237,
|
|
"kl": 0.004290223121643066,
|
|
"learning_rate": 9.487916106540465e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.7369925267994404,
|
|
"reward_std": 0.7727241404354572,
|
|
"rewards/cosine_scaled_reward": 0.0768295917659998,
|
|
"rewards/format_reward": 0.5833333395421505,
|
|
"step": 119
|
|
},
|
|
{
|
|
"completion_length": 2289.1667289733887,
|
|
"epoch": 0.13714285714285715,
|
|
"grad_norm": 0.2960655391216278,
|
|
"kl": 0.0022792816162109375,
|
|
"learning_rate": 9.473264167865171e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.513806963339448,
|
|
"reward_std": 0.7356006652116776,
|
|
"rewards/cosine_scaled_reward": -0.045179841690696776,
|
|
"rewards/format_reward": 0.6041666772216558,
|
|
"step": 120
|
|
},
|
|
{
|
|
"completion_length": 1657.1041870117188,
|
|
"epoch": 0.1382857142857143,
|
|
"grad_norm": 0.27528315782546997,
|
|
"kl": 0.00405430793762207,
|
|
"learning_rate": 9.458418577899774e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.7713621072471142,
|
|
"reward_std": 0.6179003790020943,
|
|
"rewards/cosine_scaled_reward": -0.010152293369174004,
|
|
"rewards/format_reward": 0.7916666716337204,
|
|
"step": 121
|
|
},
|
|
{
|
|
"completion_length": 2898.375030517578,
|
|
"epoch": 0.13942857142857143,
|
|
"grad_norm": 0.20366472005844116,
|
|
"kl": 0.0014312267303466797,
|
|
"learning_rate": 9.443380060197385e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.47324386797845364,
|
|
"reward_std": 0.8266710359603167,
|
|
"rewards/cosine_scaled_reward": 0.02828860469162464,
|
|
"rewards/format_reward": 0.4166666753590107,
|
|
"step": 122
|
|
},
|
|
{
|
|
"completion_length": 2754.791717529297,
|
|
"epoch": 0.14057142857142857,
|
|
"grad_norm": 0.17461104691028595,
|
|
"kl": 0.0015136003494262695,
|
|
"learning_rate": 9.428149347714143e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.28382661822251976,
|
|
"reward_std": 0.5956688185688108,
|
|
"rewards/cosine_scaled_reward": -0.12892001681029797,
|
|
"rewards/format_reward": 0.541666679084301,
|
|
"step": 123
|
|
},
|
|
{
|
|
"completion_length": 2220.0833587646484,
|
|
"epoch": 0.1417142857142857,
|
|
"grad_norm": 0.2452717274427414,
|
|
"kl": 0.0047130584716796875,
|
|
"learning_rate": 9.412727182773486e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5766889279475436,
|
|
"reward_std": 0.7685924731194973,
|
|
"rewards/cosine_scaled_reward": -0.03457220923155546,
|
|
"rewards/format_reward": 0.6458333414047956,
|
|
"step": 124
|
|
},
|
|
{
|
|
"completion_length": 2875.895866394043,
|
|
"epoch": 0.14285714285714285,
|
|
"grad_norm": 0.17229042947292328,
|
|
"kl": 0.0020313262939453125,
|
|
"learning_rate": 9.397114317029974e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.23444998264312744,
|
|
"reward_std": 0.5397426411509514,
|
|
"rewards/cosine_scaled_reward": -0.028608346357941628,
|
|
"rewards/format_reward": 0.2916666679084301,
|
|
"step": 125
|
|
},
|
|
{
|
|
"completion_length": 2846.5833587646484,
|
|
"epoch": 0.144,
|
|
"grad_norm": 0.1771528422832489,
|
|
"kl": 0.0011348724365234375,
|
|
"learning_rate": 9.381311511432658e-07,
|
|
"loss": 0.0,
|
|
"reward": 0.2827608957886696,
|
|
"reward_std": 0.6769377700984478,
|
|
"rewards/cosine_scaled_reward": -0.11903620883822441,
|
|
"rewards/format_reward": 0.5208333414047956,
|
|
"step": 126
|
|
},
|
|
{
|
|
"completion_length": 3026.437545776367,
|
|
"epoch": 0.14514285714285713,
|
|
"grad_norm": 0.23673483729362488,
|
|
"kl": 0.0022172927856445312,
|
|
"learning_rate": 9.36531953618799e-07,
|
|
"loss": 0.0001,
|
|
"reward": -0.08590636402368546,
|
|
"reward_std": 0.6378746330738068,
|
|
"rewards/cosine_scaled_reward": -0.18878651410341263,
|
|
"rewards/format_reward": 0.2916666716337204,
|
|
"step": 127
|
|
},
|
|
{
|
|
"completion_length": 2956.1458892822266,
|
|
"epoch": 0.1462857142857143,
|
|
"grad_norm": 0.1844361275434494,
|
|
"kl": 0.002949953079223633,
|
|
"learning_rate": 9.34913917072228e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.39643347449600697,
|
|
"reward_std": 0.7595263011753559,
|
|
"rewards/cosine_scaled_reward": 0.010716758668422699,
|
|
"rewards/format_reward": 0.37500000186264515,
|
|
"step": 128
|
|
},
|
|
{
|
|
"completion_length": 3379.1666870117188,
|
|
"epoch": 0.14742857142857144,
|
|
"grad_norm": 0.40960681438446045,
|
|
"kl": 0.02291560173034668,
|
|
"learning_rate": 9.332771203643714e-07,
|
|
"loss": 0.0009,
|
|
"reward": -0.011747716460376978,
|
|
"reward_std": 0.9172870293259621,
|
|
"rewards/cosine_scaled_reward": -0.09962387196719646,
|
|
"rewards/format_reward": 0.1875000037252903,
|
|
"step": 129
|
|
},
|
|
{
|
|
"completion_length": 2683.5416870117188,
|
|
"epoch": 0.14857142857142858,
|
|
"grad_norm": 0.18997545540332794,
|
|
"kl": 0.0017414093017578125,
|
|
"learning_rate": 9.316216432703916e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.2956298217177391,
|
|
"reward_std": 0.7938947193324566,
|
|
"rewards/cosine_scaled_reward": -0.05010176869109273,
|
|
"rewards/format_reward": 0.39583334140479565,
|
|
"step": 130
|
|
},
|
|
{
|
|
"completion_length": 2938.958396911621,
|
|
"epoch": 0.14971428571428572,
|
|
"grad_norm": 0.1829785704612732,
|
|
"kl": 0.004005908966064453,
|
|
"learning_rate": 9.299475664759068e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.6047782748937607,
|
|
"reward_std": 0.8474699303042144,
|
|
"rewards/cosine_scaled_reward": 0.10447247140109539,
|
|
"rewards/format_reward": 0.39583334140479565,
|
|
"step": 131
|
|
},
|
|
{
|
|
"completion_length": 2620.854179382324,
|
|
"epoch": 0.15085714285714286,
|
|
"grad_norm": 0.17881910502910614,
|
|
"kl": 0.0018241405487060547,
|
|
"learning_rate": 9.282549715730579e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.3893936946988106,
|
|
"reward_std": 0.687925798818469,
|
|
"rewards/cosine_scaled_reward": -0.013636493356898427,
|
|
"rewards/format_reward": 0.41666667349636555,
|
|
"step": 132
|
|
},
|
|
{
|
|
"completion_length": 3095.3958740234375,
|
|
"epoch": 0.152,
|
|
"grad_norm": 0.20941434800624847,
|
|
"kl": 0.0031735897064208984,
|
|
"learning_rate": 9.265439410565328e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.05856896564364433,
|
|
"reward_std": 0.7605378944426775,
|
|
"rewards/cosine_scaled_reward": -0.12696552043780684,
|
|
"rewards/format_reward": 0.31250000558793545,
|
|
"step": 133
|
|
},
|
|
{
|
|
"completion_length": 2441.6458740234375,
|
|
"epoch": 0.15314285714285714,
|
|
"grad_norm": 0.18976929783821106,
|
|
"kl": 0.0040435791015625,
|
|
"learning_rate": 9.248145583195447e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5501680783927441,
|
|
"reward_std": 0.618685107678175,
|
|
"rewards/cosine_scaled_reward": -0.026999298483133316,
|
|
"rewards/format_reward": 0.6041666697710752,
|
|
"step": 134
|
|
},
|
|
{
|
|
"completion_length": 2050.6041870117188,
|
|
"epoch": 0.15428571428571428,
|
|
"grad_norm": 0.2555489242076874,
|
|
"kl": 0.004626750946044922,
|
|
"learning_rate": 9.230669076497687e-07,
|
|
"loss": 0.0002,
|
|
"reward": 1.1215206906199455,
|
|
"reward_std": 0.8544092262163758,
|
|
"rewards/cosine_scaled_reward": 0.22742698714137077,
|
|
"rewards/format_reward": 0.6666666772216558,
|
|
"step": 135
|
|
},
|
|
{
|
|
"completion_length": 2620.791717529297,
|
|
"epoch": 0.15542857142857142,
|
|
"grad_norm": 0.18313182890415192,
|
|
"kl": 0.003920078277587891,
|
|
"learning_rate": 9.213010742252327e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5208159852772951,
|
|
"reward_std": 0.9554536268115044,
|
|
"rewards/cosine_scaled_reward": 0.010407987982034683,
|
|
"rewards/format_reward": 0.5000000111758709,
|
|
"step": 136
|
|
},
|
|
{
|
|
"completion_length": 2811.666702270508,
|
|
"epoch": 0.15657142857142858,
|
|
"grad_norm": 0.17304006218910217,
|
|
"kl": 0.0026001930236816406,
|
|
"learning_rate": 9.195171441101668e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.03694445453584194,
|
|
"reward_std": 0.6093163676559925,
|
|
"rewards/cosine_scaled_reward": -0.15861110761761665,
|
|
"rewards/format_reward": 0.35416666977107525,
|
|
"step": 137
|
|
},
|
|
{
|
|
"completion_length": 2228.2500762939453,
|
|
"epoch": 0.15771428571428572,
|
|
"grad_norm": 0.19729673862457275,
|
|
"kl": 0.0018553733825683594,
|
|
"learning_rate": 9.177152042508077e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.4917922643944621,
|
|
"reward_std": 0.5503780655562878,
|
|
"rewards/cosine_scaled_reward": -0.08743719174526632,
|
|
"rewards/format_reward": 0.6666666734963655,
|
|
"step": 138
|
|
},
|
|
{
|
|
"completion_length": 3121.8750915527344,
|
|
"epoch": 0.15885714285714286,
|
|
"grad_norm": 0.17099516093730927,
|
|
"kl": 0.0035991668701171875,
|
|
"learning_rate": 9.158953424711624e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.20771604776382446,
|
|
"reward_std": 0.8068908657878637,
|
|
"rewards/cosine_scaled_reward": -0.08364198234630749,
|
|
"rewards/format_reward": 0.37500000931322575,
|
|
"step": 139
|
|
},
|
|
{
|
|
"completion_length": 2868.979217529297,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.22809311747550964,
|
|
"kl": 0.0057888031005859375,
|
|
"learning_rate": 9.140576474687263e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.10171525273472071,
|
|
"reward_std": 0.42831680551171303,
|
|
"rewards/cosine_scaled_reward": -0.11580904200673103,
|
|
"rewards/format_reward": 0.33333333767950535,
|
|
"step": 140
|
|
},
|
|
{
|
|
"completion_length": 2534.354263305664,
|
|
"epoch": 0.16114285714285714,
|
|
"grad_norm": 0.1993127316236496,
|
|
"kl": 0.0041599273681640625,
|
|
"learning_rate": 9.122022088101613e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5355201922357082,
|
|
"reward_std": 0.7652903571724892,
|
|
"rewards/cosine_scaled_reward": -0.04473992623388767,
|
|
"rewards/format_reward": 0.6250000149011612,
|
|
"step": 141
|
|
},
|
|
{
|
|
"completion_length": 2745.250015258789,
|
|
"epoch": 0.16228571428571428,
|
|
"grad_norm": 0.20162560045719147,
|
|
"kl": 0.0030961036682128906,
|
|
"learning_rate": 9.103291169269299e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.3946248684078455,
|
|
"reward_std": 0.7829636707901955,
|
|
"rewards/cosine_scaled_reward": -0.09435425186529756,
|
|
"rewards/format_reward": 0.583333345130086,
|
|
"step": 142
|
|
},
|
|
{
|
|
"completion_length": 2417.6250610351562,
|
|
"epoch": 0.16342857142857142,
|
|
"grad_norm": 0.4018979072570801,
|
|
"kl": 0.004683494567871094,
|
|
"learning_rate": 9.084384631108882e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.19592272117733955,
|
|
"reward_std": 0.5982137080281973,
|
|
"rewards/cosine_scaled_reward": -0.1832886370830238,
|
|
"rewards/format_reward": 0.5625000111758709,
|
|
"step": 143
|
|
},
|
|
{
|
|
"completion_length": 2747.145881652832,
|
|
"epoch": 0.16457142857142856,
|
|
"grad_norm": 0.2238093614578247,
|
|
"kl": 0.0034952163696289062,
|
|
"learning_rate": 9.065303395098358e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.3479926884174347,
|
|
"reward_std": 0.8791730478405952,
|
|
"rewards/cosine_scaled_reward": -0.04475365893449634,
|
|
"rewards/format_reward": 0.43750001303851604,
|
|
"step": 144
|
|
},
|
|
{
|
|
"completion_length": 2055.895866394043,
|
|
"epoch": 0.1657142857142857,
|
|
"grad_norm": 0.2830770015716553,
|
|
"kl": 0.0055332183837890625,
|
|
"learning_rate": 9.046048391230247e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5167977176606655,
|
|
"reward_std": 0.5320884976536036,
|
|
"rewards/cosine_scaled_reward": -0.043684473261237144,
|
|
"rewards/format_reward": 0.6041666716337204,
|
|
"step": 145
|
|
},
|
|
{
|
|
"completion_length": 1917.2917251586914,
|
|
"epoch": 0.16685714285714287,
|
|
"grad_norm": 0.18457695841789246,
|
|
"kl": 0.0024001598358154297,
|
|
"learning_rate": 9.026620557966279e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5474055055528879,
|
|
"reward_std": 0.7067083790898323,
|
|
"rewards/cosine_scaled_reward": -0.11171392910182476,
|
|
"rewards/format_reward": 0.7708333395421505,
|
|
"step": 146
|
|
},
|
|
{
|
|
"completion_length": 2394.2500534057617,
|
|
"epoch": 0.168,
|
|
"grad_norm": 0.26179954409599304,
|
|
"kl": 0.004405975341796875,
|
|
"learning_rate": 9.007020842191634e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.39791956916451454,
|
|
"reward_std": 1.0572543032467365,
|
|
"rewards/cosine_scaled_reward": -0.07187354937195778,
|
|
"rewards/format_reward": 0.5416666697710752,
|
|
"step": 147
|
|
},
|
|
{
|
|
"completion_length": 1997.3125534057617,
|
|
"epoch": 0.16914285714285715,
|
|
"grad_norm": 0.22162283957004547,
|
|
"kl": 0.0032930374145507812,
|
|
"learning_rate": 8.987250199168808e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.6760393833974376,
|
|
"reward_std": 0.7666322588920593,
|
|
"rewards/cosine_scaled_reward": -0.01614698488265276,
|
|
"rewards/format_reward": 0.7083333358168602,
|
|
"step": 148
|
|
},
|
|
{
|
|
"completion_length": 2594.416717529297,
|
|
"epoch": 0.1702857142857143,
|
|
"grad_norm": 0.17498500645160675,
|
|
"kl": 0.0034232139587402344,
|
|
"learning_rate": 8.967309592491052e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5365147553384304,
|
|
"reward_std": 0.6859856657683849,
|
|
"rewards/cosine_scaled_reward": -0.012992626056075096,
|
|
"rewards/format_reward": 0.5625000186264515,
|
|
"step": 149
|
|
},
|
|
{
|
|
"completion_length": 2656.104202270508,
|
|
"epoch": 0.17142857142857143,
|
|
"grad_norm": 0.22585879266262054,
|
|
"kl": 0.0047016143798828125,
|
|
"learning_rate": 8.9471999940354e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.2979818880558014,
|
|
"reward_std": 0.7468250878155231,
|
|
"rewards/cosine_scaled_reward": -0.08017573575489223,
|
|
"rewards/format_reward": 0.45833333767950535,
|
|
"step": 150
|
|
},
|
|
{
|
|
"completion_length": 2433.208396911621,
|
|
"epoch": 0.17257142857142857,
|
|
"grad_norm": 0.2490771859884262,
|
|
"kl": 0.0044155120849609375,
|
|
"learning_rate": 8.926922383915315e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5584549466148019,
|
|
"reward_std": 0.8048359379172325,
|
|
"rewards/cosine_scaled_reward": -0.022855868563055992,
|
|
"rewards/format_reward": 0.6041666809469461,
|
|
"step": 151
|
|
},
|
|
{
|
|
"completion_length": 2724.979217529297,
|
|
"epoch": 0.1737142857142857,
|
|
"grad_norm": 0.2223489135503769,
|
|
"kl": 0.003587007522583008,
|
|
"learning_rate": 8.906477750432903e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.020049065351486206,
|
|
"reward_std": 0.6065222397446632,
|
|
"rewards/cosine_scaled_reward": -0.1774754635989666,
|
|
"rewards/format_reward": 0.37500000558793545,
|
|
"step": 152
|
|
},
|
|
{
|
|
"completion_length": 2770.2708740234375,
|
|
"epoch": 0.17485714285714285,
|
|
"grad_norm": 0.19104580581188202,
|
|
"kl": 0.006191253662109375,
|
|
"learning_rate": 8.88586709003076e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.004709841683506966,
|
|
"reward_std": 0.47944022715091705,
|
|
"rewards/cosine_scaled_reward": -0.19556174334138632,
|
|
"rewards/format_reward": 0.3958333358168602,
|
|
"step": 153
|
|
},
|
|
{
|
|
"completion_length": 3214.062530517578,
|
|
"epoch": 0.176,
|
|
"grad_norm": 0.186995267868042,
|
|
"kl": 0.0031766891479492188,
|
|
"learning_rate": 8.865091407243394e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.3578900098800659,
|
|
"reward_std": 1.006507821381092,
|
|
"rewards/cosine_scaled_reward": 0.03311167098581791,
|
|
"rewards/format_reward": 0.29166666977107525,
|
|
"step": 154
|
|
},
|
|
{
|
|
"completion_length": 2554.666732788086,
|
|
"epoch": 0.17714285714285713,
|
|
"grad_norm": 0.20163290202617645,
|
|
"kl": 0.004076957702636719,
|
|
"learning_rate": 8.844151714648274e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5557569041848183,
|
|
"reward_std": 0.6795310117304325,
|
|
"rewards/cosine_scaled_reward": 0.02787843905389309,
|
|
"rewards/format_reward": 0.5000000037252903,
|
|
"step": 155
|
|
},
|
|
{
|
|
"completion_length": 2689.2292098999023,
|
|
"epoch": 0.1782857142857143,
|
|
"grad_norm": 0.18030290305614471,
|
|
"kl": 0.003159046173095703,
|
|
"learning_rate": 8.823049032816478e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.4783413279801607,
|
|
"reward_std": 0.8905918002128601,
|
|
"rewards/cosine_scaled_reward": 0.02042064256966114,
|
|
"rewards/format_reward": 0.4375,
|
|
"step": 156
|
|
},
|
|
{
|
|
"completion_length": 2461.875015258789,
|
|
"epoch": 0.17942857142857144,
|
|
"grad_norm": 0.22876004874706268,
|
|
"kl": 0.003749370574951172,
|
|
"learning_rate": 8.801784390262943e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.15547512285411358,
|
|
"reward_std": 0.5757812671363354,
|
|
"rewards/cosine_scaled_reward": -0.19309578835964203,
|
|
"rewards/format_reward": 0.541666679084301,
|
|
"step": 157
|
|
},
|
|
{
|
|
"completion_length": 2797.8333740234375,
|
|
"epoch": 0.18057142857142858,
|
|
"grad_norm": 0.2126486450433731,
|
|
"kl": 0.0034036636352539062,
|
|
"learning_rate": 8.780358823396352e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5735968193039298,
|
|
"reward_std": 0.8958301469683647,
|
|
"rewards/cosine_scaled_reward": 0.02638173568993807,
|
|
"rewards/format_reward": 0.5208333469927311,
|
|
"step": 158
|
|
},
|
|
{
|
|
"completion_length": 2499.937515258789,
|
|
"epoch": 0.18171428571428572,
|
|
"grad_norm": 0.17875580489635468,
|
|
"kl": 0.003962516784667969,
|
|
"learning_rate": 8.758773376468604e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.11497697234153748,
|
|
"reward_std": 0.49865414947271347,
|
|
"rewards/cosine_scaled_reward": -0.1820948626846075,
|
|
"rewards/format_reward": 0.4791666753590107,
|
|
"step": 159
|
|
},
|
|
{
|
|
"completion_length": 2463.708381652832,
|
|
"epoch": 0.18285714285714286,
|
|
"grad_norm": 0.23183724284172058,
|
|
"kl": 0.006978034973144531,
|
|
"learning_rate": 8.737029101523929e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.7383107729256153,
|
|
"reward_std": 0.9244415387511253,
|
|
"rewards/cosine_scaled_reward": 0.05665536457672715,
|
|
"rewards/format_reward": 0.6250000055879354,
|
|
"step": 160
|
|
},
|
|
{
|
|
"completion_length": 2109.3333740234375,
|
|
"epoch": 0.184,
|
|
"grad_norm": 0.2291431725025177,
|
|
"kl": 0.0051670074462890625,
|
|
"learning_rate": 8.715127058347614e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.43655015155673027,
|
|
"reward_std": 0.5879729185253382,
|
|
"rewards/cosine_scaled_reward": -0.11505825724452734,
|
|
"rewards/format_reward": 0.6666666772216558,
|
|
"step": 161
|
|
},
|
|
{
|
|
"completion_length": 2962.1458740234375,
|
|
"epoch": 0.18514285714285714,
|
|
"grad_norm": 0.18190893530845642,
|
|
"kl": 0.008568763732910156,
|
|
"learning_rate": 8.693068314414344e-07,
|
|
"loss": 0.0003,
|
|
"reward": -0.018576149828732014,
|
|
"reward_std": 0.6142776571214199,
|
|
"rewards/cosine_scaled_reward": -0.1759547544643283,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 162
|
|
},
|
|
{
|
|
"completion_length": 2259.770851135254,
|
|
"epoch": 0.18628571428571428,
|
|
"grad_norm": 0.2146298736333847,
|
|
"kl": 0.004612922668457031,
|
|
"learning_rate": 8.670853944836176e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.9779267348349094,
|
|
"reward_std": 0.5618617758154869,
|
|
"rewards/cosine_scaled_reward": 0.18687998969107866,
|
|
"rewards/format_reward": 0.6041666716337204,
|
|
"step": 163
|
|
},
|
|
{
|
|
"completion_length": 2226.6042098999023,
|
|
"epoch": 0.18742857142857142,
|
|
"grad_norm": 0.2856714725494385,
|
|
"kl": 0.0065479278564453125,
|
|
"learning_rate": 8.648485032310144e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.5088155679404736,
|
|
"reward_std": 0.6196571066975594,
|
|
"rewards/cosine_scaled_reward": -0.05809224210679531,
|
|
"rewards/format_reward": 0.6250000167638063,
|
|
"step": 164
|
|
},
|
|
{
|
|
"completion_length": 2562.2083892822266,
|
|
"epoch": 0.18857142857142858,
|
|
"grad_norm": 0.2433558702468872,
|
|
"kl": 0.0056896209716796875,
|
|
"learning_rate": 8.625962667065487e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.17038288246840239,
|
|
"reward_std": 0.8414329029619694,
|
|
"rewards/cosine_scaled_reward": -0.14397523301886395,
|
|
"rewards/format_reward": 0.4583333432674408,
|
|
"step": 165
|
|
},
|
|
{
|
|
"completion_length": 2418.8958892822266,
|
|
"epoch": 0.18971428571428572,
|
|
"grad_norm": 0.1904870867729187,
|
|
"kl": 0.0036516189575195312,
|
|
"learning_rate": 8.603287946810513e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.37238264083862305,
|
|
"reward_std": 0.7455781847238541,
|
|
"rewards/cosine_scaled_reward": -0.08464201167225838,
|
|
"rewards/format_reward": 0.5416666772216558,
|
|
"step": 166
|
|
},
|
|
{
|
|
"completion_length": 1871.6042175292969,
|
|
"epoch": 0.19085714285714286,
|
|
"grad_norm": 0.21718546748161316,
|
|
"kl": 0.003131866455078125,
|
|
"learning_rate": 8.580461976679099e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5296564288437366,
|
|
"reward_std": 0.6462862379848957,
|
|
"rewards/cosine_scaled_reward": -0.1414217846468091,
|
|
"rewards/format_reward": 0.8125000204890966,
|
|
"step": 167
|
|
},
|
|
{
|
|
"completion_length": 2663.8750534057617,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.22513794898986816,
|
|
"kl": 0.0043582916259765625,
|
|
"learning_rate": 8.557485869176825e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5514265485107899,
|
|
"reward_std": 0.9005202539265156,
|
|
"rewards/cosine_scaled_reward": -0.01595340482890606,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 168
|
|
},
|
|
{
|
|
"completion_length": 1646.375015258789,
|
|
"epoch": 0.19314285714285714,
|
|
"grad_norm": 0.26874470710754395,
|
|
"kl": 0.0048503875732421875,
|
|
"learning_rate": 8.534360744126753e-07,
|
|
"loss": 0.0002,
|
|
"reward": 1.3640152756124735,
|
|
"reward_std": 0.753107562661171,
|
|
"rewards/cosine_scaled_reward": 0.2549243066459894,
|
|
"rewards/format_reward": 0.8541666753590107,
|
|
"step": 169
|
|
},
|
|
{
|
|
"completion_length": 2181.6667289733887,
|
|
"epoch": 0.19428571428571428,
|
|
"grad_norm": 0.2248026579618454,
|
|
"kl": 0.0046710968017578125,
|
|
"learning_rate": 8.511087728614862e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5494927279651165,
|
|
"reward_std": 0.5512229539453983,
|
|
"rewards/cosine_scaled_reward": -0.006503628566861153,
|
|
"rewards/format_reward": 0.5625000111758709,
|
|
"step": 170
|
|
},
|
|
{
|
|
"completion_length": 2032.6458587646484,
|
|
"epoch": 0.19542857142857142,
|
|
"grad_norm": 0.2114606499671936,
|
|
"kl": 0.0031061172485351562,
|
|
"learning_rate": 8.487667956935087e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5898782406002283,
|
|
"reward_std": 0.7169837113469839,
|
|
"rewards/cosine_scaled_reward": -0.0175608959980309,
|
|
"rewards/format_reward": 0.6250000037252903,
|
|
"step": 171
|
|
},
|
|
{
|
|
"completion_length": 2407.6875610351562,
|
|
"epoch": 0.19657142857142856,
|
|
"grad_norm": 0.295926034450531,
|
|
"kl": 0.0061702728271484375,
|
|
"learning_rate": 8.464102570534061e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.8869556821882725,
|
|
"reward_std": 0.7517024762928486,
|
|
"rewards/cosine_scaled_reward": 0.1518111675977707,
|
|
"rewards/format_reward": 0.583333345130086,
|
|
"step": 172
|
|
},
|
|
{
|
|
"completion_length": 1608.0833702087402,
|
|
"epoch": 0.1977142857142857,
|
|
"grad_norm": 0.28166893124580383,
|
|
"kl": 0.00473785400390625,
|
|
"learning_rate": 8.440392717955475e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.49864733405411243,
|
|
"reward_std": 0.6368578001856804,
|
|
"rewards/cosine_scaled_reward": -0.12567635625600815,
|
|
"rewards/format_reward": 0.7500000037252903,
|
|
"step": 173
|
|
},
|
|
{
|
|
"completion_length": 1789.645866394043,
|
|
"epoch": 0.19885714285714284,
|
|
"grad_norm": 0.2832661271095276,
|
|
"kl": 0.006890296936035156,
|
|
"learning_rate": 8.416539554784089e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.717628687620163,
|
|
"reward_std": 0.6769092865288258,
|
|
"rewards/cosine_scaled_reward": -0.026602333411574364,
|
|
"rewards/format_reward": 0.7708333414047956,
|
|
"step": 174
|
|
},
|
|
{
|
|
"completion_length": 2314.1042098999023,
|
|
"epoch": 0.2,
|
|
"grad_norm": 0.2392614483833313,
|
|
"kl": 0.004840850830078125,
|
|
"learning_rate": 8.392544243589427e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.7497927155345678,
|
|
"reward_std": 0.7398151978850365,
|
|
"rewards/cosine_scaled_reward": 0.0728130005300045,
|
|
"rewards/format_reward": 0.6041666753590107,
|
|
"step": 175
|
|
},
|
|
{
|
|
"completion_length": 1971.0417213439941,
|
|
"epoch": 0.20114285714285715,
|
|
"grad_norm": 0.23342494666576385,
|
|
"kl": 0.006039619445800781,
|
|
"learning_rate": 8.368407953869103e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5884247794747353,
|
|
"reward_std": 0.859587823972106,
|
|
"rewards/cosine_scaled_reward": -0.10162095539271832,
|
|
"rewards/format_reward": 0.7916666716337204,
|
|
"step": 176
|
|
},
|
|
{
|
|
"completion_length": 2159.5417098999023,
|
|
"epoch": 0.2022857142857143,
|
|
"grad_norm": 0.24055302143096924,
|
|
"kl": 0.004790306091308594,
|
|
"learning_rate": 8.344131861991828e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.6115904222242534,
|
|
"reward_std": 0.7329309619963169,
|
|
"rewards/cosine_scaled_reward": -0.04837145563215017,
|
|
"rewards/format_reward": 0.7083333432674408,
|
|
"step": 177
|
|
},
|
|
{
|
|
"completion_length": 1962.1875457763672,
|
|
"epoch": 0.20342857142857143,
|
|
"grad_norm": 0.24433988332748413,
|
|
"kl": 0.007831573486328125,
|
|
"learning_rate": 8.319717151140072e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.5400892496109009,
|
|
"reward_std": 0.5809228383004665,
|
|
"rewards/cosine_scaled_reward": -0.07370538869872689,
|
|
"rewards/format_reward": 0.6875000111758709,
|
|
"step": 178
|
|
},
|
|
{
|
|
"completion_length": 2294.9375228881836,
|
|
"epoch": 0.20457142857142857,
|
|
"grad_norm": 0.21765747666358948,
|
|
"kl": 0.00481414794921875,
|
|
"learning_rate": 8.295165011252396e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.3706574998795986,
|
|
"reward_std": 0.711703471839428,
|
|
"rewards/cosine_scaled_reward": -0.08550458867102861,
|
|
"rewards/format_reward": 0.5416666679084301,
|
|
"step": 179
|
|
},
|
|
{
|
|
"completion_length": 1782.8333587646484,
|
|
"epoch": 0.2057142857142857,
|
|
"grad_norm": 0.2759692668914795,
|
|
"kl": 0.009118080139160156,
|
|
"learning_rate": 8.270476638965461e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.8141425922513008,
|
|
"reward_std": 0.8202670086175203,
|
|
"rewards/cosine_scaled_reward": 0.06332127377390862,
|
|
"rewards/format_reward": 0.6875000018626451,
|
|
"step": 180
|
|
},
|
|
{
|
|
"completion_length": 2508.3750534057617,
|
|
"epoch": 0.20685714285714285,
|
|
"grad_norm": 0.2136908620595932,
|
|
"kl": 0.005702972412109375,
|
|
"learning_rate": 8.245653237555705e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.53153170022415,
|
|
"reward_std": 0.6961524914950132,
|
|
"rewards/cosine_scaled_reward": -0.02590082644019276,
|
|
"rewards/format_reward": 0.5833333469927311,
|
|
"step": 181
|
|
},
|
|
{
|
|
"completion_length": 1820.1875228881836,
|
|
"epoch": 0.208,
|
|
"grad_norm": 0.1944979727268219,
|
|
"kl": 0.0024585723876953125,
|
|
"learning_rate": 8.220696016880687e-07,
|
|
"loss": 0.0001,
|
|
"reward": 0.5819809623062611,
|
|
"reward_std": 0.6004299521446228,
|
|
"rewards/cosine_scaled_reward": -0.07359286700375378,
|
|
"rewards/format_reward": 0.7291666716337204,
|
|
"step": 182
|
|
},
|
|
{
|
|
"completion_length": 1418.458351135254,
|
|
"epoch": 0.20914285714285713,
|
|
"grad_norm": 0.2479531317949295,
|
|
"kl": 0.00787353515625,
|
|
"learning_rate": 8.195606193320136e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0037191323935986,
|
|
"reward_std": 0.7833016626536846,
|
|
"rewards/cosine_scaled_reward": 0.04352622898295522,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 183
|
|
},
|
|
{
|
|
"completion_length": 2061.06254196167,
|
|
"epoch": 0.2102857142857143,
|
|
"grad_norm": 0.2752915620803833,
|
|
"kl": 0.0069942474365234375,
|
|
"learning_rate": 8.170384989716657e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.39811624586582184,
|
|
"reward_std": 0.612327728420496,
|
|
"rewards/cosine_scaled_reward": -0.11344187799841166,
|
|
"rewards/format_reward": 0.6250000093132257,
|
|
"step": 184
|
|
},
|
|
{
|
|
"completion_length": 1844.6667098999023,
|
|
"epoch": 0.21142857142857144,
|
|
"grad_norm": 0.2239396721124649,
|
|
"kl": 0.0045299530029296875,
|
|
"learning_rate": 8.145033635316128e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.32189831510186195,
|
|
"reward_std": 0.5676768328994513,
|
|
"rewards/cosine_scaled_reward": -0.18280084058642387,
|
|
"rewards/format_reward": 0.6875000149011612,
|
|
"step": 185
|
|
},
|
|
{
|
|
"completion_length": 2218.791702270508,
|
|
"epoch": 0.21257142857142858,
|
|
"grad_norm": 0.20360872149467468,
|
|
"kl": 0.0063877105712890625,
|
|
"learning_rate": 8.119553365707802e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.48006572760641575,
|
|
"reward_std": 0.5926500409841537,
|
|
"rewards/cosine_scaled_reward": -0.030800477601587772,
|
|
"rewards/format_reward": 0.5416666734963655,
|
|
"step": 186
|
|
},
|
|
{
|
|
"completion_length": 1770.9791870117188,
|
|
"epoch": 0.21371428571428572,
|
|
"grad_norm": 0.23867206275463104,
|
|
"kl": 0.0055904388427734375,
|
|
"learning_rate": 8.093945422764069e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.6042033806443214,
|
|
"reward_std": 0.4485644996166229,
|
|
"rewards/cosine_scaled_reward": -0.08331498829647899,
|
|
"rewards/format_reward": 0.7708333395421505,
|
|
"step": 187
|
|
},
|
|
{
|
|
"completion_length": 2462.854202270508,
|
|
"epoch": 0.21485714285714286,
|
|
"grad_norm": 0.24813704192638397,
|
|
"kl": 0.0071868896484375,
|
|
"learning_rate": 8.068211054579943e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.014021937269717455,
|
|
"reward_std": 0.5901096761226654,
|
|
"rewards/cosine_scaled_reward": -0.22215570323169231,
|
|
"rewards/format_reward": 0.4583333469927311,
|
|
"step": 188
|
|
},
|
|
{
|
|
"completion_length": 1708.5000534057617,
|
|
"epoch": 0.216,
|
|
"grad_norm": 0.2578124403953552,
|
|
"kl": 0.006272315979003906,
|
|
"learning_rate": 8.04235151541222e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.6445069871842861,
|
|
"reward_std": 0.7020072378218174,
|
|
"rewards/cosine_scaled_reward": -0.052746512461453676,
|
|
"rewards/format_reward": 0.7500000111758709,
|
|
"step": 189
|
|
},
|
|
{
|
|
"completion_length": 1291.5833892822266,
|
|
"epoch": 0.21714285714285714,
|
|
"grad_norm": 0.2704342007637024,
|
|
"kl": 0.0067882537841796875,
|
|
"learning_rate": 8.01636806561836e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.9361954592168331,
|
|
"reward_std": 0.6467648409307003,
|
|
"rewards/cosine_scaled_reward": -0.0006522866897284985,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 190
|
|
},
|
|
{
|
|
"completion_length": 1299.5833892822266,
|
|
"epoch": 0.21828571428571428,
|
|
"grad_norm": 0.24868877232074738,
|
|
"kl": 0.0057735443115234375,
|
|
"learning_rate": 7.990261971595048e-07,
|
|
"loss": 0.0002,
|
|
"reward": 1.0458338633179665,
|
|
"reward_std": 0.7703647427260876,
|
|
"rewards/cosine_scaled_reward": 0.08541689871344715,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 191
|
|
},
|
|
{
|
|
"completion_length": 1983.1042404174805,
|
|
"epoch": 0.21942857142857142,
|
|
"grad_norm": 0.24434901773929596,
|
|
"kl": 0.006023406982421875,
|
|
"learning_rate": 7.964034505716476e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5690425429493189,
|
|
"reward_std": 0.7882693633437157,
|
|
"rewards/cosine_scaled_reward": -0.09047873830422759,
|
|
"rewards/format_reward": 0.750000013038516,
|
|
"step": 192
|
|
},
|
|
{
|
|
"completion_length": 2579.333366394043,
|
|
"epoch": 0.22057142857142858,
|
|
"grad_norm": 0.26767823100090027,
|
|
"kl": 0.008405685424804688,
|
|
"learning_rate": 7.93768694627233e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.08483336865901947,
|
|
"reward_std": 0.5648258291184902,
|
|
"rewards/cosine_scaled_reward": -0.19716665521264076,
|
|
"rewards/format_reward": 0.4791666753590107,
|
|
"step": 193
|
|
},
|
|
{
|
|
"completion_length": 2240.562568664551,
|
|
"epoch": 0.22171428571428572,
|
|
"grad_norm": 0.19335110485553741,
|
|
"kl": 0.00666046142578125,
|
|
"learning_rate": 7.911220577405484e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0503641180694103,
|
|
"reward_std": 0.5451640971004963,
|
|
"rewards/cosine_scaled_reward": 0.16059872414916754,
|
|
"rewards/format_reward": 0.7291666772216558,
|
|
"step": 194
|
|
},
|
|
{
|
|
"completion_length": 1826.2708892822266,
|
|
"epoch": 0.22285714285714286,
|
|
"grad_norm": 0.24450555443763733,
|
|
"kl": 0.006824493408203125,
|
|
"learning_rate": 7.884636689049422e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.7163921128958464,
|
|
"reward_std": 1.0118936747312546,
|
|
"rewards/cosine_scaled_reward": -0.03763728140620515,
|
|
"rewards/format_reward": 0.7916666828095913,
|
|
"step": 195
|
|
},
|
|
{
|
|
"completion_length": 2620.041778564453,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.2696687877178192,
|
|
"kl": 0.00728607177734375,
|
|
"learning_rate": 7.857936576865356e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.21067072823643684,
|
|
"reward_std": 0.859990905970335,
|
|
"rewards/cosine_scaled_reward": -0.13424798846244812,
|
|
"rewards/format_reward": 0.4791666716337204,
|
|
"step": 196
|
|
},
|
|
{
|
|
"completion_length": 1049.0208587646484,
|
|
"epoch": 0.22514285714285714,
|
|
"grad_norm": 0.27546316385269165,
|
|
"kl": 0.00786590576171875,
|
|
"learning_rate": 7.831121542179086e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.9739863537251949,
|
|
"reward_std": 0.8307300806045532,
|
|
"rewards/cosine_scaled_reward": 0.01824316382408142,
|
|
"rewards/format_reward": 0.9375,
|
|
"step": 197
|
|
},
|
|
{
|
|
"completion_length": 1475.4583892822266,
|
|
"epoch": 0.22628571428571428,
|
|
"grad_norm": 0.2635370194911957,
|
|
"kl": 0.008056640625,
|
|
"learning_rate": 7.804192891917571e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.8781713657081127,
|
|
"reward_std": 0.8930405229330063,
|
|
"rewards/cosine_scaled_reward": 0.012002333998680115,
|
|
"rewards/format_reward": 0.8541666679084301,
|
|
"step": 198
|
|
},
|
|
{
|
|
"completion_length": 1526.4583587646484,
|
|
"epoch": 0.22742857142857142,
|
|
"grad_norm": 0.29261499643325806,
|
|
"kl": 0.007879257202148438,
|
|
"learning_rate": 7.777151938545235e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.42517440766096115,
|
|
"reward_std": 0.6142569780349731,
|
|
"rewards/cosine_scaled_reward": -0.22491280548274517,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 199
|
|
},
|
|
{
|
|
"completion_length": 1371.458366394043,
|
|
"epoch": 0.22857142857142856,
|
|
"grad_norm": 0.23450589179992676,
|
|
"kl": 0.0063762664794921875,
|
|
"learning_rate": 7.75e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.134487384930253,
|
|
"reward_std": 0.7592474110424519,
|
|
"rewards/cosine_scaled_reward": 0.10891036130487919,
|
|
"rewards/format_reward": 0.9166666679084301,
|
|
"step": 200
|
|
},
|
|
{
|
|
"completion_length": 1873.7500686645508,
|
|
"epoch": 0.2297142857142857,
|
|
"grad_norm": 0.20057973265647888,
|
|
"kl": 0.0047245025634765625,
|
|
"learning_rate": 7.72273839962904e-07,
|
|
"loss": 0.0002,
|
|
"reward": 1.3440011143684387,
|
|
"reward_std": 0.6584898792207241,
|
|
"rewards/cosine_scaled_reward": 0.2657505361130461,
|
|
"rewards/format_reward": 0.8125000074505806,
|
|
"step": 201
|
|
},
|
|
{
|
|
"completion_length": 1296.6041870117188,
|
|
"epoch": 0.23085714285714284,
|
|
"grad_norm": 0.2126869261264801,
|
|
"kl": 0.006069183349609375,
|
|
"learning_rate": 7.695368466124296e-07,
|
|
"loss": 0.0002,
|
|
"reward": 1.2371138632297516,
|
|
"reward_std": 0.45892489701509476,
|
|
"rewards/cosine_scaled_reward": 0.18105687946081161,
|
|
"rewards/format_reward": 0.8750000055879354,
|
|
"step": 202
|
|
},
|
|
{
|
|
"completion_length": 1661.5000457763672,
|
|
"epoch": 0.232,
|
|
"grad_norm": 0.25018230080604553,
|
|
"kl": 0.008678436279296875,
|
|
"learning_rate": 7.667891533457718e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.8724356088787317,
|
|
"reward_std": 0.8623361866921186,
|
|
"rewards/cosine_scaled_reward": 0.040384437423199415,
|
|
"rewards/format_reward": 0.7916666939854622,
|
|
"step": 203
|
|
},
|
|
{
|
|
"completion_length": 1537.708381652832,
|
|
"epoch": 0.23314285714285715,
|
|
"grad_norm": 0.34546995162963867,
|
|
"kl": 0.007953643798828125,
|
|
"learning_rate": 7.640308940816239e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.7893814034759998,
|
|
"reward_std": 0.7430329732596874,
|
|
"rewards/cosine_scaled_reward": -0.03239264711737633,
|
|
"rewards/format_reward": 0.8541666697710752,
|
|
"step": 204
|
|
},
|
|
{
|
|
"completion_length": 1645.1250305175781,
|
|
"epoch": 0.2342857142857143,
|
|
"grad_norm": 0.24868625402450562,
|
|
"kl": 0.0064334869384765625,
|
|
"learning_rate": 7.612622032536507e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.1130757508799434,
|
|
"reward_std": 0.9551872611045837,
|
|
"rewards/cosine_scaled_reward": 0.15028784982860088,
|
|
"rewards/format_reward": 0.8125000055879354,
|
|
"step": 205
|
|
},
|
|
{
|
|
"completion_length": 2075.187545776367,
|
|
"epoch": 0.23542857142857143,
|
|
"grad_norm": 0.25428035855293274,
|
|
"kl": 0.0065708160400390625,
|
|
"learning_rate": 7.584832158039378e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.233387497253716,
|
|
"reward_std": 0.5259231552481651,
|
|
"rewards/cosine_scaled_reward": -0.22705625742673874,
|
|
"rewards/format_reward": 0.6875000260770321,
|
|
"step": 206
|
|
},
|
|
{
|
|
"completion_length": 1656.3750686645508,
|
|
"epoch": 0.23657142857142857,
|
|
"grad_norm": 0.26431310176849365,
|
|
"kl": 0.0085601806640625,
|
|
"learning_rate": 7.556940671764124e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.6532822176814079,
|
|
"reward_std": 0.6114402338862419,
|
|
"rewards/cosine_scaled_reward": -0.13169224187731743,
|
|
"rewards/format_reward": 0.9166666716337204,
|
|
"step": 207
|
|
},
|
|
{
|
|
"completion_length": 1265.3542022705078,
|
|
"epoch": 0.2377142857142857,
|
|
"grad_norm": 0.241045743227005,
|
|
"kl": 0.00737762451171875,
|
|
"learning_rate": 7.528948933102438e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.9077612403780222,
|
|
"reward_std": 0.5354608930647373,
|
|
"rewards/cosine_scaled_reward": -0.01486940123140812,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 208
|
|
},
|
|
{
|
|
"completion_length": 1344.3750305175781,
|
|
"epoch": 0.23885714285714285,
|
|
"grad_norm": 0.3023705780506134,
|
|
"kl": 0.010385513305664062,
|
|
"learning_rate": 7.500858306332172e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.0339510310441256,
|
|
"reward_std": 0.7570701129734516,
|
|
"rewards/cosine_scaled_reward": 0.07947551319375634,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 209
|
|
},
|
|
{
|
|
"completion_length": 1667.3541793823242,
|
|
"epoch": 0.24,
|
|
"grad_norm": 0.18748997151851654,
|
|
"kl": 0.006000518798828125,
|
|
"learning_rate": 7.472670160550848e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.666348340921104,
|
|
"reward_std": 0.5869894400238991,
|
|
"rewards/cosine_scaled_reward": -0.09390917886048555,
|
|
"rewards/format_reward": 0.8541666716337204,
|
|
"step": 210
|
|
},
|
|
{
|
|
"completion_length": 1571.8958740234375,
|
|
"epoch": 0.24114285714285713,
|
|
"grad_norm": 0.25766894221305847,
|
|
"kl": 0.0088043212890625,
|
|
"learning_rate": 7.444385869608921e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.9365372620522976,
|
|
"reward_std": 0.5268308343365788,
|
|
"rewards/cosine_scaled_reward": 0.07243526913225651,
|
|
"rewards/format_reward": 0.7916666734963655,
|
|
"step": 211
|
|
},
|
|
{
|
|
"completion_length": 967.6458549499512,
|
|
"epoch": 0.2422857142857143,
|
|
"grad_norm": 0.2648831009864807,
|
|
"kl": 0.00879669189453125,
|
|
"learning_rate": 7.416006812042827e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.0848271660506725,
|
|
"reward_std": 0.44052685238420963,
|
|
"rewards/cosine_scaled_reward": 0.08408024348318577,
|
|
"rewards/format_reward": 0.9166666679084301,
|
|
"step": 212
|
|
},
|
|
{
|
|
"completion_length": 1431.3959121704102,
|
|
"epoch": 0.24342857142857144,
|
|
"grad_norm": 0.33580857515335083,
|
|
"kl": 0.0103302001953125,
|
|
"learning_rate": 7.387534371007797e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.239082669839263,
|
|
"reward_std": 0.9362837933003902,
|
|
"rewards/cosine_scaled_reward": 0.1820413067471236,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 213
|
|
},
|
|
{
|
|
"completion_length": 1762.3125457763672,
|
|
"epoch": 0.24457142857142858,
|
|
"grad_norm": 0.24269923567771912,
|
|
"kl": 0.00745391845703125,
|
|
"learning_rate": 7.358969934210438e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0624176412820816,
|
|
"reward_std": 0.7310810908675194,
|
|
"rewards/cosine_scaled_reward": 0.11454213625984266,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 214
|
|
},
|
|
{
|
|
"completion_length": 1427.3542022705078,
|
|
"epoch": 0.24571428571428572,
|
|
"grad_norm": 0.25518307089805603,
|
|
"kl": 0.0060749053955078125,
|
|
"learning_rate": 7.330314893841101e-07,
|
|
"loss": 0.0002,
|
|
"reward": 0.5486622415482998,
|
|
"reward_std": 0.42411663569509983,
|
|
"rewards/cosine_scaled_reward": -0.14233557134866714,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 215
|
|
},
|
|
{
|
|
"completion_length": 1129.395851135254,
|
|
"epoch": 0.24685714285714286,
|
|
"grad_norm": 0.28439247608184814,
|
|
"kl": 0.007411956787109375,
|
|
"learning_rate": 7.301570646506027e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0450292900204659,
|
|
"reward_std": 0.6436006706207991,
|
|
"rewards/cosine_scaled_reward": 0.08501462638378143,
|
|
"rewards/format_reward": 0.8750000055879354,
|
|
"step": 216
|
|
},
|
|
{
|
|
"completion_length": 1375.4791946411133,
|
|
"epoch": 0.248,
|
|
"grad_norm": 0.22866062819957733,
|
|
"kl": 0.0072784423828125,
|
|
"learning_rate": 7.27273859315928e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0294950436800718,
|
|
"reward_std": 0.7613518834114075,
|
|
"rewards/cosine_scaled_reward": 0.08766416925936937,
|
|
"rewards/format_reward": 0.8541666679084301,
|
|
"step": 217
|
|
},
|
|
{
|
|
"completion_length": 1546.6041946411133,
|
|
"epoch": 0.24914285714285714,
|
|
"grad_norm": 0.2400980442762375,
|
|
"kl": 0.00803375244140625,
|
|
"learning_rate": 7.243820139034464e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.6402788404375315,
|
|
"reward_std": 0.7851009257137775,
|
|
"rewards/cosine_scaled_reward": -0.10694391108700074,
|
|
"rewards/format_reward": 0.8541666716337204,
|
|
"step": 218
|
|
},
|
|
{
|
|
"completion_length": 1293.1042098999023,
|
|
"epoch": 0.2502857142857143,
|
|
"grad_norm": 0.3005291521549225,
|
|
"kl": 0.0071582794189453125,
|
|
"learning_rate": 7.214816693576234e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.2072506174445152,
|
|
"reward_std": 0.8155215214937925,
|
|
"rewards/cosine_scaled_reward": 0.1452919525327161,
|
|
"rewards/format_reward": 0.9166666679084301,
|
|
"step": 219
|
|
},
|
|
{
|
|
"completion_length": 1585.4583740234375,
|
|
"epoch": 0.25142857142857145,
|
|
"grad_norm": 0.29073673486709595,
|
|
"kl": 0.009313583374023438,
|
|
"learning_rate": 7.185729670371604e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.25497015565633774,
|
|
"reward_std": 0.43767623975872993,
|
|
"rewards/cosine_scaled_reward": -0.2787649389356375,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 220
|
|
},
|
|
{
|
|
"completion_length": 1091.9583625793457,
|
|
"epoch": 0.25257142857142856,
|
|
"grad_norm": 0.2504833936691284,
|
|
"kl": 0.006580352783203125,
|
|
"learning_rate": 7.156560487081051e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.1241591908037663,
|
|
"reward_std": 0.3971919184550643,
|
|
"rewards/cosine_scaled_reward": 0.09332958236336708,
|
|
"rewards/format_reward": 0.9375,
|
|
"step": 221
|
|
},
|
|
{
|
|
"completion_length": 1431.5000305175781,
|
|
"epoch": 0.2537142857142857,
|
|
"grad_norm": 0.2770059108734131,
|
|
"kl": 0.008672714233398438,
|
|
"learning_rate": 7.127310565369415e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0303391478955746,
|
|
"reward_std": 0.711064089089632,
|
|
"rewards/cosine_scaled_reward": 0.09850288555026054,
|
|
"rewards/format_reward": 0.8333333395421505,
|
|
"step": 222
|
|
},
|
|
{
|
|
"completion_length": 1270.583351135254,
|
|
"epoch": 0.25485714285714284,
|
|
"grad_norm": 0.22273501753807068,
|
|
"kl": 0.00675201416015625,
|
|
"learning_rate": 7.097981330836616e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.8901128210127354,
|
|
"reward_std": 0.560223001986742,
|
|
"rewards/cosine_scaled_reward": 0.0075564137659966946,
|
|
"rewards/format_reward": 0.875,
|
|
"step": 223
|
|
},
|
|
{
|
|
"completion_length": 1732.1458587646484,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.2018723040819168,
|
|
"kl": 0.008203506469726562,
|
|
"learning_rate": 7.068574212948169e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.7687601521611214,
|
|
"reward_std": 0.7839281968772411,
|
|
"rewards/cosine_scaled_reward": -0.03228660812601447,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 224
|
|
},
|
|
{
|
|
"completion_length": 2117.18758392334,
|
|
"epoch": 0.2571428571428571,
|
|
"grad_norm": 0.2617863416671753,
|
|
"kl": 0.01264190673828125,
|
|
"learning_rate": 7.039090644965509e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.8388488199561834,
|
|
"reward_std": 0.7986405938863754,
|
|
"rewards/cosine_scaled_reward": 0.044424411840736866,
|
|
"rewards/format_reward": 0.750000013038516,
|
|
"step": 225
|
|
},
|
|
{
|
|
"completion_length": 1414.020881652832,
|
|
"epoch": 0.2582857142857143,
|
|
"grad_norm": 0.19753257930278778,
|
|
"kl": 0.0067138671875,
|
|
"learning_rate": 7.009532063876148e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.1793978083878756,
|
|
"reward_std": 0.5940953148528934,
|
|
"rewards/cosine_scaled_reward": 0.11053222604095936,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 226
|
|
},
|
|
{
|
|
"completion_length": 1173.1666870117188,
|
|
"epoch": 0.25942857142857145,
|
|
"grad_norm": 0.3516033887863159,
|
|
"kl": 0.01033782958984375,
|
|
"learning_rate": 6.979899910323624e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.7449045367538929,
|
|
"reward_std": 0.5567011050879955,
|
|
"rewards/cosine_scaled_reward": -0.10671441350132227,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 227
|
|
},
|
|
{
|
|
"completion_length": 1190.1250305175781,
|
|
"epoch": 0.26057142857142856,
|
|
"grad_norm": 0.31133556365966797,
|
|
"kl": 0.00760650634765625,
|
|
"learning_rate": 6.950195628537299e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.191257143393159,
|
|
"reward_std": 0.6371707431972027,
|
|
"rewards/cosine_scaled_reward": 0.1581285521388054,
|
|
"rewards/format_reward": 0.875,
|
|
"step": 228
|
|
},
|
|
{
|
|
"completion_length": 1375.6250305175781,
|
|
"epoch": 0.26171428571428573,
|
|
"grad_norm": 0.2500562369823456,
|
|
"kl": 0.0087432861328125,
|
|
"learning_rate": 6.920420666261961e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.7764038797467947,
|
|
"reward_std": 0.5216951882466674,
|
|
"rewards/cosine_scaled_reward": -0.04929806664586067,
|
|
"rewards/format_reward": 0.8750000055879354,
|
|
"step": 229
|
|
},
|
|
{
|
|
"completion_length": 1970.1458587646484,
|
|
"epoch": 0.26285714285714284,
|
|
"grad_norm": 0.23503738641738892,
|
|
"kl": 0.009944915771484375,
|
|
"learning_rate": 6.890576474687263e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.531811726745218,
|
|
"reward_std": 0.7926243953406811,
|
|
"rewards/cosine_scaled_reward": -0.10909414570778608,
|
|
"rewards/format_reward": 0.7500000204890966,
|
|
"step": 230
|
|
},
|
|
{
|
|
"completion_length": 1569.3750457763672,
|
|
"epoch": 0.264,
|
|
"grad_norm": 0.2320818156003952,
|
|
"kl": 0.008174896240234375,
|
|
"learning_rate": 6.860664508377001e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.0276812557131052,
|
|
"reward_std": 0.789770107716322,
|
|
"rewards/cosine_scaled_reward": 0.05550729110836983,
|
|
"rewards/format_reward": 0.9166666716337204,
|
|
"step": 231
|
|
},
|
|
{
|
|
"completion_length": 1607.0625305175781,
|
|
"epoch": 0.2651428571428571,
|
|
"grad_norm": 0.23815037310123444,
|
|
"kl": 0.008876800537109375,
|
|
"learning_rate": 6.83068622519821e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.352162616327405,
|
|
"reward_std": 0.5658496394753456,
|
|
"rewards/cosine_scaled_reward": -0.2405853734817356,
|
|
"rewards/format_reward": 0.8333333414047956,
|
|
"step": 232
|
|
},
|
|
{
|
|
"completion_length": 1084.5833740234375,
|
|
"epoch": 0.2662857142857143,
|
|
"grad_norm": 0.2646292746067047,
|
|
"kl": 0.007411956787109375,
|
|
"learning_rate": 6.800643086250121e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.6558086425065994,
|
|
"reward_std": 0.7230100817978382,
|
|
"rewards/cosine_scaled_reward": -0.1408456964418292,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 233
|
|
},
|
|
{
|
|
"completion_length": 1680.520881652832,
|
|
"epoch": 0.2674285714285714,
|
|
"grad_norm": 0.2598845660686493,
|
|
"kl": 0.009510040283203125,
|
|
"learning_rate": 6.770536555792944e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.6849129311740398,
|
|
"reward_std": 0.5895892381668091,
|
|
"rewards/cosine_scaled_reward": -0.042960209771990776,
|
|
"rewards/format_reward": 0.7708333432674408,
|
|
"step": 234
|
|
},
|
|
{
|
|
"completion_length": 1212.5208549499512,
|
|
"epoch": 0.26857142857142857,
|
|
"grad_norm": 0.281758576631546,
|
|
"kl": 0.007808685302734375,
|
|
"learning_rate": 6.740368101176495e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.2133875098079443,
|
|
"reward_std": 0.634877560660243,
|
|
"rewards/cosine_scaled_reward": 0.14836042001843452,
|
|
"rewards/format_reward": 0.9166666716337204,
|
|
"step": 235
|
|
},
|
|
{
|
|
"completion_length": 2002.2500534057617,
|
|
"epoch": 0.26971428571428574,
|
|
"grad_norm": 0.21561594307422638,
|
|
"kl": 0.008035659790039062,
|
|
"learning_rate": 6.710139192768694e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.5754090845584869,
|
|
"reward_std": 0.681129951030016,
|
|
"rewards/cosine_scaled_reward": -0.07687881146557629,
|
|
"rewards/format_reward": 0.7291666716337204,
|
|
"step": 236
|
|
},
|
|
{
|
|
"completion_length": 1571.7291946411133,
|
|
"epoch": 0.27085714285714285,
|
|
"grad_norm": 0.23634596168994904,
|
|
"kl": 0.006927490234375,
|
|
"learning_rate": 6.679851303883891e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.8946835342794657,
|
|
"reward_std": 0.5508107021450996,
|
|
"rewards/cosine_scaled_reward": 0.03067508526146412,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 237
|
|
},
|
|
{
|
|
"completion_length": 1162.2292098999023,
|
|
"epoch": 0.272,
|
|
"grad_norm": 0.23457442224025726,
|
|
"kl": 0.0085906982421875,
|
|
"learning_rate": 6.649505910711058e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.158431712538004,
|
|
"reward_std": 0.7775964550673962,
|
|
"rewards/cosine_scaled_reward": 0.08963251765817404,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 238
|
|
},
|
|
{
|
|
"completion_length": 1398.8750457763672,
|
|
"epoch": 0.27314285714285713,
|
|
"grad_norm": 0.22179792821407318,
|
|
"kl": 0.005764007568359375,
|
|
"learning_rate": 6.619104492241847e-07,
|
|
"loss": 0.0002,
|
|
"reward": 1.289793978095986,
|
|
"reward_std": 0.5327765932306647,
|
|
"rewards/cosine_scaled_reward": 0.23864697851240635,
|
|
"rewards/format_reward": 0.8125,
|
|
"step": 239
|
|
},
|
|
{
|
|
"completion_length": 1420.9375381469727,
|
|
"epoch": 0.2742857142857143,
|
|
"grad_norm": 0.33549872040748596,
|
|
"kl": 0.01104736328125,
|
|
"learning_rate": 6.588648530198504e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.3914220330771059,
|
|
"reward_std": 0.6205890811979771,
|
|
"rewards/cosine_scaled_reward": -0.2209556633606553,
|
|
"rewards/format_reward": 0.8333333395421505,
|
|
"step": 240
|
|
},
|
|
{
|
|
"completion_length": 1621.7292098999023,
|
|
"epoch": 0.2754285714285714,
|
|
"grad_norm": 0.28596657514572144,
|
|
"kl": 0.010402679443359375,
|
|
"learning_rate": 6.558139508961654e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.44276779890060425,
|
|
"reward_std": 0.5154628418385983,
|
|
"rewards/cosine_scaled_reward": -0.1848661107942462,
|
|
"rewards/format_reward": 0.8125000111758709,
|
|
"step": 241
|
|
},
|
|
{
|
|
"completion_length": 1138.5000457763672,
|
|
"epoch": 0.2765714285714286,
|
|
"grad_norm": 0.4309399724006653,
|
|
"kl": 0.0121612548828125,
|
|
"learning_rate": 6.527578915497951e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.7130568971624598,
|
|
"reward_std": 0.4758261194219813,
|
|
"rewards/cosine_scaled_reward": -0.12263822788372636,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 242
|
|
},
|
|
{
|
|
"completion_length": 1494.687515258789,
|
|
"epoch": 0.2777142857142857,
|
|
"grad_norm": 0.21594373881816864,
|
|
"kl": 0.00745391845703125,
|
|
"learning_rate": 6.496968239287603e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.7156112641096115,
|
|
"reward_std": 0.6859831623733044,
|
|
"rewards/cosine_scaled_reward": -0.06927772145718336,
|
|
"rewards/format_reward": 0.8541666772216558,
|
|
"step": 243
|
|
},
|
|
{
|
|
"completion_length": 1628.1667137145996,
|
|
"epoch": 0.27885714285714286,
|
|
"grad_norm": 0.2714637219905853,
|
|
"kl": 0.007848739624023438,
|
|
"learning_rate": 6.466308972251785e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.8751203082501888,
|
|
"reward_std": 0.9190315119922161,
|
|
"rewards/cosine_scaled_reward": 0.010476819472387433,
|
|
"rewards/format_reward": 0.8541666753590107,
|
|
"step": 244
|
|
},
|
|
{
|
|
"completion_length": 1648.875015258789,
|
|
"epoch": 0.28,
|
|
"grad_norm": 0.2101169377565384,
|
|
"kl": 0.0067119598388671875,
|
|
"learning_rate": 6.435602608679916e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.1979175508022308,
|
|
"reward_std": 0.7641258873045444,
|
|
"rewards/cosine_scaled_reward": 0.1718754144385457,
|
|
"rewards/format_reward": 0.8541666697710752,
|
|
"step": 245
|
|
},
|
|
{
|
|
"completion_length": 1389.895881652832,
|
|
"epoch": 0.28114285714285714,
|
|
"grad_norm": 0.21795134246349335,
|
|
"kl": 0.009159088134765625,
|
|
"learning_rate": 6.404850645156841e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.7879314236342907,
|
|
"reward_std": 0.6879578605294228,
|
|
"rewards/cosine_scaled_reward": -0.07478431053459644,
|
|
"rewards/format_reward": 0.9375,
|
|
"step": 246
|
|
},
|
|
{
|
|
"completion_length": 2069.1458892822266,
|
|
"epoch": 0.2822857142857143,
|
|
"grad_norm": 0.2720467150211334,
|
|
"kl": 0.012157440185546875,
|
|
"learning_rate": 6.374054580489873e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.3278281930834055,
|
|
"reward_std": 0.6504724733531475,
|
|
"rewards/cosine_scaled_reward": -0.16941926488652825,
|
|
"rewards/format_reward": 0.6666666753590107,
|
|
"step": 247
|
|
},
|
|
{
|
|
"completion_length": 1404.020866394043,
|
|
"epoch": 0.2834285714285714,
|
|
"grad_norm": 0.29227957129478455,
|
|
"kl": 0.009120941162109375,
|
|
"learning_rate": 6.343215915635761e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.3239782322198153,
|
|
"reward_std": 0.6399974799714983,
|
|
"rewards/cosine_scaled_reward": 0.24532245565205812,
|
|
"rewards/format_reward": 0.8333333395421505,
|
|
"step": 248
|
|
},
|
|
{
|
|
"completion_length": 1421.7500610351562,
|
|
"epoch": 0.2845714285714286,
|
|
"grad_norm": 0.25874021649360657,
|
|
"kl": 0.010471343994140625,
|
|
"learning_rate": 6.31233615362752e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.373143790755421,
|
|
"reward_std": 0.5737661384046078,
|
|
"rewards/cosine_scaled_reward": 0.23865519277751446,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 249
|
|
},
|
|
{
|
|
"completion_length": 1404.6667022705078,
|
|
"epoch": 0.2857142857142857,
|
|
"grad_norm": 0.6512049436569214,
|
|
"kl": 0.011989593505859375,
|
|
"learning_rate": 6.281416799501187e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.8663649614900351,
|
|
"reward_std": 0.6013601124286652,
|
|
"rewards/cosine_scaled_reward": -0.035567532293498516,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 250
|
|
},
|
|
{
|
|
"completion_length": 1101.1250305175781,
|
|
"epoch": 0.28685714285714287,
|
|
"grad_norm": 0.40293389558792114,
|
|
"kl": 0.013065338134765625,
|
|
"learning_rate": 6.25045936022246e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.7222131416201591,
|
|
"reward_std": 0.6657759360969067,
|
|
"rewards/cosine_scaled_reward": -0.07639344967901707,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 251
|
|
},
|
|
{
|
|
"completion_length": 1508.4583702087402,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.2395544797182083,
|
|
"kl": 0.010028839111328125,
|
|
"learning_rate": 6.219465344613258e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.7061197310686111,
|
|
"reward_std": 0.5144665259867907,
|
|
"rewards/cosine_scaled_reward": -0.07402347587049007,
|
|
"rewards/format_reward": 0.8541666697710752,
|
|
"step": 252
|
|
},
|
|
{
|
|
"completion_length": 1397.2291984558105,
|
|
"epoch": 0.28914285714285715,
|
|
"grad_norm": 0.32397717237472534,
|
|
"kl": 0.013702392578125,
|
|
"learning_rate": 6.188436263278172e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.6706481170840561,
|
|
"reward_std": 0.8627937883138657,
|
|
"rewards/cosine_scaled_reward": -0.09175930079072714,
|
|
"rewards/format_reward": 0.8541666716337204,
|
|
"step": 253
|
|
},
|
|
{
|
|
"completion_length": 1515.0000839233398,
|
|
"epoch": 0.29028571428571426,
|
|
"grad_norm": 0.3457793891429901,
|
|
"kl": 0.010593414306640625,
|
|
"learning_rate": 6.157373628530852e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.5583736216649413,
|
|
"reward_std": 0.7457092814147472,
|
|
"rewards/cosine_scaled_reward": -0.12706319894641638,
|
|
"rewards/format_reward": 0.8125000074505806,
|
|
"step": 254
|
|
},
|
|
{
|
|
"completion_length": 2169.166702270508,
|
|
"epoch": 0.2914285714285714,
|
|
"grad_norm": 0.2815548777580261,
|
|
"kl": 0.0111541748046875,
|
|
"learning_rate": 6.126278954320294e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.2357357144355774,
|
|
"reward_std": 0.6888550817966461,
|
|
"rewards/cosine_scaled_reward": -0.20504882326349616,
|
|
"rewards/format_reward": 0.6458333358168602,
|
|
"step": 255
|
|
},
|
|
{
|
|
"completion_length": 1219.937515258789,
|
|
"epoch": 0.2925714285714286,
|
|
"grad_norm": 0.2393861711025238,
|
|
"kl": 0.00894927978515625,
|
|
"learning_rate": 6.095153756157051e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.0311391949653625,
|
|
"reward_std": 0.6041121408343315,
|
|
"rewards/cosine_scaled_reward": 0.036402929574251175,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 256
|
|
},
|
|
{
|
|
"completion_length": 1972.0000228881836,
|
|
"epoch": 0.2937142857142857,
|
|
"grad_norm": 0.18130330741405487,
|
|
"kl": 0.009616851806640625,
|
|
"learning_rate": 6.06399955103937e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.9688311172649264,
|
|
"reward_std": 0.8995218388736248,
|
|
"rewards/cosine_scaled_reward": 0.10941554605960846,
|
|
"rewards/format_reward": 0.7500000055879354,
|
|
"step": 257
|
|
},
|
|
{
|
|
"completion_length": 1788.7917098999023,
|
|
"epoch": 0.2948571428571429,
|
|
"grad_norm": 0.24320876598358154,
|
|
"kl": 0.00939178466796875,
|
|
"learning_rate": 6.032817857379256e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.7169995531439781,
|
|
"reward_std": 0.7408483605831861,
|
|
"rewards/cosine_scaled_reward": -0.05816690996289253,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 258
|
|
},
|
|
{
|
|
"completion_length": 1297.375015258789,
|
|
"epoch": 0.296,
|
|
"grad_norm": 0.30073729157447815,
|
|
"kl": 0.011287689208984375,
|
|
"learning_rate": 6.001610194928464e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.8080965355038643,
|
|
"reward_std": 0.5971200875937939,
|
|
"rewards/cosine_scaled_reward": -0.0230350773781538,
|
|
"rewards/format_reward": 0.8541666772216558,
|
|
"step": 259
|
|
},
|
|
{
|
|
"completion_length": 1015.0000190734863,
|
|
"epoch": 0.29714285714285715,
|
|
"grad_norm": 0.29709815979003906,
|
|
"kl": 0.00806427001953125,
|
|
"learning_rate": 5.97037808470444e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.2091553770005703,
|
|
"reward_std": 0.7044631829485297,
|
|
"rewards/cosine_scaled_reward": 0.12541100312955678,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 260
|
|
},
|
|
{
|
|
"completion_length": 2179.1666870117188,
|
|
"epoch": 0.29828571428571427,
|
|
"grad_norm": 0.20508523285388947,
|
|
"kl": 0.01059722900390625,
|
|
"learning_rate": 5.939123048916173e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.47496978752315044,
|
|
"reward_std": 0.5501383896917105,
|
|
"rewards/cosine_scaled_reward": -0.07501510810106993,
|
|
"rewards/format_reward": 0.625,
|
|
"step": 261
|
|
},
|
|
{
|
|
"completion_length": 1553.3542175292969,
|
|
"epoch": 0.29942857142857143,
|
|
"grad_norm": 0.41438964009284973,
|
|
"kl": 0.01168060302734375,
|
|
"learning_rate": 5.907846610890011e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.3764358746702783,
|
|
"reward_std": 0.6005188822746277,
|
|
"rewards/cosine_scaled_reward": -0.20761540438979864,
|
|
"rewards/format_reward": 0.7916666753590107,
|
|
"step": 262
|
|
},
|
|
{
|
|
"completion_length": 1263.2917098999023,
|
|
"epoch": 0.30057142857142854,
|
|
"grad_norm": 0.26419222354888916,
|
|
"kl": 0.007190704345703125,
|
|
"learning_rate": 5.87655029499542e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.5402636826038361,
|
|
"reward_std": 0.5199831649661064,
|
|
"rewards/cosine_scaled_reward": -0.1986181689426303,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 263
|
|
},
|
|
{
|
|
"completion_length": 1310.7292404174805,
|
|
"epoch": 0.3017142857142857,
|
|
"grad_norm": 0.22596046328544617,
|
|
"kl": 0.008066177368164062,
|
|
"learning_rate": 5.845235626570683e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.8366705775260925,
|
|
"reward_std": 0.7488792147487402,
|
|
"rewards/cosine_scaled_reward": -0.0504147283063503,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 264
|
|
},
|
|
{
|
|
"completion_length": 1436.1458587646484,
|
|
"epoch": 0.3028571428571429,
|
|
"grad_norm": 0.24842660129070282,
|
|
"kl": 0.009662628173828125,
|
|
"learning_rate": 5.813904131848564e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.0776810441166162,
|
|
"reward_std": 0.6058933921158314,
|
|
"rewards/cosine_scaled_reward": 0.07009050995111465,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 265
|
|
},
|
|
{
|
|
"completion_length": 1513.9167022705078,
|
|
"epoch": 0.304,
|
|
"grad_norm": 0.22984397411346436,
|
|
"kl": 0.0084075927734375,
|
|
"learning_rate": 5.78255733788191e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.9025900475680828,
|
|
"reward_std": 0.5533245950937271,
|
|
"rewards/cosine_scaled_reward": -0.007038334384560585,
|
|
"rewards/format_reward": 0.9166666679084301,
|
|
"step": 266
|
|
},
|
|
{
|
|
"completion_length": 1806.5000228881836,
|
|
"epoch": 0.30514285714285716,
|
|
"grad_norm": 0.34077852964401245,
|
|
"kl": 0.013782501220703125,
|
|
"learning_rate": 5.751196772469237e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.22909173252992332,
|
|
"reward_std": 0.6235861741006374,
|
|
"rewards/cosine_scaled_reward": -0.2292041452601552,
|
|
"rewards/format_reward": 0.6875000074505806,
|
|
"step": 267
|
|
},
|
|
{
|
|
"completion_length": 1182.5625228881836,
|
|
"epoch": 0.3062857142857143,
|
|
"grad_norm": 0.29332637786865234,
|
|
"kl": 0.01444244384765625,
|
|
"learning_rate": 5.71982396408026e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.6916986927390099,
|
|
"reward_std": 0.5937565844506025,
|
|
"rewards/cosine_scaled_reward": -0.11248400900512934,
|
|
"rewards/format_reward": 0.9166666716337204,
|
|
"step": 268
|
|
},
|
|
{
|
|
"completion_length": 1484.3750228881836,
|
|
"epoch": 0.30742857142857144,
|
|
"grad_norm": 0.28696972131729126,
|
|
"kl": 0.010364532470703125,
|
|
"learning_rate": 5.688440441781398e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.6794960014522076,
|
|
"reward_std": 0.5427012406289577,
|
|
"rewards/cosine_scaled_reward": -0.08733535185456276,
|
|
"rewards/format_reward": 0.8541666772216558,
|
|
"step": 269
|
|
},
|
|
{
|
|
"completion_length": 1590.645851135254,
|
|
"epoch": 0.30857142857142855,
|
|
"grad_norm": 0.22989951074123383,
|
|
"kl": 0.01123046875,
|
|
"learning_rate": 5.657047735161255e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.8947062492370605,
|
|
"reward_std": 1.0069415792822838,
|
|
"rewards/cosine_scaled_reward": 0.0202697841450572,
|
|
"rewards/format_reward": 0.8541666716337204,
|
|
"step": 270
|
|
},
|
|
{
|
|
"completion_length": 1183.0000457763672,
|
|
"epoch": 0.3097142857142857,
|
|
"grad_norm": 0.29653027653694153,
|
|
"kl": 0.0086669921875,
|
|
"learning_rate": 5.625647374256061e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.1815486252307892,
|
|
"reward_std": 0.7092056274414062,
|
|
"rewards/cosine_scaled_reward": 0.11160763050429523,
|
|
"rewards/format_reward": 0.9583333358168602,
|
|
"step": 271
|
|
},
|
|
{
|
|
"completion_length": 1719.6875457763672,
|
|
"epoch": 0.31085714285714283,
|
|
"grad_norm": 0.2264571636915207,
|
|
"kl": 0.010009765625,
|
|
"learning_rate": 5.594240889475106e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.6460251696407795,
|
|
"reward_std": 0.6881718635559082,
|
|
"rewards/cosine_scaled_reward": -0.0832374356687069,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 272
|
|
},
|
|
{
|
|
"completion_length": 1260.9166870117188,
|
|
"epoch": 0.312,
|
|
"grad_norm": 0.2590281367301941,
|
|
"kl": 0.01006317138671875,
|
|
"learning_rate": 5.562829811526154e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.0608440730720758,
|
|
"reward_std": 0.49590713158249855,
|
|
"rewards/cosine_scaled_reward": 0.06167200347408652,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 273
|
|
},
|
|
{
|
|
"completion_length": 1077.4583587646484,
|
|
"epoch": 0.31314285714285717,
|
|
"grad_norm": 0.29721972346305847,
|
|
"kl": 0.012844085693359375,
|
|
"learning_rate": 5.531415671340826e-07,
|
|
"loss": 0.0005,
|
|
"reward": 1.1142795570194721,
|
|
"reward_std": 0.61124661937356,
|
|
"rewards/cosine_scaled_reward": 0.057139765471220016,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 274
|
|
},
|
|
{
|
|
"completion_length": 1554.9167175292969,
|
|
"epoch": 0.3142857142857143,
|
|
"grad_norm": 0.23796315491199493,
|
|
"kl": 0.0102996826171875,
|
|
"learning_rate": 5.5e-07,
|
|
"loss": 0.0004,
|
|
"reward": 0.974131946451962,
|
|
"reward_std": 0.6788717601448298,
|
|
"rewards/cosine_scaled_reward": 0.08081597136333585,
|
|
"rewards/format_reward": 0.8125,
|
|
"step": 275
|
|
},
|
|
{
|
|
"completion_length": 1287.2916870117188,
|
|
"epoch": 0.31542857142857145,
|
|
"grad_norm": 0.35233473777770996,
|
|
"kl": 0.012027740478515625,
|
|
"learning_rate": 5.468584328659172e-07,
|
|
"loss": 0.0005,
|
|
"reward": 1.0395971853286028,
|
|
"reward_std": 0.770504854619503,
|
|
"rewards/cosine_scaled_reward": 0.07188189588487148,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 276
|
|
},
|
|
{
|
|
"completion_length": 1217.3958549499512,
|
|
"epoch": 0.31657142857142856,
|
|
"grad_norm": 0.423069030046463,
|
|
"kl": 0.01209259033203125,
|
|
"learning_rate": 5.437170188473847e-07,
|
|
"loss": 0.0005,
|
|
"reward": 1.1905713304877281,
|
|
"reward_std": 0.7023467533290386,
|
|
"rewards/cosine_scaled_reward": 0.1369522949680686,
|
|
"rewards/format_reward": 0.9166666865348816,
|
|
"step": 277
|
|
},
|
|
{
|
|
"completion_length": 1309.5417098999023,
|
|
"epoch": 0.3177142857142857,
|
|
"grad_norm": 0.28700801730155945,
|
|
"kl": 0.007678985595703125,
|
|
"learning_rate": 5.405759110524894e-07,
|
|
"loss": 0.0003,
|
|
"reward": 1.08343615103513,
|
|
"reward_std": 0.47201682440936565,
|
|
"rewards/cosine_scaled_reward": 0.07296805875375867,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 278
|
|
},
|
|
{
|
|
"completion_length": 1422.8958740234375,
|
|
"epoch": 0.31885714285714284,
|
|
"grad_norm": 0.48463860154151917,
|
|
"kl": 0.01280975341796875,
|
|
"learning_rate": 5.37435262574394e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.8561514802277088,
|
|
"reward_std": 0.6483948938548565,
|
|
"rewards/cosine_scaled_reward": -0.04067427571862936,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 279
|
|
},
|
|
{
|
|
"completion_length": 1659.3333778381348,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.289205402135849,
|
|
"kl": 0.016315460205078125,
|
|
"learning_rate": 5.342952264838747e-07,
|
|
"loss": 0.0007,
|
|
"reward": 1.3687419444322586,
|
|
"reward_std": 0.9038915932178497,
|
|
"rewards/cosine_scaled_reward": 0.27812093193642795,
|
|
"rewards/format_reward": 0.8125000186264515,
|
|
"step": 280
|
|
},
|
|
{
|
|
"completion_length": 2221.645851135254,
|
|
"epoch": 0.3211428571428571,
|
|
"grad_norm": 0.2718718945980072,
|
|
"kl": 0.015529632568359375,
|
|
"learning_rate": 5.311559558218603e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.3475176487118006,
|
|
"reward_std": 0.7998633496463299,
|
|
"rewards/cosine_scaled_reward": -0.11790786002529785,
|
|
"rewards/format_reward": 0.5833333414047956,
|
|
"step": 281
|
|
},
|
|
{
|
|
"completion_length": 1343.0625381469727,
|
|
"epoch": 0.3222857142857143,
|
|
"grad_norm": 0.28914305567741394,
|
|
"kl": 0.010540008544921875,
|
|
"learning_rate": 5.28017603591974e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.117010936141014,
|
|
"reward_std": 0.6344310864806175,
|
|
"rewards/cosine_scaled_reward": 0.0897554587572813,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 282
|
|
},
|
|
{
|
|
"completion_length": 1945.6250686645508,
|
|
"epoch": 0.32342857142857145,
|
|
"grad_norm": 0.1946076899766922,
|
|
"kl": 0.0126190185546875,
|
|
"learning_rate": 5.248803227530763e-07,
|
|
"loss": 0.0005,
|
|
"reward": 1.111644510179758,
|
|
"reward_std": 0.7122819889336824,
|
|
"rewards/cosine_scaled_reward": 0.15998891461640596,
|
|
"rewards/format_reward": 0.7916666716337204,
|
|
"step": 283
|
|
},
|
|
{
|
|
"completion_length": 1269.0000381469727,
|
|
"epoch": 0.32457142857142857,
|
|
"grad_norm": 0.26954689621925354,
|
|
"kl": 0.006649017333984375,
|
|
"learning_rate": 5.21744266211809e-07,
|
|
"loss": 0.0003,
|
|
"reward": 0.9363188669085503,
|
|
"reward_std": 0.4825965305790305,
|
|
"rewards/cosine_scaled_reward": -0.011007236316800117,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 284
|
|
},
|
|
{
|
|
"completion_length": 881.7916946411133,
|
|
"epoch": 0.32571428571428573,
|
|
"grad_norm": 0.3140123784542084,
|
|
"kl": 0.01016998291015625,
|
|
"learning_rate": 5.186095868151436e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.102295933291316,
|
|
"reward_std": 0.8087750803679228,
|
|
"rewards/cosine_scaled_reward": 0.061564626172184944,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 285
|
|
},
|
|
{
|
|
"completion_length": 1551.270881652832,
|
|
"epoch": 0.32685714285714285,
|
|
"grad_norm": 0.4618990421295166,
|
|
"kl": 0.01885223388671875,
|
|
"learning_rate": 5.154764373429315e-07,
|
|
"loss": 0.0008,
|
|
"reward": 0.8153204774716869,
|
|
"reward_std": 0.8457888886332512,
|
|
"rewards/cosine_scaled_reward": 0.0014102212153375149,
|
|
"rewards/format_reward": 0.8125000111758709,
|
|
"step": 286
|
|
},
|
|
{
|
|
"completion_length": 1342.687515258789,
|
|
"epoch": 0.328,
|
|
"grad_norm": 0.3015505373477936,
|
|
"kl": 0.01461029052734375,
|
|
"learning_rate": 5.123449705004581e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.8659841865301132,
|
|
"reward_std": 0.7081486638635397,
|
|
"rewards/cosine_scaled_reward": 0.047575398966728244,
|
|
"rewards/format_reward": 0.7708333358168602,
|
|
"step": 287
|
|
},
|
|
{
|
|
"completion_length": 1528.8542251586914,
|
|
"epoch": 0.3291428571428571,
|
|
"grad_norm": 0.33641308546066284,
|
|
"kl": 0.014873504638671875,
|
|
"learning_rate": 5.09215338910999e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.765910281566903,
|
|
"reward_std": 0.7056640759110451,
|
|
"rewards/cosine_scaled_reward": -0.07537820562720299,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 288
|
|
},
|
|
{
|
|
"completion_length": 1399.9583587646484,
|
|
"epoch": 0.3302857142857143,
|
|
"grad_norm": 0.4806149899959564,
|
|
"kl": 0.017198562622070312,
|
|
"learning_rate": 5.060876951083828e-07,
|
|
"loss": 0.0007,
|
|
"reward": 0.7978585977107286,
|
|
"reward_std": 0.40477965772151947,
|
|
"rewards/cosine_scaled_reward": -0.038570704869925976,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 289
|
|
},
|
|
{
|
|
"completion_length": 1012.0625228881836,
|
|
"epoch": 0.3314285714285714,
|
|
"grad_norm": 0.335376113653183,
|
|
"kl": 0.009782791137695312,
|
|
"learning_rate": 5.02962191529556e-07,
|
|
"loss": 0.0004,
|
|
"reward": 1.1996897123754025,
|
|
"reward_std": 0.8074947744607925,
|
|
"rewards/cosine_scaled_reward": 0.09984485851600766,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"completion_length": 1318.7500457763672,
|
|
"epoch": 0.3325714285714286,
|
|
"grad_norm": 0.24284976720809937,
|
|
"kl": 0.014574050903320312,
|
|
"learning_rate": 4.998389805071536e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.9642973355948925,
|
|
"reward_std": 0.7317453175783157,
|
|
"rewards/cosine_scaled_reward": 0.034231980331242085,
|
|
"rewards/format_reward": 0.895833333954215,
|
|
"step": 291
|
|
},
|
|
{
|
|
"completion_length": 1687.9166946411133,
|
|
"epoch": 0.33371428571428574,
|
|
"grad_norm": 0.28374290466308594,
|
|
"kl": 0.017087936401367188,
|
|
"learning_rate": 4.967182142620745e-07,
|
|
"loss": 0.0007,
|
|
"reward": 0.6350435484200716,
|
|
"reward_std": 0.6143127456307411,
|
|
"rewards/cosine_scaled_reward": -0.1199782375479117,
|
|
"rewards/format_reward": 0.8750000111758709,
|
|
"step": 292
|
|
},
|
|
{
|
|
"completion_length": 1271.3750381469727,
|
|
"epoch": 0.33485714285714285,
|
|
"grad_norm": 0.38170552253723145,
|
|
"kl": 0.023435592651367188,
|
|
"learning_rate": 4.93600044896063e-07,
|
|
"loss": 0.0009,
|
|
"reward": 0.8408510126173496,
|
|
"reward_std": 0.6498479042202234,
|
|
"rewards/cosine_scaled_reward": -0.03790782764554024,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 293
|
|
},
|
|
{
|
|
"completion_length": 1540.2708740234375,
|
|
"epoch": 0.336,
|
|
"grad_norm": 0.29528045654296875,
|
|
"kl": 0.012912750244140625,
|
|
"learning_rate": 4.904846243842949e-07,
|
|
"loss": 0.0005,
|
|
"reward": 0.8604166656732559,
|
|
"reward_std": 0.5733331702649593,
|
|
"rewards/cosine_scaled_reward": 0.003124975599348545,
|
|
"rewards/format_reward": 0.8541666716337204,
|
|
"step": 294
|
|
},
|
|
{
|
|
"completion_length": 1394.4792098999023,
|
|
"epoch": 0.33714285714285713,
|
|
"grad_norm": 0.3902963399887085,
|
|
"kl": 0.014739990234375,
|
|
"learning_rate": 4.873721045679706e-07,
|
|
"loss": 0.0006,
|
|
"reward": 1.0252277310937643,
|
|
"reward_std": 0.5337934233248234,
|
|
"rewards/cosine_scaled_reward": 0.06469716504216194,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 295
|
|
},
|
|
{
|
|
"completion_length": 1718.8958892822266,
|
|
"epoch": 0.3382857142857143,
|
|
"grad_norm": 0.609682559967041,
|
|
"kl": 0.02099609375,
|
|
"learning_rate": 4.842626371469149e-07,
|
|
"loss": 0.0008,
|
|
"reward": 0.6263405880890787,
|
|
"reward_std": 0.6606453433632851,
|
|
"rewards/cosine_scaled_reward": -0.10349638154730201,
|
|
"rewards/format_reward": 0.833333358168602,
|
|
"step": 296
|
|
},
|
|
{
|
|
"completion_length": 1873.7083892822266,
|
|
"epoch": 0.3394285714285714,
|
|
"grad_norm": 0.29421982169151306,
|
|
"kl": 0.01636505126953125,
|
|
"learning_rate": 4.811563736721829e-07,
|
|
"loss": 0.0007,
|
|
"reward": 0.8774769939482212,
|
|
"reward_std": 0.8447716347873211,
|
|
"rewards/cosine_scaled_reward": 0.04290514811873436,
|
|
"rewards/format_reward": 0.791666679084301,
|
|
"step": 297
|
|
},
|
|
{
|
|
"completion_length": 1471.958381652832,
|
|
"epoch": 0.3405714285714286,
|
|
"grad_norm": 0.37182632088661194,
|
|
"kl": 0.017574310302734375,
|
|
"learning_rate": 4.780534655386743e-07,
|
|
"loss": 0.0007,
|
|
"reward": 0.7408417947590351,
|
|
"reward_std": 0.7095592878758907,
|
|
"rewards/cosine_scaled_reward": -0.0462457868270576,
|
|
"rewards/format_reward": 0.8333333488553762,
|
|
"step": 298
|
|
},
|
|
{
|
|
"completion_length": 1818.8958892822266,
|
|
"epoch": 0.3417142857142857,
|
|
"grad_norm": 0.4747914671897888,
|
|
"kl": 0.03339385986328125,
|
|
"learning_rate": 4.749540639777539e-07,
|
|
"loss": 0.0013,
|
|
"reward": 0.6715311715379357,
|
|
"reward_std": 0.638892836868763,
|
|
"rewards/cosine_scaled_reward": -0.02881775365676731,
|
|
"rewards/format_reward": 0.7291666846722364,
|
|
"step": 299
|
|
},
|
|
{
|
|
"completion_length": 1775.7083740234375,
|
|
"epoch": 0.34285714285714286,
|
|
"grad_norm": 0.8008535504341125,
|
|
"kl": 0.03324127197265625,
|
|
"learning_rate": 4.7185832004988133e-07,
|
|
"loss": 0.0013,
|
|
"reward": 0.503076022490859,
|
|
"reward_std": 0.6447076573967934,
|
|
"rewards/cosine_scaled_reward": -0.08179534040391445,
|
|
"rewards/format_reward": 0.6666666846722364,
|
|
"step": 300
|
|
},
|
|
{
|
|
"completion_length": 1681.7500457763672,
|
|
"epoch": 0.344,
|
|
"grad_norm": 0.6473574042320251,
|
|
"kl": 0.0345306396484375,
|
|
"learning_rate": 4.68766384637248e-07,
|
|
"loss": 0.0014,
|
|
"reward": 0.47819951456040144,
|
|
"reward_std": 0.7737650983035564,
|
|
"rewards/cosine_scaled_reward": -0.16715026053134352,
|
|
"rewards/format_reward": 0.8125000074505806,
|
|
"step": 301
|
|
},
|
|
{
|
|
"completion_length": 1704.6250228881836,
|
|
"epoch": 0.34514285714285714,
|
|
"grad_norm": 0.32545921206474304,
|
|
"kl": 0.03167724609375,
|
|
"learning_rate": 4.656784084364238e-07,
|
|
"loss": 0.0013,
|
|
"reward": 0.9550731834024191,
|
|
"reward_std": 0.7380726649425924,
|
|
"rewards/cosine_scaled_reward": 0.11295327357947826,
|
|
"rewards/format_reward": 0.7291666716337204,
|
|
"step": 302
|
|
},
|
|
{
|
|
"completion_length": 1360.1458740234375,
|
|
"epoch": 0.3462857142857143,
|
|
"grad_norm": 0.35140302777290344,
|
|
"kl": 0.02618408203125,
|
|
"learning_rate": 4.6259454195101267e-07,
|
|
"loss": 0.001,
|
|
"reward": 0.7957173548638821,
|
|
"reward_std": 0.7143728174269199,
|
|
"rewards/cosine_scaled_reward": -0.018808012828230858,
|
|
"rewards/format_reward": 0.8333333358168602,
|
|
"step": 303
|
|
},
|
|
{
|
|
"completion_length": 1244.3750381469727,
|
|
"epoch": 0.3474285714285714,
|
|
"grad_norm": 0.38197869062423706,
|
|
"kl": 0.0144195556640625,
|
|
"learning_rate": 4.59514935484316e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.6606247052550316,
|
|
"reward_std": 0.5714793428778648,
|
|
"rewards/cosine_scaled_reward": -0.11760433949530125,
|
|
"rewards/format_reward": 0.8958333395421505,
|
|
"step": 304
|
|
},
|
|
{
|
|
"completion_length": 1325.6667022705078,
|
|
"epoch": 0.3485714285714286,
|
|
"grad_norm": 0.3902123272418976,
|
|
"kl": 0.016117095947265625,
|
|
"learning_rate": 4.5643973913200837e-07,
|
|
"loss": 0.0006,
|
|
"reward": 0.6409200690686703,
|
|
"reward_std": 0.6203774958848953,
|
|
"rewards/cosine_scaled_reward": -0.1587066389620304,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 305
|
|
},
|
|
{
|
|
"completion_length": 1282.3542175292969,
|
|
"epoch": 0.3497142857142857,
|
|
"grad_norm": 0.6988428235054016,
|
|
"kl": 0.026388168334960938,
|
|
"learning_rate": 4.5336910277482155e-07,
|
|
"loss": 0.0011,
|
|
"reward": 1.0727143473923206,
|
|
"reward_std": 0.6053726552054286,
|
|
"rewards/cosine_scaled_reward": 0.11969051510095596,
|
|
"rewards/format_reward": 0.833333333954215,
|
|
"step": 306
|
|
},
|
|
{
|
|
"completion_length": 1716.9583702087402,
|
|
"epoch": 0.35085714285714287,
|
|
"grad_norm": 0.36837834119796753,
|
|
"kl": 0.040142059326171875,
|
|
"learning_rate": 4.503031760712397e-07,
|
|
"loss": 0.0016,
|
|
"reward": 0.8561188094317913,
|
|
"reward_std": 0.9516715090721846,
|
|
"rewards/cosine_scaled_reward": 0.011392734944820404,
|
|
"rewards/format_reward": 0.8333333395421505,
|
|
"step": 307
|
|
},
|
|
{
|
|
"completion_length": 2273.4584045410156,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.5795905590057373,
|
|
"kl": 0.03869056701660156,
|
|
"learning_rate": 4.4724210845020494e-07,
|
|
"loss": 0.0015,
|
|
"reward": 0.7666318230330944,
|
|
"reward_std": 0.9787959046661854,
|
|
"rewards/cosine_scaled_reward": -0.002100769430398941,
|
|
"rewards/format_reward": 0.7708333432674408,
|
|
"step": 308
|
|
},
|
|
{
|
|
"completion_length": 2063.3542556762695,
|
|
"epoch": 0.35314285714285715,
|
|
"grad_norm": 0.3654479384422302,
|
|
"kl": 0.05999183654785156,
|
|
"learning_rate": 4.441860491038345e-07,
|
|
"loss": 0.0024,
|
|
"reward": 0.6858750740066171,
|
|
"reward_std": 0.760811198502779,
|
|
"rewards/cosine_scaled_reward": -0.0528958085924387,
|
|
"rewards/format_reward": 0.7916666734963655,
|
|
"step": 309
|
|
},
|
|
{
|
|
"completion_length": 1340.1250305175781,
|
|
"epoch": 0.35428571428571426,
|
|
"grad_norm": 0.49283814430236816,
|
|
"kl": 0.033687591552734375,
|
|
"learning_rate": 4.4113514698014953e-07,
|
|
"loss": 0.0013,
|
|
"reward": 0.7110094074159861,
|
|
"reward_std": 0.7351307831704617,
|
|
"rewards/cosine_scaled_reward": -0.06116198655217886,
|
|
"rewards/format_reward": 0.8333333395421505,
|
|
"step": 310
|
|
},
|
|
{
|
|
"completion_length": 1418.187515258789,
|
|
"epoch": 0.3554285714285714,
|
|
"grad_norm": 0.44990670680999756,
|
|
"kl": 0.04097747802734375,
|
|
"learning_rate": 4.3808955077581546e-07,
|
|
"loss": 0.0016,
|
|
"reward": 1.2232839856296778,
|
|
"reward_std": 0.9500982649624348,
|
|
"rewards/cosine_scaled_reward": 0.15330865047872066,
|
|
"rewards/format_reward": 0.9166666716337204,
|
|
"step": 311
|
|
},
|
|
{
|
|
"completion_length": 1471.708381652832,
|
|
"epoch": 0.3565714285714286,
|
|
"grad_norm": 0.6391323804855347,
|
|
"kl": 0.04137420654296875,
|
|
"learning_rate": 4.350494089288943e-07,
|
|
"loss": 0.0017,
|
|
"reward": 1.1680905930697918,
|
|
"reward_std": 0.37351681664586067,
|
|
"rewards/cosine_scaled_reward": 0.16737862676382065,
|
|
"rewards/format_reward": 0.8333333395421505,
|
|
"step": 312
|
|
},
|
|
{
|
|
"completion_length": 2079.6667137145996,
|
|
"epoch": 0.3577142857142857,
|
|
"grad_norm": 0.9562482833862305,
|
|
"kl": 0.07365036010742188,
|
|
"learning_rate": 4.3201486961161093e-07,
|
|
"loss": 0.0029,
|
|
"reward": 0.6807579882442951,
|
|
"reward_std": 0.8096479428932071,
|
|
"rewards/cosine_scaled_reward": 0.017462321557104588,
|
|
"rewards/format_reward": 0.6458333488553762,
|
|
"step": 313
|
|
},
|
|
{
|
|
"completion_length": 1646.2500305175781,
|
|
"epoch": 0.3588571428571429,
|
|
"grad_norm": 0.582167387008667,
|
|
"kl": 0.06851959228515625,
|
|
"learning_rate": 4.2898608072313045e-07,
|
|
"loss": 0.0027,
|
|
"reward": 0.823919128626585,
|
|
"reward_std": 0.5355783794075251,
|
|
"rewards/cosine_scaled_reward": 0.0369595680385828,
|
|
"rewards/format_reward": 0.7500000111758709,
|
|
"step": 314
|
|
},
|
|
{
|
|
"completion_length": 2116.229217529297,
|
|
"epoch": 0.36,
|
|
"grad_norm": 0.6685879230499268,
|
|
"kl": 0.1165008544921875,
|
|
"learning_rate": 4.2596318988235037e-07,
|
|
"loss": 0.0047,
|
|
"reward": 0.8504741322249174,
|
|
"reward_std": 0.5491553768515587,
|
|
"rewards/cosine_scaled_reward": 0.06065371725708246,
|
|
"rewards/format_reward": 0.7291666734963655,
|
|
"step": 315
|
|
},
|
|
{
|
|
"completion_length": 2242.4375610351562,
|
|
"epoch": 0.36114285714285715,
|
|
"grad_norm": 0.7561622262001038,
|
|
"kl": 0.11125946044921875,
|
|
"learning_rate": 4.2294634442070553e-07,
|
|
"loss": 0.0045,
|
|
"reward": -0.009940480813384056,
|
|
"reward_std": 0.5052176639437675,
|
|
"rewards/cosine_scaled_reward": -0.22372024692595005,
|
|
"rewards/format_reward": 0.43750000931322575,
|
|
"step": 316
|
|
},
|
|
{
|
|
"completion_length": 1841.1875457763672,
|
|
"epoch": 0.36228571428571427,
|
|
"grad_norm": 1.0075013637542725,
|
|
"kl": 0.0679473876953125,
|
|
"learning_rate": 4.1993569137498776e-07,
|
|
"loss": 0.0027,
|
|
"reward": 0.5096511642332189,
|
|
"reward_std": 0.8513825722038746,
|
|
"rewards/cosine_scaled_reward": -0.057674430310726166,
|
|
"rewards/format_reward": 0.625000013038516,
|
|
"step": 317
|
|
},
|
|
{
|
|
"completion_length": 1029.7916870117188,
|
|
"epoch": 0.36342857142857143,
|
|
"grad_norm": 0.38250845670700073,
|
|
"kl": 0.039661407470703125,
|
|
"learning_rate": 4.1693137748017915e-07,
|
|
"loss": 0.0016,
|
|
"reward": 0.8797074742615223,
|
|
"reward_std": 0.4655795283615589,
|
|
"rewards/cosine_scaled_reward": -0.0601462684571743,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 318
|
|
},
|
|
{
|
|
"completion_length": 1612.9167098999023,
|
|
"epoch": 0.36457142857142855,
|
|
"grad_norm": 0.7825446724891663,
|
|
"kl": 0.05585479736328125,
|
|
"learning_rate": 4.1393354916230005e-07,
|
|
"loss": 0.0022,
|
|
"reward": 0.5894158203154802,
|
|
"reward_std": 0.7856029607355595,
|
|
"rewards/cosine_scaled_reward": -0.13237543310970068,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 319
|
|
},
|
|
{
|
|
"completion_length": 1328.7916831970215,
|
|
"epoch": 0.3657142857142857,
|
|
"grad_norm": 0.7866749167442322,
|
|
"kl": 0.07510757446289062,
|
|
"learning_rate": 4.1094235253127374e-07,
|
|
"loss": 0.003,
|
|
"reward": 1.0613620709627867,
|
|
"reward_std": 0.6303130388259888,
|
|
"rewards/cosine_scaled_reward": 0.07234767638146877,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 320
|
|
},
|
|
{
|
|
"completion_length": 1117.3125457763672,
|
|
"epoch": 0.3668571428571429,
|
|
"grad_norm": 0.5064549446105957,
|
|
"kl": 0.04396820068359375,
|
|
"learning_rate": 4.079579333738039e-07,
|
|
"loss": 0.0018,
|
|
"reward": 1.1430763825774193,
|
|
"reward_std": 0.6700709462165833,
|
|
"rewards/cosine_scaled_reward": 0.1340381633490324,
|
|
"rewards/format_reward": 0.8750000111758709,
|
|
"step": 321
|
|
},
|
|
{
|
|
"completion_length": 1828.0417175292969,
|
|
"epoch": 0.368,
|
|
"grad_norm": 0.7172619104385376,
|
|
"kl": 0.133880615234375,
|
|
"learning_rate": 4.0498043714627006e-07,
|
|
"loss": 0.0054,
|
|
"reward": 0.645214811898768,
|
|
"reward_std": 1.0024632290005684,
|
|
"rewards/cosine_scaled_reward": -0.04197592940181494,
|
|
"rewards/format_reward": 0.7291666753590107,
|
|
"step": 322
|
|
},
|
|
{
|
|
"completion_length": 1734.2916946411133,
|
|
"epoch": 0.36914285714285716,
|
|
"grad_norm": 0.7203247547149658,
|
|
"kl": 0.1142730712890625,
|
|
"learning_rate": 4.020100089676376e-07,
|
|
"loss": 0.0046,
|
|
"reward": 0.6648443900048733,
|
|
"reward_std": 0.6985526494681835,
|
|
"rewards/cosine_scaled_reward": 0.009505534544587135,
|
|
"rewards/format_reward": 0.6458333414047956,
|
|
"step": 323
|
|
},
|
|
{
|
|
"completion_length": 1589.1667022705078,
|
|
"epoch": 0.3702857142857143,
|
|
"grad_norm": 0.8060963153839111,
|
|
"kl": 0.10352706909179688,
|
|
"learning_rate": 3.9904679361238526e-07,
|
|
"loss": 0.0041,
|
|
"reward": 0.27571332873776555,
|
|
"reward_std": 0.6113120466470718,
|
|
"rewards/cosine_scaled_reward": -0.20589334331452847,
|
|
"rewards/format_reward": 0.6875000167638063,
|
|
"step": 324
|
|
},
|
|
{
|
|
"completion_length": 1642.0625686645508,
|
|
"epoch": 0.37142857142857144,
|
|
"grad_norm": 0.603640079498291,
|
|
"kl": 0.059417724609375,
|
|
"learning_rate": 3.9609093550344907e-07,
|
|
"loss": 0.0024,
|
|
"reward": 0.9450785778462887,
|
|
"reward_std": 0.8715181350708008,
|
|
"rewards/cosine_scaled_reward": 0.045455962885171175,
|
|
"rewards/format_reward": 0.8541666865348816,
|
|
"step": 325
|
|
},
|
|
{
|
|
"completion_length": 1383.208351135254,
|
|
"epoch": 0.37257142857142855,
|
|
"grad_norm": 0.8735449314117432,
|
|
"kl": 0.08353424072265625,
|
|
"learning_rate": 3.931425787051832e-07,
|
|
"loss": 0.0033,
|
|
"reward": 1.0190029181540012,
|
|
"reward_std": 0.7654371298849583,
|
|
"rewards/cosine_scaled_reward": 0.07200142601504922,
|
|
"rewards/format_reward": 0.8750000111758709,
|
|
"step": 326
|
|
},
|
|
{
|
|
"completion_length": 1609.208366394043,
|
|
"epoch": 0.3737142857142857,
|
|
"grad_norm": 0.40844622254371643,
|
|
"kl": 0.056499481201171875,
|
|
"learning_rate": 3.902018669163384e-07,
|
|
"loss": 0.0023,
|
|
"reward": 1.1053630914539099,
|
|
"reward_std": 0.6595817804336548,
|
|
"rewards/cosine_scaled_reward": 0.1360148610547185,
|
|
"rewards/format_reward": 0.8333333358168602,
|
|
"step": 327
|
|
},
|
|
{
|
|
"completion_length": 1788.4375534057617,
|
|
"epoch": 0.37485714285714283,
|
|
"grad_norm": 0.6934894919395447,
|
|
"kl": 0.11262893676757812,
|
|
"learning_rate": 3.872689434630585e-07,
|
|
"loss": 0.0045,
|
|
"reward": 0.47454674541950226,
|
|
"reward_std": 0.7443269528448582,
|
|
"rewards/cosine_scaled_reward": -0.12730997893959284,
|
|
"rewards/format_reward": 0.7291666753590107,
|
|
"step": 328
|
|
},
|
|
{
|
|
"completion_length": 1103.7083587646484,
|
|
"epoch": 0.376,
|
|
"grad_norm": 0.9421954154968262,
|
|
"kl": 0.05080413818359375,
|
|
"learning_rate": 3.843439512918949e-07,
|
|
"loss": 0.002,
|
|
"reward": 1.2518079336732626,
|
|
"reward_std": 0.5773205179721117,
|
|
"rewards/cosine_scaled_reward": 0.15715394588187337,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 329
|
|
},
|
|
{
|
|
"completion_length": 1171.9791946411133,
|
|
"epoch": 0.37714285714285717,
|
|
"grad_norm": 0.7696932554244995,
|
|
"kl": 0.10460662841796875,
|
|
"learning_rate": 3.8142703296283953e-07,
|
|
"loss": 0.0042,
|
|
"reward": 0.5345208197832108,
|
|
"reward_std": 0.7594005465507507,
|
|
"rewards/cosine_scaled_reward": -0.15982293151319027,
|
|
"rewards/format_reward": 0.8541666716337204,
|
|
"step": 330
|
|
},
|
|
{
|
|
"completion_length": 1748.4167098999023,
|
|
"epoch": 0.3782857142857143,
|
|
"grad_norm": 1.497854232788086,
|
|
"kl": 0.12451934814453125,
|
|
"learning_rate": 3.785183306423767e-07,
|
|
"loss": 0.005,
|
|
"reward": 0.5829995409585536,
|
|
"reward_std": 0.8618629835546017,
|
|
"rewards/cosine_scaled_reward": -0.0626668983604759,
|
|
"rewards/format_reward": 0.7083333432674408,
|
|
"step": 331
|
|
},
|
|
{
|
|
"completion_length": 1661.1667022705078,
|
|
"epoch": 0.37942857142857145,
|
|
"grad_norm": 0.7761502861976624,
|
|
"kl": 0.08023452758789062,
|
|
"learning_rate": 3.7561798609655373e-07,
|
|
"loss": 0.0032,
|
|
"reward": 0.5321935811080039,
|
|
"reward_std": 0.5660249888896942,
|
|
"rewards/cosine_scaled_reward": -0.11931989248842001,
|
|
"rewards/format_reward": 0.7708333432674408,
|
|
"step": 332
|
|
},
|
|
{
|
|
"completion_length": 1305.7500228881836,
|
|
"epoch": 0.38057142857142856,
|
|
"grad_norm": 0.7130544185638428,
|
|
"kl": 0.0748291015625,
|
|
"learning_rate": 3.72726140684072e-07,
|
|
"loss": 0.003,
|
|
"reward": 0.7820851001888514,
|
|
"reward_std": 0.671183954924345,
|
|
"rewards/cosine_scaled_reward": -0.06729080062359571,
|
|
"rewards/format_reward": 0.9166666865348816,
|
|
"step": 333
|
|
},
|
|
{
|
|
"completion_length": 2046.4375610351562,
|
|
"epoch": 0.38171428571428573,
|
|
"grad_norm": 0.9568099975585938,
|
|
"kl": 0.18133544921875,
|
|
"learning_rate": 3.6984293534939737e-07,
|
|
"loss": 0.0072,
|
|
"reward": 0.355112224817276,
|
|
"reward_std": 0.6977374590933323,
|
|
"rewards/cosine_scaled_reward": -0.1870272308588028,
|
|
"rewards/format_reward": 0.7291666828095913,
|
|
"step": 334
|
|
},
|
|
{
|
|
"completion_length": 1462.2292022705078,
|
|
"epoch": 0.38285714285714284,
|
|
"grad_norm": 1.2614269256591797,
|
|
"kl": 0.07938385009765625,
|
|
"learning_rate": 3.6696851061588994e-07,
|
|
"loss": 0.0032,
|
|
"reward": 0.9349308745004237,
|
|
"reward_std": 0.6556266993284225,
|
|
"rewards/cosine_scaled_reward": 0.04038208909332752,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 335
|
|
},
|
|
{
|
|
"completion_length": 1641.4375534057617,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.8289753198623657,
|
|
"kl": 0.11359786987304688,
|
|
"learning_rate": 3.641030065789562e-07,
|
|
"loss": 0.0045,
|
|
"reward": 0.8081842958927155,
|
|
"reward_std": 0.931253258138895,
|
|
"rewards/cosine_scaled_reward": 0.08117546886205673,
|
|
"rewards/format_reward": 0.6458333469927311,
|
|
"step": 336
|
|
},
|
|
{
|
|
"completion_length": 1747.2500534057617,
|
|
"epoch": 0.3851428571428571,
|
|
"grad_norm": 1.1650760173797607,
|
|
"kl": 0.12990570068359375,
|
|
"learning_rate": 3.612465628992203e-07,
|
|
"loss": 0.0052,
|
|
"reward": 0.7834707293659449,
|
|
"reward_std": 0.8634283617138863,
|
|
"rewards/cosine_scaled_reward": -0.04576464742422104,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 337
|
|
},
|
|
{
|
|
"completion_length": 1280.7708740234375,
|
|
"epoch": 0.3862857142857143,
|
|
"grad_norm": 0.5302877426147461,
|
|
"kl": 0.062366485595703125,
|
|
"learning_rate": 3.5839931879571725e-07,
|
|
"loss": 0.0025,
|
|
"reward": 1.1318869441747665,
|
|
"reward_std": 0.7834238409996033,
|
|
"rewards/cosine_scaled_reward": 0.09719344391487539,
|
|
"rewards/format_reward": 0.9375,
|
|
"step": 338
|
|
},
|
|
{
|
|
"completion_length": 1575.9375381469727,
|
|
"epoch": 0.38742857142857146,
|
|
"grad_norm": 3.0021309852600098,
|
|
"kl": 0.14781951904296875,
|
|
"learning_rate": 3.555614130391079e-07,
|
|
"loss": 0.0059,
|
|
"reward": 0.435087047284469,
|
|
"reward_std": 0.43532489985227585,
|
|
"rewards/cosine_scaled_reward": -0.13662315905094147,
|
|
"rewards/format_reward": 0.7083333563059568,
|
|
"step": 339
|
|
},
|
|
{
|
|
"completion_length": 1496.6250534057617,
|
|
"epoch": 0.38857142857142857,
|
|
"grad_norm": 1.7084987163543701,
|
|
"kl": 0.12076950073242188,
|
|
"learning_rate": 3.5273298394491515e-07,
|
|
"loss": 0.0048,
|
|
"reward": 0.7966768400510773,
|
|
"reward_std": 0.5643632663413882,
|
|
"rewards/cosine_scaled_reward": -0.007911591790616512,
|
|
"rewards/format_reward": 0.8125000111758709,
|
|
"step": 340
|
|
},
|
|
{
|
|
"completion_length": 1304.9792098999023,
|
|
"epoch": 0.38971428571428574,
|
|
"grad_norm": 5.948554039001465,
|
|
"kl": 0.22769927978515625,
|
|
"learning_rate": 3.4991416936678276e-07,
|
|
"loss": 0.0091,
|
|
"reward": 1.1839299397543073,
|
|
"reward_std": 0.643942728638649,
|
|
"rewards/cosine_scaled_reward": 0.175298273563385,
|
|
"rewards/format_reward": 0.8333333507180214,
|
|
"step": 341
|
|
},
|
|
{
|
|
"completion_length": 1663.6875610351562,
|
|
"epoch": 0.39085714285714285,
|
|
"grad_norm": 1.0961518287658691,
|
|
"kl": 0.18028640747070312,
|
|
"learning_rate": 3.471051066897562e-07,
|
|
"loss": 0.0072,
|
|
"reward": 0.8878756612539291,
|
|
"reward_std": 0.9449864365160465,
|
|
"rewards/cosine_scaled_reward": 0.06893783865962178,
|
|
"rewards/format_reward": 0.750000013038516,
|
|
"step": 342
|
|
},
|
|
{
|
|
"completion_length": 1583.0000686645508,
|
|
"epoch": 0.392,
|
|
"grad_norm": 1.2760441303253174,
|
|
"kl": 0.15594482421875,
|
|
"learning_rate": 3.4430593282358777e-07,
|
|
"loss": 0.0062,
|
|
"reward": 0.8547459337860346,
|
|
"reward_std": 0.6034443583339453,
|
|
"rewards/cosine_scaled_reward": 0.0002896404330385849,
|
|
"rewards/format_reward": 0.8541666939854622,
|
|
"step": 343
|
|
},
|
|
{
|
|
"completion_length": 1706.4375457763672,
|
|
"epoch": 0.3931428571428571,
|
|
"grad_norm": 1.06587553024292,
|
|
"kl": 0.281158447265625,
|
|
"learning_rate": 3.4151678419606233e-07,
|
|
"loss": 0.0113,
|
|
"reward": 1.1242186180315912,
|
|
"reward_std": 0.6546860057860613,
|
|
"rewards/cosine_scaled_reward": 0.17669263062998652,
|
|
"rewards/format_reward": 0.7708333432674408,
|
|
"step": 344
|
|
},
|
|
{
|
|
"completion_length": 1465.4583587646484,
|
|
"epoch": 0.3942857142857143,
|
|
"grad_norm": 1.2281720638275146,
|
|
"kl": 0.1438140869140625,
|
|
"learning_rate": 3.387377967463493e-07,
|
|
"loss": 0.0058,
|
|
"reward": 0.9402041547000408,
|
|
"reward_std": 0.6865711808204651,
|
|
"rewards/cosine_scaled_reward": 0.011768726049922407,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 345
|
|
},
|
|
{
|
|
"completion_length": 1591.0833740234375,
|
|
"epoch": 0.3954285714285714,
|
|
"grad_norm": 0.8186521530151367,
|
|
"kl": 0.10266876220703125,
|
|
"learning_rate": 3.359691059183761e-07,
|
|
"loss": 0.0041,
|
|
"reward": 0.6260932851582766,
|
|
"reward_std": 0.5609011054039001,
|
|
"rewards/cosine_scaled_reward": -0.15570336702512577,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 346
|
|
},
|
|
{
|
|
"completion_length": 1574.2083892822266,
|
|
"epoch": 0.3965714285714286,
|
|
"grad_norm": 0.7894213795661926,
|
|
"kl": 0.1241302490234375,
|
|
"learning_rate": 3.3321084665422803e-07,
|
|
"loss": 0.005,
|
|
"reward": 0.5836004763841629,
|
|
"reward_std": 0.6906316690146923,
|
|
"rewards/cosine_scaled_reward": -0.15611644479213282,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 347
|
|
},
|
|
{
|
|
"completion_length": 1624.1667022705078,
|
|
"epoch": 0.3977142857142857,
|
|
"grad_norm": 1.4086353778839111,
|
|
"kl": 0.22664642333984375,
|
|
"learning_rate": 3.3046315338757026e-07,
|
|
"loss": 0.0091,
|
|
"reward": 0.7276029635686427,
|
|
"reward_std": 0.6486309953033924,
|
|
"rewards/cosine_scaled_reward": -0.04244852438569069,
|
|
"rewards/format_reward": 0.8125000074505806,
|
|
"step": 348
|
|
},
|
|
{
|
|
"completion_length": 1256.1875228881836,
|
|
"epoch": 0.39885714285714285,
|
|
"grad_norm": 1.4020756483078003,
|
|
"kl": 0.1046600341796875,
|
|
"learning_rate": 3.2772616003709616e-07,
|
|
"loss": 0.0042,
|
|
"reward": 0.5811667609959841,
|
|
"reward_std": 0.7433434501290321,
|
|
"rewards/cosine_scaled_reward": -0.09483329905197024,
|
|
"rewards/format_reward": 0.7708333469927311,
|
|
"step": 349
|
|
},
|
|
{
|
|
"completion_length": 1057.7291946411133,
|
|
"epoch": 0.4,
|
|
"grad_norm": 0.9083346128463745,
|
|
"kl": 0.0474395751953125,
|
|
"learning_rate": 3.250000000000001e-07,
|
|
"loss": 0.0019,
|
|
"reward": 0.8259809445589781,
|
|
"reward_std": 0.7229878939688206,
|
|
"rewards/cosine_scaled_reward": -0.06617620773613453,
|
|
"rewards/format_reward": 0.9583333358168602,
|
|
"step": 350
|
|
},
|
|
{
|
|
"completion_length": 1219.6458740234375,
|
|
"epoch": 0.40114285714285713,
|
|
"grad_norm": 1.6471617221832275,
|
|
"kl": 0.12386322021484375,
|
|
"learning_rate": 3.222848061454764e-07,
|
|
"loss": 0.0049,
|
|
"reward": 0.7785749807953835,
|
|
"reward_std": 0.8159589394927025,
|
|
"rewards/cosine_scaled_reward": -0.0377958663739264,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 351
|
|
},
|
|
{
|
|
"completion_length": 1422.1875534057617,
|
|
"epoch": 0.4022857142857143,
|
|
"grad_norm": 1.3077303171157837,
|
|
"kl": 0.2323760986328125,
|
|
"learning_rate": 3.195807108082429e-07,
|
|
"loss": 0.0093,
|
|
"reward": 0.9625241123139858,
|
|
"reward_std": 0.8021371066570282,
|
|
"rewards/cosine_scaled_reward": 0.06459538266062737,
|
|
"rewards/format_reward": 0.8333333469927311,
|
|
"step": 352
|
|
},
|
|
{
|
|
"completion_length": 1283.333351135254,
|
|
"epoch": 0.4034285714285714,
|
|
"grad_norm": 1.1020612716674805,
|
|
"kl": 0.125701904296875,
|
|
"learning_rate": 3.168878457820915e-07,
|
|
"loss": 0.005,
|
|
"reward": 1.0344783924520016,
|
|
"reward_std": 0.8279353678226471,
|
|
"rewards/cosine_scaled_reward": 0.05890584830194712,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 353
|
|
},
|
|
{
|
|
"completion_length": 1127.6667022705078,
|
|
"epoch": 0.4045714285714286,
|
|
"grad_norm": 1.216342568397522,
|
|
"kl": 0.12079620361328125,
|
|
"learning_rate": 3.142063423134644e-07,
|
|
"loss": 0.0048,
|
|
"reward": 1.099512368440628,
|
|
"reward_std": 0.5979024097323418,
|
|
"rewards/cosine_scaled_reward": 0.0914228311739862,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 354
|
|
},
|
|
{
|
|
"completion_length": 1001.6458549499512,
|
|
"epoch": 0.4057142857142857,
|
|
"grad_norm": 0.9325054287910461,
|
|
"kl": 0.059314727783203125,
|
|
"learning_rate": 3.115363310950578e-07,
|
|
"loss": 0.0024,
|
|
"reward": 1.2166800498962402,
|
|
"reward_std": 0.675847515463829,
|
|
"rewards/cosine_scaled_reward": 0.12917334213852882,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 355
|
|
},
|
|
{
|
|
"completion_length": 1459.2292251586914,
|
|
"epoch": 0.40685714285714286,
|
|
"grad_norm": 2.4190444946289062,
|
|
"kl": 0.2304840087890625,
|
|
"learning_rate": 3.0887794225945143e-07,
|
|
"loss": 0.0092,
|
|
"reward": 0.4463031552731991,
|
|
"reward_std": 0.6662746425718069,
|
|
"rewards/cosine_scaled_reward": -0.17268177028745413,
|
|
"rewards/format_reward": 0.7916666865348816,
|
|
"step": 356
|
|
},
|
|
{
|
|
"completion_length": 1725.6875457763672,
|
|
"epoch": 0.408,
|
|
"grad_norm": 2.1936678886413574,
|
|
"kl": 0.295989990234375,
|
|
"learning_rate": 3.062313053727671e-07,
|
|
"loss": 0.0118,
|
|
"reward": 0.3668034356087446,
|
|
"reward_std": 0.5282239988446236,
|
|
"rewards/cosine_scaled_reward": -0.26451496221125126,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 357
|
|
},
|
|
{
|
|
"completion_length": 1472.312515258789,
|
|
"epoch": 0.40914285714285714,
|
|
"grad_norm": 1.637854814529419,
|
|
"kl": 0.2021331787109375,
|
|
"learning_rate": 3.0359654942835247e-07,
|
|
"loss": 0.0081,
|
|
"reward": 1.2237652689218521,
|
|
"reward_std": 0.6800592541694641,
|
|
"rewards/cosine_scaled_reward": 0.1535492818802595,
|
|
"rewards/format_reward": 0.9166666716337204,
|
|
"step": 358
|
|
},
|
|
{
|
|
"completion_length": 929.5416793823242,
|
|
"epoch": 0.4102857142857143,
|
|
"grad_norm": 1.4736838340759277,
|
|
"kl": 0.1471099853515625,
|
|
"learning_rate": 3.0097380284049523e-07,
|
|
"loss": 0.0059,
|
|
"reward": 0.8410445712506771,
|
|
"reward_std": 0.5730547439306974,
|
|
"rewards/cosine_scaled_reward": -0.06906107859686017,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 359
|
|
},
|
|
{
|
|
"completion_length": 1209.8125457763672,
|
|
"epoch": 0.4114285714285714,
|
|
"grad_norm": 2.1532845497131348,
|
|
"kl": 0.35944366455078125,
|
|
"learning_rate": 2.9836319343816397e-07,
|
|
"loss": 0.0144,
|
|
"reward": 0.769681841135025,
|
|
"reward_std": 0.8142721727490425,
|
|
"rewards/cosine_scaled_reward": -0.06307575106620789,
|
|
"rewards/format_reward": 0.895833358168602,
|
|
"step": 360
|
|
},
|
|
{
|
|
"completion_length": 1173.6041946411133,
|
|
"epoch": 0.4125714285714286,
|
|
"grad_norm": 2.002321481704712,
|
|
"kl": 0.2322998046875,
|
|
"learning_rate": 2.9576484845877793e-07,
|
|
"loss": 0.0093,
|
|
"reward": 0.8964769653975964,
|
|
"reward_std": 0.7383791692554951,
|
|
"rewards/cosine_scaled_reward": -0.041344886645674706,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 361
|
|
},
|
|
{
|
|
"completion_length": 1014.0625305175781,
|
|
"epoch": 0.4137142857142857,
|
|
"grad_norm": 1.2043986320495605,
|
|
"kl": 0.2470703125,
|
|
"learning_rate": 2.931788945420058e-07,
|
|
"loss": 0.0099,
|
|
"reward": 1.00144612044096,
|
|
"reward_std": 0.47917743772268295,
|
|
"rewards/cosine_scaled_reward": 0.03197303228080273,
|
|
"rewards/format_reward": 0.9375,
|
|
"step": 362
|
|
},
|
|
{
|
|
"completion_length": 896.8541870117188,
|
|
"epoch": 0.41485714285714287,
|
|
"grad_norm": 1.703730583190918,
|
|
"kl": 0.1836700439453125,
|
|
"learning_rate": 2.9060545772359305e-07,
|
|
"loss": 0.0074,
|
|
"reward": 1.373911328613758,
|
|
"reward_std": 0.7581704575568438,
|
|
"rewards/cosine_scaled_reward": 0.21820560842752457,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 363
|
|
},
|
|
{
|
|
"completion_length": 1474.4167251586914,
|
|
"epoch": 0.416,
|
|
"grad_norm": 2.327479839324951,
|
|
"kl": 0.5921630859375,
|
|
"learning_rate": 2.8804466342921987e-07,
|
|
"loss": 0.0237,
|
|
"reward": 0.48526396974921227,
|
|
"reward_std": 0.5558421425521374,
|
|
"rewards/cosine_scaled_reward": -0.20528469607234,
|
|
"rewards/format_reward": 0.895833358168602,
|
|
"step": 364
|
|
},
|
|
{
|
|
"completion_length": 1866.6875610351562,
|
|
"epoch": 0.41714285714285715,
|
|
"grad_norm": 2.2852516174316406,
|
|
"kl": 0.80853271484375,
|
|
"learning_rate": 2.854966364683872e-07,
|
|
"loss": 0.0324,
|
|
"reward": 0.7523469850420952,
|
|
"reward_std": 0.6609731055796146,
|
|
"rewards/cosine_scaled_reward": 0.03242346830666065,
|
|
"rewards/format_reward": 0.6875000111758709,
|
|
"step": 365
|
|
},
|
|
{
|
|
"completion_length": 1085.2083740234375,
|
|
"epoch": 0.41828571428571426,
|
|
"grad_norm": 2.795609951019287,
|
|
"kl": 0.29659271240234375,
|
|
"learning_rate": 2.829615010283344e-07,
|
|
"loss": 0.0119,
|
|
"reward": 0.9424293376505375,
|
|
"reward_std": 0.763162437826395,
|
|
"rewards/cosine_scaled_reward": 0.023297980427742004,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 366
|
|
},
|
|
{
|
|
"completion_length": 1567.9167175292969,
|
|
"epoch": 0.41942857142857143,
|
|
"grad_norm": 1.7122759819030762,
|
|
"kl": 0.5317840576171875,
|
|
"learning_rate": 2.8043938066798645e-07,
|
|
"loss": 0.0213,
|
|
"reward": 0.8573304824531078,
|
|
"reward_std": 0.7073214678093791,
|
|
"rewards/cosine_scaled_reward": 0.01199856773018837,
|
|
"rewards/format_reward": 0.8333333507180214,
|
|
"step": 367
|
|
},
|
|
{
|
|
"completion_length": 1578.2083549499512,
|
|
"epoch": 0.4205714285714286,
|
|
"grad_norm": 4.118416786193848,
|
|
"kl": 0.380859375,
|
|
"learning_rate": 2.7793039831193133e-07,
|
|
"loss": 0.0152,
|
|
"reward": 0.5520367622375488,
|
|
"reward_std": 0.651011124253273,
|
|
"rewards/cosine_scaled_reward": -0.13023164262995124,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 368
|
|
},
|
|
{
|
|
"completion_length": 1570.8333702087402,
|
|
"epoch": 0.4217142857142857,
|
|
"grad_norm": 3.169609785079956,
|
|
"kl": 0.53338623046875,
|
|
"learning_rate": 2.7543467624442956e-07,
|
|
"loss": 0.0213,
|
|
"reward": 0.7706317435950041,
|
|
"reward_std": 0.9069001339375973,
|
|
"rewards/cosine_scaled_reward": -0.020934134838171303,
|
|
"rewards/format_reward": 0.8125000223517418,
|
|
"step": 369
|
|
},
|
|
{
|
|
"completion_length": 1200.7500228881836,
|
|
"epoch": 0.4228571428571429,
|
|
"grad_norm": 1.968066930770874,
|
|
"kl": 0.415557861328125,
|
|
"learning_rate": 2.729523361034538e-07,
|
|
"loss": 0.0166,
|
|
"reward": 0.761927604675293,
|
|
"reward_std": 0.5050818659365177,
|
|
"rewards/cosine_scaled_reward": -0.05653620883822441,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 370
|
|
},
|
|
{
|
|
"completion_length": 638.2708549499512,
|
|
"epoch": 0.424,
|
|
"grad_norm": 1.2254022359848022,
|
|
"kl": 0.0458984375,
|
|
"learning_rate": 2.7048349887476037e-07,
|
|
"loss": 0.0018,
|
|
"reward": 1.2654454093426466,
|
|
"reward_std": 0.4487613644450903,
|
|
"rewards/cosine_scaled_reward": 0.13272269815206528,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 371
|
|
},
|
|
{
|
|
"completion_length": 1610.770896911621,
|
|
"epoch": 0.42514285714285716,
|
|
"grad_norm": 1.422675371170044,
|
|
"kl": 0.4040679931640625,
|
|
"learning_rate": 2.6802828488599294e-07,
|
|
"loss": 0.0162,
|
|
"reward": 0.9225452449172735,
|
|
"reward_std": 0.8723856098949909,
|
|
"rewards/cosine_scaled_reward": 0.03418927453458309,
|
|
"rewards/format_reward": 0.8541666865348816,
|
|
"step": 372
|
|
},
|
|
{
|
|
"completion_length": 916.0416870117188,
|
|
"epoch": 0.42628571428571427,
|
|
"grad_norm": 4.1732659339904785,
|
|
"kl": 0.23648834228515625,
|
|
"learning_rate": 2.655868138008171e-07,
|
|
"loss": 0.0095,
|
|
"reward": 0.753505942877382,
|
|
"reward_std": 0.5106633007526398,
|
|
"rewards/cosine_scaled_reward": -0.1024136976338923,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 373
|
|
},
|
|
{
|
|
"completion_length": 1235.333396911621,
|
|
"epoch": 0.42742857142857144,
|
|
"grad_norm": 0.9994511008262634,
|
|
"kl": 0.1927032470703125,
|
|
"learning_rate": 2.631592046130896e-07,
|
|
"loss": 0.0077,
|
|
"reward": 1.0162720493972301,
|
|
"reward_std": 0.4486389271914959,
|
|
"rewards/cosine_scaled_reward": 0.03938601026311517,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 374
|
|
},
|
|
{
|
|
"completion_length": 1341.0625305175781,
|
|
"epoch": 0.42857142857142855,
|
|
"grad_norm": 2.8108980655670166,
|
|
"kl": 0.382843017578125,
|
|
"learning_rate": 2.6074557564105724e-07,
|
|
"loss": 0.0153,
|
|
"reward": 0.9311719611287117,
|
|
"reward_std": 0.6638787761330605,
|
|
"rewards/cosine_scaled_reward": 0.038502639159560204,
|
|
"rewards/format_reward": 0.8541666865348816,
|
|
"step": 375
|
|
},
|
|
{
|
|
"completion_length": 1282.5000534057617,
|
|
"epoch": 0.4297142857142857,
|
|
"grad_norm": 1.4323093891143799,
|
|
"kl": 0.21431732177734375,
|
|
"learning_rate": 2.583460445215911e-07,
|
|
"loss": 0.0086,
|
|
"reward": 0.8344197571277618,
|
|
"reward_std": 0.7551028467714787,
|
|
"rewards/cosine_scaled_reward": -0.04112347261980176,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 376
|
|
},
|
|
{
|
|
"completion_length": 1487.5625495910645,
|
|
"epoch": 0.4308571428571429,
|
|
"grad_norm": 2.224385976791382,
|
|
"kl": 0.4652557373046875,
|
|
"learning_rate": 2.5596072820445254e-07,
|
|
"loss": 0.0186,
|
|
"reward": 0.49483706802129745,
|
|
"reward_std": 0.7973055504262447,
|
|
"rewards/cosine_scaled_reward": -0.13799815066158772,
|
|
"rewards/format_reward": 0.7708333432674408,
|
|
"step": 377
|
|
},
|
|
{
|
|
"completion_length": 1158.9791851043701,
|
|
"epoch": 0.432,
|
|
"grad_norm": 1.5131416320800781,
|
|
"kl": 0.283233642578125,
|
|
"learning_rate": 2.5358974294659373e-07,
|
|
"loss": 0.0113,
|
|
"reward": 1.1355108618736267,
|
|
"reward_std": 0.7891764938831329,
|
|
"rewards/cosine_scaled_reward": 0.1198387467302382,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 378
|
|
},
|
|
{
|
|
"completion_length": 1687.2292213439941,
|
|
"epoch": 0.43314285714285716,
|
|
"grad_norm": 2.515700340270996,
|
|
"kl": 0.5447998046875,
|
|
"learning_rate": 2.512332043064913e-07,
|
|
"loss": 0.0218,
|
|
"reward": 0.488259082660079,
|
|
"reward_std": 0.6213226802647114,
|
|
"rewards/cosine_scaled_reward": -0.16212046705186367,
|
|
"rewards/format_reward": 0.8125000186264515,
|
|
"step": 379
|
|
},
|
|
{
|
|
"completion_length": 1145.1250343322754,
|
|
"epoch": 0.4342857142857143,
|
|
"grad_norm": 2.491877555847168,
|
|
"kl": 0.34210205078125,
|
|
"learning_rate": 2.488912271385139e-07,
|
|
"loss": 0.0137,
|
|
"reward": 0.6510128956288099,
|
|
"reward_std": 0.764140423387289,
|
|
"rewards/cosine_scaled_reward": -0.09116021171212196,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 380
|
|
},
|
|
{
|
|
"completion_length": 1576.9791946411133,
|
|
"epoch": 0.43542857142857144,
|
|
"grad_norm": 2.5349020957946777,
|
|
"kl": 0.4387664794921875,
|
|
"learning_rate": 2.465639255873246e-07,
|
|
"loss": 0.0176,
|
|
"reward": 0.45536880288273096,
|
|
"reward_std": 0.6865225993096828,
|
|
"rewards/cosine_scaled_reward": -0.17856561671942472,
|
|
"rewards/format_reward": 0.8125000223517418,
|
|
"step": 381
|
|
},
|
|
{
|
|
"completion_length": 1223.1667022705078,
|
|
"epoch": 0.43657142857142855,
|
|
"grad_norm": 3.171522378921509,
|
|
"kl": 0.3052978515625,
|
|
"learning_rate": 2.4425141308231765e-07,
|
|
"loss": 0.0122,
|
|
"reward": 0.47218240704387426,
|
|
"reward_std": 0.6197453737258911,
|
|
"rewards/cosine_scaled_reward": -0.20140881277620792,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 382
|
|
},
|
|
{
|
|
"completion_length": 1093.8750343322754,
|
|
"epoch": 0.4377142857142857,
|
|
"grad_norm": 1.554950475692749,
|
|
"kl": 0.307342529296875,
|
|
"learning_rate": 2.4195380233209006e-07,
|
|
"loss": 0.0123,
|
|
"reward": 1.3121628165245056,
|
|
"reward_std": 0.7447218038141727,
|
|
"rewards/cosine_scaled_reward": 0.1873313980177045,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 383
|
|
},
|
|
{
|
|
"completion_length": 1102.0833778381348,
|
|
"epoch": 0.43885714285714283,
|
|
"grad_norm": 3.9803340435028076,
|
|
"kl": 0.32332611083984375,
|
|
"learning_rate": 2.3967120531894857e-07,
|
|
"loss": 0.0129,
|
|
"reward": 1.2209933521226048,
|
|
"reward_std": 0.9811634235084057,
|
|
"rewards/cosine_scaled_reward": 0.1834133416414261,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 384
|
|
},
|
|
{
|
|
"completion_length": 1301.0625381469727,
|
|
"epoch": 0.44,
|
|
"grad_norm": 1.6203845739364624,
|
|
"kl": 0.3782196044921875,
|
|
"learning_rate": 2.374037332934512e-07,
|
|
"loss": 0.0151,
|
|
"reward": 0.6873492915183306,
|
|
"reward_std": 0.7231754045933485,
|
|
"rewards/cosine_scaled_reward": -0.09382536727935076,
|
|
"rewards/format_reward": 0.8750000111758709,
|
|
"step": 385
|
|
},
|
|
{
|
|
"completion_length": 1025.8125305175781,
|
|
"epoch": 0.44114285714285717,
|
|
"grad_norm": 2.561079978942871,
|
|
"kl": 0.351043701171875,
|
|
"learning_rate": 2.3515149676898552e-07,
|
|
"loss": 0.014,
|
|
"reward": 1.1378429010510445,
|
|
"reward_std": 0.8463675826787949,
|
|
"rewards/cosine_scaled_reward": 0.14183809887617826,
|
|
"rewards/format_reward": 0.8541666865348816,
|
|
"step": 386
|
|
},
|
|
{
|
|
"completion_length": 1460.0417175292969,
|
|
"epoch": 0.4422857142857143,
|
|
"grad_norm": 3.437443494796753,
|
|
"kl": 0.76416015625,
|
|
"learning_rate": 2.3291460551638237e-07,
|
|
"loss": 0.0306,
|
|
"reward": 0.6168055031448603,
|
|
"reward_std": 0.6576487310230732,
|
|
"rewards/cosine_scaled_reward": -0.1290972474962473,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 387
|
|
},
|
|
{
|
|
"completion_length": 1255.8958740234375,
|
|
"epoch": 0.44342857142857145,
|
|
"grad_norm": 3.211789846420288,
|
|
"kl": 0.52313232421875,
|
|
"learning_rate": 2.306931685585657e-07,
|
|
"loss": 0.0209,
|
|
"reward": 0.8223461015149951,
|
|
"reward_std": 0.6507496982812881,
|
|
"rewards/cosine_scaled_reward": -0.015910295769572258,
|
|
"rewards/format_reward": 0.8541666939854622,
|
|
"step": 388
|
|
},
|
|
{
|
|
"completion_length": 1253.7917022705078,
|
|
"epoch": 0.44457142857142856,
|
|
"grad_norm": 5.535946846008301,
|
|
"kl": 0.503692626953125,
|
|
"learning_rate": 2.2848729416523859e-07,
|
|
"loss": 0.0201,
|
|
"reward": 0.7248962745070457,
|
|
"reward_std": 0.5912930071353912,
|
|
"rewards/cosine_scaled_reward": -0.07505187718197703,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 389
|
|
},
|
|
{
|
|
"completion_length": 1278.0417022705078,
|
|
"epoch": 0.44571428571428573,
|
|
"grad_norm": 3.257326364517212,
|
|
"kl": 0.6408615112304688,
|
|
"learning_rate": 2.2629708984760706e-07,
|
|
"loss": 0.0256,
|
|
"reward": 0.6384771540760994,
|
|
"reward_std": 0.7342811785638332,
|
|
"rewards/cosine_scaled_reward": -0.11826143972575665,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 390
|
|
},
|
|
{
|
|
"completion_length": 1312.6042022705078,
|
|
"epoch": 0.44685714285714284,
|
|
"grad_norm": 5.830494403839111,
|
|
"kl": 1.293975830078125,
|
|
"learning_rate": 2.2412266235313973e-07,
|
|
"loss": 0.0519,
|
|
"reward": 0.8241331037133932,
|
|
"reward_std": 0.9150289222598076,
|
|
"rewards/cosine_scaled_reward": 0.016233190894126892,
|
|
"rewards/format_reward": 0.791666679084301,
|
|
"step": 391
|
|
},
|
|
{
|
|
"completion_length": 1468.937557220459,
|
|
"epoch": 0.448,
|
|
"grad_norm": 1.9504032135009766,
|
|
"kl": 0.7166290283203125,
|
|
"learning_rate": 2.2196411766036487e-07,
|
|
"loss": 0.0287,
|
|
"reward": 0.5725310668349266,
|
|
"reward_std": 0.7125820200890303,
|
|
"rewards/cosine_scaled_reward": -0.1512344698421657,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 392
|
|
},
|
|
{
|
|
"completion_length": 1273.1250267028809,
|
|
"epoch": 0.4491428571428571,
|
|
"grad_norm": 6.959710597991943,
|
|
"kl": 1.0224609375,
|
|
"learning_rate": 2.1982156097370557e-07,
|
|
"loss": 0.0409,
|
|
"reward": 0.8387455176562071,
|
|
"reward_std": 0.8648201785981655,
|
|
"rewards/cosine_scaled_reward": 0.013122743383519264,
|
|
"rewards/format_reward": 0.8125000204890966,
|
|
"step": 393
|
|
},
|
|
{
|
|
"completion_length": 1102.4583854675293,
|
|
"epoch": 0.4502857142857143,
|
|
"grad_norm": 1.945556402206421,
|
|
"kl": 0.54296875,
|
|
"learning_rate": 2.1769509671835223e-07,
|
|
"loss": 0.0217,
|
|
"reward": 0.32192777935415506,
|
|
"reward_std": 0.5174819845706224,
|
|
"rewards/cosine_scaled_reward": -0.24528613314032555,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 394
|
|
},
|
|
{
|
|
"completion_length": 937.8958549499512,
|
|
"epoch": 0.4514285714285714,
|
|
"grad_norm": 5.496992588043213,
|
|
"kl": 0.67437744140625,
|
|
"learning_rate": 2.1558482853517253e-07,
|
|
"loss": 0.027,
|
|
"reward": 0.9939035659190267,
|
|
"reward_std": 0.7032170053571463,
|
|
"rewards/cosine_scaled_reward": 0.049035104806534946,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 395
|
|
},
|
|
{
|
|
"completion_length": 987.7292022705078,
|
|
"epoch": 0.45257142857142857,
|
|
"grad_norm": 4.864074230194092,
|
|
"kl": 0.49527740478515625,
|
|
"learning_rate": 2.134908592756607e-07,
|
|
"loss": 0.0198,
|
|
"reward": 0.7741489135660231,
|
|
"reward_std": 0.5997458174824715,
|
|
"rewards/cosine_scaled_reward": -0.06084221974015236,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 396
|
|
},
|
|
{
|
|
"completion_length": 1146.437515258789,
|
|
"epoch": 0.45371428571428574,
|
|
"grad_norm": 4.235805511474609,
|
|
"kl": 0.8509521484375,
|
|
"learning_rate": 2.1141329099692406e-07,
|
|
"loss": 0.034,
|
|
"reward": 0.8896873220801353,
|
|
"reward_std": 0.6099906638264656,
|
|
"rewards/cosine_scaled_reward": -0.04473969340324402,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 397
|
|
},
|
|
{
|
|
"completion_length": 1264.4167022705078,
|
|
"epoch": 0.45485714285714285,
|
|
"grad_norm": 2.6962742805480957,
|
|
"kl": 0.738128662109375,
|
|
"learning_rate": 2.0935222495670968e-07,
|
|
"loss": 0.0296,
|
|
"reward": 0.5152244120836258,
|
|
"reward_std": 0.8842574842274189,
|
|
"rewards/cosine_scaled_reward": -0.14863780653104186,
|
|
"rewards/format_reward": 0.8125000186264515,
|
|
"step": 398
|
|
},
|
|
{
|
|
"completion_length": 1035.7292022705078,
|
|
"epoch": 0.456,
|
|
"grad_norm": 2.1586830615997314,
|
|
"kl": 0.4624786376953125,
|
|
"learning_rate": 2.0730776160846853e-07,
|
|
"loss": 0.0185,
|
|
"reward": 0.9472854379564524,
|
|
"reward_std": 0.6391190886497498,
|
|
"rewards/cosine_scaled_reward": 0.025726054795086384,
|
|
"rewards/format_reward": 0.8958333358168602,
|
|
"step": 399
|
|
},
|
|
{
|
|
"completion_length": 913.2500343322754,
|
|
"epoch": 0.45714285714285713,
|
|
"grad_norm": 3.6302056312561035,
|
|
"kl": 0.26483154296875,
|
|
"learning_rate": 2.0528000059645995e-07,
|
|
"loss": 0.0106,
|
|
"reward": 1.4475815817713737,
|
|
"reward_std": 0.7946172095835209,
|
|
"rewards/cosine_scaled_reward": 0.26545744470786303,
|
|
"rewards/format_reward": 0.916666679084301,
|
|
"step": 400
|
|
},
|
|
{
|
|
"completion_length": 1466.1667175292969,
|
|
"epoch": 0.4582857142857143,
|
|
"grad_norm": 3.406836986541748,
|
|
"kl": 0.783599853515625,
|
|
"learning_rate": 2.032690407508949e-07,
|
|
"loss": 0.0314,
|
|
"reward": 0.6450461961794645,
|
|
"reward_std": 0.541100338101387,
|
|
"rewards/cosine_scaled_reward": -0.1149769127368927,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 401
|
|
},
|
|
{
|
|
"completion_length": 1107.4375228881836,
|
|
"epoch": 0.4594285714285714,
|
|
"grad_norm": 5.733443737030029,
|
|
"kl": 0.68023681640625,
|
|
"learning_rate": 2.0127498008311922e-07,
|
|
"loss": 0.0272,
|
|
"reward": 0.7256474066525698,
|
|
"reward_std": 0.6739897206425667,
|
|
"rewards/cosine_scaled_reward": -0.07467629760503769,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 402
|
|
},
|
|
{
|
|
"completion_length": 961.5000381469727,
|
|
"epoch": 0.4605714285714286,
|
|
"grad_norm": 2.457207441329956,
|
|
"kl": 0.1911468505859375,
|
|
"learning_rate": 1.9929791578083655e-07,
|
|
"loss": 0.0076,
|
|
"reward": 1.1732404585927725,
|
|
"reward_std": 0.589294470846653,
|
|
"rewards/cosine_scaled_reward": 0.08662020694464445,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 403
|
|
},
|
|
{
|
|
"completion_length": 1151.1667175292969,
|
|
"epoch": 0.4617142857142857,
|
|
"grad_norm": 4.529026508331299,
|
|
"kl": 0.5990447998046875,
|
|
"learning_rate": 1.9733794420337213e-07,
|
|
"loss": 0.024,
|
|
"reward": 0.7381823370233178,
|
|
"reward_std": 0.6230232007801533,
|
|
"rewards/cosine_scaled_reward": -0.04757549986243248,
|
|
"rewards/format_reward": 0.8333333507180214,
|
|
"step": 404
|
|
},
|
|
{
|
|
"completion_length": 1068.1667251586914,
|
|
"epoch": 0.46285714285714286,
|
|
"grad_norm": 7.2086262702941895,
|
|
"kl": 0.4109954833984375,
|
|
"learning_rate": 1.9539516087697517e-07,
|
|
"loss": 0.0164,
|
|
"reward": 1.3128372617065907,
|
|
"reward_std": 0.8704881519079208,
|
|
"rewards/cosine_scaled_reward": 0.20850196853280067,
|
|
"rewards/format_reward": 0.895833358168602,
|
|
"step": 405
|
|
},
|
|
{
|
|
"completion_length": 1452.145851135254,
|
|
"epoch": 0.464,
|
|
"grad_norm": 6.928232192993164,
|
|
"kl": 1.5478515625,
|
|
"learning_rate": 1.934696604901642e-07,
|
|
"loss": 0.062,
|
|
"reward": 0.8689113333821297,
|
|
"reward_std": 1.0427627116441727,
|
|
"rewards/cosine_scaled_reward": 0.017788991099223495,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 406
|
|
},
|
|
{
|
|
"completion_length": 1271.6041984558105,
|
|
"epoch": 0.46514285714285714,
|
|
"grad_norm": 2.267540216445923,
|
|
"kl": 0.8037109375,
|
|
"learning_rate": 1.915615368891117e-07,
|
|
"loss": 0.0321,
|
|
"reward": 0.8099214821122587,
|
|
"reward_std": 0.6622566767036915,
|
|
"rewards/cosine_scaled_reward": -0.0012892577797174454,
|
|
"rewards/format_reward": 0.8125000111758709,
|
|
"step": 407
|
|
},
|
|
{
|
|
"completion_length": 1401.8333625793457,
|
|
"epoch": 0.4662857142857143,
|
|
"grad_norm": 2.2123162746429443,
|
|
"kl": 0.6725006103515625,
|
|
"learning_rate": 1.8967088307307e-07,
|
|
"loss": 0.0269,
|
|
"reward": 1.0615033656358719,
|
|
"reward_std": 0.8494626618921757,
|
|
"rewards/cosine_scaled_reward": 0.10366834327578545,
|
|
"rewards/format_reward": 0.8541666865348816,
|
|
"step": 408
|
|
},
|
|
{
|
|
"completion_length": 1703.6875610351562,
|
|
"epoch": 0.4674285714285714,
|
|
"grad_norm": 3.3070085048675537,
|
|
"kl": 1.30078125,
|
|
"learning_rate": 1.8779779118983867e-07,
|
|
"loss": 0.052,
|
|
"reward": 0.682604375295341,
|
|
"reward_std": 0.7020121356472373,
|
|
"rewards/cosine_scaled_reward": -0.0649478193372488,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 409
|
|
},
|
|
{
|
|
"completion_length": 1503.2500343322754,
|
|
"epoch": 0.4685714285714286,
|
|
"grad_norm": 7.976802349090576,
|
|
"kl": 1.97076416015625,
|
|
"learning_rate": 1.8594235253127372e-07,
|
|
"loss": 0.0788,
|
|
"reward": 0.46726590394973755,
|
|
"reward_std": 0.8286273218691349,
|
|
"rewards/cosine_scaled_reward": -0.07886704918928444,
|
|
"rewards/format_reward": 0.6250000260770321,
|
|
"step": 410
|
|
},
|
|
{
|
|
"completion_length": 1640.1458778381348,
|
|
"epoch": 0.4697142857142857,
|
|
"grad_norm": 3.6351277828216553,
|
|
"kl": 0.9295196533203125,
|
|
"learning_rate": 1.8410465752883758e-07,
|
|
"loss": 0.0372,
|
|
"reward": 0.6482336856424809,
|
|
"reward_std": 0.7643021754920483,
|
|
"rewards/cosine_scaled_reward": -0.0821331706829369,
|
|
"rewards/format_reward": 0.8125000186264515,
|
|
"step": 411
|
|
},
|
|
{
|
|
"completion_length": 1004.1458587646484,
|
|
"epoch": 0.47085714285714286,
|
|
"grad_norm": 2.904054641723633,
|
|
"kl": 0.413330078125,
|
|
"learning_rate": 1.822847957491922e-07,
|
|
"loss": 0.0165,
|
|
"reward": 0.9712292104959488,
|
|
"reward_std": 0.9014318101108074,
|
|
"rewards/cosine_scaled_reward": 0.048114595003426075,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 412
|
|
},
|
|
{
|
|
"completion_length": 1190.9583740234375,
|
|
"epoch": 0.472,
|
|
"grad_norm": 2.611903667449951,
|
|
"kl": 0.674652099609375,
|
|
"learning_rate": 1.804828558898332e-07,
|
|
"loss": 0.027,
|
|
"reward": 0.8645992483943701,
|
|
"reward_std": 0.8248982280492783,
|
|
"rewards/cosine_scaled_reward": 0.005216277204453945,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 413
|
|
},
|
|
{
|
|
"completion_length": 1804.0000228881836,
|
|
"epoch": 0.47314285714285714,
|
|
"grad_norm": 3.9129860401153564,
|
|
"kl": 1.2784423828125,
|
|
"learning_rate": 1.7869892577476722e-07,
|
|
"loss": 0.0512,
|
|
"reward": 0.5207854583859444,
|
|
"reward_std": 0.8007166758179665,
|
|
"rewards/cosine_scaled_reward": -0.10419062164146453,
|
|
"rewards/format_reward": 0.7291666753590107,
|
|
"step": 414
|
|
},
|
|
{
|
|
"completion_length": 1269.1667022705078,
|
|
"epoch": 0.4742857142857143,
|
|
"grad_norm": 6.273374080657959,
|
|
"kl": 0.8948516845703125,
|
|
"learning_rate": 1.7693309235023127e-07,
|
|
"loss": 0.0358,
|
|
"reward": 0.6741074454039335,
|
|
"reward_std": 0.7499744538217783,
|
|
"rewards/cosine_scaled_reward": -0.05877961404621601,
|
|
"rewards/format_reward": 0.7916666865348816,
|
|
"step": 415
|
|
},
|
|
{
|
|
"completion_length": 1129.3125534057617,
|
|
"epoch": 0.4754285714285714,
|
|
"grad_norm": 1.7560322284698486,
|
|
"kl": 0.2455902099609375,
|
|
"learning_rate": 1.7518544168045524e-07,
|
|
"loss": 0.0098,
|
|
"reward": 1.1211753264069557,
|
|
"reward_std": 0.7526138452813029,
|
|
"rewards/cosine_scaled_reward": 0.07100430876016617,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 416
|
|
},
|
|
{
|
|
"completion_length": 1839.1875762939453,
|
|
"epoch": 0.4765714285714286,
|
|
"grad_norm": 4.114261627197266,
|
|
"kl": 1.20947265625,
|
|
"learning_rate": 1.7345605894346726e-07,
|
|
"loss": 0.0484,
|
|
"reward": 0.6709015071392059,
|
|
"reward_std": 0.874784380197525,
|
|
"rewards/cosine_scaled_reward": -0.049965920858085155,
|
|
"rewards/format_reward": 0.7708333544433117,
|
|
"step": 417
|
|
},
|
|
{
|
|
"completion_length": 896.5625305175781,
|
|
"epoch": 0.4777142857142857,
|
|
"grad_norm": 4.675693511962891,
|
|
"kl": 0.6626663208007812,
|
|
"learning_rate": 1.7174502842694212e-07,
|
|
"loss": 0.0265,
|
|
"reward": 1.091725105419755,
|
|
"reward_std": 0.7902912050485611,
|
|
"rewards/cosine_scaled_reward": 0.13961253920570016,
|
|
"rewards/format_reward": 0.8125000186264515,
|
|
"step": 418
|
|
},
|
|
{
|
|
"completion_length": 1493.6250457763672,
|
|
"epoch": 0.47885714285714287,
|
|
"grad_norm": 2.2568399906158447,
|
|
"kl": 0.5430755615234375,
|
|
"learning_rate": 1.7005243352409333e-07,
|
|
"loss": 0.0217,
|
|
"reward": 0.8138027191162109,
|
|
"reward_std": 0.8418096788227558,
|
|
"rewards/cosine_scaled_reward": -0.04101531347259879,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 419
|
|
},
|
|
{
|
|
"completion_length": 1029.1875228881836,
|
|
"epoch": 0.48,
|
|
"grad_norm": 3.587470293045044,
|
|
"kl": 0.508331298828125,
|
|
"learning_rate": 1.6837835672960831e-07,
|
|
"loss": 0.0204,
|
|
"reward": 0.7593546295538545,
|
|
"reward_std": 0.41525958105921745,
|
|
"rewards/cosine_scaled_reward": -0.057822706177830696,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 420
|
|
},
|
|
{
|
|
"completion_length": 1540.0833625793457,
|
|
"epoch": 0.48114285714285715,
|
|
"grad_norm": 2.360013484954834,
|
|
"kl": 0.88397216796875,
|
|
"learning_rate": 1.6672287963562852e-07,
|
|
"loss": 0.0354,
|
|
"reward": 0.4944930747151375,
|
|
"reward_std": 0.6833166517317295,
|
|
"rewards/cosine_scaled_reward": -0.19025347288697958,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 421
|
|
},
|
|
{
|
|
"completion_length": 1501.7292556762695,
|
|
"epoch": 0.48228571428571426,
|
|
"grad_norm": 2.635981798171997,
|
|
"kl": 0.740234375,
|
|
"learning_rate": 1.6508608292777203e-07,
|
|
"loss": 0.0296,
|
|
"reward": 0.9262944171205163,
|
|
"reward_std": 0.754150040447712,
|
|
"rewards/cosine_scaled_reward": 0.025647209025919437,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 422
|
|
},
|
|
{
|
|
"completion_length": 1964.8750762939453,
|
|
"epoch": 0.48342857142857143,
|
|
"grad_norm": 4.063174247741699,
|
|
"kl": 1.255462646484375,
|
|
"learning_rate": 1.6346804638120098e-07,
|
|
"loss": 0.0503,
|
|
"reward": 0.3909978587180376,
|
|
"reward_std": 0.8100734055042267,
|
|
"rewards/cosine_scaled_reward": -0.13783442322164774,
|
|
"rewards/format_reward": 0.6666666865348816,
|
|
"step": 423
|
|
},
|
|
{
|
|
"completion_length": 1720.937515258789,
|
|
"epoch": 0.4845714285714286,
|
|
"grad_norm": 3.836009979248047,
|
|
"kl": 0.89697265625,
|
|
"learning_rate": 1.6186884885673413e-07,
|
|
"loss": 0.0358,
|
|
"reward": 0.29376337490975857,
|
|
"reward_std": 0.5368039496243,
|
|
"rewards/cosine_scaled_reward": -0.2489516567438841,
|
|
"rewards/format_reward": 0.7916666865348816,
|
|
"step": 424
|
|
},
|
|
{
|
|
"completion_length": 1298.8125495910645,
|
|
"epoch": 0.4857142857142857,
|
|
"grad_norm": 1.9228633642196655,
|
|
"kl": 0.457244873046875,
|
|
"learning_rate": 1.6028856829700258e-07,
|
|
"loss": 0.0183,
|
|
"reward": 1.3913816176354885,
|
|
"reward_std": 0.6065534967929125,
|
|
"rewards/cosine_scaled_reward": 0.2477741353213787,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 425
|
|
},
|
|
{
|
|
"completion_length": 973.8541870117188,
|
|
"epoch": 0.4868571428571429,
|
|
"grad_norm": 2.929983615875244,
|
|
"kl": 0.32680511474609375,
|
|
"learning_rate": 1.5872728172265146e-07,
|
|
"loss": 0.0131,
|
|
"reward": 0.8921321593225002,
|
|
"reward_std": 0.4427370298653841,
|
|
"rewards/cosine_scaled_reward": -0.04351727291941643,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 426
|
|
},
|
|
{
|
|
"completion_length": 1596.7708740234375,
|
|
"epoch": 0.488,
|
|
"grad_norm": 3.161161422729492,
|
|
"kl": 0.484588623046875,
|
|
"learning_rate": 1.5718506522858572e-07,
|
|
"loss": 0.0194,
|
|
"reward": 0.973305675201118,
|
|
"reward_std": 0.8661460429430008,
|
|
"rewards/cosine_scaled_reward": 0.06998615153133869,
|
|
"rewards/format_reward": 0.8333333507180214,
|
|
"step": 427
|
|
},
|
|
{
|
|
"completion_length": 1573.3125610351562,
|
|
"epoch": 0.48914285714285716,
|
|
"grad_norm": 2.7298057079315186,
|
|
"kl": 0.6797714233398438,
|
|
"learning_rate": 1.5566199398026147e-07,
|
|
"loss": 0.0272,
|
|
"reward": 0.833369608346402,
|
|
"reward_std": 0.8424164094030857,
|
|
"rewards/cosine_scaled_reward": -0.020815202966332436,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 428
|
|
},
|
|
{
|
|
"completion_length": 972.3958435058594,
|
|
"epoch": 0.49028571428571427,
|
|
"grad_norm": 3.271073579788208,
|
|
"kl": 0.5417633056640625,
|
|
"learning_rate": 1.5415814221002265e-07,
|
|
"loss": 0.0216,
|
|
"reward": 0.6053271126002073,
|
|
"reward_std": 0.7208438105881214,
|
|
"rewards/cosine_scaled_reward": -0.15566978510469198,
|
|
"rewards/format_reward": 0.9166666865348816,
|
|
"step": 429
|
|
},
|
|
{
|
|
"completion_length": 1340.9167098999023,
|
|
"epoch": 0.49142857142857144,
|
|
"grad_norm": 4.3969879150390625,
|
|
"kl": 0.57080078125,
|
|
"learning_rate": 1.5267358321348285e-07,
|
|
"loss": 0.0228,
|
|
"reward": 0.9481127467006445,
|
|
"reward_std": 0.8026427961885929,
|
|
"rewards/cosine_scaled_reward": 0.0365563714876771,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 430
|
|
},
|
|
{
|
|
"completion_length": 1008.6250267028809,
|
|
"epoch": 0.49257142857142855,
|
|
"grad_norm": 2.9247236251831055,
|
|
"kl": 0.3984832763671875,
|
|
"learning_rate": 1.5120838934595337e-07,
|
|
"loss": 0.016,
|
|
"reward": 0.7810343587771058,
|
|
"reward_std": 0.6039218865334988,
|
|
"rewards/cosine_scaled_reward": -0.06781616434454918,
|
|
"rewards/format_reward": 0.9166666865348816,
|
|
"step": 431
|
|
},
|
|
{
|
|
"completion_length": 1682.0417022705078,
|
|
"epoch": 0.4937142857142857,
|
|
"grad_norm": 3.6658926010131836,
|
|
"kl": 0.94122314453125,
|
|
"learning_rate": 1.4976263201891613e-07,
|
|
"loss": 0.0376,
|
|
"reward": 0.37208714336156845,
|
|
"reward_std": 0.7206946834921837,
|
|
"rewards/cosine_scaled_reward": -0.17853978439234197,
|
|
"rewards/format_reward": 0.7291666846722364,
|
|
"step": 432
|
|
},
|
|
{
|
|
"completion_length": 1573.8125610351562,
|
|
"epoch": 0.4948571428571429,
|
|
"grad_norm": 6.710811138153076,
|
|
"kl": 0.98748779296875,
|
|
"learning_rate": 1.483363816965435e-07,
|
|
"loss": 0.0395,
|
|
"reward": 0.84003546833992,
|
|
"reward_std": 0.6314461715519428,
|
|
"rewards/cosine_scaled_reward": 0.0033510662615299225,
|
|
"rewards/format_reward": 0.833333358168602,
|
|
"step": 433
|
|
},
|
|
{
|
|
"completion_length": 1738.7292175292969,
|
|
"epoch": 0.496,
|
|
"grad_norm": 4.559843063354492,
|
|
"kl": 1.0400390625,
|
|
"learning_rate": 1.469297078922642e-07,
|
|
"loss": 0.0415,
|
|
"reward": 0.4310444425791502,
|
|
"reward_std": 0.6072977893054485,
|
|
"rewards/cosine_scaled_reward": -0.15947778831468895,
|
|
"rewards/format_reward": 0.7500000223517418,
|
|
"step": 434
|
|
},
|
|
{
|
|
"completion_length": 935.7708511352539,
|
|
"epoch": 0.49714285714285716,
|
|
"grad_norm": 1.8925296068191528,
|
|
"kl": 0.5841522216796875,
|
|
"learning_rate": 1.4554267916537495e-07,
|
|
"loss": 0.0234,
|
|
"reward": 0.6217253655195236,
|
|
"reward_std": 0.5926484689116478,
|
|
"rewards/cosine_scaled_reward": -0.1578873231774196,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 435
|
|
},
|
|
{
|
|
"completion_length": 1107.6875495910645,
|
|
"epoch": 0.4982857142857143,
|
|
"grad_norm": 2.4335594177246094,
|
|
"kl": 0.6590576171875,
|
|
"learning_rate": 1.4417536311769885e-07,
|
|
"loss": 0.0264,
|
|
"reward": 1.1079479418694973,
|
|
"reward_std": 0.7582566514611244,
|
|
"rewards/cosine_scaled_reward": 0.11647394431201974,
|
|
"rewards/format_reward": 0.8750000111758709,
|
|
"step": 436
|
|
},
|
|
{
|
|
"completion_length": 1200.9375381469727,
|
|
"epoch": 0.49942857142857144,
|
|
"grad_norm": 3.8739376068115234,
|
|
"kl": 0.56396484375,
|
|
"learning_rate": 1.4282782639029128e-07,
|
|
"loss": 0.0225,
|
|
"reward": 0.7988617792725563,
|
|
"reward_std": 0.6751260980963707,
|
|
"rewards/cosine_scaled_reward": -0.04848578106611967,
|
|
"rewards/format_reward": 0.895833358168602,
|
|
"step": 437
|
|
},
|
|
{
|
|
"completion_length": 1821.7708892822266,
|
|
"epoch": 0.5005714285714286,
|
|
"grad_norm": 3.8501853942871094,
|
|
"kl": 1.15753173828125,
|
|
"learning_rate": 1.4150013466019114e-07,
|
|
"loss": 0.0462,
|
|
"reward": 0.45935247000306845,
|
|
"reward_std": 0.6324943359941244,
|
|
"rewards/cosine_scaled_reward": -0.16615711338818073,
|
|
"rewards/format_reward": 0.7916666846722364,
|
|
"step": 438
|
|
},
|
|
{
|
|
"completion_length": 1447.7916793823242,
|
|
"epoch": 0.5017142857142857,
|
|
"grad_norm": 13.093454360961914,
|
|
"kl": 1.1785888671875,
|
|
"learning_rate": 1.4019235263722034e-07,
|
|
"loss": 0.0471,
|
|
"reward": 0.6338995918631554,
|
|
"reward_std": 0.6636701337993145,
|
|
"rewards/cosine_scaled_reward": -0.12055021477863193,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 439
|
|
},
|
|
{
|
|
"completion_length": 1387.4792022705078,
|
|
"epoch": 0.5028571428571429,
|
|
"grad_norm": 4.15952205657959,
|
|
"kl": 1.02880859375,
|
|
"learning_rate": 1.3890454406082956e-07,
|
|
"loss": 0.0411,
|
|
"reward": 0.5482727251946926,
|
|
"reward_std": 0.6047806814312935,
|
|
"rewards/cosine_scaled_reward": -0.15294698532670736,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 440
|
|
},
|
|
{
|
|
"completion_length": 1395.708381652832,
|
|
"epoch": 0.504,
|
|
"grad_norm": 3.0667598247528076,
|
|
"kl": 0.73974609375,
|
|
"learning_rate": 1.3763677169699217e-07,
|
|
"loss": 0.0296,
|
|
"reward": 0.9216288132593036,
|
|
"reward_std": 0.7844664789736271,
|
|
"rewards/cosine_scaled_reward": 0.04414770007133484,
|
|
"rewards/format_reward": 0.8333333507180214,
|
|
"step": 441
|
|
},
|
|
{
|
|
"completion_length": 1071.7917022705078,
|
|
"epoch": 0.5051428571428571,
|
|
"grad_norm": 2.833597421646118,
|
|
"kl": 0.763397216796875,
|
|
"learning_rate": 1.3638909733514452e-07,
|
|
"loss": 0.0305,
|
|
"reward": 0.886376628652215,
|
|
"reward_std": 0.8642673939466476,
|
|
"rewards/cosine_scaled_reward": 0.02652162907179445,
|
|
"rewards/format_reward": 0.8333333469927311,
|
|
"step": 442
|
|
},
|
|
{
|
|
"completion_length": 1647.8542404174805,
|
|
"epoch": 0.5062857142857143,
|
|
"grad_norm": 2.424894332885742,
|
|
"kl": 1.24749755859375,
|
|
"learning_rate": 1.351615817851748e-07,
|
|
"loss": 0.0499,
|
|
"reward": 0.5579078826121986,
|
|
"reward_std": 0.6206741183996201,
|
|
"rewards/cosine_scaled_reward": -0.12729605846107006,
|
|
"rewards/format_reward": 0.8125000111758709,
|
|
"step": 443
|
|
},
|
|
{
|
|
"completion_length": 1236.9166946411133,
|
|
"epoch": 0.5074285714285715,
|
|
"grad_norm": 4.010989665985107,
|
|
"kl": 0.55133056640625,
|
|
"learning_rate": 1.3395428487445914e-07,
|
|
"loss": 0.0221,
|
|
"reward": 0.581955180503428,
|
|
"reward_std": 0.7338661774992943,
|
|
"rewards/cosine_scaled_reward": -0.1569390781223774,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 444
|
|
},
|
|
{
|
|
"completion_length": 1259.9166793823242,
|
|
"epoch": 0.5085714285714286,
|
|
"grad_norm": 2.583313465118408,
|
|
"kl": 0.74591064453125,
|
|
"learning_rate": 1.3276726544494571e-07,
|
|
"loss": 0.0298,
|
|
"reward": 0.5723795741796494,
|
|
"reward_std": 0.788989819586277,
|
|
"rewards/cosine_scaled_reward": -0.13047689152881503,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 445
|
|
},
|
|
{
|
|
"completion_length": 1332.9792098999023,
|
|
"epoch": 0.5097142857142857,
|
|
"grad_norm": 16.281204223632812,
|
|
"kl": 1.068878173828125,
|
|
"learning_rate": 1.316005813502869e-07,
|
|
"loss": 0.0428,
|
|
"reward": 0.7542771156877279,
|
|
"reward_std": 0.7587186098098755,
|
|
"rewards/cosine_scaled_reward": -0.008278121706098318,
|
|
"rewards/format_reward": 0.770833358168602,
|
|
"step": 446
|
|
},
|
|
{
|
|
"completion_length": 1513.3958930969238,
|
|
"epoch": 0.5108571428571429,
|
|
"grad_norm": 3.205157518386841,
|
|
"kl": 1.26171875,
|
|
"learning_rate": 1.3045428945301953e-07,
|
|
"loss": 0.0505,
|
|
"reward": 0.7465504482388496,
|
|
"reward_std": 0.57644097879529,
|
|
"rewards/cosine_scaled_reward": -0.04339144751429558,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 447
|
|
},
|
|
{
|
|
"completion_length": 1335.083366394043,
|
|
"epoch": 0.512,
|
|
"grad_norm": 3.9793076515197754,
|
|
"kl": 1.251495361328125,
|
|
"learning_rate": 1.2932844562179352e-07,
|
|
"loss": 0.0501,
|
|
"reward": 0.6771020290179877,
|
|
"reward_std": 0.7358664702624083,
|
|
"rewards/cosine_scaled_reward": -0.026032326743006706,
|
|
"rewards/format_reward": 0.729166679084301,
|
|
"step": 448
|
|
},
|
|
{
|
|
"completion_length": 1115.187515258789,
|
|
"epoch": 0.5131428571428571,
|
|
"grad_norm": 3.228578805923462,
|
|
"kl": 0.47408294677734375,
|
|
"learning_rate": 1.2822310472864885e-07,
|
|
"loss": 0.0189,
|
|
"reward": 0.422957434784621,
|
|
"reward_std": 0.5728885792195797,
|
|
"rewards/cosine_scaled_reward": -0.22602130100131035,
|
|
"rewards/format_reward": 0.8750000223517418,
|
|
"step": 449
|
|
},
|
|
{
|
|
"completion_length": 1225.1458740234375,
|
|
"epoch": 0.5142857142857142,
|
|
"grad_norm": 2.933077335357666,
|
|
"kl": 0.64776611328125,
|
|
"learning_rate": 1.2713832064634125e-07,
|
|
"loss": 0.0259,
|
|
"reward": 0.790303866029717,
|
|
"reward_std": 0.622036661952734,
|
|
"rewards/cosine_scaled_reward": -0.04234808124601841,
|
|
"rewards/format_reward": 0.8750000149011612,
|
|
"step": 450
|
|
},
|
|
{
|
|
"completion_length": 1283.5208892822266,
|
|
"epoch": 0.5154285714285715,
|
|
"grad_norm": 4.34015417098999,
|
|
"kl": 0.707855224609375,
|
|
"learning_rate": 1.260741462457165e-07,
|
|
"loss": 0.0283,
|
|
"reward": 0.7061118334531784,
|
|
"reward_std": 0.6641687378287315,
|
|
"rewards/cosine_scaled_reward": -0.06361076328903437,
|
|
"rewards/format_reward": 0.833333358168602,
|
|
"step": 451
|
|
},
|
|
{
|
|
"completion_length": 1390.333366394043,
|
|
"epoch": 0.5165714285714286,
|
|
"grad_norm": 3.791003942489624,
|
|
"kl": 0.733154296875,
|
|
"learning_rate": 1.2503063339313356e-07,
|
|
"loss": 0.0294,
|
|
"reward": 0.8051364235579967,
|
|
"reward_std": 0.9408066868782043,
|
|
"rewards/cosine_scaled_reward": -0.014098492218181491,
|
|
"rewards/format_reward": 0.8333333507180214,
|
|
"step": 452
|
|
},
|
|
{
|
|
"completion_length": 1164.7916984558105,
|
|
"epoch": 0.5177142857142857,
|
|
"grad_norm": 2.2949931621551514,
|
|
"kl": 0.513427734375,
|
|
"learning_rate": 1.2400783294793668e-07,
|
|
"loss": 0.0205,
|
|
"reward": 0.8233426138758659,
|
|
"reward_std": 0.5944336298853159,
|
|
"rewards/cosine_scaled_reward": -0.06749536748975515,
|
|
"rewards/format_reward": 0.9583333432674408,
|
|
"step": 453
|
|
},
|
|
{
|
|
"completion_length": 1308.4167022705078,
|
|
"epoch": 0.5188571428571429,
|
|
"grad_norm": 3.0252280235290527,
|
|
"kl": 0.7433929443359375,
|
|
"learning_rate": 1.2300579475997657e-07,
|
|
"loss": 0.0298,
|
|
"reward": 0.5274054184556007,
|
|
"reward_std": 0.6061475053429604,
|
|
"rewards/cosine_scaled_reward": -0.14254729636013508,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 454
|
|
},
|
|
{
|
|
"completion_length": 1739.1458702087402,
|
|
"epoch": 0.52,
|
|
"grad_norm": 4.987695217132568,
|
|
"kl": 1.2000732421875,
|
|
"learning_rate": 1.220245676671809e-07,
|
|
"loss": 0.048,
|
|
"reward": 0.2408284079283476,
|
|
"reward_std": 0.6372124627232552,
|
|
"rewards/cosine_scaled_reward": -0.27541913744062185,
|
|
"rewards/format_reward": 0.7916666865348816,
|
|
"step": 455
|
|
},
|
|
{
|
|
"completion_length": 1663.187557220459,
|
|
"epoch": 0.5211428571428571,
|
|
"grad_norm": 3.197329044342041,
|
|
"kl": 0.941253662109375,
|
|
"learning_rate": 1.2106419949317388e-07,
|
|
"loss": 0.0376,
|
|
"reward": 0.47842175513505936,
|
|
"reward_std": 0.7478823028504848,
|
|
"rewards/cosine_scaled_reward": -0.15662246476858854,
|
|
"rewards/format_reward": 0.791666679084301,
|
|
"step": 456
|
|
},
|
|
{
|
|
"completion_length": 1417.3542289733887,
|
|
"epoch": 0.5222857142857142,
|
|
"grad_norm": 2.964496612548828,
|
|
"kl": 0.938079833984375,
|
|
"learning_rate": 1.2012473704494537e-07,
|
|
"loss": 0.0375,
|
|
"reward": 0.6622494223265676,
|
|
"reward_std": 0.5692349448800087,
|
|
"rewards/cosine_scaled_reward": -0.03345862403512001,
|
|
"rewards/format_reward": 0.7291666828095913,
|
|
"step": 457
|
|
},
|
|
{
|
|
"completion_length": 1527.2916946411133,
|
|
"epoch": 0.5234285714285715,
|
|
"grad_norm": 4.469851493835449,
|
|
"kl": 1.12396240234375,
|
|
"learning_rate": 1.1920622611056974e-07,
|
|
"loss": 0.0449,
|
|
"reward": 0.5578488986939192,
|
|
"reward_std": 0.693300411105156,
|
|
"rewards/cosine_scaled_reward": -0.10649222880601883,
|
|
"rewards/format_reward": 0.7708333469927311,
|
|
"step": 458
|
|
},
|
|
{
|
|
"completion_length": 1051.7708740234375,
|
|
"epoch": 0.5245714285714286,
|
|
"grad_norm": 3.3350539207458496,
|
|
"kl": 0.466766357421875,
|
|
"learning_rate": 1.1830871145697412e-07,
|
|
"loss": 0.0187,
|
|
"reward": 0.9517181403934956,
|
|
"reward_std": 0.8380660861730576,
|
|
"rewards/cosine_scaled_reward": 0.0071090515702962875,
|
|
"rewards/format_reward": 0.9375000074505806,
|
|
"step": 459
|
|
},
|
|
{
|
|
"completion_length": 1639.1250610351562,
|
|
"epoch": 0.5257142857142857,
|
|
"grad_norm": 7.242498874664307,
|
|
"kl": 1.00091552734375,
|
|
"learning_rate": 1.1743223682775649e-07,
|
|
"loss": 0.04,
|
|
"reward": 0.3730992656201124,
|
|
"reward_std": 0.726275160908699,
|
|
"rewards/cosine_scaled_reward": -0.16761704441159964,
|
|
"rewards/format_reward": 0.7083333432674408,
|
|
"step": 460
|
|
},
|
|
{
|
|
"completion_length": 1452.6250305175781,
|
|
"epoch": 0.5268571428571428,
|
|
"grad_norm": 3.2622721195220947,
|
|
"kl": 1.1546630859375,
|
|
"learning_rate": 1.1657684494105386e-07,
|
|
"loss": 0.0461,
|
|
"reward": 0.8234957940876484,
|
|
"reward_std": 0.6311179846525192,
|
|
"rewards/cosine_scaled_reward": -0.0049187901604454964,
|
|
"rewards/format_reward": 0.8333333488553762,
|
|
"step": 461
|
|
},
|
|
{
|
|
"completion_length": 1285.4166946411133,
|
|
"epoch": 0.528,
|
|
"grad_norm": 3.8556363582611084,
|
|
"kl": 0.8689117431640625,
|
|
"learning_rate": 1.1574257748745986e-07,
|
|
"loss": 0.0348,
|
|
"reward": 0.4587779585272074,
|
|
"reward_std": 0.6436084322631359,
|
|
"rewards/cosine_scaled_reward": -0.17686102783773094,
|
|
"rewards/format_reward": 0.8125000223517418,
|
|
"step": 462
|
|
},
|
|
{
|
|
"completion_length": 1802.8125381469727,
|
|
"epoch": 0.5291428571428571,
|
|
"grad_norm": 5.279976844787598,
|
|
"kl": 0.799560546875,
|
|
"learning_rate": 1.1492947512799328e-07,
|
|
"loss": 0.032,
|
|
"reward": 0.6291724583134055,
|
|
"reward_std": 0.8651260025799274,
|
|
"rewards/cosine_scaled_reward": -0.04999712225981057,
|
|
"rewards/format_reward": 0.7291666939854622,
|
|
"step": 463
|
|
},
|
|
{
|
|
"completion_length": 1047.145851135254,
|
|
"epoch": 0.5302857142857142,
|
|
"grad_norm": 2.4638097286224365,
|
|
"kl": 0.605743408203125,
|
|
"learning_rate": 1.1413757749211602e-07,
|
|
"loss": 0.0242,
|
|
"reward": 1.302383467555046,
|
|
"reward_std": 0.4087425358593464,
|
|
"rewards/cosine_scaled_reward": 0.1616083886474371,
|
|
"rewards/format_reward": 0.9791666716337204,
|
|
"step": 464
|
|
},
|
|
{
|
|
"completion_length": 1426.8542022705078,
|
|
"epoch": 0.5314285714285715,
|
|
"grad_norm": 2.563239336013794,
|
|
"kl": 0.8856201171875,
|
|
"learning_rate": 1.1336692317580158e-07,
|
|
"loss": 0.0354,
|
|
"reward": 0.514048907905817,
|
|
"reward_std": 0.7893399521708488,
|
|
"rewards/cosine_scaled_reward": -0.14922555815428495,
|
|
"rewards/format_reward": 0.8125000223517418,
|
|
"step": 465
|
|
},
|
|
{
|
|
"completion_length": 1289.5625381469727,
|
|
"epoch": 0.5325714285714286,
|
|
"grad_norm": 2.4319827556610107,
|
|
"kl": 0.561614990234375,
|
|
"learning_rate": 1.1261754973965422e-07,
|
|
"loss": 0.0225,
|
|
"reward": 1.006679143756628,
|
|
"reward_std": 0.8896586894989014,
|
|
"rewards/cosine_scaled_reward": 0.045006227446720004,
|
|
"rewards/format_reward": 0.9166666865348816,
|
|
"step": 466
|
|
},
|
|
{
|
|
"completion_length": 1679.6666870117188,
|
|
"epoch": 0.5337142857142857,
|
|
"grad_norm": 3.1560564041137695,
|
|
"kl": 1.129669189453125,
|
|
"learning_rate": 1.1188949370707787e-07,
|
|
"loss": 0.0453,
|
|
"reward": 0.34113773331046104,
|
|
"reward_std": 0.6508898884057999,
|
|
"rewards/cosine_scaled_reward": -0.19401447381824255,
|
|
"rewards/format_reward": 0.7291666846722364,
|
|
"step": 467
|
|
},
|
|
{
|
|
"completion_length": 1565.2083778381348,
|
|
"epoch": 0.5348571428571428,
|
|
"grad_norm": 181.5196990966797,
|
|
"kl": 7.007781982421875,
|
|
"learning_rate": 1.1118279056249653e-07,
|
|
"loss": 0.2808,
|
|
"reward": 0.6900721359997988,
|
|
"reward_std": 0.8350103311240673,
|
|
"rewards/cosine_scaled_reward": -0.07163060246966779,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 468
|
|
},
|
|
{
|
|
"completion_length": 1385.6458892822266,
|
|
"epoch": 0.536,
|
|
"grad_norm": 3.4314520359039307,
|
|
"kl": 0.941375732421875,
|
|
"learning_rate": 1.1049747474962444e-07,
|
|
"loss": 0.0377,
|
|
"reward": 0.5675666080787778,
|
|
"reward_std": 0.6128499489277601,
|
|
"rewards/cosine_scaled_reward": -0.12246670690365136,
|
|
"rewards/format_reward": 0.8125000149011612,
|
|
"step": 469
|
|
},
|
|
{
|
|
"completion_length": 1621.4375305175781,
|
|
"epoch": 0.5371428571428571,
|
|
"grad_norm": 4.546476364135742,
|
|
"kl": 1.376953125,
|
|
"learning_rate": 1.0983357966978745e-07,
|
|
"loss": 0.0551,
|
|
"reward": 0.30739592137979344,
|
|
"reward_std": 0.725900623947382,
|
|
"rewards/cosine_scaled_reward": -0.15880204178392887,
|
|
"rewards/format_reward": 0.625000013038516,
|
|
"step": 470
|
|
},
|
|
{
|
|
"completion_length": 1291.0208587646484,
|
|
"epoch": 0.5382857142857143,
|
|
"grad_norm": 2.4413740634918213,
|
|
"kl": 0.525848388671875,
|
|
"learning_rate": 1.0919113768029517e-07,
|
|
"loss": 0.021,
|
|
"reward": 0.7892501968890429,
|
|
"reward_std": 0.7734898887574673,
|
|
"rewards/cosine_scaled_reward": -0.03245823457837105,
|
|
"rewards/format_reward": 0.8541666939854622,
|
|
"step": 471
|
|
},
|
|
{
|
|
"completion_length": 1400.1667022705078,
|
|
"epoch": 0.5394285714285715,
|
|
"grad_norm": 3.6039347648620605,
|
|
"kl": 0.58624267578125,
|
|
"learning_rate": 1.0857018009286381e-07,
|
|
"loss": 0.0234,
|
|
"reward": 0.3975699208676815,
|
|
"reward_std": 0.6745503656566143,
|
|
"rewards/cosine_scaled_reward": -0.22829839028418064,
|
|
"rewards/format_reward": 0.8541666865348816,
|
|
"step": 472
|
|
},
|
|
{
|
|
"completion_length": 1205.7500381469727,
|
|
"epoch": 0.5405714285714286,
|
|
"grad_norm": 2.652244806289673,
|
|
"kl": 0.7998046875,
|
|
"learning_rate": 1.0797073717209013e-07,
|
|
"loss": 0.032,
|
|
"reward": 0.5399867701344192,
|
|
"reward_std": 0.6538946256041527,
|
|
"rewards/cosine_scaled_reward": -0.17792330123484135,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 473
|
|
},
|
|
{
|
|
"completion_length": 1451.083351135254,
|
|
"epoch": 0.5417142857142857,
|
|
"grad_norm": 4.507306098937988,
|
|
"kl": 0.9193878173828125,
|
|
"learning_rate": 1.0739283813397639e-07,
|
|
"loss": 0.0368,
|
|
"reward": 1.300966864451766,
|
|
"reward_std": 0.745520330965519,
|
|
"rewards/cosine_scaled_reward": 0.2546501159667969,
|
|
"rewards/format_reward": 0.7916666828095913,
|
|
"step": 474
|
|
},
|
|
{
|
|
"completion_length": 1491.0000686645508,
|
|
"epoch": 0.5428571428571428,
|
|
"grad_norm": 2.09909987449646,
|
|
"kl": 0.9030914306640625,
|
|
"learning_rate": 1.068365111445064e-07,
|
|
"loss": 0.0361,
|
|
"reward": 0.6428233720362186,
|
|
"reward_std": 0.8779257349669933,
|
|
"rewards/cosine_scaled_reward": -0.06400499166920781,
|
|
"rewards/format_reward": 0.7708333469927311,
|
|
"step": 475
|
|
},
|
|
{
|
|
"completion_length": 1502.0208740234375,
|
|
"epoch": 0.544,
|
|
"grad_norm": 3.001065969467163,
|
|
"kl": 0.85467529296875,
|
|
"learning_rate": 1.063017833182728e-07,
|
|
"loss": 0.0342,
|
|
"reward": 0.9168616086244583,
|
|
"reward_std": 0.9465513862669468,
|
|
"rewards/cosine_scaled_reward": 0.020930795930325985,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 476
|
|
},
|
|
{
|
|
"completion_length": 1313.4791946411133,
|
|
"epoch": 0.5451428571428572,
|
|
"grad_norm": 4.386512756347656,
|
|
"kl": 0.9460906982421875,
|
|
"learning_rate": 1.0578868071715544e-07,
|
|
"loss": 0.0379,
|
|
"reward": 0.8154587242752314,
|
|
"reward_std": 0.8511052504181862,
|
|
"rewards/cosine_scaled_reward": 0.022312658838927746,
|
|
"rewards/format_reward": 0.7708333544433117,
|
|
"step": 477
|
|
},
|
|
{
|
|
"completion_length": 1496.3125305175781,
|
|
"epoch": 0.5462857142857143,
|
|
"grad_norm": 3.565664768218994,
|
|
"kl": 0.8566741943359375,
|
|
"learning_rate": 1.0529722834905125e-07,
|
|
"loss": 0.0342,
|
|
"reward": 0.8052469007670879,
|
|
"reward_std": 0.7007171474397182,
|
|
"rewards/cosine_scaled_reward": -0.003626542165875435,
|
|
"rewards/format_reward": 0.8125000223517418,
|
|
"step": 478
|
|
},
|
|
{
|
|
"completion_length": 1708.4167098999023,
|
|
"epoch": 0.5474285714285714,
|
|
"grad_norm": 2.6371641159057617,
|
|
"kl": 1.12823486328125,
|
|
"learning_rate": 1.0482745016665526e-07,
|
|
"loss": 0.0451,
|
|
"reward": 0.5828341413289309,
|
|
"reward_std": 0.6526912562549114,
|
|
"rewards/cosine_scaled_reward": -0.1148329358547926,
|
|
"rewards/format_reward": 0.8125000074505806,
|
|
"step": 479
|
|
},
|
|
{
|
|
"completion_length": 1512.4166946411133,
|
|
"epoch": 0.5485714285714286,
|
|
"grad_norm": 4.293049335479736,
|
|
"kl": 0.984771728515625,
|
|
"learning_rate": 1.0437936906629334e-07,
|
|
"loss": 0.0394,
|
|
"reward": 0.8314212337136269,
|
|
"reward_std": 0.5310982428491116,
|
|
"rewards/cosine_scaled_reward": -0.03220607154071331,
|
|
"rewards/format_reward": 0.895833358168602,
|
|
"step": 480
|
|
},
|
|
{
|
|
"completion_length": 1671.5417022705078,
|
|
"epoch": 0.5497142857142857,
|
|
"grad_norm": 5.241688251495361,
|
|
"kl": 0.995025634765625,
|
|
"learning_rate": 1.0395300688680625e-07,
|
|
"loss": 0.0397,
|
|
"reward": 0.5629774704575539,
|
|
"reward_std": 0.6768053583800793,
|
|
"rewards/cosine_scaled_reward": -0.1351779391989112,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 481
|
|
},
|
|
{
|
|
"completion_length": 1726.7500267028809,
|
|
"epoch": 0.5508571428571428,
|
|
"grad_norm": 4.3753886222839355,
|
|
"kl": 1.4754638671875,
|
|
"learning_rate": 1.0354838440848501e-07,
|
|
"loss": 0.059,
|
|
"reward": 0.5926886834204197,
|
|
"reward_std": 0.6844369061291218,
|
|
"rewards/cosine_scaled_reward": -0.01615567714907229,
|
|
"rewards/format_reward": 0.6250000279396772,
|
|
"step": 482
|
|
},
|
|
{
|
|
"completion_length": 1887.2500610351562,
|
|
"epoch": 0.552,
|
|
"grad_norm": 3.9196157455444336,
|
|
"kl": 1.394287109375,
|
|
"learning_rate": 1.0316552135205837e-07,
|
|
"loss": 0.0558,
|
|
"reward": 0.5968103185296059,
|
|
"reward_std": 0.8982166573405266,
|
|
"rewards/cosine_scaled_reward": -0.08701153006404638,
|
|
"rewards/format_reward": 0.770833358168602,
|
|
"step": 483
|
|
},
|
|
{
|
|
"completion_length": 1416.1458930969238,
|
|
"epoch": 0.5531428571428572,
|
|
"grad_norm": 4.831650257110596,
|
|
"kl": 0.6915283203125,
|
|
"learning_rate": 1.0280443637773163e-07,
|
|
"loss": 0.0276,
|
|
"reward": 0.6882726345211267,
|
|
"reward_std": 0.781242698431015,
|
|
"rewards/cosine_scaled_reward": -0.06211370480014011,
|
|
"rewards/format_reward": 0.8125000223517418,
|
|
"step": 484
|
|
},
|
|
{
|
|
"completion_length": 1175.083351135254,
|
|
"epoch": 0.5542857142857143,
|
|
"grad_norm": 2.7525644302368164,
|
|
"kl": 0.708404541015625,
|
|
"learning_rate": 1.0246514708427701e-07,
|
|
"loss": 0.0283,
|
|
"reward": 0.5069214683026075,
|
|
"reward_std": 0.4130475576967001,
|
|
"rewards/cosine_scaled_reward": -0.16320594353601336,
|
|
"rewards/format_reward": 0.8333333414047956,
|
|
"step": 485
|
|
},
|
|
{
|
|
"completion_length": 945.3750343322754,
|
|
"epoch": 0.5554285714285714,
|
|
"grad_norm": 1.641010046005249,
|
|
"kl": 0.423370361328125,
|
|
"learning_rate": 1.0214767000817596e-07,
|
|
"loss": 0.0169,
|
|
"reward": 0.7414621282368898,
|
|
"reward_std": 0.5828515980392694,
|
|
"rewards/cosine_scaled_reward": -0.0771856140345335,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 486
|
|
},
|
|
{
|
|
"completion_length": 1054.7708625793457,
|
|
"epoch": 0.5565714285714286,
|
|
"grad_norm": 2.9195919036865234,
|
|
"kl": 0.4309539794921875,
|
|
"learning_rate": 1.0185202062281336e-07,
|
|
"loss": 0.0173,
|
|
"reward": 1.2239743052050471,
|
|
"reward_std": 0.5242457445710897,
|
|
"rewards/cosine_scaled_reward": 0.14323717169463634,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 487
|
|
},
|
|
{
|
|
"completion_length": 1278.1667022705078,
|
|
"epoch": 0.5577142857142857,
|
|
"grad_norm": 2.495598316192627,
|
|
"kl": 0.9759674072265625,
|
|
"learning_rate": 1.0157821333772304e-07,
|
|
"loss": 0.039,
|
|
"reward": 0.6595983803272247,
|
|
"reward_std": 0.5137137211859226,
|
|
"rewards/cosine_scaled_reward": -0.08686749078333378,
|
|
"rewards/format_reward": 0.8333333544433117,
|
|
"step": 488
|
|
},
|
|
{
|
|
"completion_length": 1564.6250381469727,
|
|
"epoch": 0.5588571428571428,
|
|
"grad_norm": 3.082076072692871,
|
|
"kl": 1.235137939453125,
|
|
"learning_rate": 1.013262614978859e-07,
|
|
"loss": 0.0494,
|
|
"reward": 0.09155605779960752,
|
|
"reward_std": 0.5432869009673595,
|
|
"rewards/cosine_scaled_reward": -0.28755532018840313,
|
|
"rewards/format_reward": 0.6666666828095913,
|
|
"step": 489
|
|
},
|
|
{
|
|
"completion_length": 1278.1250381469727,
|
|
"epoch": 0.56,
|
|
"grad_norm": 1.8155015707015991,
|
|
"kl": 0.6814727783203125,
|
|
"learning_rate": 1.0109617738307911e-07,
|
|
"loss": 0.0273,
|
|
"reward": 0.6778741236776114,
|
|
"reward_std": 0.7777648419141769,
|
|
"rewards/cosine_scaled_reward": -0.08814628981053829,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 490
|
|
},
|
|
{
|
|
"completion_length": 1448.9375381469727,
|
|
"epoch": 0.5611428571428572,
|
|
"grad_norm": 2.183570623397827,
|
|
"kl": 0.358306884765625,
|
|
"learning_rate": 1.0088797220727779e-07,
|
|
"loss": 0.0143,
|
|
"reward": 1.1492541544139385,
|
|
"reward_std": 1.02406694740057,
|
|
"rewards/cosine_scaled_reward": 0.12671040603891015,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 491
|
|
},
|
|
{
|
|
"completion_length": 1325.7708740234375,
|
|
"epoch": 0.5622857142857143,
|
|
"grad_norm": 3.939805030822754,
|
|
"kl": 0.6011505126953125,
|
|
"learning_rate": 1.0070165611810855e-07,
|
|
"loss": 0.024,
|
|
"reward": 0.8053325146902353,
|
|
"reward_std": 0.5690103769302368,
|
|
"rewards/cosine_scaled_reward": -0.045250434428453445,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 492
|
|
},
|
|
{
|
|
"completion_length": 1136.2291946411133,
|
|
"epoch": 0.5634285714285714,
|
|
"grad_norm": 3.2057812213897705,
|
|
"kl": 0.6187744140625,
|
|
"learning_rate": 1.005372381963547e-07,
|
|
"loss": 0.0248,
|
|
"reward": 0.8366872314363718,
|
|
"reward_std": 0.8825880065560341,
|
|
"rewards/cosine_scaled_reward": -0.019156392896547914,
|
|
"rewards/format_reward": 0.8750000074505806,
|
|
"step": 493
|
|
},
|
|
{
|
|
"completion_length": 1201.3125305175781,
|
|
"epoch": 0.5645714285714286,
|
|
"grad_norm": 3.475240707397461,
|
|
"kl": 0.5660400390625,
|
|
"learning_rate": 1.0039472645551372e-07,
|
|
"loss": 0.0226,
|
|
"reward": 0.8446337506175041,
|
|
"reward_std": 0.9212733060121536,
|
|
"rewards/cosine_scaled_reward": -0.025599811924621463,
|
|
"rewards/format_reward": 0.8958333432674408,
|
|
"step": 494
|
|
},
|
|
{
|
|
"completion_length": 1859.8333892822266,
|
|
"epoch": 0.5657142857142857,
|
|
"grad_norm": 5.380873680114746,
|
|
"kl": 1.025390625,
|
|
"learning_rate": 1.002741278414069e-07,
|
|
"loss": 0.0411,
|
|
"reward": 0.7559722196310759,
|
|
"reward_std": 1.0365862026810646,
|
|
"rewards/cosine_scaled_reward": -0.007430561818182468,
|
|
"rewards/format_reward": 0.7708333656191826,
|
|
"step": 495
|
|
},
|
|
{
|
|
"completion_length": 1412.4375610351562,
|
|
"epoch": 0.5668571428571428,
|
|
"grad_norm": 3.017749547958374,
|
|
"kl": 1.0056610107421875,
|
|
"learning_rate": 1.0017544823184055e-07,
|
|
"loss": 0.0403,
|
|
"reward": 0.8477246034890413,
|
|
"reward_std": 0.7844705618917942,
|
|
"rewards/cosine_scaled_reward": 0.038445642217993736,
|
|
"rewards/format_reward": 0.7708333488553762,
|
|
"step": 496
|
|
},
|
|
{
|
|
"completion_length": 1259.4583702087402,
|
|
"epoch": 0.568,
|
|
"grad_norm": 3.237330436706543,
|
|
"kl": 0.850830078125,
|
|
"learning_rate": 1.0009869243631952e-07,
|
|
"loss": 0.034,
|
|
"reward": 1.0958917308598757,
|
|
"reward_std": 0.5949588976800442,
|
|
"rewards/cosine_scaled_reward": 0.12086252495646477,
|
|
"rewards/format_reward": 0.854166679084301,
|
|
"step": 497
|
|
},
|
|
{
|
|
"completion_length": 1587.083351135254,
|
|
"epoch": 0.5691428571428572,
|
|
"grad_norm": 4.424108028411865,
|
|
"kl": 1.174072265625,
|
|
"learning_rate": 1.000438641958131e-07,
|
|
"loss": 0.047,
|
|
"reward": 0.5771038420498371,
|
|
"reward_std": 0.7162478044629097,
|
|
"rewards/cosine_scaled_reward": -0.08644808363169432,
|
|
"rewards/format_reward": 0.7500000149011612,
|
|
"step": 498
|
|
},
|
|
{
|
|
"completion_length": 1515.1667022705078,
|
|
"epoch": 0.5702857142857143,
|
|
"grad_norm": 1.4978034496307373,
|
|
"kl": 0.6841812133789062,
|
|
"learning_rate": 1.0001096618257236e-07,
|
|
"loss": 0.0274,
|
|
"reward": 0.8714520921930671,
|
|
"reward_std": 0.7796698864549398,
|
|
"rewards/cosine_scaled_reward": -0.012190633453428745,
|
|
"rewards/format_reward": 0.8958333507180214,
|
|
"step": 499
|
|
},
|
|
{
|
|
"completion_length": 1229.7083740234375,
|
|
"epoch": 0.5714285714285714,
|
|
"grad_norm": 2.3225278854370117,
|
|
"kl": 0.565399169921875,
|
|
"learning_rate": 1e-07,
|
|
"loss": 0.0227,
|
|
"reward": 0.6700912415981293,
|
|
"reward_std": 0.7157665528357029,
|
|
"rewards/cosine_scaled_reward": -0.1337043906096369,
|
|
"rewards/format_reward": 0.9375000149011612,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.5714285714285714,
|
|
"step": 500,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.009290046402068698,
|
|
"train_runtime": 55313.9986,
|
|
"train_samples_per_second": 0.434,
|
|
"train_steps_per_second": 0.009
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 500,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 6,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|