11544 lines
433 KiB
JSON
11544 lines
433 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5714285714285714,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2523.270866394043,
|
|
"epoch": 0.001142857142857143,
|
|
"grad_norm": 0.06654668599367142,
|
|
"kl": 0.0,
|
|
"lambda_div_used": 0.6164634302258492,
|
|
"learning_rate": 0.0,
|
|
"loss": -0.0258,
|
|
"reward": -0.14551701629534364,
|
|
"reward_after_mean": -0.14551701629534364,
|
|
"reward_after_std": 0.6225011153146625,
|
|
"reward_before_mean": 0.17862090840935707,
|
|
"reward_before_std": 0.5394803490489721,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3241379093378782,
|
|
"reward_change_min": -0.5330121107399464,
|
|
"reward_change_std": 0.19750467501580715,
|
|
"reward_std": 0.622501116245985,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": -0.0713790925219655,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2684.583366394043,
|
|
"epoch": 0.002285714285714286,
|
|
"grad_norm": 0.07638121396303177,
|
|
"kl": 0.0,
|
|
"lambda_div_used": 0.5910675376653671,
|
|
"learning_rate": 5e-08,
|
|
"loss": 0.001,
|
|
"reward": -0.07136534340679646,
|
|
"reward_after_mean": -0.07136534340679646,
|
|
"reward_after_std": 0.5026816055178642,
|
|
"reward_before_mean": 0.33918463438749313,
|
|
"reward_before_std": 0.41114553064107895,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4105500001460314,
|
|
"reward_change_min": -0.619081187993288,
|
|
"reward_change_std": 0.23483010660856962,
|
|
"reward_std": 0.5026816166937351,
|
|
"rewards/accuracy_reward": 0.29166667349636555,
|
|
"rewards/cosine_scaled_reward": 0.047517990693449974,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2966.9166870117188,
|
|
"epoch": 0.0034285714285714284,
|
|
"grad_norm": 0.07826078683137894,
|
|
"kl": 5.805492401123047e-05,
|
|
"lambda_div_used": 0.5747007578611374,
|
|
"learning_rate": 1e-07,
|
|
"loss": 0.0118,
|
|
"reward": -0.3506452329456806,
|
|
"reward_after_mean": -0.3506452329456806,
|
|
"reward_after_std": 0.40795029513537884,
|
|
"reward_before_mean": -0.06986081041395664,
|
|
"reward_before_std": 0.3391748256981373,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.280784422531724,
|
|
"reward_change_min": -0.45085202157497406,
|
|
"reward_change_std": 0.16326917707920074,
|
|
"reward_std": 0.4079503044486046,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.1323608043603599,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1377.000015258789,
|
|
"epoch": 0.004571428571428572,
|
|
"grad_norm": 0.10993114858865738,
|
|
"kl": 3.951042890548706e-05,
|
|
"lambda_div_used": 0.5788158848881721,
|
|
"learning_rate": 1.5e-07,
|
|
"loss": 0.0214,
|
|
"reward": -0.24558139964938164,
|
|
"reward_after_mean": -0.24558139964938164,
|
|
"reward_after_std": 0.3998244144022465,
|
|
"reward_before_mean": 0.06915237568318844,
|
|
"reward_before_std": 0.36195528600364923,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3147337753325701,
|
|
"reward_change_min": -0.4869973622262478,
|
|
"reward_change_std": 0.1927295122295618,
|
|
"reward_std": 0.3998244162648916,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.09751429967582226,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3310.4583587646484,
|
|
"epoch": 0.005714285714285714,
|
|
"grad_norm": 0.06377790123224258,
|
|
"kl": 5.4255127906799316e-05,
|
|
"lambda_div_used": 0.61885916441679,
|
|
"learning_rate": 2e-07,
|
|
"loss": 0.0428,
|
|
"reward": -0.25358792673796415,
|
|
"reward_after_mean": -0.25358792673796415,
|
|
"reward_after_std": 0.5858059301972389,
|
|
"reward_before_mean": 0.0003254720941185951,
|
|
"reward_before_std": 0.5446656532585621,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25391339510679245,
|
|
"reward_change_min": -0.45586229115724564,
|
|
"reward_change_std": 0.16313681472092867,
|
|
"reward_std": 0.5858059376478195,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.12467453628778458,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2887.8541717529297,
|
|
"epoch": 0.006857142857142857,
|
|
"grad_norm": 0.06263996660709381,
|
|
"kl": 5.511939525604248e-05,
|
|
"lambda_div_used": 0.5606320649385452,
|
|
"learning_rate": 2.5e-07,
|
|
"loss": 0.0618,
|
|
"reward": -0.4456054698675871,
|
|
"reward_after_mean": -0.4456054698675871,
|
|
"reward_after_std": 0.3509902711957693,
|
|
"reward_before_mean": -0.1859958479180932,
|
|
"reward_before_std": 0.2702419590204954,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2596096061170101,
|
|
"reward_change_min": -0.38109203428030014,
|
|
"reward_change_std": 0.14125030301511288,
|
|
"reward_std": 0.3509902749210596,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.2276625158265233,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2572.3542098999023,
|
|
"epoch": 0.008,
|
|
"grad_norm": 0.06695970892906189,
|
|
"kl": 4.0218234062194824e-05,
|
|
"lambda_div_used": 0.6162339821457863,
|
|
"learning_rate": 3e-07,
|
|
"loss": 0.0453,
|
|
"reward": -0.10290078446269035,
|
|
"reward_after_mean": -0.10290078446269035,
|
|
"reward_after_std": 0.566169198602438,
|
|
"reward_before_mean": 0.22045390354469419,
|
|
"reward_before_std": 0.5333282891660929,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32335469499230385,
|
|
"reward_change_min": -0.5540991388261318,
|
|
"reward_change_std": 0.20453903079032898,
|
|
"reward_std": 0.566169211640954,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": 0.012120556086301804,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2173.7500228881836,
|
|
"epoch": 0.009142857142857144,
|
|
"grad_norm": 0.07779484242200851,
|
|
"kl": 3.185681998729706e-05,
|
|
"lambda_div_used": 0.63105358928442,
|
|
"learning_rate": 3.5e-07,
|
|
"loss": 0.0071,
|
|
"reward": 0.09723488846793771,
|
|
"reward_after_mean": 0.09723488846793771,
|
|
"reward_after_std": 0.6893859151750803,
|
|
"reward_before_mean": 0.5079077246482484,
|
|
"reward_before_std": 0.6077435212209821,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.410672839730978,
|
|
"reward_change_min": -0.638349361717701,
|
|
"reward_change_std": 0.2494220733642578,
|
|
"reward_std": 0.6893859468400478,
|
|
"rewards/accuracy_reward": 0.3750000074505806,
|
|
"rewards/cosine_scaled_reward": 0.1329077403061092,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2828.229179382324,
|
|
"epoch": 0.010285714285714285,
|
|
"grad_norm": 0.09342571347951889,
|
|
"kl": 4.9620866775512695e-05,
|
|
"lambda_div_used": 0.6150719821453094,
|
|
"learning_rate": 4e-07,
|
|
"loss": -0.0132,
|
|
"reward": -0.18431122601032257,
|
|
"reward_after_mean": -0.18431122601032257,
|
|
"reward_after_std": 0.5804070886224508,
|
|
"reward_before_mean": 0.09753156686201692,
|
|
"reward_before_std": 0.5302032623440027,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2818427961319685,
|
|
"reward_change_min": -0.46669338643550873,
|
|
"reward_change_std": 0.18001185078173876,
|
|
"reward_std": 0.5804071053862572,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.0691350968554616,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2469.7083587646484,
|
|
"epoch": 0.011428571428571429,
|
|
"grad_norm": 0.09024360775947571,
|
|
"kl": 3.8331374526023865e-05,
|
|
"lambda_div_used": 0.63355503231287,
|
|
"learning_rate": 4.5e-07,
|
|
"loss": -0.001,
|
|
"reward": -0.04430920258164406,
|
|
"reward_after_mean": -0.04430920258164406,
|
|
"reward_after_std": 0.6170141901820898,
|
|
"reward_before_mean": 0.2625937759876251,
|
|
"reward_before_std": 0.6259458847343922,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3069029748439789,
|
|
"reward_change_min": -0.5734602250158787,
|
|
"reward_change_std": 0.22167105227708817,
|
|
"reward_std": 0.6170141994953156,
|
|
"rewards/accuracy_reward": 0.2500000074505806,
|
|
"rewards/cosine_scaled_reward": 0.012593759223818779,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3367.500015258789,
|
|
"epoch": 0.012571428571428572,
|
|
"grad_norm": 0.05533275753259659,
|
|
"kl": 5.245208740234375e-05,
|
|
"lambda_div_used": 0.5749180987477303,
|
|
"learning_rate": 5e-07,
|
|
"loss": -0.0467,
|
|
"reward": -0.35878527723252773,
|
|
"reward_after_mean": -0.35878527723252773,
|
|
"reward_after_std": 0.4030665699392557,
|
|
"reward_before_mean": -0.09556051343679428,
|
|
"reward_before_std": 0.3429161449894309,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2632247470319271,
|
|
"reward_change_min": -0.4096112735569477,
|
|
"reward_change_std": 0.1540710162371397,
|
|
"reward_std": 0.40306657925248146,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.19972719065845013,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2016.958351135254,
|
|
"epoch": 0.013714285714285714,
|
|
"grad_norm": 0.08992662280797958,
|
|
"kl": 3.9637088775634766e-05,
|
|
"lambda_div_used": 0.6247389540076256,
|
|
"learning_rate": 5.5e-07,
|
|
"loss": 0.0429,
|
|
"reward": -0.10162727534770966,
|
|
"reward_after_mean": -0.10162727534770966,
|
|
"reward_after_std": 0.594907833263278,
|
|
"reward_before_mean": 0.19102710485458374,
|
|
"reward_before_std": 0.5805315412580967,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.292654387652874,
|
|
"reward_change_min": -0.5117304362356663,
|
|
"reward_change_std": 0.20085694547742605,
|
|
"reward_std": 0.5949078388512135,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.03813957702368498,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2905.0833587646484,
|
|
"epoch": 0.014857142857142857,
|
|
"grad_norm": 0.0641385093331337,
|
|
"kl": 4.194676876068115e-05,
|
|
"lambda_div_used": 0.5766140297055244,
|
|
"learning_rate": 6e-07,
|
|
"loss": 0.0437,
|
|
"reward": -0.3046809285879135,
|
|
"reward_after_mean": -0.3046809285879135,
|
|
"reward_after_std": 0.399059085175395,
|
|
"reward_before_mean": -0.018187658861279488,
|
|
"reward_before_std": 0.35123275220394135,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28649328649044037,
|
|
"reward_change_min": -0.4807186797261238,
|
|
"reward_change_std": 0.17643271200358868,
|
|
"reward_std": 0.399059085175395,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.1431876514106989,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2520.312530517578,
|
|
"epoch": 0.016,
|
|
"grad_norm": 0.07475250959396362,
|
|
"kl": 3.432855010032654e-05,
|
|
"lambda_div_used": 0.5991310104727745,
|
|
"learning_rate": 6.5e-07,
|
|
"loss": -0.0332,
|
|
"reward": -0.22936711832880974,
|
|
"reward_after_mean": -0.22936711832880974,
|
|
"reward_after_std": 0.5019008349627256,
|
|
"reward_before_mean": 0.04907496925443411,
|
|
"reward_before_std": 0.4585955021902919,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27844210527837276,
|
|
"reward_change_min": -0.4685846194624901,
|
|
"reward_change_std": 0.17394172679632902,
|
|
"reward_std": 0.5019008629024029,
|
|
"rewards/accuracy_reward": 0.14583333767950535,
|
|
"rewards/cosine_scaled_reward": -0.09675835724920034,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2717.0416984558105,
|
|
"epoch": 0.017142857142857144,
|
|
"grad_norm": 0.08986662328243256,
|
|
"kl": 4.3585896492004395e-05,
|
|
"lambda_div_used": 0.5765259638428688,
|
|
"learning_rate": 7e-07,
|
|
"loss": 0.0355,
|
|
"reward": -0.16787780448794365,
|
|
"reward_after_mean": -0.16787780448794365,
|
|
"reward_after_std": 0.4601633083075285,
|
|
"reward_before_mean": 0.22437161579728127,
|
|
"reward_before_std": 0.3496173685416579,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3922494389116764,
|
|
"reward_change_min": -0.5602193549275398,
|
|
"reward_change_std": 0.2197399353608489,
|
|
"reward_std": 0.46016331762075424,
|
|
"rewards/accuracy_reward": 0.20833333395421505,
|
|
"rewards/cosine_scaled_reward": 0.016038289293646812,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3395.1875,
|
|
"epoch": 0.018285714285714287,
|
|
"grad_norm": 0.05402011796832085,
|
|
"kl": 4.659593105316162e-05,
|
|
"lambda_div_used": 0.5638558194041252,
|
|
"learning_rate": 7.5e-07,
|
|
"loss": 0.032,
|
|
"reward": -0.432980858720839,
|
|
"reward_after_mean": -0.432980858720839,
|
|
"reward_after_std": 0.3415633924305439,
|
|
"reward_before_mean": -0.16444075386971235,
|
|
"reward_before_std": 0.2898183651268482,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26854010485112667,
|
|
"reward_change_min": -0.45487307757139206,
|
|
"reward_change_std": 0.16323526203632355,
|
|
"reward_std": 0.34156340546905994,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.20610742270946503,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2164.020854949951,
|
|
"epoch": 0.019428571428571427,
|
|
"grad_norm": 0.1185457855463028,
|
|
"kl": 4.2147934436798096e-05,
|
|
"lambda_div_used": 0.5763917565345764,
|
|
"learning_rate": 8e-07,
|
|
"loss": 0.0302,
|
|
"reward": -0.2335935328155756,
|
|
"reward_after_mean": -0.2335935328155756,
|
|
"reward_after_std": 0.4716028142720461,
|
|
"reward_before_mean": 0.13111361488699913,
|
|
"reward_before_std": 0.3423871146515012,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36470715142786503,
|
|
"reward_change_min": -0.5119686089456081,
|
|
"reward_change_std": 0.19370271731168032,
|
|
"reward_std": 0.471602825447917,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.05638639978133142,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3079.4583740234375,
|
|
"epoch": 0.02057142857142857,
|
|
"grad_norm": 0.05137661099433899,
|
|
"kl": 3.637373447418213e-05,
|
|
"lambda_div_used": 0.6073044687509537,
|
|
"learning_rate": 8.499999999999999e-07,
|
|
"loss": 0.0478,
|
|
"reward": -0.057250723242759705,
|
|
"reward_after_mean": -0.057250723242759705,
|
|
"reward_after_std": 0.5715843215584755,
|
|
"reward_before_mean": 0.3262214660644531,
|
|
"reward_before_std": 0.5018989769741893,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38347217813134193,
|
|
"reward_change_min": -0.6260374560952187,
|
|
"reward_change_std": 0.24400013033300638,
|
|
"reward_std": 0.571584340184927,
|
|
"rewards/accuracy_reward": 0.2916666679084301,
|
|
"rewards/cosine_scaled_reward": 0.034554785932414234,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2889.8542289733887,
|
|
"epoch": 0.021714285714285714,
|
|
"grad_norm": 0.07495337724685669,
|
|
"kl": 4.096329212188721e-05,
|
|
"lambda_div_used": 0.6219106838107109,
|
|
"learning_rate": 9e-07,
|
|
"loss": 0.036,
|
|
"reward": 0.1660416293889284,
|
|
"reward_after_mean": 0.1660416293889284,
|
|
"reward_after_std": 0.6783627849072218,
|
|
"reward_before_mean": 0.6544073540717363,
|
|
"reward_before_std": 0.5640754774212837,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.488365713506937,
|
|
"reward_change_min": -0.758969146758318,
|
|
"reward_change_std": 0.2900321548804641,
|
|
"reward_std": 0.6783628333359957,
|
|
"rewards/accuracy_reward": 0.45833334140479565,
|
|
"rewards/cosine_scaled_reward": 0.19607400865061209,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2114.8333587646484,
|
|
"epoch": 0.022857142857142857,
|
|
"grad_norm": 0.0912039652466774,
|
|
"kl": 2.7514994144439697e-05,
|
|
"lambda_div_used": 0.6171177923679352,
|
|
"learning_rate": 9.499999999999999e-07,
|
|
"loss": -0.0338,
|
|
"reward": -0.12753646727651358,
|
|
"reward_after_mean": -0.12753646727651358,
|
|
"reward_after_std": 0.6479649767279625,
|
|
"reward_before_mean": 0.2073977841064334,
|
|
"reward_before_std": 0.5446468549780548,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.33493425138294697,
|
|
"reward_change_min": -0.5443090125918388,
|
|
"reward_change_std": 0.20240973494946957,
|
|
"reward_std": 0.6479649972170591,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": -0.0426022089086473,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2578.3125,
|
|
"epoch": 0.024,
|
|
"grad_norm": 0.1021808609366417,
|
|
"kl": 4.0903687477111816e-05,
|
|
"lambda_div_used": 0.5696183741092682,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0243,
|
|
"reward": -0.16086186096072197,
|
|
"reward_after_mean": -0.16086186096072197,
|
|
"reward_after_std": 0.4245347697287798,
|
|
"reward_before_mean": 0.23232585442019626,
|
|
"reward_before_std": 0.3138282438740134,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3931876849383116,
|
|
"reward_change_min": -0.5524330623447895,
|
|
"reward_change_std": 0.2125744568184018,
|
|
"reward_std": 0.424534784629941,
|
|
"rewards/accuracy_reward": 0.25000000558793545,
|
|
"rewards/cosine_scaled_reward": -0.01767415925860405,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1644.437572479248,
|
|
"epoch": 0.025142857142857144,
|
|
"grad_norm": 0.11418648809194565,
|
|
"kl": 2.1685846149921417e-05,
|
|
"lambda_div_used": 0.5976943448185921,
|
|
"learning_rate": 9.99931462820376e-07,
|
|
"loss": -0.0304,
|
|
"reward": -0.10828415304422379,
|
|
"reward_after_mean": -0.10828415304422379,
|
|
"reward_after_std": 0.5476059578359127,
|
|
"reward_before_mean": 0.2575272931717336,
|
|
"reward_before_std": 0.4531834872905165,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36581144109368324,
|
|
"reward_change_min": -0.5498910807073116,
|
|
"reward_change_std": 0.21405170671641827,
|
|
"reward_std": 0.5476059839129448,
|
|
"rewards/accuracy_reward": 0.29166667349636555,
|
|
"rewards/cosine_scaled_reward": -0.03413938358426094,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2188.687530517578,
|
|
"epoch": 0.026285714285714287,
|
|
"grad_norm": 0.0898994579911232,
|
|
"kl": 3.7088990211486816e-05,
|
|
"lambda_div_used": 0.6402537003159523,
|
|
"learning_rate": 9.997258721585931e-07,
|
|
"loss": -0.0037,
|
|
"reward": -0.13877022732049227,
|
|
"reward_after_mean": -0.13877022732049227,
|
|
"reward_after_std": 0.6969181355088949,
|
|
"reward_before_mean": 0.11387635703431442,
|
|
"reward_before_std": 0.6502415342256427,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2526465691626072,
|
|
"reward_change_min": -0.45175985619425774,
|
|
"reward_change_std": 0.1627071350812912,
|
|
"reward_std": 0.6969181522727013,
|
|
"rewards/accuracy_reward": 0.16666666977107525,
|
|
"rewards/cosine_scaled_reward": -0.05279031861573458,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2394.3958740234375,
|
|
"epoch": 0.027428571428571427,
|
|
"grad_norm": 0.09361977875232697,
|
|
"kl": 2.0720064640045166e-05,
|
|
"lambda_div_used": 0.6309552267193794,
|
|
"learning_rate": 9.993832906395582e-07,
|
|
"loss": 0.0666,
|
|
"reward": 0.0798279655573424,
|
|
"reward_after_mean": 0.0798279655573424,
|
|
"reward_after_std": 0.6638341955840588,
|
|
"reward_before_mean": 0.4744788520038128,
|
|
"reward_before_std": 0.6099728401750326,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3946509025990963,
|
|
"reward_change_min": -0.6427228525280952,
|
|
"reward_change_std": 0.25018193013966084,
|
|
"reward_std": 0.6638342067599297,
|
|
"rewards/accuracy_reward": 0.35416667349636555,
|
|
"rewards/cosine_scaled_reward": 0.12031219294294715,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2358.583366394043,
|
|
"epoch": 0.02857142857142857,
|
|
"grad_norm": 0.07725252956151962,
|
|
"kl": 4.092603921890259e-05,
|
|
"lambda_div_used": 0.6378427669405937,
|
|
"learning_rate": 9.989038226169207e-07,
|
|
"loss": -0.0557,
|
|
"reward": -0.15363326482474804,
|
|
"reward_after_mean": -0.15363326482474804,
|
|
"reward_after_std": 0.6477887704968452,
|
|
"reward_before_mean": 0.10259643197059631,
|
|
"reward_before_std": 0.6456613540649414,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25622970052063465,
|
|
"reward_change_min": -0.5558900497853756,
|
|
"reward_change_std": 0.19845533184707165,
|
|
"reward_std": 0.6477888077497482,
|
|
"rewards/accuracy_reward": 0.1875000037252903,
|
|
"rewards/cosine_scaled_reward": -0.08490357082337141,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2850.937530517578,
|
|
"epoch": 0.029714285714285714,
|
|
"grad_norm": 0.06806578487157822,
|
|
"kl": 3.5643577575683594e-05,
|
|
"lambda_div_used": 0.5781113430857658,
|
|
"learning_rate": 9.982876141412855e-07,
|
|
"loss": 0.005,
|
|
"reward": -0.25517464708536863,
|
|
"reward_after_mean": -0.25517464708536863,
|
|
"reward_after_std": 0.47686139307916164,
|
|
"reward_before_mean": 0.09097394905984402,
|
|
"reward_before_std": 0.3564971052110195,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3461485952138901,
|
|
"reward_change_min": -0.4805891402065754,
|
|
"reward_change_std": 0.18107541371136904,
|
|
"reward_std": 0.476861409842968,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.07569271977990866,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2795.3750534057617,
|
|
"epoch": 0.030857142857142857,
|
|
"grad_norm": 0.0670829713344574,
|
|
"kl": 5.0067901611328125e-05,
|
|
"lambda_div_used": 0.5909813195466995,
|
|
"learning_rate": 9.975348529157229e-07,
|
|
"loss": 0.038,
|
|
"reward": -0.30352935567498207,
|
|
"reward_after_mean": -0.30352935567498207,
|
|
"reward_after_std": 0.4782011900097132,
|
|
"reward_before_mean": -0.04106062464416027,
|
|
"reward_before_std": 0.42168071679770947,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2624687273055315,
|
|
"reward_change_min": -0.441840048879385,
|
|
"reward_change_std": 0.16586182732135057,
|
|
"reward_std": 0.47820119373500347,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.16606062231585383,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2662.562530517578,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.07541470974683762,
|
|
"kl": 4.6879053115844727e-05,
|
|
"lambda_div_used": 0.6455363035202026,
|
|
"learning_rate": 9.96645768238595e-07,
|
|
"loss": -0.0548,
|
|
"reward": -0.01765006221830845,
|
|
"reward_after_mean": -0.01765006221830845,
|
|
"reward_after_std": 0.7017391249537468,
|
|
"reward_before_mean": 0.26933106034994125,
|
|
"reward_before_std": 0.6768321208655834,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2869811188429594,
|
|
"reward_change_min": -0.5125514604151249,
|
|
"reward_change_std": 0.19719962775707245,
|
|
"reward_std": 0.7017391547560692,
|
|
"rewards/accuracy_reward": 0.29166667349636555,
|
|
"rewards/cosine_scaled_reward": -0.022335614077746868,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2988.1041717529297,
|
|
"epoch": 0.03314285714285714,
|
|
"grad_norm": 0.07968341559171677,
|
|
"kl": 4.7832727432250977e-05,
|
|
"lambda_div_used": 0.5851128473877907,
|
|
"learning_rate": 9.956206309337066e-07,
|
|
"loss": -0.1054,
|
|
"reward": -0.38509707152843475,
|
|
"reward_after_mean": -0.38509707152843475,
|
|
"reward_after_std": 0.4501145612448454,
|
|
"reward_before_mean": -0.14269733056426048,
|
|
"reward_before_std": 0.3910446595400572,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24239975214004517,
|
|
"reward_change_min": -0.4103321433067322,
|
|
"reward_change_std": 0.1474100910127163,
|
|
"reward_std": 0.4501145798712969,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.20519732707180083,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2777.1042404174805,
|
|
"epoch": 0.03428571428571429,
|
|
"grad_norm": 0.0785035490989685,
|
|
"kl": 4.0784478187561035e-05,
|
|
"lambda_div_used": 0.616574227809906,
|
|
"learning_rate": 9.944597532678119e-07,
|
|
"loss": 0.0306,
|
|
"reward": 0.014547215774655342,
|
|
"reward_after_mean": 0.014547215774655342,
|
|
"reward_after_std": 0.6317574586719275,
|
|
"reward_before_mean": 0.40835407795384526,
|
|
"reward_before_std": 0.5376694360747933,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3938068598508835,
|
|
"reward_change_min": -0.5825920477509499,
|
|
"reward_change_std": 0.22909015510231256,
|
|
"reward_std": 0.631757466122508,
|
|
"rewards/accuracy_reward": 0.3125000037252903,
|
|
"rewards/cosine_scaled_reward": 0.09585406119003892,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3100.9166870117188,
|
|
"epoch": 0.03542857142857143,
|
|
"grad_norm": 0.05876823514699936,
|
|
"kl": 5.622208118438721e-05,
|
|
"lambda_div_used": 0.5746422484517097,
|
|
"learning_rate": 9.931634888554935e-07,
|
|
"loss": -0.0301,
|
|
"reward": -0.373639321886003,
|
|
"reward_after_mean": -0.373639321886003,
|
|
"reward_after_std": 0.40832165256142616,
|
|
"reward_before_mean": -0.11681870371103287,
|
|
"reward_before_std": 0.3402246618643403,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2568206209689379,
|
|
"reward_change_min": -0.4000260457396507,
|
|
"reward_change_std": 0.149553001858294,
|
|
"reward_std": 0.4083216693252325,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.20015203207731247,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2879.625030517578,
|
|
"epoch": 0.036571428571428574,
|
|
"grad_norm": 0.060988157987594604,
|
|
"kl": 4.863739013671875e-05,
|
|
"lambda_div_used": 0.583471342921257,
|
|
"learning_rate": 9.917322325514487e-07,
|
|
"loss": 0.0143,
|
|
"reward": -0.09911171346902847,
|
|
"reward_after_mean": -0.09911171346902847,
|
|
"reward_after_std": 0.46612042374908924,
|
|
"reward_before_mean": 0.2912287414073944,
|
|
"reward_before_std": 0.3850706424564123,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.39034045673906803,
|
|
"reward_change_min": -0.5949138030409813,
|
|
"reward_change_std": 0.23070038296282291,
|
|
"reward_std": 0.46612043119966984,
|
|
"rewards/accuracy_reward": 0.2916666716337204,
|
|
"rewards/cosine_scaled_reward": -0.0004379265010356903,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3136.750030517578,
|
|
"epoch": 0.037714285714285714,
|
|
"grad_norm": 0.07551854848861694,
|
|
"kl": 4.787743091583252e-05,
|
|
"lambda_div_used": 0.6242911070585251,
|
|
"learning_rate": 9.901664203302124e-07,
|
|
"loss": -0.0446,
|
|
"reward": -0.06627794913947582,
|
|
"reward_after_mean": -0.06627794913947582,
|
|
"reward_after_std": 0.5915219262242317,
|
|
"reward_before_mean": 0.24627447500824928,
|
|
"reward_before_std": 0.5734489392489195,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3125524502247572,
|
|
"reward_change_min": -0.5326853170990944,
|
|
"reward_change_std": 0.2051102453842759,
|
|
"reward_std": 0.5915219560265541,
|
|
"rewards/accuracy_reward": 0.25000000931322575,
|
|
"rewards/cosine_scaled_reward": -0.0037255147472023964,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2379.541702270508,
|
|
"epoch": 0.038857142857142854,
|
|
"grad_norm": 0.07498405873775482,
|
|
"kl": 4.60892915725708e-05,
|
|
"lambda_div_used": 0.6475758850574493,
|
|
"learning_rate": 9.88466529153356e-07,
|
|
"loss": 0.0814,
|
|
"reward": 0.054736172780394554,
|
|
"reward_after_mean": 0.054736172780394554,
|
|
"reward_after_std": 0.7529329154640436,
|
|
"reward_before_mean": 0.4133173357695341,
|
|
"reward_before_std": 0.6894109938293695,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3585811499506235,
|
|
"reward_change_min": -0.6234904117882252,
|
|
"reward_change_std": 0.23517589084804058,
|
|
"reward_std": 0.7529329396784306,
|
|
"rewards/accuracy_reward": 0.3333333358168602,
|
|
"rewards/cosine_scaled_reward": 0.07998399529606104,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3113.1875610351562,
|
|
"epoch": 0.04,
|
|
"grad_norm": 0.1044503003358841,
|
|
"kl": 5.8710575103759766e-05,
|
|
"lambda_div_used": 0.5754920393228531,
|
|
"learning_rate": 9.866330768241983e-07,
|
|
"loss": -0.0058,
|
|
"reward": -0.36803475581109524,
|
|
"reward_after_mean": -0.36803475581109524,
|
|
"reward_after_std": 0.4120363052934408,
|
|
"reward_before_mean": -0.1010703444480896,
|
|
"reward_before_std": 0.3434353759512305,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2669644057750702,
|
|
"reward_change_min": -0.42853060737252235,
|
|
"reward_change_std": 0.1540099997073412,
|
|
"reward_std": 0.41203631460666656,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.16357035003602505,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3350.1666870117188,
|
|
"epoch": 0.04114285714285714,
|
|
"grad_norm": 0.07969332486391068,
|
|
"kl": 6.110966205596924e-05,
|
|
"lambda_div_used": 0.5744208693504333,
|
|
"learning_rate": 9.846666218300807e-07,
|
|
"loss": -0.0514,
|
|
"reward": -0.4380789175629616,
|
|
"reward_after_mean": -0.4380789175629616,
|
|
"reward_after_std": 0.41310197673738003,
|
|
"reward_before_mean": -0.20339234871789813,
|
|
"reward_before_std": 0.337322598323226,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23468656465411186,
|
|
"reward_change_min": -0.3448420464992523,
|
|
"reward_change_std": 0.12617942783981562,
|
|
"reward_std": 0.4131019860506058,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.24505900964140892,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3394.3958435058594,
|
|
"epoch": 0.04228571428571429,
|
|
"grad_norm": 0.05401439592242241,
|
|
"kl": 4.9054622650146484e-05,
|
|
"lambda_div_used": 0.5599935948848724,
|
|
"learning_rate": 9.825677631722435e-07,
|
|
"loss": 0.0086,
|
|
"reward": -0.3875775597989559,
|
|
"reward_after_mean": -0.3875775597989559,
|
|
"reward_after_std": 0.32731432281434536,
|
|
"reward_before_mean": -0.10551332868635654,
|
|
"reward_before_std": 0.2694389373064041,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28206423483788967,
|
|
"reward_change_min": -0.4370834305882454,
|
|
"reward_change_std": 0.16284553799778223,
|
|
"reward_std": 0.3273143321275711,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.1888466626405716,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3311.0833435058594,
|
|
"epoch": 0.04342857142857143,
|
|
"grad_norm": 0.05246276035904884,
|
|
"kl": 4.945695400238037e-05,
|
|
"lambda_div_used": 0.5770035535097122,
|
|
"learning_rate": 9.80337140183366e-07,
|
|
"loss": -0.0117,
|
|
"reward": -0.3118477761745453,
|
|
"reward_after_mean": -0.3118477761745453,
|
|
"reward_after_std": 0.38849758356809616,
|
|
"reward_before_mean": -0.021045896457508206,
|
|
"reward_before_std": 0.3480216721072793,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2908018734306097,
|
|
"reward_change_min": -0.44894198328256607,
|
|
"reward_change_std": 0.17141664400696754,
|
|
"reward_std": 0.3884976040571928,
|
|
"rewards/accuracy_reward": 0.12500000558793545,
|
|
"rewards/cosine_scaled_reward": -0.14604590460658073,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2919.604179382324,
|
|
"epoch": 0.044571428571428574,
|
|
"grad_norm": 0.08766351640224457,
|
|
"kl": 3.2648444175720215e-05,
|
|
"lambda_div_used": 0.5717808604240417,
|
|
"learning_rate": 9.779754323328192e-07,
|
|
"loss": 0.032,
|
|
"reward": -0.14589639008045197,
|
|
"reward_after_mean": -0.14589639008045197,
|
|
"reward_after_std": 0.4250806160271168,
|
|
"reward_before_mean": 0.26390516571700573,
|
|
"reward_before_std": 0.3228347860276699,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.40980157628655434,
|
|
"reward_change_min": -0.5692479386925697,
|
|
"reward_change_std": 0.22337355464696884,
|
|
"reward_std": 0.42508063092827797,
|
|
"rewards/accuracy_reward": 0.25000000558793545,
|
|
"rewards/cosine_scaled_reward": 0.013905153144150972,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2468.7916870117188,
|
|
"epoch": 0.045714285714285714,
|
|
"grad_norm": 0.07357986271381378,
|
|
"kl": 4.18052077293396e-05,
|
|
"lambda_div_used": 0.5706272348761559,
|
|
"learning_rate": 9.754833590196926e-07,
|
|
"loss": 0.0148,
|
|
"reward": -0.2578982161357999,
|
|
"reward_after_mean": -0.2578982161357999,
|
|
"reward_after_std": 0.4464099854230881,
|
|
"reward_before_mean": 0.1052470114082098,
|
|
"reward_before_std": 0.31700289947912097,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3631452303379774,
|
|
"reward_change_min": -0.5201860442757607,
|
|
"reward_change_std": 0.1936064399778843,
|
|
"reward_std": 0.4464099947363138,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.06141966814175248,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3203.312515258789,
|
|
"epoch": 0.046857142857142854,
|
|
"grad_norm": 0.05290238931775093,
|
|
"kl": 5.0440430641174316e-05,
|
|
"lambda_div_used": 0.6480180770158768,
|
|
"learning_rate": 9.728616793536587e-07,
|
|
"loss": -0.0199,
|
|
"reward": -0.04493038635700941,
|
|
"reward_after_mean": -0.04493038635700941,
|
|
"reward_after_std": 0.7092454191297293,
|
|
"reward_before_mean": 0.23497815802693367,
|
|
"reward_before_std": 0.693053056485951,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2799085471779108,
|
|
"reward_change_min": -0.5064845345914364,
|
|
"reward_change_std": 0.19558908697217703,
|
|
"reward_std": 0.7092454340308905,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": 0.005811510724015534,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2890.9791927337646,
|
|
"epoch": 0.048,
|
|
"grad_norm": 0.09494666010141373,
|
|
"kl": 5.85019588470459e-05,
|
|
"lambda_div_used": 0.5329261645674706,
|
|
"learning_rate": 9.701111919237408e-07,
|
|
"loss": 0.0335,
|
|
"reward": -0.5650591850280762,
|
|
"reward_after_mean": -0.5650591850280762,
|
|
"reward_after_std": 0.225538931787014,
|
|
"reward_before_mean": -0.32472414150834084,
|
|
"reward_before_std": 0.14451794046908617,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24033503979444504,
|
|
"reward_change_min": -0.358134388923645,
|
|
"reward_change_std": 0.12634030357003212,
|
|
"reward_std": 0.2255389392375946,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.32472414895892143,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3027.9583587646484,
|
|
"epoch": 0.04914285714285714,
|
|
"grad_norm": 0.05899634212255478,
|
|
"kl": 4.252791404724121e-05,
|
|
"lambda_div_used": 0.595756284892559,
|
|
"learning_rate": 9.672327345550543e-07,
|
|
"loss": -0.017,
|
|
"reward": -0.31989429891109467,
|
|
"reward_after_mean": -0.31989429891109467,
|
|
"reward_after_std": 0.5033334400504827,
|
|
"reward_before_mean": -0.07388713955879211,
|
|
"reward_before_std": 0.43908379040658474,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24600715190172195,
|
|
"reward_change_min": -0.4015818126499653,
|
|
"reward_change_std": 0.143897395581007,
|
|
"reward_std": 0.5033334512263536,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1780538223683834,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2372.5833587646484,
|
|
"epoch": 0.05028571428571429,
|
|
"grad_norm": 0.08952542394399643,
|
|
"kl": 4.1332095861434937e-05,
|
|
"lambda_div_used": 0.5768283307552338,
|
|
"learning_rate": 9.64227184053598e-07,
|
|
"loss": 0.0238,
|
|
"reward": -0.09475147165358067,
|
|
"reward_after_mean": -0.09475147165358067,
|
|
"reward_after_std": 0.4600023180246353,
|
|
"reward_before_mean": 0.33489263616502285,
|
|
"reward_before_std": 0.35124383692163974,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4296441040933132,
|
|
"reward_change_min": -0.6358921378850937,
|
|
"reward_change_std": 0.24556122440844774,
|
|
"reward_std": 0.4600023254752159,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": 0.06405930174514651,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3178.125,
|
|
"epoch": 0.05142857142857143,
|
|
"grad_norm": 0.05793697386980057,
|
|
"kl": 4.953145980834961e-05,
|
|
"lambda_div_used": 0.6092750132083893,
|
|
"learning_rate": 9.610954559391704e-07,
|
|
"loss": 0.0375,
|
|
"reward": -0.24964951165020466,
|
|
"reward_after_mean": -0.24964951165020466,
|
|
"reward_after_std": 0.575963044539094,
|
|
"reward_before_mean": 0.010031561367213726,
|
|
"reward_before_std": 0.5040898034349084,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25968106649816036,
|
|
"reward_change_min": -0.41585399955511093,
|
|
"reward_change_std": 0.15605208091437817,
|
|
"reward_std": 0.5759630724787712,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.114968441426754,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3160.895866394043,
|
|
"epoch": 0.052571428571428575,
|
|
"grad_norm": 0.06750793755054474,
|
|
"kl": 4.884600639343262e-05,
|
|
"lambda_div_used": 0.5543409436941147,
|
|
"learning_rate": 9.578385041664925e-07,
|
|
"loss": 0.0442,
|
|
"reward": -0.47163831628859043,
|
|
"reward_after_mean": -0.47163831628859043,
|
|
"reward_after_std": 0.32501144520938396,
|
|
"reward_before_mean": -0.21356616588309407,
|
|
"reward_before_std": 0.24316862598061562,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25807216577231884,
|
|
"reward_change_min": -0.3883417770266533,
|
|
"reward_change_std": 0.13798328768461943,
|
|
"reward_std": 0.3250114619731903,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.23439949192106724,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2624.270881652832,
|
|
"epoch": 0.053714285714285714,
|
|
"grad_norm": 0.10712098330259323,
|
|
"kl": 5.3569674491882324e-05,
|
|
"lambda_div_used": 0.6219898834824562,
|
|
"learning_rate": 9.54457320834625e-07,
|
|
"loss": 0.0102,
|
|
"reward": -0.10781855694949627,
|
|
"reward_after_mean": -0.10781855694949627,
|
|
"reward_after_std": 0.5886904541403055,
|
|
"reward_before_mean": 0.19626505579799414,
|
|
"reward_before_std": 0.5641138087958097,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3040836229920387,
|
|
"reward_change_min": -0.5314439944922924,
|
|
"reward_change_std": 0.2001748401671648,
|
|
"reward_std": 0.5886904690414667,
|
|
"rewards/accuracy_reward": 0.22916667349636555,
|
|
"rewards/cosine_scaled_reward": -0.03290162514895201,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2589.458396911621,
|
|
"epoch": 0.054857142857142854,
|
|
"grad_norm": 0.07933427393436432,
|
|
"kl": 4.4988468289375305e-05,
|
|
"lambda_div_used": 0.6262567341327667,
|
|
"learning_rate": 9.509529358847654e-07,
|
|
"loss": -0.0358,
|
|
"reward": -0.13860276085324585,
|
|
"reward_after_mean": -0.13860276085324585,
|
|
"reward_after_std": 0.5960155855864286,
|
|
"reward_before_mean": 0.13786590658128262,
|
|
"reward_before_std": 0.5872014760971069,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2764686793088913,
|
|
"reward_change_min": -0.5099791921675205,
|
|
"reward_change_std": 0.19576375279575586,
|
|
"reward_std": 0.5960155855864286,
|
|
"rewards/accuracy_reward": 0.20833333767950535,
|
|
"rewards/cosine_scaled_reward": -0.07046743389219046,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1985.2291946411133,
|
|
"epoch": 0.056,
|
|
"grad_norm": 0.12161526083946228,
|
|
"kl": 4.811584949493408e-05,
|
|
"lambda_div_used": 0.5859075263142586,
|
|
"learning_rate": 9.473264167865171e-07,
|
|
"loss": 0.0065,
|
|
"reward": -0.21219712868332863,
|
|
"reward_after_mean": -0.21219712868332863,
|
|
"reward_after_std": 0.439599821344018,
|
|
"reward_before_mean": 0.10536342021077871,
|
|
"reward_before_std": 0.39430073963012546,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3175605833530426,
|
|
"reward_change_min": -0.4657192714512348,
|
|
"reward_change_std": 0.19229275174438953,
|
|
"reward_std": 0.4395998399704695,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.10296990955248475,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2880.250011444092,
|
|
"epoch": 0.05714285714285714,
|
|
"grad_norm": 0.07723147422075272,
|
|
"kl": 3.2588839530944824e-05,
|
|
"lambda_div_used": 0.5741998106241226,
|
|
"learning_rate": 9.43578868212728e-07,
|
|
"loss": 0.0182,
|
|
"reward": -0.0913238637149334,
|
|
"reward_after_mean": -0.0913238637149334,
|
|
"reward_after_std": 0.42474013939499855,
|
|
"reward_before_mean": 0.3248649761080742,
|
|
"reward_before_std": 0.3397149038501084,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41618884168565273,
|
|
"reward_change_min": -0.6034708395600319,
|
|
"reward_change_std": 0.23950859624892473,
|
|
"reward_std": 0.4247401561588049,
|
|
"rewards/accuracy_reward": 0.3125000074505806,
|
|
"rewards/cosine_scaled_reward": 0.012364983558654785,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2433.020854949951,
|
|
"epoch": 0.05828571428571429,
|
|
"grad_norm": 0.09581972658634186,
|
|
"kl": 5.4717063903808594e-05,
|
|
"lambda_div_used": 0.5894461125135422,
|
|
"learning_rate": 9.397114317029974e-07,
|
|
"loss": -0.0308,
|
|
"reward": -0.28266510320827365,
|
|
"reward_after_mean": -0.28266510320827365,
|
|
"reward_after_std": 0.44258156418800354,
|
|
"reward_before_mean": 0.012647990137338638,
|
|
"reward_before_std": 0.40818152111023664,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29531308077275753,
|
|
"reward_change_min": -0.4761137217283249,
|
|
"reward_change_std": 0.1856073010712862,
|
|
"reward_std": 0.4425815735012293,
|
|
"rewards/accuracy_reward": 0.16666666977107525,
|
|
"rewards/cosine_scaled_reward": -0.15401867777109146,
|
|
"step": 51
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2707.250030517578,
|
|
"epoch": 0.05942857142857143,
|
|
"grad_norm": 0.14702580869197845,
|
|
"kl": 4.030205309391022e-05,
|
|
"lambda_div_used": 0.6197419166564941,
|
|
"learning_rate": 9.357252853159505e-07,
|
|
"loss": 0.0298,
|
|
"reward": 0.17821598052978516,
|
|
"reward_after_mean": 0.17821598052978516,
|
|
"reward_after_std": 0.6063419748097658,
|
|
"reward_before_mean": 0.629100788384676,
|
|
"reward_before_std": 0.5578461596742272,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4508847985416651,
|
|
"reward_change_min": -0.6935725994408131,
|
|
"reward_change_std": 0.2830996550619602,
|
|
"reward_std": 0.6063419822603464,
|
|
"rewards/accuracy_reward": 0.4583333432674408,
|
|
"rewards/cosine_scaled_reward": 0.17076744884252548,
|
|
"step": 52
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2799.8125610351562,
|
|
"epoch": 0.060571428571428575,
|
|
"grad_norm": 0.07566332817077637,
|
|
"kl": 5.0321221351623535e-05,
|
|
"lambda_div_used": 0.6389462202787399,
|
|
"learning_rate": 9.316216432703916e-07,
|
|
"loss": 0.0088,
|
|
"reward": -0.0043678306974470615,
|
|
"reward_after_mean": -0.0043678306974470615,
|
|
"reward_after_std": 0.6600480955094099,
|
|
"reward_before_mean": 0.31353550031781197,
|
|
"reward_before_std": 0.6461421558633447,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3179033286869526,
|
|
"reward_change_min": -0.5801984183490276,
|
|
"reward_change_std": 0.22101250104606152,
|
|
"reward_std": 0.6600481104105711,
|
|
"rewards/accuracy_reward": 0.29166667722165585,
|
|
"rewards/cosine_scaled_reward": 0.0218688128516078,
|
|
"step": 53
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2297.9583778381348,
|
|
"epoch": 0.061714285714285715,
|
|
"grad_norm": 0.09625467658042908,
|
|
"kl": 4.2378902435302734e-05,
|
|
"lambda_div_used": 0.6194662973284721,
|
|
"learning_rate": 9.274017555754407e-07,
|
|
"loss": 0.0224,
|
|
"reward": 0.0802488662302494,
|
|
"reward_after_mean": 0.0802488662302494,
|
|
"reward_after_std": 0.6323203574866056,
|
|
"reward_before_mean": 0.4949483387172222,
|
|
"reward_before_std": 0.5504263024777174,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41469944082200527,
|
|
"reward_change_min": -0.6165112145245075,
|
|
"reward_change_std": 0.2416615542024374,
|
|
"reward_std": 0.632320374250412,
|
|
"rewards/accuracy_reward": 0.3541666753590107,
|
|
"rewards/cosine_scaled_reward": 0.1407816605642438,
|
|
"step": 54
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2896.416702270508,
|
|
"epoch": 0.06285714285714286,
|
|
"grad_norm": 0.05907125398516655,
|
|
"kl": 3.5256147384643555e-05,
|
|
"lambda_div_used": 0.602072462439537,
|
|
"learning_rate": 9.230669076497687e-07,
|
|
"loss": 0.0467,
|
|
"reward": -0.13429339788854122,
|
|
"reward_after_mean": -0.13429339788854122,
|
|
"reward_after_std": 0.57946902140975,
|
|
"reward_before_mean": 0.22700263070873916,
|
|
"reward_before_std": 0.4767341245897114,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36129603534936905,
|
|
"reward_change_min": -0.5782174952328205,
|
|
"reward_change_std": 0.21661545429378748,
|
|
"reward_std": 0.5794690307229757,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": -0.02299738209694624,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3075.187545776367,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.06310205906629562,
|
|
"kl": 4.1544437408447266e-05,
|
|
"lambda_div_used": 0.5549901723861694,
|
|
"learning_rate": 9.186184199300463e-07,
|
|
"loss": -0.0108,
|
|
"reward": -0.4740895018912852,
|
|
"reward_after_mean": -0.4740895018912852,
|
|
"reward_after_std": 0.32212498411536217,
|
|
"reward_before_mean": -0.2245362438261509,
|
|
"reward_before_std": 0.24502124171704054,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24955323711037636,
|
|
"reward_change_min": -0.3650141842663288,
|
|
"reward_change_std": 0.13405024446547031,
|
|
"reward_std": 0.3221249897032976,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.24536957405507565,
|
|
"step": 56
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3013.8541870117188,
|
|
"epoch": 0.06514285714285714,
|
|
"grad_norm": 0.052018001675605774,
|
|
"kl": 3.096461296081543e-05,
|
|
"lambda_div_used": 0.5674270242452621,
|
|
"learning_rate": 9.140576474687263e-07,
|
|
"loss": 0.0192,
|
|
"reward": -0.45193217881023884,
|
|
"reward_after_mean": -0.45193217881023884,
|
|
"reward_after_std": 0.37017360515892506,
|
|
"reward_before_mean": -0.21255622059106827,
|
|
"reward_before_std": 0.3090948835015297,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23937593773007393,
|
|
"reward_change_min": -0.42212119325995445,
|
|
"reward_change_std": 0.14615233521908522,
|
|
"reward_std": 0.3701736144721508,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.2542228833772242,
|
|
"step": 57
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2257.8750534057617,
|
|
"epoch": 0.06628571428571428,
|
|
"grad_norm": 0.09038142114877701,
|
|
"kl": 3.3486634492874146e-05,
|
|
"lambda_div_used": 0.6153116375207901,
|
|
"learning_rate": 9.093859795212817e-07,
|
|
"loss": 0.049,
|
|
"reward": -0.07369695231318474,
|
|
"reward_after_mean": -0.07369695231318474,
|
|
"reward_after_std": 0.5549236796796322,
|
|
"reward_before_mean": 0.2532845102250576,
|
|
"reward_before_std": 0.5286985114216805,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3269814867526293,
|
|
"reward_change_min": -0.537376407533884,
|
|
"reward_change_std": 0.20782853197306395,
|
|
"reward_std": 0.5549236983060837,
|
|
"rewards/accuracy_reward": 0.27083334513008595,
|
|
"rewards/cosine_scaled_reward": -0.017548808827996254,
|
|
"step": 58
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2807.6250381469727,
|
|
"epoch": 0.06742857142857143,
|
|
"grad_norm": 0.06863339245319366,
|
|
"kl": 3.577768802642822e-05,
|
|
"lambda_div_used": 0.6041718497872353,
|
|
"learning_rate": 9.046048391230247e-07,
|
|
"loss": -0.005,
|
|
"reward": -0.25026911310851574,
|
|
"reward_after_mean": -0.25026911310851574,
|
|
"reward_after_std": 0.5205795764923096,
|
|
"reward_before_mean": 0.026796480640769005,
|
|
"reward_before_std": 0.4837551396340132,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27706561237573624,
|
|
"reward_change_min": -0.5239802338182926,
|
|
"reward_change_std": 0.18853904772549868,
|
|
"reward_std": 0.5205795876681805,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.11903684702701867,
|
|
"step": 59
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2797.5833740234375,
|
|
"epoch": 0.06857142857142857,
|
|
"grad_norm": 0.07598927617073059,
|
|
"kl": 3.738701343536377e-05,
|
|
"lambda_div_used": 0.6269034072756767,
|
|
"learning_rate": 8.997156826556369e-07,
|
|
"loss": 0.0313,
|
|
"reward": -0.2369612492620945,
|
|
"reward_after_mean": -0.2369612492620945,
|
|
"reward_after_std": 0.6222166065126657,
|
|
"reward_before_mean": 0.005125788040459156,
|
|
"reward_before_std": 0.5912308068946004,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24208704754710197,
|
|
"reward_change_min": -0.48471353203058243,
|
|
"reward_change_std": 0.17278119549155235,
|
|
"reward_std": 0.6222166288644075,
|
|
"rewards/accuracy_reward": 0.14583333767950535,
|
|
"rewards/cosine_scaled_reward": -0.14070754405111074,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2917.812515258789,
|
|
"epoch": 0.06971428571428571,
|
|
"grad_norm": 0.06223488971590996,
|
|
"kl": 3.0603259801864624e-05,
|
|
"lambda_div_used": 0.6177623867988586,
|
|
"learning_rate": 8.9471999940354e-07,
|
|
"loss": 0.0259,
|
|
"reward": -0.1357121616601944,
|
|
"reward_after_mean": -0.1357121616601944,
|
|
"reward_after_std": 0.5694319568574429,
|
|
"reward_before_mean": 0.16569078899919987,
|
|
"reward_before_std": 0.5441586868837476,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3014029450714588,
|
|
"reward_change_min": -0.5209132842719555,
|
|
"reward_change_std": 0.20053375512361526,
|
|
"reward_std": 0.5694319736212492,
|
|
"rewards/accuracy_reward": 0.20833333767950535,
|
|
"rewards/cosine_scaled_reward": -0.042642548214644194,
|
|
"step": 61
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2507.8958854675293,
|
|
"epoch": 0.07085714285714285,
|
|
"grad_norm": 0.0747612789273262,
|
|
"kl": 2.6777386665344238e-05,
|
|
"lambda_div_used": 0.61732517182827,
|
|
"learning_rate": 8.896193111002475e-07,
|
|
"loss": 0.0366,
|
|
"reward": 0.010478481650352478,
|
|
"reward_after_mean": 0.010478481650352478,
|
|
"reward_after_std": 0.6334415413439274,
|
|
"reward_before_mean": 0.4018659461289644,
|
|
"reward_before_std": 0.5490978918969631,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.39138743840157986,
|
|
"reward_change_min": -0.6315983533859253,
|
|
"reward_change_std": 0.24404066987335682,
|
|
"reward_std": 0.6334415581077337,
|
|
"rewards/accuracy_reward": 0.33333333767950535,
|
|
"rewards/cosine_scaled_reward": 0.06853258889168501,
|
|
"step": 62
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1902.5000228881836,
|
|
"epoch": 0.072,
|
|
"grad_norm": 0.07997458428144455,
|
|
"kl": 2.5197863578796387e-05,
|
|
"lambda_div_used": 0.6046535074710846,
|
|
"learning_rate": 8.844151714648274e-07,
|
|
"loss": 0.059,
|
|
"reward": 0.0217137411236763,
|
|
"reward_after_mean": 0.0217137411236763,
|
|
"reward_after_std": 0.5547807831317186,
|
|
"reward_before_mean": 0.42593196779489517,
|
|
"reward_before_std": 0.49076249497011304,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4042182229459286,
|
|
"reward_change_min": -0.6314646527171135,
|
|
"reward_change_std": 0.25159123074263334,
|
|
"reward_std": 0.5547807849943638,
|
|
"rewards/accuracy_reward": 0.3541666716337204,
|
|
"rewards/cosine_scaled_reward": 0.07176528126001358,
|
|
"step": 63
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2790.979217529297,
|
|
"epoch": 0.07314285714285715,
|
|
"grad_norm": 0.07697053253650665,
|
|
"kl": 4.419684410095215e-05,
|
|
"lambda_div_used": 0.6203151121735573,
|
|
"learning_rate": 8.791091657286267e-07,
|
|
"loss": 0.061,
|
|
"reward": -0.1100537832826376,
|
|
"reward_after_mean": -0.1100537832826376,
|
|
"reward_after_std": 0.5783387050032616,
|
|
"reward_before_mean": 0.19534806534647942,
|
|
"reward_before_std": 0.5548522733151913,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3054018337279558,
|
|
"reward_change_min": -0.5362299680709839,
|
|
"reward_change_std": 0.2037976048886776,
|
|
"reward_std": 0.5783387236297131,
|
|
"rewards/accuracy_reward": 0.20833333767950535,
|
|
"rewards/cosine_scaled_reward": -0.012985273322556168,
|
|
"step": 64
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2605.2708587646484,
|
|
"epoch": 0.07428571428571429,
|
|
"grad_norm": 0.0772913321852684,
|
|
"kl": 3.089010715484619e-05,
|
|
"lambda_div_used": 0.5563548430800438,
|
|
"learning_rate": 8.737029101523929e-07,
|
|
"loss": -0.0861,
|
|
"reward": -0.33125742711126804,
|
|
"reward_after_mean": -0.33125742711126804,
|
|
"reward_after_std": 0.3626741226762533,
|
|
"reward_before_mean": 0.005530592054128647,
|
|
"reward_before_std": 0.2566492212936282,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.33678802102804184,
|
|
"reward_change_min": -0.5241431556642056,
|
|
"reward_change_std": 0.18918895348906517,
|
|
"reward_std": 0.36267412453889847,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.161136067006737,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2088.562511444092,
|
|
"epoch": 0.07542857142857143,
|
|
"grad_norm": 0.10951534658670425,
|
|
"kl": 2.690870314836502e-05,
|
|
"lambda_div_used": 0.5969364494085312,
|
|
"learning_rate": 8.681980515339463e-07,
|
|
"loss": 0.0287,
|
|
"reward": 0.012385480105876923,
|
|
"reward_after_mean": 0.012385480105876923,
|
|
"reward_after_std": 0.5128760654479265,
|
|
"reward_before_mean": 0.44090043008327484,
|
|
"reward_before_std": 0.4437296399846673,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4285149369388819,
|
|
"reward_change_min": -0.660201895982027,
|
|
"reward_change_std": 0.25701938942074776,
|
|
"reward_std": 0.512876084074378,
|
|
"rewards/accuracy_reward": 0.3750000111758709,
|
|
"rewards/cosine_scaled_reward": 0.0659004095941782,
|
|
"step": 66
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3402.8958740234375,
|
|
"epoch": 0.07657142857142857,
|
|
"grad_norm": 0.047973256558179855,
|
|
"kl": 3.771483898162842e-05,
|
|
"lambda_div_used": 0.5712955147027969,
|
|
"learning_rate": 8.625962667065487e-07,
|
|
"loss": -0.0027,
|
|
"reward": -0.4502652711234987,
|
|
"reward_after_mean": -0.4502652711234987,
|
|
"reward_after_std": 0.39950996078550816,
|
|
"reward_before_mean": -0.22067994717508554,
|
|
"reward_before_std": 0.3211147477850318,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.229585325345397,
|
|
"reward_change_min": -0.3470626436173916,
|
|
"reward_change_std": 0.12443333957344294,
|
|
"reward_std": 0.39950997941195965,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.26234661415219307,
|
|
"step": 67
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1642.9166984558105,
|
|
"epoch": 0.07771428571428571,
|
|
"grad_norm": 0.11412619799375534,
|
|
"kl": 2.551823854446411e-05,
|
|
"lambda_div_used": 0.6189040914177895,
|
|
"learning_rate": 8.568992620281243e-07,
|
|
"loss": -0.0912,
|
|
"reward": -0.22925877509987913,
|
|
"reward_after_mean": -0.22925877509987913,
|
|
"reward_after_std": 0.6055262424051762,
|
|
"reward_before_mean": 0.03262739907950163,
|
|
"reward_before_std": 0.5506744375452399,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2618861813098192,
|
|
"reward_change_min": -0.4676270857453346,
|
|
"reward_change_std": 0.16872322466224432,
|
|
"reward_std": 0.6055262610316277,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.09237260185182095,
|
|
"step": 68
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2018.6666946411133,
|
|
"epoch": 0.07885714285714286,
|
|
"grad_norm": 0.11094633489847183,
|
|
"kl": 4.1466206312179565e-05,
|
|
"lambda_div_used": 0.5734386518597603,
|
|
"learning_rate": 8.511087728614862e-07,
|
|
"loss": -0.0618,
|
|
"reward": -0.4316765144467354,
|
|
"reward_after_mean": -0.4316765144467354,
|
|
"reward_after_std": 0.4006939698010683,
|
|
"reward_before_mean": -0.1882182292174548,
|
|
"reward_before_std": 0.331131674349308,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24345828033983707,
|
|
"reward_change_min": -0.3763142116367817,
|
|
"reward_change_std": 0.13501812983304262,
|
|
"reward_std": 0.40069398283958435,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.22988490015268326,
|
|
"step": 69
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2860.8958587646484,
|
|
"epoch": 0.08,
|
|
"grad_norm": 0.06797178089618683,
|
|
"kl": 4.0102750062942505e-05,
|
|
"lambda_div_used": 0.6089852601289749,
|
|
"learning_rate": 8.452265630457282e-07,
|
|
"loss": 0.0416,
|
|
"reward": -0.16451344639062881,
|
|
"reward_after_mean": -0.16451344639062881,
|
|
"reward_after_std": 0.5175492316484451,
|
|
"reward_before_mean": 0.1350960824638605,
|
|
"reward_before_std": 0.5058744940906763,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2996095381677151,
|
|
"reward_change_min": -0.5449608005583286,
|
|
"reward_change_std": 0.2059446070343256,
|
|
"reward_std": 0.5175492409616709,
|
|
"rewards/accuracy_reward": 0.1875000037252903,
|
|
"rewards/cosine_scaled_reward": -0.05240389332175255,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2700.2708587646484,
|
|
"epoch": 0.08114285714285714,
|
|
"grad_norm": 0.07971946895122528,
|
|
"kl": 4.7519803047180176e-05,
|
|
"lambda_div_used": 0.6114047914743423,
|
|
"learning_rate": 8.392544243589427e-07,
|
|
"loss": 0.0248,
|
|
"reward": -0.17357509583234787,
|
|
"reward_after_mean": -0.17357509583234787,
|
|
"reward_after_std": 0.5260436423122883,
|
|
"reward_before_mean": 0.1154884397983551,
|
|
"reward_before_std": 0.5224206217098981,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2890635374933481,
|
|
"reward_change_min": -0.5293963067233562,
|
|
"reward_change_std": 0.20269155222922564,
|
|
"reward_std": 0.5260436479002237,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.051178233698010445,
|
|
"step": 71
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2651.1250610351562,
|
|
"epoch": 0.08228571428571428,
|
|
"grad_norm": 0.1173771321773529,
|
|
"kl": 5.626678466796875e-05,
|
|
"lambda_div_used": 0.5625430718064308,
|
|
"learning_rate": 8.331941759724268e-07,
|
|
"loss": -0.0294,
|
|
"reward": -0.4364648088812828,
|
|
"reward_after_mean": -0.4364648088812828,
|
|
"reward_after_std": 0.3423657324165106,
|
|
"reward_before_mean": -0.17360325902700424,
|
|
"reward_before_std": 0.28492841869592667,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2628615368157625,
|
|
"reward_change_min": -0.44537778943777084,
|
|
"reward_change_std": 0.15708798076957464,
|
|
"reward_std": 0.3423657491803169,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.2152699390426278,
|
|
"step": 72
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3339.9583740234375,
|
|
"epoch": 0.08342857142857144,
|
|
"grad_norm": 0.04790791496634483,
|
|
"kl": 4.526972770690918e-05,
|
|
"lambda_div_used": 0.6265770718455315,
|
|
"learning_rate": 8.270476638965461e-07,
|
|
"loss": 0.0626,
|
|
"reward": -0.12956268154084682,
|
|
"reward_after_mean": -0.12956268154084682,
|
|
"reward_after_std": 0.6143265012651682,
|
|
"reward_before_mean": 0.15906396182253957,
|
|
"reward_before_std": 0.5886987801641226,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.288626654073596,
|
|
"reward_change_min": -0.5101466961205006,
|
|
"reward_change_std": 0.1920691430568695,
|
|
"reward_std": 0.6143265273422003,
|
|
"rewards/accuracy_reward": 0.20833333767950535,
|
|
"rewards/cosine_scaled_reward": -0.049269367940723896,
|
|
"step": 73
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2385.7500381469727,
|
|
"epoch": 0.08457142857142858,
|
|
"grad_norm": 0.06600002944469452,
|
|
"kl": 3.542378544807434e-05,
|
|
"lambda_div_used": 0.6220900639891624,
|
|
"learning_rate": 8.208167604184217e-07,
|
|
"loss": -0.0312,
|
|
"reward": -0.0998319232603535,
|
|
"reward_after_mean": -0.0998319232603535,
|
|
"reward_after_std": 0.5906735248863697,
|
|
"reward_before_mean": 0.20307225361466408,
|
|
"reward_before_std": 0.566557977348566,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30290416814386845,
|
|
"reward_change_min": -0.5291841961443424,
|
|
"reward_change_std": 0.20113296527415514,
|
|
"reward_std": 0.5906735453754663,
|
|
"rewards/accuracy_reward": 0.20833333767950535,
|
|
"rewards/cosine_scaled_reward": -0.005261080645141192,
|
|
"step": 74
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2732.458335876465,
|
|
"epoch": 0.08571428571428572,
|
|
"grad_norm": 0.059586018323898315,
|
|
"kl": 3.4804921597242355e-05,
|
|
"lambda_div_used": 0.5497717335820198,
|
|
"learning_rate": 8.145033635316128e-07,
|
|
"loss": -0.026,
|
|
"reward": -0.16143636964261532,
|
|
"reward_after_mean": -0.16143636964261532,
|
|
"reward_after_std": 0.41724423691630363,
|
|
"reward_before_mean": 0.31992355175316334,
|
|
"reward_before_std": 0.22394374571740627,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4813598971813917,
|
|
"reward_change_min": -0.6314556114375591,
|
|
"reward_change_std": 0.24685372970998287,
|
|
"reward_std": 0.4172442499548197,
|
|
"rewards/accuracy_reward": 0.27083333395421505,
|
|
"rewards/cosine_scaled_reward": 0.0490901879966259,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2649.9791870117188,
|
|
"epoch": 0.08685714285714285,
|
|
"grad_norm": 0.07793173938989639,
|
|
"kl": 3.8176774978637695e-05,
|
|
"lambda_div_used": 0.5533522665500641,
|
|
"learning_rate": 8.081093963579707e-07,
|
|
"loss": 0.0329,
|
|
"reward": -0.4524771338328719,
|
|
"reward_after_mean": -0.4524771338328719,
|
|
"reward_after_std": 0.32186589390039444,
|
|
"reward_before_mean": -0.18054483737796545,
|
|
"reward_before_std": 0.23863591719418764,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2719323057681322,
|
|
"reward_change_min": -0.402019876986742,
|
|
"reward_change_std": 0.14489794615656137,
|
|
"reward_std": 0.32186589762568474,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.20137817226350307,
|
|
"step": 76
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2847.8333587646484,
|
|
"epoch": 0.088,
|
|
"grad_norm": 0.06544006615877151,
|
|
"kl": 4.89354133605957e-05,
|
|
"lambda_div_used": 0.5558431893587112,
|
|
"learning_rate": 8.01636806561836e-07,
|
|
"loss": -0.0184,
|
|
"reward": -0.46523131616413593,
|
|
"reward_after_mean": -0.46523131616413593,
|
|
"reward_after_std": 0.3304946720600128,
|
|
"reward_before_mean": -0.20099198445677757,
|
|
"reward_before_std": 0.2519041560590267,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26423933170735836,
|
|
"reward_change_min": -0.37857379391789436,
|
|
"reward_change_std": 0.140173084102571,
|
|
"reward_std": 0.3304946757853031,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.22182531282305717,
|
|
"step": 77
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3269.8333740234375,
|
|
"epoch": 0.08914285714285715,
|
|
"grad_norm": 0.05014213174581528,
|
|
"kl": 3.695487976074219e-05,
|
|
"lambda_div_used": 0.6150171384215355,
|
|
"learning_rate": 7.950875657567621e-07,
|
|
"loss": -0.0153,
|
|
"reward": -0.23963370453566313,
|
|
"reward_after_mean": -0.23963370453566313,
|
|
"reward_after_std": 0.5818531475961208,
|
|
"reward_before_mean": 0.01525909942574799,
|
|
"reward_before_std": 0.5286716222763062,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2548927925527096,
|
|
"reward_change_min": -0.411530327051878,
|
|
"reward_change_std": 0.1550313262268901,
|
|
"reward_std": 0.5818531513214111,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.10974090336821973,
|
|
"step": 78
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2262.062511444092,
|
|
"epoch": 0.09028571428571429,
|
|
"grad_norm": 0.06913956999778748,
|
|
"kl": 3.7103891372680664e-05,
|
|
"lambda_div_used": 0.5935603529214859,
|
|
"learning_rate": 7.884636689049422e-07,
|
|
"loss": 0.0312,
|
|
"reward": -0.23656679573468864,
|
|
"reward_after_mean": -0.23656679573468864,
|
|
"reward_after_std": 0.5612265523523092,
|
|
"reward_before_mean": 0.08522074297070503,
|
|
"reward_before_std": 0.4243400124832988,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3217875510454178,
|
|
"reward_change_min": -0.4353032112121582,
|
|
"reward_change_std": 0.16354462038725615,
|
|
"reward_std": 0.5612265765666962,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.10227925609797239,
|
|
"step": 79
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3292.3333587646484,
|
|
"epoch": 0.09142857142857143,
|
|
"grad_norm": 0.0538838729262352,
|
|
"kl": 4.382804036140442e-05,
|
|
"lambda_div_used": 0.5688970535993576,
|
|
"learning_rate": 7.817671337095244e-07,
|
|
"loss": 0.0288,
|
|
"reward": -0.3089722655713558,
|
|
"reward_after_mean": -0.3089722655713558,
|
|
"reward_after_std": 0.3733787778764963,
|
|
"reward_before_mean": 0.0013715587556362152,
|
|
"reward_before_std": 0.30914535373449326,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31034383550286293,
|
|
"reward_change_min": -0.46327832341194153,
|
|
"reward_change_std": 0.17456937301903963,
|
|
"reward_std": 0.3733787890523672,
|
|
"rewards/accuracy_reward": 0.12500000558793545,
|
|
"rewards/cosine_scaled_reward": -0.12362844927702099,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3043.270866394043,
|
|
"epoch": 0.09257142857142857,
|
|
"grad_norm": 0.08238786458969116,
|
|
"kl": 5.1021575927734375e-05,
|
|
"lambda_div_used": 0.5923707559704781,
|
|
"learning_rate": 7.75e-07,
|
|
"loss": 0.0152,
|
|
"reward": -0.26676796935498714,
|
|
"reward_after_mean": -0.26676796935498714,
|
|
"reward_after_std": 0.4838532619178295,
|
|
"reward_before_mean": 0.01490369625389576,
|
|
"reward_before_std": 0.4254928780719638,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28167167864739895,
|
|
"reward_change_min": -0.4777062237262726,
|
|
"reward_change_std": 0.17371418420225382,
|
|
"reward_std": 0.4838532619178295,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.11009630188345909,
|
|
"step": 81
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2630.625030517578,
|
|
"epoch": 0.09371428571428571,
|
|
"grad_norm": 0.06656210124492645,
|
|
"kl": 4.544854164123535e-05,
|
|
"lambda_div_used": 0.6220841184258461,
|
|
"learning_rate": 7.681643291108517e-07,
|
|
"loss": 0.0658,
|
|
"reward": -0.2009956305846572,
|
|
"reward_after_mean": -0.2009956305846572,
|
|
"reward_after_std": 0.6119217481464148,
|
|
"reward_before_mean": 0.05983926076442003,
|
|
"reward_before_std": 0.5658847019076347,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2608348857611418,
|
|
"reward_change_min": -0.45365019887685776,
|
|
"reward_change_std": 0.16699018515646458,
|
|
"reward_std": 0.6119217481464148,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.08599409856833518,
|
|
"step": 82
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2535.208381652832,
|
|
"epoch": 0.09485714285714286,
|
|
"grad_norm": 0.09338229149580002,
|
|
"kl": 5.167722702026367e-05,
|
|
"lambda_div_used": 0.5914888307452202,
|
|
"learning_rate": 7.612622032536507e-07,
|
|
"loss": 0.0644,
|
|
"reward": -0.14509151689708233,
|
|
"reward_after_mean": -0.14509151689708233,
|
|
"reward_after_std": 0.47357647120952606,
|
|
"reward_before_mean": 0.1998078590258956,
|
|
"reward_before_std": 0.4206119291484356,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3448993805795908,
|
|
"reward_change_min": -0.5211166813969612,
|
|
"reward_change_std": 0.20530468598008156,
|
|
"reward_std": 0.4735764730721712,
|
|
"rewards/accuracy_reward": 0.18750000558793545,
|
|
"rewards/cosine_scaled_reward": 0.012307855300605297,
|
|
"step": 83
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2915.8750076293945,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.07025005668401718,
|
|
"kl": 4.2825937271118164e-05,
|
|
"lambda_div_used": 0.6353590413928032,
|
|
"learning_rate": 7.54295724882796e-07,
|
|
"loss": -0.0466,
|
|
"reward": -0.1503741154447198,
|
|
"reward_after_mean": -0.1503741154447198,
|
|
"reward_after_std": 0.6686306204646826,
|
|
"reward_before_mean": 0.10233869170770049,
|
|
"reward_before_std": 0.6240420090034604,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25271278992295265,
|
|
"reward_change_min": -0.43359045311808586,
|
|
"reward_change_std": 0.1594225950539112,
|
|
"reward_std": 0.6686306446790695,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.10599466459825635,
|
|
"step": 84
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2968.9375610351562,
|
|
"epoch": 0.09714285714285714,
|
|
"grad_norm": 0.05030859634280205,
|
|
"kl": 2.9653310775756836e-05,
|
|
"lambda_div_used": 0.6120947226881981,
|
|
"learning_rate": 7.472670160550848e-07,
|
|
"loss": 0.02,
|
|
"reward": -0.31006562570109963,
|
|
"reward_after_mean": -0.31006562570109963,
|
|
"reward_after_std": 0.5445140562951565,
|
|
"reward_before_mean": -0.0690727960318327,
|
|
"reward_before_std": 0.5210688021034002,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2409928422421217,
|
|
"reward_change_min": -0.46906592324376106,
|
|
"reward_change_std": 0.1683354014530778,
|
|
"reward_std": 0.5445140581578016,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.17323945881798863,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2726.9167098999023,
|
|
"epoch": 0.09828571428571428,
|
|
"grad_norm": 0.08248579502105713,
|
|
"kl": 5.6862831115722656e-05,
|
|
"lambda_div_used": 0.5853299722075462,
|
|
"learning_rate": 7.401782177833147e-07,
|
|
"loss": -0.015,
|
|
"reward": -0.32714659720659256,
|
|
"reward_after_mean": -0.32714659720659256,
|
|
"reward_after_std": 0.4505470525473356,
|
|
"reward_before_mean": -0.05685961013659835,
|
|
"reward_before_std": 0.39618763769976795,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2702869772911072,
|
|
"reward_change_min": -0.49061961844563484,
|
|
"reward_change_std": 0.17353465128690004,
|
|
"reward_std": 0.4505470544099808,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.14019294315949082,
|
|
"step": 86
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2504.604232788086,
|
|
"epoch": 0.09942857142857142,
|
|
"grad_norm": 0.07946456968784332,
|
|
"kl": 5.137920379638672e-05,
|
|
"lambda_div_used": 0.5860441103577614,
|
|
"learning_rate": 7.330314893841101e-07,
|
|
"loss": -0.0658,
|
|
"reward": -0.25398197025060654,
|
|
"reward_after_mean": -0.25398197025060654,
|
|
"reward_after_std": 0.4263457953929901,
|
|
"reward_before_mean": 0.039198137819767,
|
|
"reward_before_std": 0.4029952948912978,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2931801360100508,
|
|
"reward_change_min": -0.46910280734300613,
|
|
"reward_change_std": 0.18813357036560774,
|
|
"reward_std": 0.4263457991182804,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.1066351905465126,
|
|
"step": 87
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2021.3958702087402,
|
|
"epoch": 0.10057142857142858,
|
|
"grad_norm": 0.08914105594158173,
|
|
"kl": 3.445148468017578e-05,
|
|
"lambda_div_used": 0.6275195479393005,
|
|
"learning_rate": 7.258290078201731e-07,
|
|
"loss": 0.0661,
|
|
"reward": -0.03901347843930125,
|
|
"reward_after_mean": -0.03901347843930125,
|
|
"reward_after_std": 0.6304188724607229,
|
|
"reward_before_mean": 0.28059265296906233,
|
|
"reward_before_std": 0.5838784109801054,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3196061383932829,
|
|
"reward_change_min": -0.4850176088511944,
|
|
"reward_change_std": 0.19270309899002314,
|
|
"reward_std": 0.6304188761860132,
|
|
"rewards/accuracy_reward": 0.25000000931322575,
|
|
"rewards/cosine_scaled_reward": 0.030592639930546284,
|
|
"step": 88
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2855.666702270508,
|
|
"epoch": 0.10171428571428572,
|
|
"grad_norm": 0.052225928753614426,
|
|
"kl": 3.810226917266846e-05,
|
|
"lambda_div_used": 0.603381521999836,
|
|
"learning_rate": 7.185729670371604e-07,
|
|
"loss": -0.0748,
|
|
"reward": -0.2931769546121359,
|
|
"reward_after_mean": -0.2931769546121359,
|
|
"reward_after_std": 0.5186727736145258,
|
|
"reward_before_mean": -0.04963091528043151,
|
|
"reward_before_std": 0.4795092437416315,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2435460388660431,
|
|
"reward_change_min": -0.4475689232349396,
|
|
"reward_change_std": 0.1609211042523384,
|
|
"reward_std": 0.5186727829277515,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.17463092133402824,
|
|
"step": 89
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2377.9375228881836,
|
|
"epoch": 0.10285714285714286,
|
|
"grad_norm": 0.1115579754114151,
|
|
"kl": 6.511807441711426e-05,
|
|
"lambda_div_used": 0.5806571692228317,
|
|
"learning_rate": 7.11265577295385e-07,
|
|
"loss": 0.1049,
|
|
"reward": -0.39063363266177475,
|
|
"reward_after_mean": -0.39063363266177475,
|
|
"reward_after_std": 0.42313366010785103,
|
|
"reward_before_mean": -0.14117828011512756,
|
|
"reward_before_std": 0.3631622865796089,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24945535697042942,
|
|
"reward_change_min": -0.38458868488669395,
|
|
"reward_change_std": 0.14130910206586123,
|
|
"reward_std": 0.4231336638331413,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1828449461609125,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2758.250015258789,
|
|
"epoch": 0.104,
|
|
"grad_norm": 0.0690622553229332,
|
|
"kl": 4.595518112182617e-05,
|
|
"lambda_div_used": 0.5834402665495872,
|
|
"learning_rate": 7.039090644965509e-07,
|
|
"loss": 0.0142,
|
|
"reward": -0.1535566644743085,
|
|
"reward_after_mean": -0.1535566644743085,
|
|
"reward_after_std": 0.4708296097815037,
|
|
"reward_before_mean": 0.22540673054754734,
|
|
"reward_before_std": 0.3833985608071089,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3789634220302105,
|
|
"reward_change_min": -0.6174517869949341,
|
|
"reward_change_std": 0.2261828240007162,
|
|
"reward_std": 0.4708296228200197,
|
|
"rewards/accuracy_reward": 0.2291666679084301,
|
|
"rewards/cosine_scaled_reward": -0.003759911749511957,
|
|
"step": 91
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2467.2083892822266,
|
|
"epoch": 0.10514285714285715,
|
|
"grad_norm": 0.07518215477466583,
|
|
"kl": 3.6522746086120605e-05,
|
|
"lambda_div_used": 0.5778974890708923,
|
|
"learning_rate": 6.965056695057204e-07,
|
|
"loss": 0.0134,
|
|
"reward": -0.324931837618351,
|
|
"reward_after_mean": -0.324931837618351,
|
|
"reward_after_std": 0.42975695990025997,
|
|
"reward_before_mean": -0.03748153988271952,
|
|
"reward_before_std": 0.35468481201678514,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2874503042548895,
|
|
"reward_change_min": -0.4382200054824352,
|
|
"reward_change_std": 0.1673103515058756,
|
|
"reward_std": 0.42975698225200176,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.14164821058511734,
|
|
"step": 92
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3584.0,
|
|
"epoch": 0.10628571428571429,
|
|
"grad_norm": 0.05293423309922218,
|
|
"kl": 5.128979682922363e-05,
|
|
"lambda_div_used": 0.5581478402018547,
|
|
"learning_rate": 6.890576474687263e-07,
|
|
"loss": -0.0,
|
|
"reward": -0.4109080731868744,
|
|
"reward_after_mean": -0.4109080731868744,
|
|
"reward_after_std": 0.33292335644364357,
|
|
"reward_before_mean": -0.12180843483656645,
|
|
"reward_before_std": 0.26414576172828674,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28909964114427567,
|
|
"reward_change_min": -0.45860791578888893,
|
|
"reward_change_std": 0.16918409056961536,
|
|
"reward_std": 0.332923362031579,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.16347510018385947,
|
|
"step": 93
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2337.0625076293945,
|
|
"epoch": 0.10742857142857143,
|
|
"grad_norm": 0.08120116591453552,
|
|
"kl": 4.356354475021362e-05,
|
|
"lambda_div_used": 0.5623572915792465,
|
|
"learning_rate": 6.815672671252315e-07,
|
|
"loss": 0.0672,
|
|
"reward": -0.2148991823196411,
|
|
"reward_after_mean": -0.2148991823196411,
|
|
"reward_after_std": 0.37764647975564003,
|
|
"reward_before_mean": 0.17423074319958687,
|
|
"reward_before_std": 0.2815079055726528,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3891299031674862,
|
|
"reward_change_min": -0.5794133953750134,
|
|
"reward_change_std": 0.21886237617582083,
|
|
"reward_std": 0.3776464983820915,
|
|
"rewards/accuracy_reward": 0.2083333358168602,
|
|
"rewards/cosine_scaled_reward": -0.034102603793144226,
|
|
"step": 94
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3268.7708587646484,
|
|
"epoch": 0.10857142857142857,
|
|
"grad_norm": 0.048412173986434937,
|
|
"kl": 3.3482909202575684e-05,
|
|
"lambda_div_used": 0.599761851131916,
|
|
"learning_rate": 6.740368101176495e-07,
|
|
"loss": 0.0138,
|
|
"reward": -0.28226344753056765,
|
|
"reward_after_mean": -0.28226344753056765,
|
|
"reward_after_std": 0.5015552807599306,
|
|
"reward_before_mean": -0.01817359635606408,
|
|
"reward_before_std": 0.4594444427639246,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2640898581594229,
|
|
"reward_change_min": -0.45775213465094566,
|
|
"reward_change_std": 0.170160255394876,
|
|
"reward_std": 0.5015552863478661,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.14317359682172537,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2311.229202270508,
|
|
"epoch": 0.10971428571428571,
|
|
"grad_norm": 0.08286502212285995,
|
|
"kl": 3.9830803871154785e-05,
|
|
"lambda_div_used": 0.6152824014425278,
|
|
"learning_rate": 6.664685702961344e-07,
|
|
"loss": 0.0394,
|
|
"reward": -0.1798182651400566,
|
|
"reward_after_mean": -0.1798182651400566,
|
|
"reward_after_std": 0.5684184953570366,
|
|
"reward_before_mean": 0.09630595671478659,
|
|
"reward_before_std": 0.5319525888189673,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2761242166161537,
|
|
"reward_change_min": -0.4500977620482445,
|
|
"reward_change_std": 0.17272682767361403,
|
|
"reward_std": 0.5684185232967138,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.11202737595885992,
|
|
"step": 96
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3088.104202270508,
|
|
"epoch": 0.11085714285714286,
|
|
"grad_norm": 0.05773423984646797,
|
|
"kl": 4.525482654571533e-05,
|
|
"lambda_div_used": 0.5830484703183174,
|
|
"learning_rate": 6.588648530198504e-07,
|
|
"loss": 0.012,
|
|
"reward": -0.24631217122077942,
|
|
"reward_after_mean": -0.24631217122077942,
|
|
"reward_after_std": 0.4177880808711052,
|
|
"reward_before_mean": 0.052551381289958954,
|
|
"reward_before_std": 0.3777043428272009,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2988635450601578,
|
|
"reward_change_min": -0.4822593107819557,
|
|
"reward_change_std": 0.18344944156706333,
|
|
"reward_std": 0.4177880957722664,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.11411529779434204,
|
|
"step": 97
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3099.6666870117188,
|
|
"epoch": 0.112,
|
|
"grad_norm": 0.05776653811335564,
|
|
"kl": 4.242360591888428e-05,
|
|
"lambda_div_used": 0.5548205152153969,
|
|
"learning_rate": 6.512279744547392e-07,
|
|
"loss": 0.0853,
|
|
"reward": -0.30739316577091813,
|
|
"reward_after_mean": -0.30739316577091813,
|
|
"reward_after_std": 0.3838734310120344,
|
|
"reward_before_mean": 0.044590696692466736,
|
|
"reward_before_std": 0.2453754236921668,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3519838694483042,
|
|
"reward_change_min": -0.49566248431801796,
|
|
"reward_change_std": 0.18073826655745506,
|
|
"reward_std": 0.38387343287467957,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.10124263912439346,
|
|
"step": 98
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2814.0416679382324,
|
|
"epoch": 0.11314285714285714,
|
|
"grad_norm": 0.10099756717681885,
|
|
"kl": 3.9711594581604004e-05,
|
|
"lambda_div_used": 0.589689776301384,
|
|
"learning_rate": 6.435602608679916e-07,
|
|
"loss": -0.0116,
|
|
"reward": -0.24916072934865952,
|
|
"reward_after_mean": -0.24916072934865952,
|
|
"reward_after_std": 0.5363867282867432,
|
|
"reward_before_mean": 0.07555947656510398,
|
|
"reward_before_std": 0.4080575914122164,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32472023367881775,
|
|
"reward_change_min": -0.46621018648147583,
|
|
"reward_change_std": 0.16967851482331753,
|
|
"reward_std": 0.5363867320120335,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.11194050963968039,
|
|
"step": 99
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2585.208366394043,
|
|
"epoch": 0.11428571428571428,
|
|
"grad_norm": 0.08013861626386642,
|
|
"kl": 3.557652235031128e-05,
|
|
"lambda_div_used": 0.6440299674868584,
|
|
"learning_rate": 6.358640479194451e-07,
|
|
"loss": 0.0233,
|
|
"reward": -0.017059004865586758,
|
|
"reward_after_mean": -0.017059004865586758,
|
|
"reward_after_std": 0.6788044832646847,
|
|
"reward_before_mean": 0.27609538938850164,
|
|
"reward_before_std": 0.672226045280695,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29315441474318504,
|
|
"reward_change_min": -0.560445386916399,
|
|
"reward_change_std": 0.2121435971930623,
|
|
"reward_std": 0.6788045018911362,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": 0.02609538659453392,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2698.687515258789,
|
|
"epoch": 0.11542857142857142,
|
|
"grad_norm": 0.08196399360895157,
|
|
"kl": 4.9740076065063477e-05,
|
|
"lambda_div_used": 0.5603137612342834,
|
|
"learning_rate": 6.281416799501187e-07,
|
|
"loss": -0.0552,
|
|
"reward": -0.2113272361457348,
|
|
"reward_after_mean": -0.2113272361457348,
|
|
"reward_after_std": 0.3802106771618128,
|
|
"reward_before_mean": 0.176470085978508,
|
|
"reward_before_std": 0.27200379874557257,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3877973351627588,
|
|
"reward_change_min": -0.5599614717066288,
|
|
"reward_change_std": 0.21361587569117546,
|
|
"reward_std": 0.38021068647503853,
|
|
"rewards/accuracy_reward": 0.2083333358168602,
|
|
"rewards/cosine_scaled_reward": -0.03186323493719101,
|
|
"step": 101
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2008.1875457763672,
|
|
"epoch": 0.11657142857142858,
|
|
"grad_norm": 0.09427309036254883,
|
|
"kl": 3.542006015777588e-05,
|
|
"lambda_div_used": 0.5826256647706032,
|
|
"learning_rate": 6.203955092681039e-07,
|
|
"loss": -0.0045,
|
|
"reward": -0.2004177300259471,
|
|
"reward_after_mean": -0.2004177300259471,
|
|
"reward_after_std": 0.498240664601326,
|
|
"reward_before_mean": 0.16466995794326067,
|
|
"reward_before_std": 0.37351171765476465,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36508770659565926,
|
|
"reward_change_min": -0.5195483043789864,
|
|
"reward_change_std": 0.19531819131225348,
|
|
"reward_std": 0.4982406720519066,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.001996707171201706,
|
|
"step": 102
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2895.250020980835,
|
|
"epoch": 0.11771428571428572,
|
|
"grad_norm": 0.06802447140216827,
|
|
"kl": 4.501640796661377e-05,
|
|
"lambda_div_used": 0.5506640374660492,
|
|
"learning_rate": 6.126278954320294e-07,
|
|
"loss": 0.0391,
|
|
"reward": -0.3214366286993027,
|
|
"reward_after_mean": -0.3214366286993027,
|
|
"reward_after_std": 0.3532369527965784,
|
|
"reward_before_mean": 0.050119780004024506,
|
|
"reward_before_std": 0.22584082814864814,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3715564049780369,
|
|
"reward_change_min": -0.520975548774004,
|
|
"reward_change_std": 0.19540261384099722,
|
|
"reward_std": 0.35323695838451385,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.09571357164531946,
|
|
"step": 103
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2521.3333435058594,
|
|
"epoch": 0.11885714285714286,
|
|
"grad_norm": 0.096355140209198,
|
|
"kl": 4.947185516357422e-05,
|
|
"lambda_div_used": 0.5742901861667633,
|
|
"learning_rate": 6.048412045323164e-07,
|
|
"loss": 0.0001,
|
|
"reward": -0.32155255414545536,
|
|
"reward_after_mean": -0.32155255414545536,
|
|
"reward_after_std": 0.39902236871421337,
|
|
"reward_before_mean": -0.03546402044594288,
|
|
"reward_before_std": 0.3418318200856447,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2860885336995125,
|
|
"reward_change_min": -0.4318212755024433,
|
|
"reward_change_std": 0.16667354479432106,
|
|
"reward_std": 0.39902237243950367,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.13963068462908268,
|
|
"step": 104
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2526.000030517578,
|
|
"epoch": 0.12,
|
|
"grad_norm": 0.07842111587524414,
|
|
"kl": 4.8801302909851074e-05,
|
|
"lambda_div_used": 0.6022165417671204,
|
|
"learning_rate": 5.97037808470444e-07,
|
|
"loss": 0.0569,
|
|
"reward": -0.12751667387783527,
|
|
"reward_after_mean": -0.12751667387783527,
|
|
"reward_after_std": 0.582199590280652,
|
|
"reward_before_mean": 0.24656505044549704,
|
|
"reward_before_std": 0.4756303254980594,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.37408171594142914,
|
|
"reward_change_min": -0.5777854695916176,
|
|
"reward_change_std": 0.22329542227089405,
|
|
"reward_std": 0.5821996051818132,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": -0.0034349607303738594,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2396.291702270508,
|
|
"epoch": 0.12114285714285715,
|
|
"grad_norm": 0.07387516647577286,
|
|
"kl": 2.6203226298093796e-05,
|
|
"lambda_div_used": 0.6128493994474411,
|
|
"learning_rate": 5.892200842364462e-07,
|
|
"loss": -0.0293,
|
|
"reward": 0.159407502040267,
|
|
"reward_after_mean": 0.159407502040267,
|
|
"reward_after_std": 0.5759131647646427,
|
|
"reward_before_mean": 0.6274341251701117,
|
|
"reward_before_std": 0.5208892030641437,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.46802657656371593,
|
|
"reward_change_min": -0.7112628631293774,
|
|
"reward_change_std": 0.2871347274631262,
|
|
"reward_std": 0.5759131908416748,
|
|
"rewards/accuracy_reward": 0.4583333469927311,
|
|
"rewards/cosine_scaled_reward": 0.16910075349733233,
|
|
"step": 106
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2806.875045776367,
|
|
"epoch": 0.12228571428571429,
|
|
"grad_norm": 0.08797896653413773,
|
|
"kl": 4.5686960220336914e-05,
|
|
"lambda_div_used": 0.5551519840955734,
|
|
"learning_rate": 5.813904131848564e-07,
|
|
"loss": -0.0206,
|
|
"reward": -0.20906101167201996,
|
|
"reward_after_mean": -0.20906101167201996,
|
|
"reward_after_std": 0.35940215550363064,
|
|
"reward_before_mean": 0.19015613943338394,
|
|
"reward_before_std": 0.24773720651865005,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.39921717159450054,
|
|
"reward_change_min": -0.5645803362131119,
|
|
"reward_change_std": 0.21728475205600262,
|
|
"reward_std": 0.35940215922892094,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.03901051543653011,
|
|
"step": 107
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2676.3125381469727,
|
|
"epoch": 0.12342857142857143,
|
|
"grad_norm": 0.08117339015007019,
|
|
"kl": 4.683062434196472e-05,
|
|
"lambda_div_used": 0.5782058760523796,
|
|
"learning_rate": 5.735511803093248e-07,
|
|
"loss": 0.0222,
|
|
"reward": -0.33698799335979857,
|
|
"reward_after_mean": -0.33698799335979857,
|
|
"reward_after_std": 0.4150846730917692,
|
|
"reward_before_mean": -0.06898907572031021,
|
|
"reward_before_std": 0.3580573983490467,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2679989282041788,
|
|
"reward_change_min": -0.39731478318572044,
|
|
"reward_change_std": 0.15669311955571175,
|
|
"reward_std": 0.41508468240499496,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.17315573990345,
|
|
"step": 108
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3091.4166870117188,
|
|
"epoch": 0.12457142857142857,
|
|
"grad_norm": 0.06771776080131531,
|
|
"kl": 4.081428050994873e-05,
|
|
"lambda_div_used": 0.5580320879817009,
|
|
"learning_rate": 5.657047735161255e-07,
|
|
"loss": -0.0162,
|
|
"reward": -0.30369802191853523,
|
|
"reward_after_mean": -0.30369802191853523,
|
|
"reward_after_std": 0.3783009462058544,
|
|
"reward_before_mean": 0.060508192516863346,
|
|
"reward_before_std": 0.2597539462149143,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36420623771846294,
|
|
"reward_change_min": -0.5151935815811157,
|
|
"reward_change_std": 0.19158014561980963,
|
|
"reward_std": 0.37830095551908016,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.08532514423131943,
|
|
"step": 109
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2727.875045776367,
|
|
"epoch": 0.12571428571428572,
|
|
"grad_norm": 0.07848876714706421,
|
|
"kl": 3.8996338844299316e-05,
|
|
"lambda_div_used": 0.5898331105709076,
|
|
"learning_rate": 5.578535828967777e-07,
|
|
"loss": 0.0059,
|
|
"reward": -0.27789773186668754,
|
|
"reward_after_mean": -0.27789773186668754,
|
|
"reward_after_std": 0.48964778520166874,
|
|
"reward_before_mean": 0.006700331810861826,
|
|
"reward_before_std": 0.40999170672148466,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28459805622696877,
|
|
"reward_change_min": -0.4541938379406929,
|
|
"reward_change_std": 0.16897330060601234,
|
|
"reward_std": 0.48964780382812023,
|
|
"rewards/accuracy_reward": 0.16666667349636555,
|
|
"rewards/cosine_scaled_reward": -0.15996634401381016,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3152.3958587646484,
|
|
"epoch": 0.12685714285714286,
|
|
"grad_norm": 0.06026454642415047,
|
|
"kl": 5.206465721130371e-05,
|
|
"lambda_div_used": 0.6035187616944313,
|
|
"learning_rate": 5.5e-07,
|
|
"loss": 0.0004,
|
|
"reward": -0.22968922927975655,
|
|
"reward_after_mean": -0.22968922927975655,
|
|
"reward_after_std": 0.5089491438120604,
|
|
"reward_before_mean": 0.050578076392412186,
|
|
"reward_before_std": 0.47721442952752113,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28026728704571724,
|
|
"reward_change_min": -0.4855511710047722,
|
|
"reward_change_std": 0.1824399819597602,
|
|
"reward_std": 0.5089491568505764,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.09525526221841574,
|
|
"step": 111
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3253.937530517578,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.054045893251895905,
|
|
"kl": 4.832446575164795e-05,
|
|
"lambda_div_used": 0.6144762486219406,
|
|
"learning_rate": 5.421464171032224e-07,
|
|
"loss": 0.0014,
|
|
"reward": -0.07707784557715058,
|
|
"reward_after_mean": -0.07707784557715058,
|
|
"reward_after_std": 0.5871799997985363,
|
|
"reward_before_mean": 0.2822006791830063,
|
|
"reward_before_std": 0.5333117246627808,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35927851870656013,
|
|
"reward_change_min": -0.6481843888759613,
|
|
"reward_change_std": 0.23641589283943176,
|
|
"reward_std": 0.5871800072491169,
|
|
"rewards/accuracy_reward": 0.2708333358168602,
|
|
"rewards/cosine_scaled_reward": 0.01136732567101717,
|
|
"step": 112
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2541.812545776367,
|
|
"epoch": 0.12914285714285714,
|
|
"grad_norm": 0.09499017894268036,
|
|
"kl": 4.720315337181091e-05,
|
|
"lambda_div_used": 0.5564066097140312,
|
|
"learning_rate": 5.342952264838747e-07,
|
|
"loss": 0.0471,
|
|
"reward": -0.3626333475112915,
|
|
"reward_after_mean": -0.3626333475112915,
|
|
"reward_after_std": 0.3075137473642826,
|
|
"reward_before_mean": -0.05370184499770403,
|
|
"reward_before_std": 0.25310691073536873,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30893150344491005,
|
|
"reward_change_min": -0.4691920429468155,
|
|
"reward_change_std": 0.1770205283537507,
|
|
"reward_std": 0.3075137585401535,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.13703518453985453,
|
|
"step": 113
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2547.5416984558105,
|
|
"epoch": 0.13028571428571428,
|
|
"grad_norm": 0.06982935965061188,
|
|
"kl": 2.650544047355652e-05,
|
|
"lambda_div_used": 0.5948992818593979,
|
|
"learning_rate": 5.264488196906752e-07,
|
|
"loss": 0.0506,
|
|
"reward": -0.33033538423478603,
|
|
"reward_after_mean": -0.33033538423478603,
|
|
"reward_after_std": 0.49076898768544197,
|
|
"reward_before_mean": -0.08067071554251015,
|
|
"reward_before_std": 0.4387869122438133,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2496646661311388,
|
|
"reward_change_min": -0.38466524705290794,
|
|
"reward_change_std": 0.15147447120398283,
|
|
"reward_std": 0.4907689895480871,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.18483738787472248,
|
|
"step": 114
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2926.3125228881836,
|
|
"epoch": 0.13142857142857142,
|
|
"grad_norm": 0.10055308789014816,
|
|
"kl": 4.2185187339782715e-05,
|
|
"lambda_div_used": 0.5866656303405762,
|
|
"learning_rate": 5.186095868151436e-07,
|
|
"loss": 0.0219,
|
|
"reward": -0.16912975907325745,
|
|
"reward_after_mean": -0.16912975907325745,
|
|
"reward_after_std": 0.4218728318810463,
|
|
"reward_before_mean": 0.17230269685387611,
|
|
"reward_before_std": 0.39482294395565987,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34143244847655296,
|
|
"reward_change_min": -0.5322228632867336,
|
|
"reward_change_std": 0.21214590221643448,
|
|
"reward_std": 0.4218728430569172,
|
|
"rewards/accuracy_reward": 0.2708333432674408,
|
|
"rewards/cosine_scaled_reward": -0.09853065386414528,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3185.895835876465,
|
|
"epoch": 0.13257142857142856,
|
|
"grad_norm": 0.06827546656131744,
|
|
"kl": 4.4345855712890625e-05,
|
|
"lambda_div_used": 0.579861506819725,
|
|
"learning_rate": 5.107799157635538e-07,
|
|
"loss": -0.017,
|
|
"reward": -0.36347829084843397,
|
|
"reward_after_mean": -0.36347829084843397,
|
|
"reward_after_std": 0.425207169726491,
|
|
"reward_before_mean": -0.10740169882774353,
|
|
"reward_before_std": 0.36591998394578695,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25607660599052906,
|
|
"reward_change_min": -0.4011174105107784,
|
|
"reward_change_std": 0.15214112866669893,
|
|
"reward_std": 0.4252071734517813,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.19073502300307155,
|
|
"step": 116
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3265.666702270508,
|
|
"epoch": 0.1337142857142857,
|
|
"grad_norm": 0.058839015662670135,
|
|
"kl": 5.224347114562988e-05,
|
|
"lambda_div_used": 0.5589891448616982,
|
|
"learning_rate": 5.02962191529556e-07,
|
|
"loss": 0.0144,
|
|
"reward": -0.47565145045518875,
|
|
"reward_after_mean": -0.47565145045518875,
|
|
"reward_after_std": 0.34672038443386555,
|
|
"reward_before_mean": -0.2341720014810562,
|
|
"reward_before_std": 0.2644944768399,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2414794433861971,
|
|
"reward_change_min": -0.3730311393737793,
|
|
"reward_change_std": 0.13014927878975868,
|
|
"reward_std": 0.3467204011976719,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.2550053410232067,
|
|
"step": 117
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3087.9375610351562,
|
|
"epoch": 0.13485714285714287,
|
|
"grad_norm": 0.05542779713869095,
|
|
"kl": 3.71783971786499e-05,
|
|
"lambda_div_used": 0.6410617902874947,
|
|
"learning_rate": 4.951587954676837e-07,
|
|
"loss": 0.0602,
|
|
"reward": 0.02640039217658341,
|
|
"reward_after_mean": 0.02640039217658341,
|
|
"reward_after_std": 0.6670121420174837,
|
|
"reward_before_mean": 0.34903212962672114,
|
|
"reward_before_std": 0.6545964349061251,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.322631748393178,
|
|
"reward_change_min": -0.5322613082826138,
|
|
"reward_change_std": 0.21465477347373962,
|
|
"reward_std": 0.66701215878129,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": 0.07819879939779639,
|
|
"step": 118
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2036.8542251586914,
|
|
"epoch": 0.136,
|
|
"grad_norm": 0.09512177854776382,
|
|
"kl": 4.920363426208496e-05,
|
|
"lambda_div_used": 0.5765073597431183,
|
|
"learning_rate": 4.873721045679706e-07,
|
|
"loss": 0.0968,
|
|
"reward": -0.1327025555074215,
|
|
"reward_after_mean": -0.1327025555074215,
|
|
"reward_after_std": 0.4383635278791189,
|
|
"reward_before_mean": 0.2736157886683941,
|
|
"reward_before_std": 0.34712607227265835,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4063183292746544,
|
|
"reward_change_min": -0.5770336836576462,
|
|
"reward_change_std": 0.22668993193656206,
|
|
"reward_std": 0.43836353346705437,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": 0.0444490984082222,
|
|
"step": 119
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2693.6458854675293,
|
|
"epoch": 0.13714285714285715,
|
|
"grad_norm": 0.0801864042878151,
|
|
"kl": 5.1856040954589844e-05,
|
|
"lambda_div_used": 0.5954036563634872,
|
|
"learning_rate": 4.79604490731896e-07,
|
|
"loss": -0.0416,
|
|
"reward": -0.10674675926566124,
|
|
"reward_after_mean": -0.10674675926566124,
|
|
"reward_after_std": 0.5345237273722887,
|
|
"reward_before_mean": 0.26713848020881414,
|
|
"reward_before_std": 0.44027570402249694,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.37388524040579796,
|
|
"reward_change_min": -0.5998699106276035,
|
|
"reward_change_std": 0.22230207175016403,
|
|
"reward_std": 0.5345237515866756,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": 0.017138468101620674,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1960.8958587646484,
|
|
"epoch": 0.1382857142857143,
|
|
"grad_norm": 0.09297072887420654,
|
|
"kl": 3.6925775930285454e-05,
|
|
"lambda_div_used": 0.5758452340960503,
|
|
"learning_rate": 4.7185832004988133e-07,
|
|
"loss": -0.0275,
|
|
"reward": -0.15540813654661179,
|
|
"reward_after_mean": -0.15540813654661179,
|
|
"reward_after_std": 0.3896722886711359,
|
|
"reward_before_mean": 0.20818629674613476,
|
|
"reward_before_std": 0.34600438084453344,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3635944165289402,
|
|
"reward_change_min": -0.5350156500935555,
|
|
"reward_change_std": 0.21344942972064018,
|
|
"reward_std": 0.38967230543494225,
|
|
"rewards/accuracy_reward": 0.1875000074505806,
|
|
"rewards/cosine_scaled_reward": 0.020686281844973564,
|
|
"step": 121
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2996.7917289733887,
|
|
"epoch": 0.13942857142857143,
|
|
"grad_norm": 0.07053450495004654,
|
|
"kl": 5.537271499633789e-05,
|
|
"lambda_div_used": 0.604009747505188,
|
|
"learning_rate": 4.641359520805548e-07,
|
|
"loss": 0.0244,
|
|
"reward": -0.18336665583774447,
|
|
"reward_after_mean": -0.18336665583774447,
|
|
"reward_after_std": 0.5550395585596561,
|
|
"reward_before_mean": 0.11408489104360342,
|
|
"reward_before_std": 0.4755072835832834,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29745154455304146,
|
|
"reward_change_min": -0.4127044528722763,
|
|
"reward_change_std": 0.16168679296970367,
|
|
"reward_std": 0.5550395771861076,
|
|
"rewards/accuracy_reward": 0.16666667349636555,
|
|
"rewards/cosine_scaled_reward": -0.05258178571239114,
|
|
"step": 122
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2965.687530517578,
|
|
"epoch": 0.14057142857142857,
|
|
"grad_norm": 0.061138082295656204,
|
|
"kl": 4.2922794818878174e-05,
|
|
"lambda_div_used": 0.5864768177270889,
|
|
"learning_rate": 4.5643973913200837e-07,
|
|
"loss": 0.0629,
|
|
"reward": -0.24433407932519913,
|
|
"reward_after_mean": -0.24433407932519913,
|
|
"reward_after_std": 0.43522679433226585,
|
|
"reward_before_mean": 0.05285754054784775,
|
|
"reward_before_std": 0.4003843404352665,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2971916198730469,
|
|
"reward_change_min": -0.4781832844018936,
|
|
"reward_change_std": 0.18817945942282677,
|
|
"reward_std": 0.435226796194911,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.09297579154372215,
|
|
"step": 123
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2273.895881652832,
|
|
"epoch": 0.1417142857142857,
|
|
"grad_norm": 0.06792795658111572,
|
|
"kl": 2.843700349330902e-05,
|
|
"lambda_div_used": 0.5992666333913803,
|
|
"learning_rate": 4.4877202554526084e-07,
|
|
"loss": 0.0724,
|
|
"reward": -0.030117375776171684,
|
|
"reward_after_mean": -0.030117375776171684,
|
|
"reward_after_std": 0.6018249355256557,
|
|
"reward_before_mean": 0.40752510842867196,
|
|
"reward_before_std": 0.4566116305068135,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4376424718648195,
|
|
"reward_change_min": -0.6283221691846848,
|
|
"reward_change_std": 0.2420077919960022,
|
|
"reward_std": 0.601824939250946,
|
|
"rewards/accuracy_reward": 0.3541666679084301,
|
|
"rewards/cosine_scaled_reward": 0.05335840582847595,
|
|
"step": 124
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2841.687515258789,
|
|
"epoch": 0.14285714285714285,
|
|
"grad_norm": 0.057179540395736694,
|
|
"kl": 3.587547689676285e-05,
|
|
"lambda_div_used": 0.5674227699637413,
|
|
"learning_rate": 4.4113514698014953e-07,
|
|
"loss": 0.0092,
|
|
"reward": -0.09743379801511765,
|
|
"reward_after_mean": -0.09743379801511765,
|
|
"reward_after_std": 0.4501750059425831,
|
|
"reward_before_mean": 0.3561172317713499,
|
|
"reward_before_std": 0.3082335013896227,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.453551035374403,
|
|
"reward_change_min": -0.6685109585523605,
|
|
"reward_change_std": 0.25046134926378727,
|
|
"reward_std": 0.4501750282943249,
|
|
"rewards/accuracy_reward": 0.3125,
|
|
"rewards/cosine_scaled_reward": 0.043617233633995056,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2814.3333892822266,
|
|
"epoch": 0.144,
|
|
"grad_norm": 0.06912728399038315,
|
|
"kl": 3.851950168609619e-05,
|
|
"lambda_div_used": 0.6011649072170258,
|
|
"learning_rate": 4.3353142970386557e-07,
|
|
"loss": 0.0079,
|
|
"reward": -0.2701383363455534,
|
|
"reward_after_mean": -0.2701383363455534,
|
|
"reward_after_std": 0.517305538058281,
|
|
"reward_before_mean": -0.00174633227288723,
|
|
"reward_before_std": 0.46759936958551407,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2683920059353113,
|
|
"reward_change_min": -0.42111679539084435,
|
|
"reward_change_std": 0.16303524654358625,
|
|
"reward_std": 0.5173055771738291,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.10591300774831325,
|
|
"step": 126
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3520.375,
|
|
"epoch": 0.14514285714285713,
|
|
"grad_norm": 0.05083320662379265,
|
|
"kl": 3.781914710998535e-05,
|
|
"lambda_div_used": 0.5308314934372902,
|
|
"learning_rate": 4.2596318988235037e-07,
|
|
"loss": 0.0094,
|
|
"reward": -0.5137354172766209,
|
|
"reward_after_mean": -0.5137354172766209,
|
|
"reward_after_std": 0.23104721494019032,
|
|
"reward_before_mean": -0.23012915067374706,
|
|
"reward_before_std": 0.13533379370346665,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28360624983906746,
|
|
"reward_change_min": -0.4074147306382656,
|
|
"reward_change_std": 0.14828919060528278,
|
|
"reward_std": 0.2310472223907709,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2301291599869728,
|
|
"step": 127
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2622.6041946411133,
|
|
"epoch": 0.1462857142857143,
|
|
"grad_norm": 0.07047850638628006,
|
|
"kl": 3.9443373680114746e-05,
|
|
"lambda_div_used": 0.5977307558059692,
|
|
"learning_rate": 4.1843273287476854e-07,
|
|
"loss": 0.0245,
|
|
"reward": 0.14617427624762058,
|
|
"reward_after_mean": 0.14617427624762058,
|
|
"reward_after_std": 0.5826170947402716,
|
|
"reward_before_mean": 0.6620007424353389,
|
|
"reward_before_std": 0.45144926803186536,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5158264562487602,
|
|
"reward_change_min": -0.7390152402222157,
|
|
"reward_change_std": 0.29064416885375977,
|
|
"reward_std": 0.5826170966029167,
|
|
"rewards/accuracy_reward": 0.41666666977107525,
|
|
"rewards/cosine_scaled_reward": 0.2453340534120798,
|
|
"step": 128
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3211.166702270508,
|
|
"epoch": 0.14742857142857144,
|
|
"grad_norm": 0.06908967345952988,
|
|
"kl": 4.245340824127197e-05,
|
|
"lambda_div_used": 0.579024501144886,
|
|
"learning_rate": 4.1094235253127374e-07,
|
|
"loss": -0.0068,
|
|
"reward": -0.41220802813768387,
|
|
"reward_after_mean": -0.41220802813768387,
|
|
"reward_after_std": 0.4235076569020748,
|
|
"reward_before_mean": -0.1792316001956351,
|
|
"reward_before_std": 0.3573149349540472,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2329764310270548,
|
|
"reward_change_min": -0.38812074810266495,
|
|
"reward_change_std": 0.13700235076248646,
|
|
"reward_std": 0.423507671803236,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.24173159897327423,
|
|
"step": 129
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3416.375030517578,
|
|
"epoch": 0.14857142857142858,
|
|
"grad_norm": 0.05305078998208046,
|
|
"kl": 3.8370490074157715e-05,
|
|
"lambda_div_used": 0.5585792362689972,
|
|
"learning_rate": 4.034943304942796e-07,
|
|
"loss": -0.0353,
|
|
"reward": -0.324653722345829,
|
|
"reward_after_mean": -0.324653722345829,
|
|
"reward_after_std": 0.30889566242694855,
|
|
"reward_before_mean": 0.0034640291705727577,
|
|
"reward_before_std": 0.2620142959058285,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32811774499714375,
|
|
"reward_change_min": -0.4815611355006695,
|
|
"reward_change_std": 0.18737321346998215,
|
|
"reward_std": 0.30889566615223885,
|
|
"rewards/accuracy_reward": 0.1041666716337204,
|
|
"rewards/cosine_scaled_reward": -0.10070264618843794,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2672.8541984558105,
|
|
"epoch": 0.14971428571428572,
|
|
"grad_norm": 0.08398960530757904,
|
|
"kl": 2.8876587748527527e-05,
|
|
"lambda_div_used": 0.6004800871014595,
|
|
"learning_rate": 3.9609093550344907e-07,
|
|
"loss": 0.0142,
|
|
"reward": 0.04825104773044586,
|
|
"reward_after_mean": 0.04825104773044586,
|
|
"reward_after_std": 0.537496130913496,
|
|
"reward_before_mean": 0.48470486514270306,
|
|
"reward_before_std": 0.4661959493532777,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.43645381927490234,
|
|
"reward_change_min": -0.650093249976635,
|
|
"reward_change_std": 0.2640222804620862,
|
|
"reward_std": 0.5374961327761412,
|
|
"rewards/accuracy_reward": 0.3750000074505806,
|
|
"rewards/cosine_scaled_reward": 0.10970486886799335,
|
|
"step": 131
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2745.8541870117188,
|
|
"epoch": 0.15085714285714286,
|
|
"grad_norm": 0.10217073559761047,
|
|
"kl": 3.91155481338501e-05,
|
|
"lambda_div_used": 0.5872602090239525,
|
|
"learning_rate": 3.8873442270461485e-07,
|
|
"loss": -0.0282,
|
|
"reward": -0.18162552546709776,
|
|
"reward_after_mean": -0.18162552546709776,
|
|
"reward_after_std": 0.4464551955461502,
|
|
"reward_before_mean": 0.14102918095886707,
|
|
"reward_before_std": 0.3960893382318318,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32265469804406166,
|
|
"reward_change_min": -0.477376826107502,
|
|
"reward_change_std": 0.18945662677288055,
|
|
"reward_std": 0.4464551992714405,
|
|
"rewards/accuracy_reward": 0.22916667722165585,
|
|
"rewards/cosine_scaled_reward": -0.08813749923137948,
|
|
"step": 132
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3396.625,
|
|
"epoch": 0.152,
|
|
"grad_norm": 0.05338770151138306,
|
|
"kl": 4.1738152503967285e-05,
|
|
"lambda_div_used": 0.5715877488255501,
|
|
"learning_rate": 3.8142703296283953e-07,
|
|
"loss": -0.0083,
|
|
"reward": -0.3019852191209793,
|
|
"reward_after_mean": -0.3019852191209793,
|
|
"reward_after_std": 0.394625848159194,
|
|
"reward_before_mean": 0.015070955269038677,
|
|
"reward_before_std": 0.3251611590385437,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3170561697334051,
|
|
"reward_change_min": -0.5240310952067375,
|
|
"reward_change_std": 0.18560713436454535,
|
|
"reward_std": 0.39462585374712944,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.15159572381526232,
|
|
"step": 133
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2655.7708435058594,
|
|
"epoch": 0.15314285714285714,
|
|
"grad_norm": 0.08679015934467316,
|
|
"kl": 8.308887481689453e-05,
|
|
"lambda_div_used": 0.5542216002941132,
|
|
"learning_rate": 3.7417099217982686e-07,
|
|
"loss": 0.0208,
|
|
"reward": -0.32178156822919846,
|
|
"reward_after_mean": -0.32178156822919846,
|
|
"reward_after_std": 0.2957470379769802,
|
|
"reward_before_mean": 0.012201536446809769,
|
|
"reward_before_std": 0.24192149471491575,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3339830953627825,
|
|
"reward_change_min": -0.4839101508259773,
|
|
"reward_change_std": 0.18742438219487667,
|
|
"reward_std": 0.2957470417022705,
|
|
"rewards/accuracy_reward": 0.1041666716337204,
|
|
"rewards/cosine_scaled_reward": -0.09196514077484608,
|
|
"step": 134
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2014.9583587646484,
|
|
"epoch": 0.15428571428571428,
|
|
"grad_norm": 0.07728467881679535,
|
|
"kl": 2.7412548661231995e-05,
|
|
"lambda_div_used": 0.6218282878398895,
|
|
"learning_rate": 3.6696851061588994e-07,
|
|
"loss": 0.0739,
|
|
"reward": 0.15841423906385899,
|
|
"reward_after_mean": 0.15841423906385899,
|
|
"reward_after_std": 0.6740392297506332,
|
|
"reward_before_mean": 0.6407645158469677,
|
|
"reward_before_std": 0.5697397403419018,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.48235028609633446,
|
|
"reward_change_min": -0.7837426699697971,
|
|
"reward_change_std": 0.3034245353192091,
|
|
"reward_std": 0.6740392409265041,
|
|
"rewards/accuracy_reward": 0.4791666716337204,
|
|
"rewards/cosine_scaled_reward": 0.161597837228328,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2602.416717529297,
|
|
"epoch": 0.15542857142857142,
|
|
"grad_norm": 0.06872163712978363,
|
|
"kl": 2.8759241104125977e-05,
|
|
"lambda_div_used": 0.6318690255284309,
|
|
"learning_rate": 3.5982178221668533e-07,
|
|
"loss": 0.0344,
|
|
"reward": 0.06710329907946289,
|
|
"reward_after_mean": 0.06710329907946289,
|
|
"reward_after_std": 0.6661366745829582,
|
|
"reward_before_mean": 0.454304663464427,
|
|
"reward_before_std": 0.6177845690399408,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38720136508345604,
|
|
"reward_change_min": -0.6763367056846619,
|
|
"reward_change_std": 0.258380358107388,
|
|
"reward_std": 0.6661366857588291,
|
|
"rewards/accuracy_reward": 0.3541666716337204,
|
|
"rewards/cosine_scaled_reward": 0.10013799648731947,
|
|
"step": 136
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3239.666679382324,
|
|
"epoch": 0.15657142857142858,
|
|
"grad_norm": 0.0593891404569149,
|
|
"kl": 3.842264413833618e-05,
|
|
"lambda_div_used": 0.5779730677604675,
|
|
"learning_rate": 3.5273298394491515e-07,
|
|
"loss": 0.0124,
|
|
"reward": -0.3537064976990223,
|
|
"reward_after_mean": -0.3537064976990223,
|
|
"reward_after_std": 0.41078851372003555,
|
|
"reward_before_mean": -0.08445308171212673,
|
|
"reward_before_std": 0.35645375214517117,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2692534364759922,
|
|
"reward_change_min": -0.4356107972562313,
|
|
"reward_change_std": 0.15999021660536528,
|
|
"reward_std": 0.4107885267585516,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.16778641194105148,
|
|
"step": 137
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2760.687526702881,
|
|
"epoch": 0.15771428571428572,
|
|
"grad_norm": 0.0790812149643898,
|
|
"kl": 2.8595328330993652e-05,
|
|
"lambda_div_used": 0.5799111127853394,
|
|
"learning_rate": 3.45704275117204e-07,
|
|
"loss": -0.0362,
|
|
"reward": -0.27829512720927596,
|
|
"reward_after_mean": -0.27829512720927596,
|
|
"reward_after_std": 0.41231589019298553,
|
|
"reward_before_mean": 0.02187724970281124,
|
|
"reward_before_std": 0.362152349203825,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3001723885536194,
|
|
"reward_change_min": -0.45827802270650864,
|
|
"reward_change_std": 0.17282870691269636,
|
|
"reward_std": 0.412315895780921,
|
|
"rewards/accuracy_reward": 0.12500000558793545,
|
|
"rewards/cosine_scaled_reward": -0.10312275495380163,
|
|
"step": 138
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2984.4583587646484,
|
|
"epoch": 0.15885714285714286,
|
|
"grad_norm": 0.06229131296277046,
|
|
"kl": 3.9517879486083984e-05,
|
|
"lambda_div_used": 0.5838895812630653,
|
|
"learning_rate": 3.387377967463493e-07,
|
|
"loss": -0.0589,
|
|
"reward": -0.3416835393290967,
|
|
"reward_after_mean": -0.3416835393290967,
|
|
"reward_after_std": 0.4420221708714962,
|
|
"reward_before_mean": -0.08248884417116642,
|
|
"reward_before_std": 0.384817186743021,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.259194690734148,
|
|
"reward_change_min": -0.42579447478055954,
|
|
"reward_change_std": 0.15513746719807386,
|
|
"reward_std": 0.44202217273414135,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.16582217533141375,
|
|
"step": 139
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2881.1250228881836,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.071082703769207,
|
|
"kl": 4.310905933380127e-05,
|
|
"lambda_div_used": 0.5740059092640877,
|
|
"learning_rate": 3.3183567088914833e-07,
|
|
"loss": 0.0019,
|
|
"reward": -0.25879082828760147,
|
|
"reward_after_mean": -0.25879082828760147,
|
|
"reward_after_std": 0.4811950568109751,
|
|
"reward_before_mean": 0.09872006997466087,
|
|
"reward_before_std": 0.3372529884800315,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35751091688871384,
|
|
"reward_change_min": -0.4904663935303688,
|
|
"reward_change_std": 0.18400432635098696,
|
|
"reward_std": 0.481195081025362,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.06794659808656434,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3180.187515258789,
|
|
"epoch": 0.16114285714285714,
|
|
"grad_norm": 0.05953420698642731,
|
|
"kl": 3.790855407714844e-05,
|
|
"lambda_div_used": 0.5660471692681313,
|
|
"learning_rate": 3.250000000000001e-07,
|
|
"loss": -0.0557,
|
|
"reward": -0.3696493972092867,
|
|
"reward_after_mean": -0.3696493972092867,
|
|
"reward_after_std": 0.37020369805395603,
|
|
"reward_before_mean": -0.1031316639855504,
|
|
"reward_before_std": 0.300534725189209,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2665177509188652,
|
|
"reward_change_min": -0.3948013670742512,
|
|
"reward_change_std": 0.151769301854074,
|
|
"reward_std": 0.3702037110924721,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.20729831932112575,
|
|
"step": 141
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2633.291732788086,
|
|
"epoch": 0.16228571428571428,
|
|
"grad_norm": 0.06993869692087173,
|
|
"kl": 4.523620009422302e-05,
|
|
"lambda_div_used": 0.643665611743927,
|
|
"learning_rate": 3.182328662904756e-07,
|
|
"loss": 0.0607,
|
|
"reward": -0.11501466785557568,
|
|
"reward_after_mean": -0.11501466785557568,
|
|
"reward_after_std": 0.7061873953789473,
|
|
"reward_before_mean": 0.1335596082135453,
|
|
"reward_before_std": 0.6679177191108465,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2485742662101984,
|
|
"reward_change_min": -0.438314501196146,
|
|
"reward_change_std": 0.16557594947516918,
|
|
"reward_std": 0.7061873972415924,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.05394040001556277,
|
|
"step": 142
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2543.854202270508,
|
|
"epoch": 0.16342857142857142,
|
|
"grad_norm": 0.09050207585096359,
|
|
"kl": 4.5418739318847656e-05,
|
|
"lambda_div_used": 0.5568482205271721,
|
|
"learning_rate": 3.115363310950578e-07,
|
|
"loss": 0.0559,
|
|
"reward": -0.44561293721199036,
|
|
"reward_after_mean": -0.44561293721199036,
|
|
"reward_after_std": 0.34061737172305584,
|
|
"reward_before_mean": -0.17733613029122353,
|
|
"reward_before_std": 0.2554386807605624,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26827680319547653,
|
|
"reward_change_min": -0.39999108389019966,
|
|
"reward_change_std": 0.14368562400341034,
|
|
"reward_std": 0.3406173773109913,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.19816946546779945,
|
|
"step": 143
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3062.2291984558105,
|
|
"epoch": 0.16457142857142856,
|
|
"grad_norm": 0.08773668855428696,
|
|
"kl": 4.3526291847229004e-05,
|
|
"lambda_div_used": 0.5946916490793228,
|
|
"learning_rate": 3.0491243424323783e-07,
|
|
"loss": 0.0214,
|
|
"reward": -0.1852840557694435,
|
|
"reward_after_mean": -0.1852840557694435,
|
|
"reward_after_std": 0.4624195992946625,
|
|
"reward_before_mean": 0.13713636994361877,
|
|
"reward_before_std": 0.4314339701086283,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3224204182624817,
|
|
"reward_change_min": -0.502110667526722,
|
|
"reward_change_std": 0.19803205784410238,
|
|
"reward_std": 0.4624196030199528,
|
|
"rewards/accuracy_reward": 0.18750000558793545,
|
|
"rewards/cosine_scaled_reward": -0.05036364169791341,
|
|
"step": 144
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2343.708381652832,
|
|
"epoch": 0.1657142857142857,
|
|
"grad_norm": 0.09618457406759262,
|
|
"kl": 3.914535045623779e-05,
|
|
"lambda_div_used": 0.6065893918275833,
|
|
"learning_rate": 2.9836319343816397e-07,
|
|
"loss": 0.0239,
|
|
"reward": -0.08643046766519547,
|
|
"reward_after_mean": -0.08643046766519547,
|
|
"reward_after_std": 0.565411014482379,
|
|
"reward_before_mean": 0.2700228439643979,
|
|
"reward_before_std": 0.4942969996482134,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35645330883562565,
|
|
"reward_change_min": -0.5621693357825279,
|
|
"reward_change_std": 0.21702316030859947,
|
|
"reward_std": 0.565411014482379,
|
|
"rewards/accuracy_reward": 0.29166666977107525,
|
|
"rewards/cosine_scaled_reward": -0.021643826737999916,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2643.750030517578,
|
|
"epoch": 0.16685714285714287,
|
|
"grad_norm": 0.06749647855758667,
|
|
"kl": 3.0444934964179993e-05,
|
|
"lambda_div_used": 0.5759064182639122,
|
|
"learning_rate": 2.918906036420294e-07,
|
|
"loss": -0.0525,
|
|
"reward": -0.3986330684274435,
|
|
"reward_after_mean": -0.3986330684274435,
|
|
"reward_after_std": 0.4206914007663727,
|
|
"reward_before_mean": -0.1481835450977087,
|
|
"reward_before_std": 0.3465117560699582,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2504495307803154,
|
|
"reward_change_min": -0.3944549411535263,
|
|
"reward_change_std": 0.14348686579614878,
|
|
"reward_std": 0.42069140635430813,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.21068353950977325,
|
|
"step": 146
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3530.8958435058594,
|
|
"epoch": 0.168,
|
|
"grad_norm": 0.05082716792821884,
|
|
"kl": 4.493445158004761e-05,
|
|
"lambda_div_used": 0.6231074929237366,
|
|
"learning_rate": 2.854966364683872e-07,
|
|
"loss": 0.0064,
|
|
"reward": -0.17444764077663422,
|
|
"reward_after_mean": -0.17444764077663422,
|
|
"reward_after_std": 0.5966980569064617,
|
|
"reward_before_mean": 0.08711316343396902,
|
|
"reward_before_std": 0.5713155549019575,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26156080327928066,
|
|
"reward_change_min": -0.42396802455186844,
|
|
"reward_change_std": 0.16937424894422293,
|
|
"reward_std": 0.5966980736702681,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.12122016958892345,
|
|
"step": 147
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2726.0625228881836,
|
|
"epoch": 0.16914285714285715,
|
|
"grad_norm": 0.05811236426234245,
|
|
"kl": 2.9481947422027588e-05,
|
|
"lambda_div_used": 0.5576904863119125,
|
|
"learning_rate": 2.791832395815782e-07,
|
|
"loss": -0.0067,
|
|
"reward": -0.3092116080224514,
|
|
"reward_after_mean": -0.3092116080224514,
|
|
"reward_after_std": 0.3737166114151478,
|
|
"reward_before_mean": 0.04292176803573966,
|
|
"reward_before_std": 0.2566564744338393,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35213335789740086,
|
|
"reward_change_min": -0.5105132721364498,
|
|
"reward_change_std": 0.18572800233960152,
|
|
"reward_std": 0.3737166281789541,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.10291159152984619,
|
|
"step": 148
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3018.4375228881836,
|
|
"epoch": 0.1702857142857143,
|
|
"grad_norm": 0.07346916198730469,
|
|
"kl": 3.395974636077881e-05,
|
|
"lambda_div_used": 0.5993617027997971,
|
|
"learning_rate": 2.729523361034538e-07,
|
|
"loss": 0.0316,
|
|
"reward": -0.23622393235564232,
|
|
"reward_after_mean": -0.23622393235564232,
|
|
"reward_after_std": 0.513044873252511,
|
|
"reward_before_mean": 0.05197374615818262,
|
|
"reward_before_std": 0.4586914679966867,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2881976682692766,
|
|
"reward_change_min": -0.4784680940210819,
|
|
"reward_change_std": 0.1794019928202033,
|
|
"reward_std": 0.5130448862910271,
|
|
"rewards/accuracy_reward": 0.14583333767950535,
|
|
"rewards/cosine_scaled_reward": -0.0938595961779356,
|
|
"step": 149
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2817.000030517578,
|
|
"epoch": 0.17142857142857143,
|
|
"grad_norm": 0.07110631465911865,
|
|
"kl": 4.3623149394989014e-05,
|
|
"lambda_div_used": 0.5749505385756493,
|
|
"learning_rate": 2.6680582402757324e-07,
|
|
"loss": 0.0143,
|
|
"reward": -0.4273622464388609,
|
|
"reward_after_mean": -0.4273622464388609,
|
|
"reward_after_std": 0.41951365023851395,
|
|
"reward_before_mean": -0.1882035918533802,
|
|
"reward_before_std": 0.34118228033185005,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23915864899754524,
|
|
"reward_change_min": -0.3481520377099514,
|
|
"reward_change_std": 0.1273820260539651,
|
|
"reward_std": 0.4195136558264494,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.22987026162445545,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2473.5208892822266,
|
|
"epoch": 0.17257142857142857,
|
|
"grad_norm": 0.06608390063047409,
|
|
"kl": 3.757700324058533e-05,
|
|
"lambda_div_used": 0.6444682851433754,
|
|
"learning_rate": 2.6074557564105724e-07,
|
|
"loss": 0.0674,
|
|
"reward": 0.013197226449847221,
|
|
"reward_after_mean": 0.013197226449847221,
|
|
"reward_after_std": 0.6638381816446781,
|
|
"reward_before_mean": 0.3273471943102777,
|
|
"reward_before_std": 0.6709178425371647,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31414996832609177,
|
|
"reward_change_min": -0.5566080771386623,
|
|
"reward_change_std": 0.22394540812820196,
|
|
"reward_std": 0.6638382077217102,
|
|
"rewards/accuracy_reward": 0.31250000558793545,
|
|
"rewards/cosine_scaled_reward": 0.014847185462713242,
|
|
"step": 151
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2980.312530517578,
|
|
"epoch": 0.1737142857142857,
|
|
"grad_norm": 0.09729959070682526,
|
|
"kl": 3.904849290847778e-05,
|
|
"lambda_div_used": 0.5731147676706314,
|
|
"learning_rate": 2.547734369542718e-07,
|
|
"loss": 0.036,
|
|
"reward": -0.42251406982541084,
|
|
"reward_after_mean": -0.42251406982541084,
|
|
"reward_after_std": 0.4107065536081791,
|
|
"reward_before_mean": -0.17358114942908287,
|
|
"reward_before_std": 0.3313267915509641,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24893292412161827,
|
|
"reward_change_min": -0.37377795577049255,
|
|
"reward_change_std": 0.13549237046390772,
|
|
"reward_std": 0.4107065834105015,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.21524782478809357,
|
|
"step": 152
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2980.4167098999023,
|
|
"epoch": 0.17485714285714285,
|
|
"grad_norm": 0.07872413098812103,
|
|
"kl": 5.060434341430664e-05,
|
|
"lambda_div_used": 0.5996033921837807,
|
|
"learning_rate": 2.488912271385139e-07,
|
|
"loss": 0.0468,
|
|
"reward": -0.25326492823660374,
|
|
"reward_after_mean": -0.25326492823660374,
|
|
"reward_after_std": 0.5015600807964802,
|
|
"reward_before_mean": 0.03005093801766634,
|
|
"reward_before_std": 0.45544715132564306,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2833158541470766,
|
|
"reward_change_min": -0.490575835108757,
|
|
"reward_change_std": 0.17807927820831537,
|
|
"reward_std": 0.5015600919723511,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.09494908014312387,
|
|
"step": 153
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3400.6250610351562,
|
|
"epoch": 0.176,
|
|
"grad_norm": 0.044398024678230286,
|
|
"kl": 3.533065319061279e-05,
|
|
"lambda_div_used": 0.6407897770404816,
|
|
"learning_rate": 2.4310073797187573e-07,
|
|
"loss": 0.001,
|
|
"reward": -0.08846104983240366,
|
|
"reward_after_mean": -0.08846104983240366,
|
|
"reward_after_std": 0.6875216029584408,
|
|
"reward_before_mean": 0.1883750823326409,
|
|
"reward_before_std": 0.6521897967904806,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27683611772954464,
|
|
"reward_change_min": -0.45937369018793106,
|
|
"reward_change_std": 0.17437504325062037,
|
|
"reward_std": 0.6875216346234083,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.019958254415541887,
|
|
"step": 154
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2453.5416717529297,
|
|
"epoch": 0.17714285714285713,
|
|
"grad_norm": 0.08257688581943512,
|
|
"kl": 4.6115368604660034e-05,
|
|
"lambda_div_used": 0.5891367048025131,
|
|
"learning_rate": 2.374037332934512e-07,
|
|
"loss": 0.0267,
|
|
"reward": -0.14017992746084929,
|
|
"reward_after_mean": -0.14017992746084929,
|
|
"reward_after_std": 0.5328723080456257,
|
|
"reward_before_mean": 0.24201755598187447,
|
|
"reward_before_std": 0.40873255487531424,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3821975253522396,
|
|
"reward_change_min": -0.5966118611395359,
|
|
"reward_change_std": 0.22055233176797628,
|
|
"reward_std": 0.532872324809432,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": 0.012850900180637836,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3058.145835876465,
|
|
"epoch": 0.1782857142857143,
|
|
"grad_norm": 0.07657067477703094,
|
|
"kl": 3.8407742977142334e-05,
|
|
"lambda_div_used": 0.5792115926742554,
|
|
"learning_rate": 2.3180194846605364e-07,
|
|
"loss": 0.0192,
|
|
"reward": -0.35964071936905384,
|
|
"reward_after_mean": -0.35964071936905384,
|
|
"reward_after_std": 0.41963592916727066,
|
|
"reward_before_mean": -0.10375695489346981,
|
|
"reward_before_std": 0.36636115331202745,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2558837812393904,
|
|
"reward_change_min": -0.40037716925144196,
|
|
"reward_change_std": 0.15252912789583206,
|
|
"reward_std": 0.41963593289256096,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.18709028977900743,
|
|
"step": 156
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3067.0833435058594,
|
|
"epoch": 0.17942857142857144,
|
|
"grad_norm": 0.0602630190551281,
|
|
"kl": 4.166364669799805e-05,
|
|
"lambda_div_used": 0.578017845749855,
|
|
"learning_rate": 2.2629708984760706e-07,
|
|
"loss": -0.0024,
|
|
"reward": -0.20843622065149248,
|
|
"reward_after_mean": -0.20843622065149248,
|
|
"reward_after_std": 0.46523306891322136,
|
|
"reward_before_mean": 0.15985593758523464,
|
|
"reward_before_std": 0.35562971234321594,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3682921752333641,
|
|
"reward_change_min": -0.5581395737826824,
|
|
"reward_change_std": 0.20453235507011414,
|
|
"reward_std": 0.46523308381438255,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.027644065208733082,
|
|
"step": 157
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2439.8125381469727,
|
|
"epoch": 0.18057142857142858,
|
|
"grad_norm": 0.07962260395288467,
|
|
"kl": 3.724917769432068e-05,
|
|
"lambda_div_used": 0.5700756460428238,
|
|
"learning_rate": 2.2089083427137329e-07,
|
|
"loss": 0.0427,
|
|
"reward": -0.09846613928675652,
|
|
"reward_after_mean": -0.09846613928675652,
|
|
"reward_after_std": 0.50458899512887,
|
|
"reward_before_mean": 0.36336963158100843,
|
|
"reward_before_std": 0.31559942476451397,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.46183576062321663,
|
|
"reward_change_min": -0.6303520426154137,
|
|
"reward_change_std": 0.23663399182260036,
|
|
"reward_std": 0.5045890025794506,
|
|
"rewards/accuracy_reward": 0.2916666679084301,
|
|
"rewards/cosine_scaled_reward": 0.07170296460390091,
|
|
"step": 158
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3283.7916717529297,
|
|
"epoch": 0.18171428571428572,
|
|
"grad_norm": 0.05873558670282364,
|
|
"kl": 4.16487455368042e-05,
|
|
"lambda_div_used": 0.565159484744072,
|
|
"learning_rate": 2.1558482853517253e-07,
|
|
"loss": -0.0183,
|
|
"reward": -0.43611275404691696,
|
|
"reward_after_mean": -0.43611275404691696,
|
|
"reward_after_std": 0.34730882942676544,
|
|
"reward_before_mean": -0.18830033391714096,
|
|
"reward_before_std": 0.2990688029676676,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.247812420129776,
|
|
"reward_change_min": -0.4116964153945446,
|
|
"reward_change_std": 0.15130725782364607,
|
|
"reward_std": 0.34730884805321693,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.2508003171533346,
|
|
"step": 159
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3132.4167098999023,
|
|
"epoch": 0.18285714285714286,
|
|
"grad_norm": 0.06507622450590134,
|
|
"kl": 5.078315734863281e-05,
|
|
"lambda_div_used": 0.5871393531560898,
|
|
"learning_rate": 2.1038068889975259e-07,
|
|
"loss": 0.0371,
|
|
"reward": -0.25536114536225796,
|
|
"reward_after_mean": -0.25536114536225796,
|
|
"reward_after_std": 0.4553165938705206,
|
|
"reward_before_mean": 0.0424603084102273,
|
|
"reward_before_std": 0.3940334524959326,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2978214379400015,
|
|
"reward_change_min": -0.43723243474960327,
|
|
"reward_change_std": 0.16810003202408552,
|
|
"reward_std": 0.45531659945845604,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.10337302926927805,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1814.145866394043,
|
|
"epoch": 0.184,
|
|
"grad_norm": 0.09199656546115875,
|
|
"kl": 3.68654727935791e-05,
|
|
"lambda_div_used": 0.6221684664487839,
|
|
"learning_rate": 2.0528000059645995e-07,
|
|
"loss": -0.1313,
|
|
"reward": -0.14752347487956285,
|
|
"reward_after_mean": -0.14752347487956285,
|
|
"reward_after_std": 0.6091006584465504,
|
|
"reward_before_mean": 0.1306287944316864,
|
|
"reward_before_std": 0.5693164113909006,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2781522646546364,
|
|
"reward_change_min": -0.4651326909661293,
|
|
"reward_change_std": 0.17978444695472717,
|
|
"reward_std": 0.6091006807982922,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.03603787627071142,
|
|
"step": 161
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3445.2291870117188,
|
|
"epoch": 0.18514285714285714,
|
|
"grad_norm": 0.054613932967185974,
|
|
"kl": 4.4696033000946045e-05,
|
|
"lambda_div_used": 0.6093570217490196,
|
|
"learning_rate": 2.0028431734436308e-07,
|
|
"loss": 0.0111,
|
|
"reward": -0.21708323806524277,
|
|
"reward_after_mean": -0.21708323806524277,
|
|
"reward_after_std": 0.5340255293995142,
|
|
"reward_before_mean": 0.05781930312514305,
|
|
"reward_before_std": 0.5134020620025694,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27490255795419216,
|
|
"reward_change_min": -0.5003471188247204,
|
|
"reward_change_std": 0.1919182576239109,
|
|
"reward_std": 0.5340255443006754,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.10884736478328705,
|
|
"step": 162
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2758.5833435058594,
|
|
"epoch": 0.18628571428571428,
|
|
"grad_norm": 0.06869502365589142,
|
|
"kl": 3.579258918762207e-05,
|
|
"lambda_div_used": 0.5997234806418419,
|
|
"learning_rate": 1.9539516087697517e-07,
|
|
"loss": 0.025,
|
|
"reward": -0.0028839372098445892,
|
|
"reward_after_mean": -0.0028839372098445892,
|
|
"reward_after_std": 0.5558424014598131,
|
|
"reward_before_mean": 0.42010904336348176,
|
|
"reward_before_std": 0.46184782730415463,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.42299298755824566,
|
|
"reward_change_min": -0.614623662084341,
|
|
"reward_change_std": 0.2444069180637598,
|
|
"reward_std": 0.5558424014598131,
|
|
"rewards/accuracy_reward": 0.33333334140479565,
|
|
"rewards/cosine_scaled_reward": 0.08677570521831512,
|
|
"step": 163
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2280.4583435058594,
|
|
"epoch": 0.18742857142857142,
|
|
"grad_norm": 0.08430740982294083,
|
|
"kl": 3.807246685028076e-05,
|
|
"lambda_div_used": 0.583802655339241,
|
|
"learning_rate": 1.9061402047871833e-07,
|
|
"loss": -0.0005,
|
|
"reward": -0.14323966577649117,
|
|
"reward_after_mean": -0.14323966577649117,
|
|
"reward_after_std": 0.48269340209662914,
|
|
"reward_before_mean": 0.24511565826833248,
|
|
"reward_before_std": 0.38433590345084667,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38835531286895275,
|
|
"reward_change_min": -0.6137315705418587,
|
|
"reward_change_std": 0.2298044739291072,
|
|
"reward_std": 0.48269341699779034,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": -0.0048843612894415855,
|
|
"step": 164
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3350.2083435058594,
|
|
"epoch": 0.18857142857142858,
|
|
"grad_norm": 0.06055794283747673,
|
|
"kl": 4.6700239181518555e-05,
|
|
"lambda_div_used": 0.5597026646137238,
|
|
"learning_rate": 1.8594235253127372e-07,
|
|
"loss": -0.0239,
|
|
"reward": -0.383526224642992,
|
|
"reward_after_mean": -0.383526224642992,
|
|
"reward_after_std": 0.3388876337558031,
|
|
"reward_before_mean": -0.09019916784018278,
|
|
"reward_before_std": 0.27148230001330376,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2933270577341318,
|
|
"reward_change_min": -0.44717802107334137,
|
|
"reward_change_std": 0.16905023716390133,
|
|
"reward_std": 0.3388876374810934,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.15269917901605368,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2993.7709045410156,
|
|
"epoch": 0.18971428571428572,
|
|
"grad_norm": 0.07070616632699966,
|
|
"kl": 3.575533628463745e-05,
|
|
"lambda_div_used": 0.6159055009484291,
|
|
"learning_rate": 1.8138158006995363e-07,
|
|
"loss": -0.1171,
|
|
"reward": -0.20591574627906084,
|
|
"reward_after_mean": -0.20591574627906084,
|
|
"reward_after_std": 0.568760309368372,
|
|
"reward_before_mean": 0.07225125166587532,
|
|
"reward_before_std": 0.5325462874025106,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2781669981777668,
|
|
"reward_change_min": -0.4841184914112091,
|
|
"reward_change_std": 0.17965693771839142,
|
|
"reward_std": 0.5687603335827589,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.0735820853151381,
|
|
"step": 166
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2243.4583740234375,
|
|
"epoch": 0.19085714285714286,
|
|
"grad_norm": 0.08954203128814697,
|
|
"kl": 3.8199592381715775e-05,
|
|
"lambda_div_used": 0.5807118713855743,
|
|
"learning_rate": 1.7693309235023127e-07,
|
|
"loss": -0.0227,
|
|
"reward": -0.12234261445701122,
|
|
"reward_after_mean": -0.12234261445701122,
|
|
"reward_after_std": 0.46423870138823986,
|
|
"reward_before_mean": 0.28887630719691515,
|
|
"reward_before_std": 0.3718760753981769,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41121890768408775,
|
|
"reward_change_min": -0.6246605962514877,
|
|
"reward_change_std": 0.24211041443049908,
|
|
"reward_std": 0.4642387069761753,
|
|
"rewards/accuracy_reward": 0.2291666679084301,
|
|
"rewards/cosine_scaled_reward": 0.0597096448764205,
|
|
"step": 167
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3159.604217529297,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.05736541375517845,
|
|
"kl": 4.6193599700927734e-05,
|
|
"lambda_div_used": 0.6025644987821579,
|
|
"learning_rate": 1.7259824442455923e-07,
|
|
"loss": 0.0616,
|
|
"reward": -0.1081604603677988,
|
|
"reward_after_mean": -0.1081604603677988,
|
|
"reward_after_std": 0.5043755322694778,
|
|
"reward_before_mean": 0.23938040900975466,
|
|
"reward_before_std": 0.4762058244086802,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.347540894523263,
|
|
"reward_change_min": -0.5797867476940155,
|
|
"reward_change_std": 0.22277529910206795,
|
|
"reward_std": 0.504375534132123,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": 0.031047106254845858,
|
|
"step": 168
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2691.166717529297,
|
|
"epoch": 0.19314285714285714,
|
|
"grad_norm": 0.05840221792459488,
|
|
"kl": 3.322819247841835e-05,
|
|
"lambda_div_used": 0.6123679727315903,
|
|
"learning_rate": 1.6837835672960831e-07,
|
|
"loss": -0.0221,
|
|
"reward": 0.15223310515284538,
|
|
"reward_after_mean": 0.15223310515284538,
|
|
"reward_after_std": 0.589836286380887,
|
|
"reward_before_mean": 0.622445510700345,
|
|
"reward_before_std": 0.5133083704859018,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4702123887836933,
|
|
"reward_change_min": -0.686709500849247,
|
|
"reward_change_std": 0.27231256756931543,
|
|
"reward_std": 0.5898362882435322,
|
|
"rewards/accuracy_reward": 0.45833334140479565,
|
|
"rewards/cosine_scaled_reward": 0.1641121432185173,
|
|
"step": 169
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2737.8542137145996,
|
|
"epoch": 0.19428571428571428,
|
|
"grad_norm": 0.08408840745687485,
|
|
"kl": 2.9318034648895264e-05,
|
|
"lambda_div_used": 0.5516072362661362,
|
|
"learning_rate": 1.6427471468404952e-07,
|
|
"loss": -0.0223,
|
|
"reward": -0.22534361481666565,
|
|
"reward_after_mean": -0.22534361481666565,
|
|
"reward_after_std": 0.34615582413971424,
|
|
"reward_before_mean": 0.17829649709165096,
|
|
"reward_before_std": 0.231270051561296,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4036401268094778,
|
|
"reward_change_min": -0.5518503561615944,
|
|
"reward_change_std": 0.21789801493287086,
|
|
"reward_std": 0.34615583159029484,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.05087016709148884,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2810.208366394043,
|
|
"epoch": 0.19542857142857142,
|
|
"grad_norm": 0.07381287217140198,
|
|
"kl": 3.468245267868042e-05,
|
|
"lambda_div_used": 0.6072571501135826,
|
|
"learning_rate": 1.6028856829700258e-07,
|
|
"loss": 0.0763,
|
|
"reward": -0.0880913995206356,
|
|
"reward_after_mean": -0.0880913995206356,
|
|
"reward_after_std": 0.4982736185193062,
|
|
"reward_before_mean": 0.2392272837460041,
|
|
"reward_before_std": 0.4966685324907303,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3273186720907688,
|
|
"reward_change_min": -0.5419761650264263,
|
|
"reward_change_std": 0.21660141553729773,
|
|
"reward_std": 0.4982736259698868,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": 0.010060586035251617,
|
|
"step": 171
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2782.833381652832,
|
|
"epoch": 0.19657142857142856,
|
|
"grad_norm": 0.08758535981178284,
|
|
"kl": 4.4949352741241455e-05,
|
|
"lambda_div_used": 0.5883520841598511,
|
|
"learning_rate": 1.5642113178727193e-07,
|
|
"loss": 0.0173,
|
|
"reward": -0.0421207002364099,
|
|
"reward_after_mean": -0.0421207002364099,
|
|
"reward_after_std": 0.5838102325797081,
|
|
"reward_before_mean": 0.41242535319179296,
|
|
"reward_before_std": 0.4052962730638683,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.45454608276486397,
|
|
"reward_change_min": -0.6413811855018139,
|
|
"reward_change_std": 0.2437124690040946,
|
|
"reward_std": 0.5838102586567402,
|
|
"rewards/accuracy_reward": 0.3333333358168602,
|
|
"rewards/cosine_scaled_reward": 0.07909201784059405,
|
|
"step": 172
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2038.3125114440918,
|
|
"epoch": 0.1977142857142857,
|
|
"grad_norm": 0.10115820169448853,
|
|
"kl": 2.4806708097457886e-05,
|
|
"lambda_div_used": 0.5800783261656761,
|
|
"learning_rate": 1.5267358321348285e-07,
|
|
"loss": -0.0379,
|
|
"reward": -0.3465901352465153,
|
|
"reward_after_mean": -0.3465901352465153,
|
|
"reward_after_std": 0.41013904474675655,
|
|
"reward_before_mean": -0.07510924944654107,
|
|
"reward_before_std": 0.3657920900732279,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27148088812828064,
|
|
"reward_change_min": -0.41459682397544384,
|
|
"reward_change_std": 0.16112061124294996,
|
|
"reward_std": 0.4101390540599823,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.17927592061460018,
|
|
"step": 173
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2429.7083740234375,
|
|
"epoch": 0.19885714285714284,
|
|
"grad_norm": 0.08579805493354797,
|
|
"kl": 4.820525646209717e-05,
|
|
"lambda_div_used": 0.5971189364790916,
|
|
"learning_rate": 1.4904706411523448e-07,
|
|
"loss": -0.0681,
|
|
"reward": -0.16348301246762276,
|
|
"reward_after_mean": -0.16348301246762276,
|
|
"reward_after_std": 0.5588766317814589,
|
|
"reward_before_mean": 0.18685074103996158,
|
|
"reward_before_std": 0.44327013567090034,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3503337763249874,
|
|
"reward_change_min": -0.5374101847410202,
|
|
"reward_change_std": 0.19548403285443783,
|
|
"reward_std": 0.5588766410946846,
|
|
"rewards/accuracy_reward": 0.2083333358168602,
|
|
"rewards/cosine_scaled_reward": -0.02148258499801159,
|
|
"step": 174
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2815.1458587646484,
|
|
"epoch": 0.2,
|
|
"grad_norm": 0.06386198848485947,
|
|
"kl": 3.6529265344142914e-05,
|
|
"lambda_div_used": 0.5798816308379173,
|
|
"learning_rate": 1.4554267916537495e-07,
|
|
"loss": 0.0362,
|
|
"reward": -0.08284463733434677,
|
|
"reward_after_mean": -0.08284463733434677,
|
|
"reward_after_std": 0.46138195879757404,
|
|
"reward_before_mean": 0.33735317550599575,
|
|
"reward_before_std": 0.3691369164735079,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4201977960765362,
|
|
"reward_change_min": -0.615710511803627,
|
|
"reward_change_std": 0.2454231232404709,
|
|
"reward_std": 0.46138197369873524,
|
|
"rewards/accuracy_reward": 0.2708333358168602,
|
|
"rewards/cosine_scaled_reward": 0.06651980988681316,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2699.916748046875,
|
|
"epoch": 0.20114285714285715,
|
|
"grad_norm": 0.1062018871307373,
|
|
"kl": 3.183633089065552e-05,
|
|
"lambda_div_used": 0.6652076914906502,
|
|
"learning_rate": 1.4216149583350755e-07,
|
|
"loss": -0.0096,
|
|
"reward": 0.01935443957336247,
|
|
"reward_after_mean": 0.01935443957336247,
|
|
"reward_after_std": 0.7740565538406372,
|
|
"reward_before_mean": 0.30571601539850235,
|
|
"reward_before_std": 0.7710783276706934,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2863615807145834,
|
|
"reward_change_min": -0.5516976863145828,
|
|
"reward_change_std": 0.20931501779705286,
|
|
"reward_std": 0.7740565687417984,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": 0.03488267329521477,
|
|
"step": 176
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2837.062530517578,
|
|
"epoch": 0.2022857142857143,
|
|
"grad_norm": 0.10419050604104996,
|
|
"kl": 5.3942203521728516e-05,
|
|
"lambda_div_used": 0.5891516581177711,
|
|
"learning_rate": 1.3890454406082956e-07,
|
|
"loss": -0.094,
|
|
"reward": -0.275299109518528,
|
|
"reward_after_mean": -0.275299109518528,
|
|
"reward_after_std": 0.4370743464678526,
|
|
"reward_before_mean": 0.019260672852396965,
|
|
"reward_before_std": 0.4135833829641342,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.294559795409441,
|
|
"reward_change_min": -0.49321193993091583,
|
|
"reward_change_std": 0.19294326566159725,
|
|
"reward_std": 0.43707435205578804,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.10573932714760303,
|
|
"step": 177
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2239.645866394043,
|
|
"epoch": 0.20342857142857143,
|
|
"grad_norm": 0.08482369780540466,
|
|
"kl": 3.471970558166504e-05,
|
|
"lambda_div_used": 0.5743999034166336,
|
|
"learning_rate": 1.3577281594640182e-07,
|
|
"loss": -0.012,
|
|
"reward": -0.2907655192539096,
|
|
"reward_after_mean": -0.2907655192539096,
|
|
"reward_after_std": 0.3906351812183857,
|
|
"reward_before_mean": 0.020154590718448162,
|
|
"reward_before_std": 0.33835357427597046,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31092010997235775,
|
|
"reward_change_min": -0.4689374789595604,
|
|
"reward_change_std": 0.18138453178107738,
|
|
"reward_std": 0.3906352035701275,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.08401207532733679,
|
|
"step": 178
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2847.187530517578,
|
|
"epoch": 0.20457142857142857,
|
|
"grad_norm": 0.06261342018842697,
|
|
"kl": 3.7359073758125305e-05,
|
|
"lambda_div_used": 0.5993921086192131,
|
|
"learning_rate": 1.3276726544494571e-07,
|
|
"loss": 0.0003,
|
|
"reward": -0.3251352347433567,
|
|
"reward_after_mean": -0.3251352347433567,
|
|
"reward_after_std": 0.507962841540575,
|
|
"reward_before_mean": -0.08432525303214788,
|
|
"reward_before_std": 0.45724861416965723,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2408099938184023,
|
|
"reward_change_min": -0.39610618352890015,
|
|
"reward_change_std": 0.14655023906379938,
|
|
"reward_std": 0.5079628489911556,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.18849190324544907,
|
|
"step": 179
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2329.562515258789,
|
|
"epoch": 0.2057142857142857,
|
|
"grad_norm": 0.10947566479444504,
|
|
"kl": 3.916025161743164e-05,
|
|
"lambda_div_used": 0.6209289953112602,
|
|
"learning_rate": 1.2988880807625927e-07,
|
|
"loss": -0.0732,
|
|
"reward": 0.0392537759616971,
|
|
"reward_after_mean": 0.0392537759616971,
|
|
"reward_after_std": 0.6374544408172369,
|
|
"reward_before_mean": 0.4333432329003699,
|
|
"reward_before_std": 0.5579905398190022,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3940894529223442,
|
|
"reward_change_min": -0.5780177563428879,
|
|
"reward_change_std": 0.229821746237576,
|
|
"reward_std": 0.6374544575810432,
|
|
"rewards/accuracy_reward": 0.3541666753590107,
|
|
"rewards/cosine_scaled_reward": 0.07917657308280468,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3174.9166870117188,
|
|
"epoch": 0.20685714285714285,
|
|
"grad_norm": 0.0638444796204567,
|
|
"kl": 4.488229751586914e-05,
|
|
"lambda_div_used": 0.5625592544674873,
|
|
"learning_rate": 1.2713832064634125e-07,
|
|
"loss": -0.0055,
|
|
"reward": -0.3669877387583256,
|
|
"reward_after_mean": -0.3669877387583256,
|
|
"reward_after_std": 0.3254594895988703,
|
|
"reward_before_mean": -0.07308395206928253,
|
|
"reward_before_std": 0.2811311110854149,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29390377551317215,
|
|
"reward_change_min": -0.47421957924962044,
|
|
"reward_change_std": 0.1733461432158947,
|
|
"reward_std": 0.3254595026373863,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.15641729161143303,
|
|
"step": 181
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2200.5000228881836,
|
|
"epoch": 0.208,
|
|
"grad_norm": 0.07948590070009232,
|
|
"kl": 3.180652856826782e-05,
|
|
"lambda_div_used": 0.5802313759922981,
|
|
"learning_rate": 1.2451664098030743e-07,
|
|
"loss": -0.0177,
|
|
"reward": -0.33155214530415833,
|
|
"reward_after_mean": -0.33155214530415833,
|
|
"reward_after_std": 0.4093964695930481,
|
|
"reward_before_mean": -0.04757622070610523,
|
|
"reward_before_std": 0.362399042584002,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2839759271591902,
|
|
"reward_change_min": -0.4600135274231434,
|
|
"reward_change_std": 0.1762974951416254,
|
|
"reward_std": 0.40939648635685444,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.19340955652296543,
|
|
"step": 182
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1919.0000381469727,
|
|
"epoch": 0.20914285714285713,
|
|
"grad_norm": 0.09855654090642929,
|
|
"kl": 3.127008676528931e-05,
|
|
"lambda_div_used": 0.5876503959298134,
|
|
"learning_rate": 1.220245676671809e-07,
|
|
"loss": 0.0354,
|
|
"reward": -0.29830155335366726,
|
|
"reward_after_mean": -0.29830155335366726,
|
|
"reward_after_std": 0.461369052529335,
|
|
"reward_before_mean": -0.01361087104305625,
|
|
"reward_before_std": 0.4003021940588951,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28469069860875607,
|
|
"reward_change_min": -0.4400797598063946,
|
|
"reward_change_std": 0.1702390005812049,
|
|
"reward_std": 0.4613690562546253,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.11777753569185734,
|
|
"step": 183
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2771.0416870117188,
|
|
"epoch": 0.2102857142857143,
|
|
"grad_norm": 0.09264298528432846,
|
|
"kl": 4.6037137508392334e-05,
|
|
"lambda_div_used": 0.5391388088464737,
|
|
"learning_rate": 1.1966285981663407e-07,
|
|
"loss": -0.001,
|
|
"reward": -0.3703397810459137,
|
|
"reward_after_mean": -0.3703397810459137,
|
|
"reward_after_std": 0.2937673106789589,
|
|
"reward_before_mean": -0.011352727189660072,
|
|
"reward_before_std": 0.17188004031777382,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35898703522980213,
|
|
"reward_change_min": -0.5206632278859615,
|
|
"reward_change_std": 0.18913730140775442,
|
|
"reward_std": 0.29376731254160404,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.13635273277759552,
|
|
"step": 184
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2770.9792098999023,
|
|
"epoch": 0.21142857142857144,
|
|
"grad_norm": 0.09669750928878784,
|
|
"kl": 4.3720006942749023e-05,
|
|
"lambda_div_used": 0.5636111497879028,
|
|
"learning_rate": 1.1743223682775649e-07,
|
|
"loss": 0.0651,
|
|
"reward": -0.3112166179344058,
|
|
"reward_after_mean": -0.3112166179344058,
|
|
"reward_after_std": 0.3695005215704441,
|
|
"reward_before_mean": -0.0006957156583666801,
|
|
"reward_before_std": 0.2874382403679192,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3105209097266197,
|
|
"reward_change_min": -0.43808290362358093,
|
|
"reward_change_std": 0.17091987561434507,
|
|
"reward_std": 0.369500532746315,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.10486237239092588,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2965.375015258789,
|
|
"epoch": 0.21257142857142858,
|
|
"grad_norm": 0.05594424530863762,
|
|
"kl": 3.8955360651016235e-05,
|
|
"lambda_div_used": 0.608224630355835,
|
|
"learning_rate": 1.1533337816991931e-07,
|
|
"loss": -0.0681,
|
|
"reward": -0.20297073479741812,
|
|
"reward_after_mean": -0.20297073479741812,
|
|
"reward_after_std": 0.5205403883010149,
|
|
"reward_before_mean": 0.07951527182012796,
|
|
"reward_before_std": 0.502502404153347,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.282486030831933,
|
|
"reward_change_min": -0.47177543863654137,
|
|
"reward_change_std": 0.18455488048493862,
|
|
"reward_std": 0.5205404032021761,
|
|
"rewards/accuracy_reward": 0.16666666977107525,
|
|
"rewards/cosine_scaled_reward": -0.08715140074491501,
|
|
"step": 186
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2639.291702270508,
|
|
"epoch": 0.21371428571428572,
|
|
"grad_norm": 0.08843009173870087,
|
|
"kl": 4.921853542327881e-05,
|
|
"lambda_div_used": 0.5908937901258469,
|
|
"learning_rate": 1.1336692317580158e-07,
|
|
"loss": -0.0048,
|
|
"reward": -0.25490788742899895,
|
|
"reward_after_mean": -0.25490788742899895,
|
|
"reward_after_std": 0.4807432759553194,
|
|
"reward_before_mean": 0.04253344633616507,
|
|
"reward_before_std": 0.4119319263845682,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29744134843349457,
|
|
"reward_change_min": -0.4517475329339504,
|
|
"reward_change_std": 0.16703799460083246,
|
|
"reward_std": 0.48074328526854515,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.10329987667500973,
|
|
"step": 187
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3541.9791870117188,
|
|
"epoch": 0.21485714285714286,
|
|
"grad_norm": 0.05120408907532692,
|
|
"kl": 4.652142524719238e-05,
|
|
"lambda_div_used": 0.5589732080698013,
|
|
"learning_rate": 1.1153347084664419e-07,
|
|
"loss": -0.0029,
|
|
"reward": -0.4253443591296673,
|
|
"reward_after_mean": -0.4253443591296673,
|
|
"reward_after_std": 0.3237063102424145,
|
|
"reward_before_mean": -0.15165027976036072,
|
|
"reward_before_std": 0.26862882915884256,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27369407564401627,
|
|
"reward_change_min": -0.437398936599493,
|
|
"reward_change_std": 0.16332428343594074,
|
|
"reward_std": 0.3237063158303499,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.2141502844169736,
|
|
"step": 188
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2377.2708892822266,
|
|
"epoch": 0.216,
|
|
"grad_norm": 0.07240846008062363,
|
|
"kl": 5.1606446504592896e-05,
|
|
"lambda_div_used": 0.5700332000851631,
|
|
"learning_rate": 1.0983357966978745e-07,
|
|
"loss": 0.0342,
|
|
"reward": -0.3627132559195161,
|
|
"reward_after_mean": -0.3627132559195161,
|
|
"reward_after_std": 0.4038649797439575,
|
|
"reward_before_mean": -0.08412502333521843,
|
|
"reward_before_std": 0.3190039964392781,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2785882242023945,
|
|
"reward_change_min": -0.4183661602437496,
|
|
"reward_change_std": 0.15456592850387096,
|
|
"reward_std": 0.4038649834692478,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.1466250205412507,
|
|
"step": 189
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3024.041717529297,
|
|
"epoch": 0.21714285714285714,
|
|
"grad_norm": 0.0596420094370842,
|
|
"kl": 2.85319983959198e-05,
|
|
"lambda_div_used": 0.5832700356841087,
|
|
"learning_rate": 1.0826776744855121e-07,
|
|
"loss": -0.0126,
|
|
"reward": -0.17394864186644554,
|
|
"reward_after_mean": -0.17394864186644554,
|
|
"reward_after_std": 0.471806388348341,
|
|
"reward_before_mean": 0.198049274738878,
|
|
"reward_before_std": 0.38090503215789795,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.37199792452156544,
|
|
"reward_change_min": -0.5356397405266762,
|
|
"reward_change_std": 0.2117150044068694,
|
|
"reward_std": 0.4718063995242119,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.031117402017116547,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2440.3541870117188,
|
|
"epoch": 0.21828571428571428,
|
|
"grad_norm": 0.0756516084074974,
|
|
"kl": 3.930646926164627e-05,
|
|
"lambda_div_used": 0.5715140625834465,
|
|
"learning_rate": 1.068365111445064e-07,
|
|
"loss": 0.0003,
|
|
"reward": -0.284542384557426,
|
|
"reward_after_mean": -0.284542384557426,
|
|
"reward_after_std": 0.447479585185647,
|
|
"reward_before_mean": 0.05547440081136301,
|
|
"reward_before_std": 0.3240885529667139,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34001679718494415,
|
|
"reward_change_min": -0.4750482700765133,
|
|
"reward_change_std": 0.1778492433950305,
|
|
"reward_std": 0.4474795889109373,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.11119226738810539,
|
|
"step": 191
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3487.6041870117188,
|
|
"epoch": 0.21942857142857142,
|
|
"grad_norm": 0.048071179538965225,
|
|
"kl": 3.250688314437866e-05,
|
|
"lambda_div_used": 0.5586921945214272,
|
|
"learning_rate": 1.0554024673218806e-07,
|
|
"loss": 0.0206,
|
|
"reward": -0.3871428966522217,
|
|
"reward_after_mean": -0.3871428966522217,
|
|
"reward_after_std": 0.33264124393463135,
|
|
"reward_before_mean": -0.11114806681871414,
|
|
"reward_before_std": 0.2715215114876628,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27599484845995903,
|
|
"reward_change_min": -0.40478406473994255,
|
|
"reward_change_std": 0.15969175938516855,
|
|
"reward_std": 0.33264124765992165,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.19448141008615494,
|
|
"step": 192
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2929.937545776367,
|
|
"epoch": 0.22057142857142858,
|
|
"grad_norm": 0.06499005109071732,
|
|
"kl": 3.759562969207764e-05,
|
|
"lambda_div_used": 0.6119573265314102,
|
|
"learning_rate": 1.0437936906629334e-07,
|
|
"loss": 0.0198,
|
|
"reward": -0.24959510192275047,
|
|
"reward_after_mean": -0.24959510192275047,
|
|
"reward_after_std": 0.5340033005923033,
|
|
"reward_before_mean": 0.009953722357749939,
|
|
"reward_before_std": 0.520604582503438,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25954881869256496,
|
|
"reward_change_min": -0.5182105302810669,
|
|
"reward_change_std": 0.186925214715302,
|
|
"reward_std": 0.5340033229440451,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.1358796115964651,
|
|
"step": 193
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3132.875030517578,
|
|
"epoch": 0.22171428571428572,
|
|
"grad_norm": 0.06673065572977066,
|
|
"kl": 4.443526268005371e-05,
|
|
"lambda_div_used": 0.6207702159881592,
|
|
"learning_rate": 1.0335423176140511e-07,
|
|
"loss": 0.0719,
|
|
"reward": 0.2215312235057354,
|
|
"reward_after_mean": 0.2215312235057354,
|
|
"reward_after_std": 0.6115883849561214,
|
|
"reward_before_mean": 0.7030764240771532,
|
|
"reward_before_std": 0.5582148376852274,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.48154521360993385,
|
|
"reward_change_min": -0.7418999075889587,
|
|
"reward_change_std": 0.29459451511502266,
|
|
"reward_std": 0.611588392406702,
|
|
"rewards/accuracy_reward": 0.4583333469927311,
|
|
"rewards/cosine_scaled_reward": 0.2447430812753737,
|
|
"step": 194
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2998.687530517578,
|
|
"epoch": 0.22285714285714286,
|
|
"grad_norm": 0.05692190304398537,
|
|
"kl": 4.340708255767822e-05,
|
|
"lambda_div_used": 0.5588866546750069,
|
|
"learning_rate": 1.0246514708427701e-07,
|
|
"loss": -0.0321,
|
|
"reward": -0.4250563494861126,
|
|
"reward_after_mean": -0.4250563494861126,
|
|
"reward_after_std": 0.3346366826444864,
|
|
"reward_before_mean": -0.16300050355494022,
|
|
"reward_before_std": 0.26834714552387595,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2620558477938175,
|
|
"reward_change_min": -0.4044179916381836,
|
|
"reward_change_std": 0.15184260439127684,
|
|
"reward_std": 0.3346366863697767,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.22550049051642418,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3551.6041870117188,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.051314327865839005,
|
|
"kl": 4.8547983169555664e-05,
|
|
"lambda_div_used": 0.5673639252781868,
|
|
"learning_rate": 1.017123858587145e-07,
|
|
"loss": 0.0144,
|
|
"reward": -0.43661339208483696,
|
|
"reward_after_mean": -0.43661339208483696,
|
|
"reward_after_std": 0.3381017968058586,
|
|
"reward_before_mean": -0.18068170547485352,
|
|
"reward_before_std": 0.3060085875913501,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25593167915940285,
|
|
"reward_change_min": -0.436182364821434,
|
|
"reward_change_std": 0.16187659837305546,
|
|
"reward_std": 0.3381018117070198,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.24318172316998243,
|
|
"step": 196
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2789.125068664551,
|
|
"epoch": 0.22514285714285714,
|
|
"grad_norm": 0.08223231136798859,
|
|
"kl": 3.402773290872574e-05,
|
|
"lambda_div_used": 0.6149426028132439,
|
|
"learning_rate": 1.0109617738307911e-07,
|
|
"loss": 0.0526,
|
|
"reward": 0.08269345387816429,
|
|
"reward_after_mean": 0.08269345387816429,
|
|
"reward_after_std": 0.5922380294650793,
|
|
"reward_before_mean": 0.4997589197009802,
|
|
"reward_before_std": 0.5276397680863738,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4170654658228159,
|
|
"reward_change_min": -0.6376415118575096,
|
|
"reward_change_std": 0.24999654106795788,
|
|
"reward_std": 0.592238049954176,
|
|
"rewards/accuracy_reward": 0.39583334140479565,
|
|
"rewards/cosine_scaled_reward": 0.10392560251057148,
|
|
"step": 197
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2770.2708435058594,
|
|
"epoch": 0.22628571428571428,
|
|
"grad_norm": 0.07601747661828995,
|
|
"kl": 3.542006015777588e-05,
|
|
"lambda_div_used": 0.5800714045763016,
|
|
"learning_rate": 1.0061670936044178e-07,
|
|
"loss": -0.0178,
|
|
"reward": -0.24220025539398193,
|
|
"reward_after_mean": -0.24220025539398193,
|
|
"reward_after_std": 0.4058863651007414,
|
|
"reward_before_mean": 0.06675046496093273,
|
|
"reward_before_std": 0.36683204025030136,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30895073898136616,
|
|
"reward_change_min": -0.4967211000621319,
|
|
"reward_change_std": 0.18911676667630672,
|
|
"reward_std": 0.40588637441396713,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.09991620294749737,
|
|
"step": 198
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3554.0416870117188,
|
|
"epoch": 0.22742857142857142,
|
|
"grad_norm": 0.04930044710636139,
|
|
"kl": 4.279613494873047e-05,
|
|
"lambda_div_used": 0.5560082867741585,
|
|
"learning_rate": 1.002741278414069e-07,
|
|
"loss": 0.0027,
|
|
"reward": -0.4229874052107334,
|
|
"reward_after_mean": -0.4229874052107334,
|
|
"reward_after_std": 0.31992789916694164,
|
|
"reward_before_mean": -0.14317837683483958,
|
|
"reward_before_std": 0.25433824164792895,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27980900928378105,
|
|
"reward_change_min": -0.4250169135630131,
|
|
"reward_change_std": 0.16277197189629078,
|
|
"reward_std": 0.31992790661752224,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.20567839220166206,
|
|
"step": 199
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2299.5209045410156,
|
|
"epoch": 0.22857142857142856,
|
|
"grad_norm": 0.0840829536318779,
|
|
"kl": 4.097074270248413e-05,
|
|
"lambda_div_used": 0.6226711273193359,
|
|
"learning_rate": 1.0006853717962393e-07,
|
|
"loss": 0.0203,
|
|
"reward": 0.062016794458031654,
|
|
"reward_after_mean": 0.062016794458031654,
|
|
"reward_after_std": 0.631916331127286,
|
|
"reward_before_mean": 0.4668920338153839,
|
|
"reward_before_std": 0.5674512181431055,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.40487524308264256,
|
|
"reward_change_min": -0.6527138948440552,
|
|
"reward_change_std": 0.250981617718935,
|
|
"reward_std": 0.6319163534790277,
|
|
"rewards/accuracy_reward": 0.35416667349636555,
|
|
"rewards/cosine_scaled_reward": 0.11272535985335708,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2760.7084197998047,
|
|
"epoch": 0.2297142857142857,
|
|
"grad_norm": 0.06477575749158859,
|
|
"kl": 1.7711427062749863e-05,
|
|
"lambda_div_used": 0.6429438292980194,
|
|
"learning_rate": 1e-07,
|
|
"loss": -0.0022,
|
|
"reward": 0.15019571036100388,
|
|
"reward_after_mean": 0.15019571036100388,
|
|
"reward_after_std": 0.7328120246529579,
|
|
"reward_before_mean": 0.5535202682949603,
|
|
"reward_before_std": 0.6601617820560932,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4033245462924242,
|
|
"reward_change_min": -0.6303750872612,
|
|
"reward_change_std": 0.24585676938295364,
|
|
"reward_std": 0.7328120358288288,
|
|
"rewards/accuracy_reward": 0.41666667349636555,
|
|
"rewards/cosine_scaled_reward": 0.13685358315706253,
|
|
"step": 201
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2511.937511444092,
|
|
"epoch": 0.23085714285714284,
|
|
"grad_norm": 0.08345558494329453,
|
|
"kl": 4.0203332901000977e-05,
|
|
"lambda_div_used": 0.5811712816357613,
|
|
"learning_rate": 7.72273839962904e-07,
|
|
"loss": 0.0587,
|
|
"reward": 0.03902309015393257,
|
|
"reward_after_mean": 0.03902309015393257,
|
|
"reward_after_std": 0.5094954669475555,
|
|
"reward_before_mean": 0.5408022310584784,
|
|
"reward_before_std": 0.37269798293709755,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5017791502177715,
|
|
"reward_change_min": -0.7236880213022232,
|
|
"reward_change_std": 0.28125342447310686,
|
|
"reward_std": 0.5094954781234264,
|
|
"rewards/accuracy_reward": 0.3958333395421505,
|
|
"rewards/cosine_scaled_reward": 0.14496888127177954,
|
|
"step": 202
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3065.3333435058594,
|
|
"epoch": 0.232,
|
|
"grad_norm": 0.05931547284126282,
|
|
"kl": 5.182623863220215e-05,
|
|
"lambda_div_used": 0.5537669658660889,
|
|
"learning_rate": 7.695368466124296e-07,
|
|
"loss": 0.0447,
|
|
"reward": -0.3224771413952112,
|
|
"reward_after_mean": -0.3224771413952112,
|
|
"reward_after_std": 0.36307925172150135,
|
|
"reward_before_mean": 0.029698201455175877,
|
|
"reward_before_std": 0.23980092909187078,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3521753493696451,
|
|
"reward_change_min": -0.49568963050842285,
|
|
"reward_change_std": 0.18385440576821566,
|
|
"reward_std": 0.36307926289737225,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.1161351166665554,
|
|
"step": 203
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2511.833366394043,
|
|
"epoch": 0.23314285714285715,
|
|
"grad_norm": 0.07267706841230392,
|
|
"kl": 5.4508447647094727e-05,
|
|
"lambda_div_used": 0.5887176766991615,
|
|
"learning_rate": 7.667891533457718e-07,
|
|
"loss": 0.0023,
|
|
"reward": -0.23160922899842262,
|
|
"reward_after_mean": -0.23160922899842262,
|
|
"reward_after_std": 0.430733734741807,
|
|
"reward_before_mean": 0.06992286071181297,
|
|
"reward_before_std": 0.4116028640419245,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.301532082259655,
|
|
"reward_change_min": -0.49793890863657,
|
|
"reward_change_std": 0.19511268101632595,
|
|
"reward_std": 0.43073374405503273,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.07591048628091812,
|
|
"step": 204
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2967.1875915527344,
|
|
"epoch": 0.2342857142857143,
|
|
"grad_norm": 0.06776981800794601,
|
|
"kl": 4.439055919647217e-05,
|
|
"lambda_div_used": 0.653583250939846,
|
|
"learning_rate": 7.640308940816239e-07,
|
|
"loss": 0.1014,
|
|
"reward": -0.08090712130069733,
|
|
"reward_after_mean": -0.08090712130069733,
|
|
"reward_after_std": 0.7262772191315889,
|
|
"reward_before_mean": 0.17835846357047558,
|
|
"reward_before_std": 0.7181071005761623,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2592655848711729,
|
|
"reward_change_min": -0.4743462074548006,
|
|
"reward_change_std": 0.18475584778934717,
|
|
"reward_std": 0.7262772284448147,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.05080821271985769,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3085.5208892822266,
|
|
"epoch": 0.23542857142857143,
|
|
"grad_norm": 0.06689873337745667,
|
|
"kl": 3.569573163986206e-05,
|
|
"lambda_div_used": 0.5862618833780289,
|
|
"learning_rate": 7.612622032536507e-07,
|
|
"loss": 0.0353,
|
|
"reward": -0.28365214727818966,
|
|
"reward_after_mean": -0.28365214727818966,
|
|
"reward_after_std": 0.44012872874736786,
|
|
"reward_before_mean": 0.004427256062626839,
|
|
"reward_before_std": 0.3988625044003129,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2880793921649456,
|
|
"reward_change_min": -0.49235254526138306,
|
|
"reward_change_std": 0.18395678792148829,
|
|
"reward_std": 0.44012873619794846,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.12057275045663118,
|
|
"step": 206
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3146.6458740234375,
|
|
"epoch": 0.23657142857142857,
|
|
"grad_norm": 0.06122450903058052,
|
|
"kl": 3.725104033946991e-05,
|
|
"lambda_div_used": 0.579835832118988,
|
|
"learning_rate": 7.584832158039378e-07,
|
|
"loss": 0.0182,
|
|
"reward": -0.28688727831467986,
|
|
"reward_after_mean": -0.28688727831467986,
|
|
"reward_after_std": 0.4150819983333349,
|
|
"reward_before_mean": 0.009919969365000725,
|
|
"reward_before_std": 0.3629662115126848,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29680725932121277,
|
|
"reward_change_min": -0.45919613167643547,
|
|
"reward_change_std": 0.17279880121350288,
|
|
"reward_std": 0.41508200392127037,
|
|
"rewards/accuracy_reward": 0.12500000558793545,
|
|
"rewards/cosine_scaled_reward": -0.11508002015762031,
|
|
"step": 207
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2780.291717529297,
|
|
"epoch": 0.2377142857142857,
|
|
"grad_norm": 0.06370716542005539,
|
|
"kl": 3.692507743835449e-05,
|
|
"lambda_div_used": 0.5807016789913177,
|
|
"learning_rate": 7.556940671764124e-07,
|
|
"loss": 0.0403,
|
|
"reward": -0.17094913870096207,
|
|
"reward_after_mean": -0.17094913870096207,
|
|
"reward_after_std": 0.46442117914557457,
|
|
"reward_before_mean": 0.20349296741187572,
|
|
"reward_before_std": 0.3671952560544014,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3744421415030956,
|
|
"reward_change_min": -0.5505415536463261,
|
|
"reward_change_std": 0.21210224367678165,
|
|
"reward_std": 0.46442119032144547,
|
|
"rewards/accuracy_reward": 0.20833333395421505,
|
|
"rewards/cosine_scaled_reward": -0.004840357229113579,
|
|
"step": 208
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2574.6042079925537,
|
|
"epoch": 0.23885714285714285,
|
|
"grad_norm": 0.08756011724472046,
|
|
"kl": 1.8547754734754562e-05,
|
|
"lambda_div_used": 0.6246868968009949,
|
|
"learning_rate": 7.528948933102438e-07,
|
|
"loss": 0.0837,
|
|
"reward": 0.028150439262390137,
|
|
"reward_after_mean": 0.028150439262390137,
|
|
"reward_after_std": 0.5636487938463688,
|
|
"reward_before_mean": 0.3838757500052452,
|
|
"reward_before_std": 0.576031070202589,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3557253200560808,
|
|
"reward_change_min": -0.5899170190095901,
|
|
"reward_change_std": 0.23946599010378122,
|
|
"reward_std": 0.5636488180607557,
|
|
"rewards/accuracy_reward": 0.3333333469927311,
|
|
"rewards/cosine_scaled_reward": 0.05054241791367531,
|
|
"step": 209
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2779.3333435058594,
|
|
"epoch": 0.24,
|
|
"grad_norm": 0.06111512333154678,
|
|
"kl": 3.0368566513061523e-05,
|
|
"lambda_div_used": 0.5897415727376938,
|
|
"learning_rate": 7.500858306332172e-07,
|
|
"loss": 0.0277,
|
|
"reward": -0.25500940857455134,
|
|
"reward_after_mean": -0.25500940857455134,
|
|
"reward_after_std": 0.47190882451832294,
|
|
"reward_before_mean": 0.03478116978658363,
|
|
"reward_before_std": 0.4098346810787916,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2897905595600605,
|
|
"reward_change_min": -0.42270801588892937,
|
|
"reward_change_std": 0.1620404813438654,
|
|
"reward_std": 0.4719088301062584,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.11105216934811324,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2558.8541946411133,
|
|
"epoch": 0.24114285714285713,
|
|
"grad_norm": 0.06923027336597443,
|
|
"kl": 3.6522746086120605e-05,
|
|
"lambda_div_used": 0.5664671063423157,
|
|
"learning_rate": 7.472670160550848e-07,
|
|
"loss": 0.0729,
|
|
"reward": -0.06898702308535576,
|
|
"reward_after_mean": -0.06898702308535576,
|
|
"reward_after_std": 0.4354093614965677,
|
|
"reward_before_mean": 0.4075740482658148,
|
|
"reward_before_std": 0.30287738889455795,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4765610620379448,
|
|
"reward_change_min": -0.6883851811289787,
|
|
"reward_change_std": 0.262312775477767,
|
|
"reward_std": 0.4354093801230192,
|
|
"rewards/accuracy_reward": 0.3125,
|
|
"rewards/cosine_scaled_reward": 0.09507404454052448,
|
|
"step": 211
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2327.750015258789,
|
|
"epoch": 0.2422857142857143,
|
|
"grad_norm": 0.08028864860534668,
|
|
"kl": 2.8003007173538208e-05,
|
|
"lambda_div_used": 0.5712975114583969,
|
|
"learning_rate": 7.444385869608921e-07,
|
|
"loss": -0.0113,
|
|
"reward": -0.09517045877873898,
|
|
"reward_after_mean": -0.09517045877873898,
|
|
"reward_after_std": 0.49211281538009644,
|
|
"reward_before_mean": 0.3566615767776966,
|
|
"reward_before_std": 0.32028379291296005,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4518320318311453,
|
|
"reward_change_min": -0.6186705827713013,
|
|
"reward_change_std": 0.23167487233877182,
|
|
"reward_std": 0.4921128321439028,
|
|
"rewards/accuracy_reward": 0.2916666679084301,
|
|
"rewards/cosine_scaled_reward": 0.06499490264104679,
|
|
"step": 212
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2145.6041946411133,
|
|
"epoch": 0.24342857142857144,
|
|
"grad_norm": 0.09073800593614578,
|
|
"kl": 4.177819937467575e-05,
|
|
"lambda_div_used": 0.6168783828616142,
|
|
"learning_rate": 7.416006812042827e-07,
|
|
"loss": 0.027,
|
|
"reward": -0.08367926510982215,
|
|
"reward_after_mean": -0.08367926510982215,
|
|
"reward_after_std": 0.5790203902870417,
|
|
"reward_before_mean": 0.24448610469698906,
|
|
"reward_before_std": 0.5364782512187958,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32816537097096443,
|
|
"reward_change_min": -0.5068525895476341,
|
|
"reward_change_std": 0.19713077135384083,
|
|
"reward_std": 0.579020407050848,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": 0.036152773071080446,
|
|
"step": 213
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2982.6458587646484,
|
|
"epoch": 0.24457142857142858,
|
|
"grad_norm": 0.06497277319431305,
|
|
"kl": 3.772228956222534e-05,
|
|
"lambda_div_used": 0.6168569102883339,
|
|
"learning_rate": 7.387534371007797e-07,
|
|
"loss": 0.0247,
|
|
"reward": -0.10639690980315208,
|
|
"reward_after_mean": -0.10639690980315208,
|
|
"reward_after_std": 0.563305439427495,
|
|
"reward_before_mean": 0.21648684330284595,
|
|
"reward_before_std": 0.5375220291316509,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3228837437927723,
|
|
"reward_change_min": -0.5641361065208912,
|
|
"reward_change_std": 0.20987980626523495,
|
|
"reward_std": 0.5633054543286562,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": 0.00815348862670362,
|
|
"step": 214
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2382.2500610351562,
|
|
"epoch": 0.24571428571428572,
|
|
"grad_norm": 0.07877103984355927,
|
|
"kl": 4.485622048377991e-05,
|
|
"lambda_div_used": 0.5695274397730827,
|
|
"learning_rate": 7.358969934210438e-07,
|
|
"loss": 0.0351,
|
|
"reward": -0.39196348818950355,
|
|
"reward_after_mean": -0.39196348818950355,
|
|
"reward_after_std": 0.39862883277237415,
|
|
"reward_before_mean": -0.128106027841568,
|
|
"reward_before_std": 0.31178954988718033,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.263857439160347,
|
|
"reward_change_min": -0.38021961972117424,
|
|
"reward_change_std": 0.13994430005550385,
|
|
"reward_std": 0.39862884022295475,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1697727027349174,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2156.3958740234375,
|
|
"epoch": 0.24685714285714286,
|
|
"grad_norm": 0.11318857222795486,
|
|
"kl": 4.297494888305664e-05,
|
|
"lambda_div_used": 0.5965342745184898,
|
|
"learning_rate": 7.330314893841101e-07,
|
|
"loss": 0.0186,
|
|
"reward": -0.22045887634158134,
|
|
"reward_after_mean": -0.22045887634158134,
|
|
"reward_after_std": 0.4805217906832695,
|
|
"reward_before_mean": 0.08258800266776234,
|
|
"reward_before_std": 0.4415160808712244,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3030468635261059,
|
|
"reward_change_min": -0.4999500457197428,
|
|
"reward_change_std": 0.185057258233428,
|
|
"reward_std": 0.4805217981338501,
|
|
"rewards/accuracy_reward": 0.16666667349636555,
|
|
"rewards/cosine_scaled_reward": -0.08407866954803467,
|
|
"step": 216
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2763.520866394043,
|
|
"epoch": 0.248,
|
|
"grad_norm": 0.067295141518116,
|
|
"kl": 3.0644237995147705e-05,
|
|
"lambda_div_used": 0.6201670467853546,
|
|
"learning_rate": 7.301570646506027e-07,
|
|
"loss": 0.0139,
|
|
"reward": -0.19884846359491348,
|
|
"reward_after_mean": -0.19884846359491348,
|
|
"reward_after_std": 0.5856006741523743,
|
|
"reward_before_mean": 0.06158541014883667,
|
|
"reward_before_std": 0.5545480605214834,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2604338899254799,
|
|
"reward_change_min": -0.503634799271822,
|
|
"reward_change_std": 0.1784855630248785,
|
|
"reward_std": 0.5856006946414709,
|
|
"rewards/accuracy_reward": 0.16666666977107525,
|
|
"rewards/cosine_scaled_reward": -0.10508125275373459,
|
|
"step": 217
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2880.7083587646484,
|
|
"epoch": 0.24914285714285714,
|
|
"grad_norm": 0.07706139236688614,
|
|
"kl": 3.7044286727905273e-05,
|
|
"lambda_div_used": 0.6074161231517792,
|
|
"learning_rate": 7.27273859315928e-07,
|
|
"loss": -0.0223,
|
|
"reward": -0.19500153325498104,
|
|
"reward_after_mean": -0.19500153325498104,
|
|
"reward_after_std": 0.5332945492118597,
|
|
"reward_before_mean": 0.08328226953744888,
|
|
"reward_before_std": 0.5006468072533607,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27828381210565567,
|
|
"reward_change_min": -0.5113113783299923,
|
|
"reward_change_std": 0.18696410488337278,
|
|
"reward_std": 0.5332945715636015,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.08338439278304577,
|
|
"step": 218
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2352.645854949951,
|
|
"epoch": 0.2502857142857143,
|
|
"grad_norm": 0.10948827862739563,
|
|
"kl": 3.1441450119018555e-05,
|
|
"lambda_div_used": 0.5561031624674797,
|
|
"learning_rate": 7.243820139034464e-07,
|
|
"loss": -0.0096,
|
|
"reward": -0.029067307710647583,
|
|
"reward_after_mean": -0.029067307710647583,
|
|
"reward_after_std": 0.41141366213560104,
|
|
"reward_before_mean": 0.4911606255918741,
|
|
"reward_before_std": 0.2519808644428849,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5202279426157475,
|
|
"reward_change_min": -0.7093416638672352,
|
|
"reward_change_std": 0.27748389169573784,
|
|
"reward_std": 0.41141367703676224,
|
|
"rewards/accuracy_reward": 0.3541666716337204,
|
|
"rewards/cosine_scaled_reward": 0.13699393905699253,
|
|
"step": 219
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2635.5833435058594,
|
|
"epoch": 0.25142857142857145,
|
|
"grad_norm": 0.08914919197559357,
|
|
"kl": 3.5672448575496674e-05,
|
|
"lambda_div_used": 0.5308618620038033,
|
|
"learning_rate": 7.214816693576234e-07,
|
|
"loss": -0.0362,
|
|
"reward": -0.5146691724658012,
|
|
"reward_after_mean": -0.5146691724658012,
|
|
"reward_after_std": 0.23347610421478748,
|
|
"reward_before_mean": -0.2369950506836176,
|
|
"reward_before_std": 0.13545648753643036,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27767411433160305,
|
|
"reward_change_min": -0.3908136747777462,
|
|
"reward_change_std": 0.14405533485114574,
|
|
"reward_std": 0.23347610607743263,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2369950469583273,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2279.562530517578,
|
|
"epoch": 0.25257142857142856,
|
|
"grad_norm": 0.0954662561416626,
|
|
"kl": 2.4788081645965576e-05,
|
|
"lambda_div_used": 0.5764833092689514,
|
|
"learning_rate": 7.185729670371604e-07,
|
|
"loss": -0.0168,
|
|
"reward": 0.023251693695783615,
|
|
"reward_after_mean": 0.023251693695783615,
|
|
"reward_after_std": 0.4863298423588276,
|
|
"reward_before_mean": 0.5212067291140556,
|
|
"reward_before_std": 0.34889572812244296,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.49795500561594963,
|
|
"reward_change_min": -0.7016883753240108,
|
|
"reward_change_std": 0.27911104913800955,
|
|
"reward_std": 0.48632985167205334,
|
|
"rewards/accuracy_reward": 0.4166666716337204,
|
|
"rewards/cosine_scaled_reward": 0.10454002395272255,
|
|
"step": 221
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2104.9583740234375,
|
|
"epoch": 0.2537142857142857,
|
|
"grad_norm": 0.0776790902018547,
|
|
"kl": 2.4726614356040955e-05,
|
|
"lambda_div_used": 0.5997953563928604,
|
|
"learning_rate": 7.156560487081051e-07,
|
|
"loss": -0.0148,
|
|
"reward": -0.04329463094472885,
|
|
"reward_after_mean": -0.04329463094472885,
|
|
"reward_after_std": 0.48106030002236366,
|
|
"reward_before_mean": 0.33003126084804535,
|
|
"reward_before_std": 0.46051184553653,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3733258917927742,
|
|
"reward_change_min": -0.5728014186024666,
|
|
"reward_change_std": 0.23287776950746775,
|
|
"reward_std": 0.48106031119823456,
|
|
"rewards/accuracy_reward": 0.2708333432674408,
|
|
"rewards/cosine_scaled_reward": 0.0591979268938303,
|
|
"step": 222
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2521.3333587646484,
|
|
"epoch": 0.25485714285714284,
|
|
"grad_norm": 0.07036174088716507,
|
|
"kl": 2.9304996132850647e-05,
|
|
"lambda_div_used": 0.5750112235546112,
|
|
"learning_rate": 7.127310565369415e-07,
|
|
"loss": 0.0216,
|
|
"reward": -0.1473498847335577,
|
|
"reward_after_mean": -0.1473498847335577,
|
|
"reward_after_std": 0.4359878208488226,
|
|
"reward_before_mean": 0.24288302287459373,
|
|
"reward_before_std": 0.3438769578933716,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.39023288898169994,
|
|
"reward_change_min": -0.588653527200222,
|
|
"reward_change_std": 0.22452317085117102,
|
|
"reward_std": 0.43598783388733864,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": -0.027950339019298553,
|
|
"step": 223
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3317.729217529297,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.05309538170695305,
|
|
"kl": 2.8050970286130905e-05,
|
|
"lambda_div_used": 0.5725216493010521,
|
|
"learning_rate": 7.097981330836616e-07,
|
|
"loss": 0.069,
|
|
"reward": -0.30242439545691013,
|
|
"reward_after_mean": -0.30242439545691013,
|
|
"reward_after_std": 0.38869454339146614,
|
|
"reward_before_mean": -0.000308917835354805,
|
|
"reward_before_std": 0.3251040354371071,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30211549811065197,
|
|
"reward_change_min": -0.4553787522017956,
|
|
"reward_change_std": 0.171081081032753,
|
|
"reward_std": 0.3886945564299822,
|
|
"rewards/accuracy_reward": 0.12500000558793545,
|
|
"rewards/cosine_scaled_reward": -0.12530891119968146,
|
|
"step": 224
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3037.208335876465,
|
|
"epoch": 0.2571428571428571,
|
|
"grad_norm": 0.0768936425447464,
|
|
"kl": 4.080682992935181e-05,
|
|
"lambda_div_used": 0.5907088667154312,
|
|
"learning_rate": 7.068574212948169e-07,
|
|
"loss": 0.0169,
|
|
"reward": -0.28160014655441046,
|
|
"reward_after_mean": -0.28160014655441046,
|
|
"reward_after_std": 0.48472702503204346,
|
|
"reward_before_mean": -0.008234186680056155,
|
|
"reward_before_std": 0.41540220472961664,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27336596697568893,
|
|
"reward_change_min": -0.4061685614287853,
|
|
"reward_change_std": 0.15673903841525316,
|
|
"reward_std": 0.48472702503204346,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.13323417864739895,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2861.416702270508,
|
|
"epoch": 0.2582857142857143,
|
|
"grad_norm": 0.07499676197767258,
|
|
"kl": 3.580749034881592e-05,
|
|
"lambda_div_used": 0.6208535805344582,
|
|
"learning_rate": 7.039090644965509e-07,
|
|
"loss": 0.0702,
|
|
"reward": -0.08291551470756531,
|
|
"reward_after_mean": -0.08291551470756531,
|
|
"reward_after_std": 0.5764818880707026,
|
|
"reward_before_mean": 0.23111886344850063,
|
|
"reward_before_std": 0.5593543313443661,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3140343725681305,
|
|
"reward_change_min": -0.5283008627593517,
|
|
"reward_change_std": 0.20683539099991322,
|
|
"reward_std": 0.5764818955212831,
|
|
"rewards/accuracy_reward": 0.27083334140479565,
|
|
"rewards/cosine_scaled_reward": -0.03971448230731767,
|
|
"step": 226
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1926.6875305175781,
|
|
"epoch": 0.25942857142857145,
|
|
"grad_norm": 0.10923109948635101,
|
|
"kl": 4.176795482635498e-05,
|
|
"lambda_div_used": 0.5834182798862457,
|
|
"learning_rate": 7.009532063876148e-07,
|
|
"loss": -0.0447,
|
|
"reward": -0.2778073139488697,
|
|
"reward_after_mean": -0.2778073139488697,
|
|
"reward_after_std": 0.4395454227924347,
|
|
"reward_before_mean": 0.009000460617244244,
|
|
"reward_before_std": 0.3901812704280019,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28680778108537197,
|
|
"reward_change_min": -0.45677450299263,
|
|
"reward_change_std": 0.18111994117498398,
|
|
"reward_std": 0.43954543210566044,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.1159995449706912,
|
|
"step": 227
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2502.562545776367,
|
|
"epoch": 0.26057142857142856,
|
|
"grad_norm": 0.08182154595851898,
|
|
"kl": 1.8984079360961914e-05,
|
|
"lambda_div_used": 0.6025907471776009,
|
|
"learning_rate": 6.979899910323624e-07,
|
|
"loss": 0.074,
|
|
"reward": 0.03976001590490341,
|
|
"reward_after_mean": 0.03976001590490341,
|
|
"reward_after_std": 0.5421929359436035,
|
|
"reward_before_mean": 0.4711545445024967,
|
|
"reward_before_std": 0.4750876808539033,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4313945174217224,
|
|
"reward_change_min": -0.6774051003158092,
|
|
"reward_change_std": 0.2637105621397495,
|
|
"reward_std": 0.5421929433941841,
|
|
"rewards/accuracy_reward": 0.3541666753590107,
|
|
"rewards/cosine_scaled_reward": 0.11698784306645393,
|
|
"step": 228
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3196.3125228881836,
|
|
"epoch": 0.26171428571428573,
|
|
"grad_norm": 0.06398586928844452,
|
|
"kl": 4.0434300899505615e-05,
|
|
"lambda_div_used": 0.5669709742069244,
|
|
"learning_rate": 6.950195628537299e-07,
|
|
"loss": 0.0135,
|
|
"reward": -0.15348458290100098,
|
|
"reward_after_mean": -0.15348458290100098,
|
|
"reward_after_std": 0.42672324273735285,
|
|
"reward_before_mean": 0.26130594592541456,
|
|
"reward_before_std": 0.3047938751988113,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41479056514799595,
|
|
"reward_change_min": -0.5674791261553764,
|
|
"reward_change_std": 0.22689451277256012,
|
|
"reward_std": 0.4267232706770301,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": 0.03213928057812154,
|
|
"step": 229
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3100.5000228881836,
|
|
"epoch": 0.26285714285714284,
|
|
"grad_norm": 0.05996527522802353,
|
|
"kl": 2.22623348236084e-05,
|
|
"lambda_div_used": 0.5811162814497948,
|
|
"learning_rate": 6.920420666261961e-07,
|
|
"loss": 0.0391,
|
|
"reward": -0.36068666726350784,
|
|
"reward_after_mean": -0.36068666726350784,
|
|
"reward_after_std": 0.43734684213995934,
|
|
"reward_before_mean": -0.08804995659738779,
|
|
"reward_before_std": 0.37305002473294735,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2726367134600878,
|
|
"reward_change_min": -0.4353605732321739,
|
|
"reward_change_std": 0.16089125256985426,
|
|
"reward_std": 0.43734684586524963,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.15054996183607727,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2678.7083587646484,
|
|
"epoch": 0.264,
|
|
"grad_norm": 0.06770680844783783,
|
|
"kl": 2.5890767574310303e-05,
|
|
"lambda_div_used": 0.5817164853215218,
|
|
"learning_rate": 6.890576474687263e-07,
|
|
"loss": 0.0722,
|
|
"reward": -0.13625335786491632,
|
|
"reward_after_mean": -0.13625335786491632,
|
|
"reward_after_std": 0.46202925965189934,
|
|
"reward_before_mean": 0.25235490314662457,
|
|
"reward_before_std": 0.37311020120978355,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3886082824319601,
|
|
"reward_change_min": -0.6004670634865761,
|
|
"reward_change_std": 0.2275555245578289,
|
|
"reward_std": 0.4620292726904154,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": -0.018478410318493843,
|
|
"step": 231
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3258.625030517578,
|
|
"epoch": 0.2651428571428571,
|
|
"grad_norm": 0.05468503385782242,
|
|
"kl": 3.515183925628662e-05,
|
|
"lambda_div_used": 0.5800251811742783,
|
|
"learning_rate": 6.860664508377001e-07,
|
|
"loss": 0.0159,
|
|
"reward": -0.3428461756557226,
|
|
"reward_after_mean": -0.3428461756557226,
|
|
"reward_after_std": 0.4266065489500761,
|
|
"reward_before_mean": -0.07336848601698875,
|
|
"reward_before_std": 0.36755594704300165,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26947769708931446,
|
|
"reward_change_min": -0.43084757775068283,
|
|
"reward_change_std": 0.1597052849829197,
|
|
"reward_std": 0.4266065787523985,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.1567018236964941,
|
|
"step": 232
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2661.2292251586914,
|
|
"epoch": 0.2662857142857143,
|
|
"grad_norm": 0.08775883167982101,
|
|
"kl": 3.3989548683166504e-05,
|
|
"lambda_div_used": 0.6155472174286842,
|
|
"learning_rate": 6.83068622519821e-07,
|
|
"loss": -0.016,
|
|
"reward": -0.22876367531716824,
|
|
"reward_after_mean": -0.22876367531716824,
|
|
"reward_after_std": 0.5766322333365679,
|
|
"reward_before_mean": 0.027781556826084852,
|
|
"reward_before_std": 0.5290827043354511,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2565452288836241,
|
|
"reward_change_min": -0.4581022933125496,
|
|
"reward_change_std": 0.16627201065421104,
|
|
"reward_std": 0.5766322445124388,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.11805178504437208,
|
|
"step": 233
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2694.3125228881836,
|
|
"epoch": 0.2674285714285714,
|
|
"grad_norm": 0.11633959412574768,
|
|
"kl": 3.643333911895752e-05,
|
|
"lambda_div_used": 0.5539941042661667,
|
|
"learning_rate": 6.800643086250121e-07,
|
|
"loss": -0.0056,
|
|
"reward": -0.22800572216510773,
|
|
"reward_after_mean": -0.22800572216510773,
|
|
"reward_after_std": 0.3484720904380083,
|
|
"reward_before_mean": 0.16009100899100304,
|
|
"reward_before_std": 0.24206165876239538,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3880967255681753,
|
|
"reward_change_min": -0.5548957660794258,
|
|
"reward_change_std": 0.21049270872026682,
|
|
"reward_std": 0.34847209975123405,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.06907567009329796,
|
|
"step": 234
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2490.4375,
|
|
"epoch": 0.26857142857142857,
|
|
"grad_norm": 0.10378725826740265,
|
|
"kl": 2.8094742447137833e-05,
|
|
"lambda_div_used": 0.5782083421945572,
|
|
"learning_rate": 6.770536555792944e-07,
|
|
"loss": -0.0425,
|
|
"reward": 0.01590564101934433,
|
|
"reward_after_mean": 0.01590564101934433,
|
|
"reward_after_std": 0.5053408965468407,
|
|
"reward_before_mean": 0.49857149738818407,
|
|
"reward_before_std": 0.3586234971880913,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.48266585171222687,
|
|
"reward_change_min": -0.7199156694114208,
|
|
"reward_change_std": 0.2697788691148162,
|
|
"reward_std": 0.505340900272131,
|
|
"rewards/accuracy_reward": 0.3750000037252903,
|
|
"rewards/cosine_scaled_reward": 0.12357146013528109,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2824.7917098999023,
|
|
"epoch": 0.26971428571428574,
|
|
"grad_norm": 0.06474865972995758,
|
|
"kl": 2.1502375602722168e-05,
|
|
"lambda_div_used": 0.5763456672430038,
|
|
"learning_rate": 6.740368101176495e-07,
|
|
"loss": 0.0011,
|
|
"reward": -0.3977004513144493,
|
|
"reward_after_mean": -0.3977004513144493,
|
|
"reward_after_std": 0.41489073634147644,
|
|
"reward_before_mean": -0.1460555698722601,
|
|
"reward_before_std": 0.347917802631855,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2516448702663183,
|
|
"reward_change_min": -0.42327918112277985,
|
|
"reward_change_std": 0.14836463797837496,
|
|
"reward_std": 0.4148907568305731,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.20855557406321168,
|
|
"step": 236
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2764.9583740234375,
|
|
"epoch": 0.27085714285714285,
|
|
"grad_norm": 0.06224860996007919,
|
|
"kl": 2.5756657123565674e-05,
|
|
"lambda_div_used": 0.5886275842785835,
|
|
"learning_rate": 6.710139192768694e-07,
|
|
"loss": -0.0228,
|
|
"reward": -0.1175742200575769,
|
|
"reward_after_mean": -0.1175742200575769,
|
|
"reward_after_std": 0.4582329224795103,
|
|
"reward_before_mean": 0.24255024455487728,
|
|
"reward_before_std": 0.40142686292529106,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36012447625398636,
|
|
"reward_change_min": -0.5257687419652939,
|
|
"reward_change_std": 0.20567627251148224,
|
|
"reward_std": 0.4582329299300909,
|
|
"rewards/accuracy_reward": 0.22916667722165585,
|
|
"rewards/cosine_scaled_reward": 0.013383567042183131,
|
|
"step": 237
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3222.479217529297,
|
|
"epoch": 0.272,
|
|
"grad_norm": 0.05841728672385216,
|
|
"kl": 3.546103835105896e-05,
|
|
"lambda_div_used": 0.6281886473298073,
|
|
"learning_rate": 6.679851303883891e-07,
|
|
"loss": 0.0655,
|
|
"reward": 0.044486068189144135,
|
|
"reward_after_mean": 0.044486068189144135,
|
|
"reward_after_std": 0.6025157757103443,
|
|
"reward_before_mean": 0.4036002438515425,
|
|
"reward_before_std": 0.6015807576477528,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.359114158898592,
|
|
"reward_change_min": -0.609935961663723,
|
|
"reward_change_std": 0.24452881701290607,
|
|
"reward_std": 0.6025157924741507,
|
|
"rewards/accuracy_reward": 0.3125000074505806,
|
|
"rewards/cosine_scaled_reward": 0.09110023453831673,
|
|
"step": 238
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1740.1666793823242,
|
|
"epoch": 0.27314285714285713,
|
|
"grad_norm": 0.09357193857431412,
|
|
"kl": 1.8790364265441895e-05,
|
|
"lambda_div_used": 0.5962883979082108,
|
|
"learning_rate": 6.649505910711058e-07,
|
|
"loss": 0.0055,
|
|
"reward": 0.07376761082559824,
|
|
"reward_after_mean": 0.07376761082559824,
|
|
"reward_after_std": 0.5891099888831377,
|
|
"reward_before_mean": 0.5742647312581539,
|
|
"reward_before_std": 0.44304153323173523,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5004971195012331,
|
|
"reward_change_min": -0.7436067499220371,
|
|
"reward_change_std": 0.2840570341795683,
|
|
"reward_std": 0.5891100075095892,
|
|
"rewards/accuracy_reward": 0.37500000186264515,
|
|
"rewards/cosine_scaled_reward": 0.19926471984945238,
|
|
"step": 239
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3087.416702270508,
|
|
"epoch": 0.2742857142857143,
|
|
"grad_norm": 0.06324295699596405,
|
|
"kl": 3.217160701751709e-05,
|
|
"lambda_div_used": 0.5339473709464073,
|
|
"learning_rate": 6.619104492241847e-07,
|
|
"loss": 0.0166,
|
|
"reward": -0.5358126908540726,
|
|
"reward_after_mean": -0.5358126908540726,
|
|
"reward_after_std": 0.23053276538848877,
|
|
"reward_before_mean": -0.2753155492246151,
|
|
"reward_before_std": 0.1490377252921462,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26049717888236046,
|
|
"reward_change_min": -0.3783254958689213,
|
|
"reward_change_std": 0.1383643699809909,
|
|
"reward_std": 0.23053277097642422,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2753155454993248,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3419.625,
|
|
"epoch": 0.2754285714285714,
|
|
"grad_norm": 0.047396283596754074,
|
|
"kl": 3.555417060852051e-05,
|
|
"lambda_div_used": 0.5585425272583961,
|
|
"learning_rate": 6.588648530198504e-07,
|
|
"loss": 0.0027,
|
|
"reward": -0.4805446630343795,
|
|
"reward_after_mean": -0.4805446630343795,
|
|
"reward_after_std": 0.3355217073112726,
|
|
"reward_before_mean": -0.23604051489382982,
|
|
"reward_before_std": 0.2620125887915492,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2445041425526142,
|
|
"reward_change_min": -0.37724288925528526,
|
|
"reward_change_std": 0.13484715577214956,
|
|
"reward_std": 0.3355217222124338,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.2568738413974643,
|
|
"step": 241
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2400.8750228881836,
|
|
"epoch": 0.2765714285714286,
|
|
"grad_norm": 0.08974741399288177,
|
|
"kl": 4.5746564865112305e-05,
|
|
"lambda_div_used": 0.5887154564261436,
|
|
"learning_rate": 6.558139508961654e-07,
|
|
"loss": -0.0452,
|
|
"reward": -0.2555653927847743,
|
|
"reward_after_mean": -0.2555653927847743,
|
|
"reward_after_std": 0.47001883387565613,
|
|
"reward_before_mean": 0.03157716616988182,
|
|
"reward_before_std": 0.4061046461574733,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28714255429804325,
|
|
"reward_change_min": -0.41172971203923225,
|
|
"reward_change_std": 0.16218022629618645,
|
|
"reward_std": 0.47001886926591396,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.1142561559099704,
|
|
"step": 242
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2828.541679382324,
|
|
"epoch": 0.2777142857142857,
|
|
"grad_norm": 0.07105688005685806,
|
|
"kl": 1.8571503460407257e-05,
|
|
"lambda_div_used": 0.5722818151116371,
|
|
"learning_rate": 6.527578915497951e-07,
|
|
"loss": 0.0039,
|
|
"reward": -0.1431608721613884,
|
|
"reward_after_mean": -0.1431608721613884,
|
|
"reward_after_std": 0.42974008433520794,
|
|
"reward_before_mean": 0.2529455106705427,
|
|
"reward_before_std": 0.3276430475525558,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.39610639959573746,
|
|
"reward_change_min": -0.5491434335708618,
|
|
"reward_change_std": 0.21527612209320068,
|
|
"reward_std": 0.42974009923636913,
|
|
"rewards/accuracy_reward": 0.25000000558793545,
|
|
"rewards/cosine_scaled_reward": 0.0029455198673531413,
|
|
"step": 243
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2882.291679382324,
|
|
"epoch": 0.27885714285714286,
|
|
"grad_norm": 0.0700126439332962,
|
|
"kl": 3.098323941230774e-05,
|
|
"lambda_div_used": 0.6161807402968407,
|
|
"learning_rate": 6.496968239287603e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.06458889320492744,
|
|
"reward_after_mean": -0.06458889320492744,
|
|
"reward_after_std": 0.5576508566737175,
|
|
"reward_before_mean": 0.2605547234416008,
|
|
"reward_before_std": 0.5402053641155362,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32514358684420586,
|
|
"reward_change_min": -0.5207333639264107,
|
|
"reward_change_std": 0.21059911139309406,
|
|
"reward_std": 0.5576508603990078,
|
|
"rewards/accuracy_reward": 0.27083334140479565,
|
|
"rewards/cosine_scaled_reward": -0.01027863984927535,
|
|
"step": 244
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2907.4583587646484,
|
|
"epoch": 0.28,
|
|
"grad_norm": 0.05816182866692543,
|
|
"kl": 3.5434961318969727e-05,
|
|
"lambda_div_used": 0.5615689232945442,
|
|
"learning_rate": 6.466308972251785e-07,
|
|
"loss": -0.0133,
|
|
"reward": -0.42641448229551315,
|
|
"reward_after_mean": -0.42641448229551315,
|
|
"reward_after_std": 0.3361923936754465,
|
|
"reward_before_mean": -0.15409247018396854,
|
|
"reward_before_std": 0.2813150165602565,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2723220158368349,
|
|
"reward_change_min": -0.445806298404932,
|
|
"reward_change_std": 0.1635848032310605,
|
|
"reward_std": 0.3361923974007368,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.19575914042070508,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2680.7083587646484,
|
|
"epoch": 0.28114285714285714,
|
|
"grad_norm": 0.07198330760002136,
|
|
"kl": 1.764507032930851e-05,
|
|
"lambda_div_used": 0.5865297466516495,
|
|
"learning_rate": 6.435602608679916e-07,
|
|
"loss": 0.0347,
|
|
"reward": -0.19568787328898907,
|
|
"reward_after_mean": -0.19568787328898907,
|
|
"reward_after_std": 0.4485197216272354,
|
|
"reward_before_mean": 0.129365224391222,
|
|
"reward_before_std": 0.393932550214231,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32505310885608196,
|
|
"reward_change_min": -0.4816659241914749,
|
|
"reward_change_std": 0.1897038472816348,
|
|
"reward_std": 0.44851974956691265,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.0789681114256382,
|
|
"step": 246
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3076.0208435058594,
|
|
"epoch": 0.2822857142857143,
|
|
"grad_norm": 0.08006418496370316,
|
|
"kl": 4.15705144405365e-05,
|
|
"lambda_div_used": 0.5735789015889168,
|
|
"learning_rate": 6.404850645156841e-07,
|
|
"loss": 0.0199,
|
|
"reward": -0.3997113136574626,
|
|
"reward_after_mean": -0.3997113136574626,
|
|
"reward_after_std": 0.41638931445777416,
|
|
"reward_before_mean": -0.1426885835826397,
|
|
"reward_before_std": 0.3330965582281351,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2570227347314358,
|
|
"reward_change_min": -0.38052143156528473,
|
|
"reward_change_std": 0.13911327440291643,
|
|
"reward_std": 0.4163893237709999,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.18435525218956172,
|
|
"step": 247
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2145.0000228881836,
|
|
"epoch": 0.2834285714285714,
|
|
"grad_norm": 0.10052043944597244,
|
|
"kl": 3.3289194107055664e-05,
|
|
"lambda_div_used": 0.6171368733048439,
|
|
"learning_rate": 6.374054580489873e-07,
|
|
"loss": 0.0514,
|
|
"reward": 0.28496517799794674,
|
|
"reward_after_mean": 0.28496517799794674,
|
|
"reward_after_std": 0.6654367055743933,
|
|
"reward_before_mean": 0.83577667362988,
|
|
"reward_before_std": 0.5420532608404756,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5508115068078041,
|
|
"reward_change_min": -0.7994196638464928,
|
|
"reward_change_std": 0.3166588256135583,
|
|
"reward_std": 0.6654367428272963,
|
|
"rewards/accuracy_reward": 0.5000000093132257,
|
|
"rewards/cosine_scaled_reward": 0.33577666338533163,
|
|
"step": 248
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2076.937515258789,
|
|
"epoch": 0.2845714285714286,
|
|
"grad_norm": 0.0908605083823204,
|
|
"kl": 2.4942681193351746e-05,
|
|
"lambda_div_used": 0.5948201194405556,
|
|
"learning_rate": 6.343215915635761e-07,
|
|
"loss": 0.0219,
|
|
"reward": -0.003894178196787834,
|
|
"reward_after_mean": -0.003894178196787834,
|
|
"reward_after_std": 0.5242901761084795,
|
|
"reward_before_mean": 0.4153097262606025,
|
|
"reward_before_std": 0.43534869560971856,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41920389235019684,
|
|
"reward_change_min": -0.5903440341353416,
|
|
"reward_change_std": 0.2394579891115427,
|
|
"reward_std": 0.524290194734931,
|
|
"rewards/accuracy_reward": 0.33333334140479565,
|
|
"rewards/cosine_scaled_reward": 0.08197638019919395,
|
|
"step": 249
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2605.0208587646484,
|
|
"epoch": 0.2857142857142857,
|
|
"grad_norm": 0.09027790278196335,
|
|
"kl": 3.804638981819153e-05,
|
|
"lambda_div_used": 0.5779460370540619,
|
|
"learning_rate": 6.31233615362752e-07,
|
|
"loss": -0.0589,
|
|
"reward": -0.2859988175332546,
|
|
"reward_after_mean": -0.2859988175332546,
|
|
"reward_after_std": 0.4080943390727043,
|
|
"reward_before_mean": 0.008792944252490997,
|
|
"reward_before_std": 0.35337654035538435,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2947917561978102,
|
|
"reward_change_min": -0.4365835040807724,
|
|
"reward_change_std": 0.16840537451207638,
|
|
"reward_std": 0.40809434466063976,
|
|
"rewards/accuracy_reward": 0.12500000558793545,
|
|
"rewards/cosine_scaled_reward": -0.11620705761015415,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2368.9167137145996,
|
|
"epoch": 0.28685714285714287,
|
|
"grad_norm": 0.09640161693096161,
|
|
"kl": 2.7060508728027344e-05,
|
|
"lambda_div_used": 0.5987462773919106,
|
|
"learning_rate": 6.281416799501187e-07,
|
|
"loss": -0.0435,
|
|
"reward": -0.028017258271574974,
|
|
"reward_after_mean": -0.028017258271574974,
|
|
"reward_after_std": 0.5376799181103706,
|
|
"reward_before_mean": 0.38438196340575814,
|
|
"reward_before_std": 0.4528377316892147,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41239920631051064,
|
|
"reward_change_min": -0.6119380593299866,
|
|
"reward_change_std": 0.23853347077965736,
|
|
"reward_std": 0.5376799292862415,
|
|
"rewards/accuracy_reward": 0.33333334140479565,
|
|
"rewards/cosine_scaled_reward": 0.05104860384017229,
|
|
"step": 251
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2798.5625228881836,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.06784425675868988,
|
|
"kl": 3.535952419042587e-05,
|
|
"lambda_div_used": 0.5580189973115921,
|
|
"learning_rate": 6.25045936022246e-07,
|
|
"loss": 0.0335,
|
|
"reward": -0.2931139934808016,
|
|
"reward_after_mean": -0.2931139934808016,
|
|
"reward_after_std": 0.38731373474001884,
|
|
"reward_before_mean": 0.06481979880481958,
|
|
"reward_before_std": 0.2658566879108548,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3579337988048792,
|
|
"reward_change_min": -0.5358714908361435,
|
|
"reward_change_std": 0.19932966493070126,
|
|
"reward_std": 0.3873137477785349,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.10184688493609428,
|
|
"step": 252
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2946.416679382324,
|
|
"epoch": 0.28914285714285715,
|
|
"grad_norm": 0.09991417825222015,
|
|
"kl": 3.4242868423461914e-05,
|
|
"lambda_div_used": 0.6016674339771271,
|
|
"learning_rate": 6.219465344613258e-07,
|
|
"loss": -0.0465,
|
|
"reward": -0.0356330550275743,
|
|
"reward_after_mean": -0.0356330550275743,
|
|
"reward_after_std": 0.5561915785074234,
|
|
"reward_before_mean": 0.353804474696517,
|
|
"reward_before_std": 0.4667428769171238,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.389437522739172,
|
|
"reward_change_min": -0.592089481651783,
|
|
"reward_change_std": 0.22914788126945496,
|
|
"reward_std": 0.5561915840953588,
|
|
"rewards/accuracy_reward": 0.31250000558793545,
|
|
"rewards/cosine_scaled_reward": 0.041304459096863866,
|
|
"step": 253
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2754.3541946411133,
|
|
"epoch": 0.29028571428571426,
|
|
"grad_norm": 0.08061102032661438,
|
|
"kl": 3.886967897415161e-05,
|
|
"lambda_div_used": 0.6251527816057205,
|
|
"learning_rate": 6.188436263278172e-07,
|
|
"loss": 0.0857,
|
|
"reward": -0.1664750911295414,
|
|
"reward_after_mean": -0.1664750911295414,
|
|
"reward_after_std": 0.599596256390214,
|
|
"reward_before_mean": 0.1026190984994173,
|
|
"reward_before_std": 0.5836509419605136,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2690941859036684,
|
|
"reward_change_min": -0.49960801005363464,
|
|
"reward_change_std": 0.19118957500904799,
|
|
"reward_std": 0.5995962955057621,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.08488089527236298,
|
|
"step": 254
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3190.083335876465,
|
|
"epoch": 0.2914285714285714,
|
|
"grad_norm": 0.08434556424617767,
|
|
"kl": 2.3322179913520813e-05,
|
|
"lambda_div_used": 0.5955284982919693,
|
|
"learning_rate": 6.157373628530852e-07,
|
|
"loss": -0.0125,
|
|
"reward": -0.3585695568472147,
|
|
"reward_after_mean": -0.3585695568472147,
|
|
"reward_after_std": 0.4996061436831951,
|
|
"reward_before_mean": -0.12086338270455599,
|
|
"reward_before_std": 0.4362535886466503,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2377061638981104,
|
|
"reward_change_min": -0.39885040931403637,
|
|
"reward_change_std": 0.1406589960679412,
|
|
"reward_std": 0.4996061585843563,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.20419671526178718,
|
|
"step": 255
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3160.8125610351562,
|
|
"epoch": 0.2925714285714286,
|
|
"grad_norm": 0.0576457642018795,
|
|
"kl": 3.143772482872009e-05,
|
|
"lambda_div_used": 0.639025017619133,
|
|
"learning_rate": 6.126278954320294e-07,
|
|
"loss": 0.0243,
|
|
"reward": -0.03833652473986149,
|
|
"reward_after_mean": -0.03833652473986149,
|
|
"reward_after_std": 0.6670235451310873,
|
|
"reward_before_mean": 0.2675688254312263,
|
|
"reward_before_std": 0.6473761759698391,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30590534023940563,
|
|
"reward_change_min": -0.511355496942997,
|
|
"reward_change_std": 0.2077637044712901,
|
|
"reward_std": 0.6670235693454742,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": 0.01756881456822157,
|
|
"step": 256
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3138.5000228881836,
|
|
"epoch": 0.2937142857142857,
|
|
"grad_norm": 0.05595054104924202,
|
|
"kl": 3.0346214771270752e-05,
|
|
"lambda_div_used": 0.6474356725811958,
|
|
"learning_rate": 6.095153756157051e-07,
|
|
"loss": 0.025,
|
|
"reward": -0.0769930558744818,
|
|
"reward_after_mean": -0.0769930558744818,
|
|
"reward_after_std": 0.7138958293944597,
|
|
"reward_before_mean": 0.18525638710707426,
|
|
"reward_before_std": 0.685269920155406,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26224944926798344,
|
|
"reward_change_min": -0.4810149297118187,
|
|
"reward_change_std": 0.18088674824684858,
|
|
"reward_std": 0.7138958312571049,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.04391028080135584,
|
|
"step": 257
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3473.187530517578,
|
|
"epoch": 0.2948571428571429,
|
|
"grad_norm": 0.05202889442443848,
|
|
"kl": 3.561750054359436e-05,
|
|
"lambda_div_used": 0.6007983982563019,
|
|
"learning_rate": 6.06399955103937e-07,
|
|
"loss": 0.0058,
|
|
"reward": -0.23316247668117285,
|
|
"reward_after_mean": -0.23316247668117285,
|
|
"reward_after_std": 0.5119184870272875,
|
|
"reward_before_mean": 0.04610642418265343,
|
|
"reward_before_std": 0.46757086645811796,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2792689222842455,
|
|
"reward_change_min": -0.4602624364197254,
|
|
"reward_change_std": 0.17828952055424452,
|
|
"reward_std": 0.5119185000658035,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.09972689533606172,
|
|
"step": 258
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2996.7500534057617,
|
|
"epoch": 0.296,
|
|
"grad_norm": 0.11353152990341187,
|
|
"kl": 4.0959566831588745e-05,
|
|
"lambda_div_used": 0.605031318962574,
|
|
"learning_rate": 6.032817857379256e-07,
|
|
"loss": -0.0572,
|
|
"reward": -0.24131755530834198,
|
|
"reward_after_mean": -0.24131755530834198,
|
|
"reward_after_std": 0.5208989772945642,
|
|
"reward_before_mean": 0.03226012596860528,
|
|
"reward_before_std": 0.4831458665430546,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27357766777276993,
|
|
"reward_change_min": -0.4753991588950157,
|
|
"reward_change_std": 0.1725512887351215,
|
|
"reward_std": 0.5208989884704351,
|
|
"rewards/accuracy_reward": 0.14583333767950535,
|
|
"rewards/cosine_scaled_reward": -0.1135732214897871,
|
|
"step": 259
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2184.8542098999023,
|
|
"epoch": 0.29714285714285715,
|
|
"grad_norm": 0.1352783739566803,
|
|
"kl": 2.3871660232543945e-05,
|
|
"lambda_div_used": 0.6604775562882423,
|
|
"learning_rate": 6.001610194928464e-07,
|
|
"loss": 0.0496,
|
|
"reward": 0.1305767484009266,
|
|
"reward_after_mean": 0.1305767484009266,
|
|
"reward_after_std": 0.7162795849144459,
|
|
"reward_before_mean": 0.48802967881783843,
|
|
"reward_before_std": 0.7462537074461579,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3574528992176056,
|
|
"reward_change_min": -0.6257632970809937,
|
|
"reward_change_std": 0.25755990110337734,
|
|
"reward_std": 0.7162796072661877,
|
|
"rewards/accuracy_reward": 0.39583334513008595,
|
|
"rewards/cosine_scaled_reward": 0.0921963145956397,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3070.3125228881836,
|
|
"epoch": 0.29828571428571427,
|
|
"grad_norm": 0.06917808949947357,
|
|
"kl": 3.3020973205566406e-05,
|
|
"lambda_div_used": 0.5845921337604523,
|
|
"learning_rate": 5.97037808470444e-07,
|
|
"loss": 0.0106,
|
|
"reward": -0.3141371374949813,
|
|
"reward_after_mean": -0.3141371374949813,
|
|
"reward_after_std": 0.43580205366015434,
|
|
"reward_before_mean": -0.03358305338770151,
|
|
"reward_before_std": 0.38949463702738285,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28055410645902157,
|
|
"reward_change_min": -0.44674795493483543,
|
|
"reward_change_std": 0.16879158467054367,
|
|
"reward_std": 0.4358020592480898,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.11691637896001339,
|
|
"step": 261
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3258.3125,
|
|
"epoch": 0.29942857142857143,
|
|
"grad_norm": 0.06239396706223488,
|
|
"kl": 4.535168409347534e-05,
|
|
"lambda_div_used": 0.5396690741181374,
|
|
"learning_rate": 5.939123048916173e-07,
|
|
"loss": 0.0153,
|
|
"reward": -0.49852147325873375,
|
|
"reward_after_mean": -0.49852147325873375,
|
|
"reward_after_std": 0.23993146419525146,
|
|
"reward_before_mean": -0.21930878423154354,
|
|
"reward_before_std": 0.17422470543533564,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2792126890271902,
|
|
"reward_change_min": -0.4082174263894558,
|
|
"reward_change_std": 0.15261581167578697,
|
|
"reward_std": 0.2399314697831869,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.21930878423154354,
|
|
"step": 262
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2837.6666717529297,
|
|
"epoch": 0.30057142857142854,
|
|
"grad_norm": 0.07131204754114151,
|
|
"kl": 2.771243453025818e-05,
|
|
"lambda_div_used": 0.5753477811813354,
|
|
"learning_rate": 5.907846610890011e-07,
|
|
"loss": 0.0159,
|
|
"reward": -0.32574891671538353,
|
|
"reward_after_mean": -0.32574891671538353,
|
|
"reward_after_std": 0.40547293052077293,
|
|
"reward_before_mean": -0.04349888768047094,
|
|
"reward_before_std": 0.3437284992542118,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28225001133978367,
|
|
"reward_change_min": -0.423517182469368,
|
|
"reward_change_std": 0.16225541010499,
|
|
"reward_std": 0.4054729398339987,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.14766556210815907,
|
|
"step": 263
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2913.812515258789,
|
|
"epoch": 0.3017142857142857,
|
|
"grad_norm": 0.07276218384504318,
|
|
"kl": 3.5703182220458984e-05,
|
|
"lambda_div_used": 0.5857644975185394,
|
|
"learning_rate": 5.87655029499542e-07,
|
|
"loss": 0.039,
|
|
"reward": -0.19073306024074554,
|
|
"reward_after_mean": -0.19073306024074554,
|
|
"reward_after_std": 0.4183583725243807,
|
|
"reward_before_mean": 0.13579276762902737,
|
|
"reward_before_std": 0.39237749949097633,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32652581483125687,
|
|
"reward_change_min": -0.512863963842392,
|
|
"reward_change_std": 0.20149299688637257,
|
|
"reward_std": 0.4183583725243807,
|
|
"rewards/accuracy_reward": 0.1875000074505806,
|
|
"rewards/cosine_scaled_reward": -0.051707250997424126,
|
|
"step": 264
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2092.333351135254,
|
|
"epoch": 0.3028571428571429,
|
|
"grad_norm": 0.09631343185901642,
|
|
"kl": 2.018176019191742e-05,
|
|
"lambda_div_used": 0.5927653685212135,
|
|
"learning_rate": 5.845235626570683e-07,
|
|
"loss": 0.0409,
|
|
"reward": -0.05978839658200741,
|
|
"reward_after_mean": -0.05978839658200741,
|
|
"reward_after_std": 0.5236263573169708,
|
|
"reward_before_mean": 0.3295632619410753,
|
|
"reward_before_std": 0.4253385625779629,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38935166224837303,
|
|
"reward_change_min": -0.5855174511671066,
|
|
"reward_change_std": 0.22452097665518522,
|
|
"reward_std": 0.5236263833940029,
|
|
"rewards/accuracy_reward": 0.29166666977107525,
|
|
"rewards/cosine_scaled_reward": 0.037896597757935524,
|
|
"step": 265
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3210.8541717529297,
|
|
"epoch": 0.304,
|
|
"grad_norm": 0.05451129376888275,
|
|
"kl": 3.138929605484009e-05,
|
|
"lambda_div_used": 0.557657316327095,
|
|
"learning_rate": 5.813904131848564e-07,
|
|
"loss": -0.0024,
|
|
"reward": -0.3950183019042015,
|
|
"reward_after_mean": -0.3950183019042015,
|
|
"reward_after_std": 0.32154187746345997,
|
|
"reward_before_mean": -0.12019951082766056,
|
|
"reward_before_std": 0.2612606221809983,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2748187892138958,
|
|
"reward_change_min": -0.42080841958522797,
|
|
"reward_change_std": 0.158494733273983,
|
|
"reward_std": 0.321541890501976,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.20353284664452076,
|
|
"step": 266
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3039.916679382324,
|
|
"epoch": 0.30514285714285716,
|
|
"grad_norm": 0.06252676248550415,
|
|
"kl": 3.5075165214948356e-05,
|
|
"lambda_div_used": 0.5528770685195923,
|
|
"learning_rate": 5.78255733788191e-07,
|
|
"loss": -0.037,
|
|
"reward": -0.4460434205830097,
|
|
"reward_after_mean": -0.4460434205830097,
|
|
"reward_after_std": 0.3107005339115858,
|
|
"reward_before_mean": -0.1744655454531312,
|
|
"reward_before_std": 0.2377403611317277,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27157786674797535,
|
|
"reward_change_min": -0.42647456377744675,
|
|
"reward_change_std": 0.1541104121133685,
|
|
"reward_std": 0.3107005413621664,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.21613221243023872,
|
|
"step": 267
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2704.8333587646484,
|
|
"epoch": 0.3062857142857143,
|
|
"grad_norm": 0.08417963981628418,
|
|
"kl": 4.7713518142700195e-05,
|
|
"lambda_div_used": 0.6233388632535934,
|
|
"learning_rate": 5.751196772469237e-07,
|
|
"loss": 0.0822,
|
|
"reward": -0.21941478177905083,
|
|
"reward_after_mean": -0.21941478177905083,
|
|
"reward_after_std": 0.6098091676831245,
|
|
"reward_before_mean": 0.03345827816519886,
|
|
"reward_before_std": 0.5743174999952316,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25287305377423763,
|
|
"reward_change_min": -0.45724478363990784,
|
|
"reward_change_std": 0.1690685572102666,
|
|
"reward_std": 0.6098091900348663,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.11237506754696369,
|
|
"step": 268
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3165.9791717529297,
|
|
"epoch": 0.30742857142857144,
|
|
"grad_norm": 0.06371315568685532,
|
|
"kl": 2.181902527809143e-05,
|
|
"lambda_div_used": 0.5798910111188889,
|
|
"learning_rate": 5.71982396408026e-07,
|
|
"loss": 0.0477,
|
|
"reward": -0.38025568798184395,
|
|
"reward_after_mean": -0.38025568798184395,
|
|
"reward_after_std": 0.4126162938773632,
|
|
"reward_before_mean": -0.13023450784385204,
|
|
"reward_before_std": 0.3629760518670082,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2500211838632822,
|
|
"reward_change_min": -0.40024495497345924,
|
|
"reward_change_std": 0.15232266392558813,
|
|
"reward_std": 0.4126163087785244,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.2135678417980671,
|
|
"step": 269
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3056.875045776367,
|
|
"epoch": 0.30857142857142855,
|
|
"grad_norm": 0.05920446664094925,
|
|
"kl": 2.387911081314087e-05,
|
|
"lambda_div_used": 0.6503699943423271,
|
|
"learning_rate": 5.688440441781398e-07,
|
|
"loss": 0.0,
|
|
"reward": -0.004786740057170391,
|
|
"reward_after_mean": -0.004786740057170391,
|
|
"reward_after_std": 0.6943319551646709,
|
|
"reward_before_mean": 0.292231020051986,
|
|
"reward_before_std": 0.7036092299968004,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2970177587121725,
|
|
"reward_change_min": -0.5634518079459667,
|
|
"reward_change_std": 0.21872046310454607,
|
|
"reward_std": 0.6943319924175739,
|
|
"rewards/accuracy_reward": 0.27083333767950535,
|
|
"rewards/cosine_scaled_reward": 0.021397670439910144,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2221.3958702087402,
|
|
"epoch": 0.3097142857142857,
|
|
"grad_norm": 0.09025074541568756,
|
|
"kl": 6.723217666149139e-06,
|
|
"lambda_div_used": 0.640851192176342,
|
|
"learning_rate": 5.657047735161255e-07,
|
|
"loss": 0.0064,
|
|
"reward": 0.11755780689418316,
|
|
"reward_after_mean": 0.11755780689418316,
|
|
"reward_after_std": 0.6356876939535141,
|
|
"reward_before_mean": 0.4748641401529312,
|
|
"reward_before_std": 0.654548792168498,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35730634443461895,
|
|
"reward_change_min": -0.5644437614828348,
|
|
"reward_change_std": 0.23989163804799318,
|
|
"reward_std": 0.6356877088546753,
|
|
"rewards/accuracy_reward": 0.39583334885537624,
|
|
"rewards/cosine_scaled_reward": 0.07903079688549042,
|
|
"step": 271
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2814.5208740234375,
|
|
"epoch": 0.31085714285714283,
|
|
"grad_norm": 0.07214022427797318,
|
|
"kl": 2.553686499595642e-05,
|
|
"lambda_div_used": 0.5532237812876701,
|
|
"learning_rate": 5.625647374256061e-07,
|
|
"loss": -0.023,
|
|
"reward": -0.33643968403339386,
|
|
"reward_after_mean": -0.33643968403339386,
|
|
"reward_after_std": 0.35168860107660294,
|
|
"reward_before_mean": 0.01186647079885006,
|
|
"reward_before_std": 0.2353415172547102,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34830615669488907,
|
|
"reward_change_min": -0.5079392194747925,
|
|
"reward_change_std": 0.1846030419692397,
|
|
"reward_std": 0.351688614115119,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.13396686874330044,
|
|
"step": 272
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2673.937515258789,
|
|
"epoch": 0.312,
|
|
"grad_norm": 0.067450150847435,
|
|
"kl": 2.347538247704506e-05,
|
|
"lambda_div_used": 0.5976931154727936,
|
|
"learning_rate": 5.594240889475106e-07,
|
|
"loss": 0.0439,
|
|
"reward": 0.060379184782505035,
|
|
"reward_after_mean": 0.060379184782505035,
|
|
"reward_after_std": 0.5129444599151611,
|
|
"reward_before_mean": 0.4946505483239889,
|
|
"reward_before_std": 0.44454328902065754,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4342713989317417,
|
|
"reward_change_min": -0.6668536812067032,
|
|
"reward_change_std": 0.25412870943546295,
|
|
"reward_std": 0.5129444822669029,
|
|
"rewards/accuracy_reward": 0.3750000111758709,
|
|
"rewards/cosine_scaled_reward": 0.1196505706757307,
|
|
"step": 273
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1866.4375076293945,
|
|
"epoch": 0.31314285714285717,
|
|
"grad_norm": 0.11823767423629761,
|
|
"kl": 2.6108697056770325e-05,
|
|
"lambda_div_used": 0.6173145174980164,
|
|
"learning_rate": 5.562829811526154e-07,
|
|
"loss": -0.0666,
|
|
"reward": 0.20245935022830963,
|
|
"reward_after_mean": 0.20245935022830963,
|
|
"reward_after_std": 0.6078098546713591,
|
|
"reward_before_mean": 0.6919816123554483,
|
|
"reward_before_std": 0.5468513960950077,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4895222559571266,
|
|
"reward_change_min": -0.7379350513219833,
|
|
"reward_change_std": 0.3024911228567362,
|
|
"reward_std": 0.6078098695725203,
|
|
"rewards/accuracy_reward": 0.4583333432674408,
|
|
"rewards/cosine_scaled_reward": 0.2336482839891687,
|
|
"step": 274
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2334.0416870117188,
|
|
"epoch": 0.3142857142857143,
|
|
"grad_norm": 0.07390157133340836,
|
|
"kl": 2.975761890411377e-05,
|
|
"lambda_div_used": 0.5953021869063377,
|
|
"learning_rate": 5.531415671340826e-07,
|
|
"loss": 0.0067,
|
|
"reward": -0.02193348854780197,
|
|
"reward_after_mean": -0.02193348854780197,
|
|
"reward_after_std": 0.5273136273026466,
|
|
"reward_before_mean": 0.3926870714276447,
|
|
"reward_before_std": 0.4364564120769501,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4146205447614193,
|
|
"reward_change_min": -0.6028474234044552,
|
|
"reward_change_std": 0.23770872876048088,
|
|
"reward_std": 0.5273136328905821,
|
|
"rewards/accuracy_reward": 0.31250000558793545,
|
|
"rewards/cosine_scaled_reward": 0.08018706925213337,
|
|
"step": 275
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2735.6458740234375,
|
|
"epoch": 0.31542857142857145,
|
|
"grad_norm": 0.07891591638326645,
|
|
"kl": 3.522634506225586e-05,
|
|
"lambda_div_used": 0.6019129753112793,
|
|
"learning_rate": 5.5e-07,
|
|
"loss": -0.0433,
|
|
"reward": 0.016597013920545578,
|
|
"reward_after_mean": 0.016597013920545578,
|
|
"reward_after_std": 0.5496951006352901,
|
|
"reward_before_mean": 0.43198024667799473,
|
|
"reward_before_std": 0.4716393407434225,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4153832569718361,
|
|
"reward_change_min": -0.6486029289662838,
|
|
"reward_change_std": 0.25036295782774687,
|
|
"reward_std": 0.5496951248496771,
|
|
"rewards/accuracy_reward": 0.3333333395421505,
|
|
"rewards/cosine_scaled_reward": 0.09864690899848938,
|
|
"step": 276
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2435.437530517578,
|
|
"epoch": 0.31657142857142856,
|
|
"grad_norm": 0.08397295325994492,
|
|
"kl": 2.7611851692199707e-05,
|
|
"lambda_div_used": 0.6150463595986366,
|
|
"learning_rate": 5.468584328659172e-07,
|
|
"loss": 0.0536,
|
|
"reward": -0.049143560230731964,
|
|
"reward_after_mean": -0.049143560230731964,
|
|
"reward_after_std": 0.5724204778671265,
|
|
"reward_before_mean": 0.29686027206480503,
|
|
"reward_before_std": 0.5315626971423626,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34600381925702095,
|
|
"reward_change_min": -0.5762605480849743,
|
|
"reward_change_std": 0.21639508474618196,
|
|
"reward_std": 0.5724204815924168,
|
|
"rewards/accuracy_reward": 0.27083333767950535,
|
|
"rewards/cosine_scaled_reward": 0.02602693811058998,
|
|
"step": 277
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2029.708381652832,
|
|
"epoch": 0.3177142857142857,
|
|
"grad_norm": 0.11247767508029938,
|
|
"kl": 3.094971179962158e-05,
|
|
"lambda_div_used": 0.6170216798782349,
|
|
"learning_rate": 5.437170188473847e-07,
|
|
"loss": 0.002,
|
|
"reward": 0.056462954729795456,
|
|
"reward_after_mean": 0.056462954729795456,
|
|
"reward_after_std": 0.6216260213404894,
|
|
"reward_before_mean": 0.48691817931830883,
|
|
"reward_before_std": 0.5461275167763233,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4304552264511585,
|
|
"reward_change_min": -0.7186719551682472,
|
|
"reward_change_std": 0.27375681325793266,
|
|
"reward_std": 0.6216260306537151,
|
|
"rewards/accuracy_reward": 0.3541666716337204,
|
|
"rewards/cosine_scaled_reward": 0.13275149278342724,
|
|
"step": 278
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3158.6875228881836,
|
|
"epoch": 0.31885714285714284,
|
|
"grad_norm": 0.06945524364709854,
|
|
"kl": 3.522634506225586e-05,
|
|
"lambda_div_used": 0.5564287528395653,
|
|
"learning_rate": 5.405759110524894e-07,
|
|
"loss": -0.0272,
|
|
"reward": -0.43442236818373203,
|
|
"reward_after_mean": -0.43442236818373203,
|
|
"reward_after_std": 0.32187592424452305,
|
|
"reward_before_mean": -0.1544840056449175,
|
|
"reward_before_std": 0.25075172632932663,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27993838116526604,
|
|
"reward_change_min": -0.41151439025998116,
|
|
"reward_change_std": 0.15283891931176186,
|
|
"reward_std": 0.32187593914568424,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.17531732900533825,
|
|
"step": 279
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2157.833335876465,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.14424921572208405,
|
|
"kl": 4.202499985694885e-05,
|
|
"lambda_div_used": 0.630903884768486,
|
|
"learning_rate": 5.37435262574394e-07,
|
|
"loss": -0.0339,
|
|
"reward": -0.14438428170979023,
|
|
"reward_after_mean": -0.14438428170979023,
|
|
"reward_after_std": 0.6414528246968985,
|
|
"reward_before_mean": 0.1313652544049546,
|
|
"reward_before_std": 0.6011563409119844,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.275749534368515,
|
|
"reward_change_min": -0.4570390097796917,
|
|
"reward_change_std": 0.1709576854482293,
|
|
"reward_std": 0.6414528302848339,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.076968085631961,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3559.000030517578,
|
|
"epoch": 0.3211428571428571,
|
|
"grad_norm": 0.0494488850235939,
|
|
"kl": 3.0197203159332275e-05,
|
|
"lambda_div_used": 0.5301951244473457,
|
|
"learning_rate": 5.342952264838747e-07,
|
|
"loss": -0.0035,
|
|
"reward": -0.5142169296741486,
|
|
"reward_after_mean": -0.5142169296741486,
|
|
"reward_after_std": 0.22926145792007446,
|
|
"reward_before_mean": -0.2300790660083294,
|
|
"reward_before_std": 0.1324998252093792,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2841378580778837,
|
|
"reward_change_min": -0.4100039228796959,
|
|
"reward_change_std": 0.14742697216570377,
|
|
"reward_std": 0.22926146537065506,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2300790660083294,
|
|
"step": 281
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2757.7708587646484,
|
|
"epoch": 0.3222857142857143,
|
|
"grad_norm": 0.07136296480894089,
|
|
"kl": 1.5236437320709229e-05,
|
|
"lambda_div_used": 0.6041858941316605,
|
|
"learning_rate": 5.311559558218603e-07,
|
|
"loss": 0.0304,
|
|
"reward": 0.016035709530115128,
|
|
"reward_after_mean": 0.016035709530115128,
|
|
"reward_after_std": 0.5573885068297386,
|
|
"reward_before_mean": 0.4244079850614071,
|
|
"reward_before_std": 0.48719789227470756,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.40837226063013077,
|
|
"reward_change_min": -0.630510251969099,
|
|
"reward_change_std": 0.253003865480423,
|
|
"reward_std": 0.5573885291814804,
|
|
"rewards/accuracy_reward": 0.3541666716337204,
|
|
"rewards/cosine_scaled_reward": 0.07024132460355759,
|
|
"step": 282
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2882.5416946411133,
|
|
"epoch": 0.32342857142857145,
|
|
"grad_norm": 0.05982055515050888,
|
|
"kl": 1.1865049600601196e-05,
|
|
"lambda_div_used": 0.6191478446125984,
|
|
"learning_rate": 5.28017603591974e-07,
|
|
"loss": -0.015,
|
|
"reward": 0.10000502690672874,
|
|
"reward_after_mean": 0.10000502690672874,
|
|
"reward_after_std": 0.6287181153893471,
|
|
"reward_before_mean": 0.5279214177280664,
|
|
"reward_before_std": 0.5531216450035572,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4279164057224989,
|
|
"reward_change_min": -0.6583010666072369,
|
|
"reward_change_std": 0.2602456407621503,
|
|
"reward_std": 0.6287181228399277,
|
|
"rewards/accuracy_reward": 0.37500000931322575,
|
|
"rewards/cosine_scaled_reward": 0.1529214084148407,
|
|
"step": 283
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2187.083381652832,
|
|
"epoch": 0.32457142857142857,
|
|
"grad_norm": 0.20831985771656036,
|
|
"kl": 5.378853529691696e-05,
|
|
"lambda_div_used": 0.617803268134594,
|
|
"learning_rate": 5.248803227530763e-07,
|
|
"loss": 0.0301,
|
|
"reward": -0.09672576747834682,
|
|
"reward_after_mean": -0.09672576747834682,
|
|
"reward_after_std": 0.5820088647305965,
|
|
"reward_before_mean": 0.21344758570194244,
|
|
"reward_before_std": 0.544846129603684,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3101733736693859,
|
|
"reward_change_min": -0.5270103476941586,
|
|
"reward_change_std": 0.20029443874955177,
|
|
"reward_std": 0.5820088759064674,
|
|
"rewards/accuracy_reward": 0.22916667349636555,
|
|
"rewards/cosine_scaled_reward": -0.0157190952450037,
|
|
"step": 284
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2571.4583435058594,
|
|
"epoch": 0.32571428571428573,
|
|
"grad_norm": 0.05750874802470207,
|
|
"kl": 2.5823712348937988e-05,
|
|
"lambda_div_used": 0.5412982106208801,
|
|
"learning_rate": 5.21744266211809e-07,
|
|
"loss": 0.0429,
|
|
"reward": -0.3610886335372925,
|
|
"reward_after_mean": -0.3610886335372925,
|
|
"reward_after_std": 0.2963402010500431,
|
|
"reward_before_mean": 0.004524916410446167,
|
|
"reward_before_std": 0.18141429405659437,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36561354249715805,
|
|
"reward_change_min": -0.5212092585861683,
|
|
"reward_change_std": 0.19485185854136944,
|
|
"reward_std": 0.29634021408855915,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.12047509290277958,
|
|
"step": 285
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2607.625030517578,
|
|
"epoch": 0.32685714285714285,
|
|
"grad_norm": 0.06422320753335953,
|
|
"kl": 2.036895602941513e-05,
|
|
"lambda_div_used": 0.593349277973175,
|
|
"learning_rate": 5.186095868151436e-07,
|
|
"loss": 0.0069,
|
|
"reward": -0.1434864792972803,
|
|
"reward_after_mean": -0.1434864792972803,
|
|
"reward_after_std": 0.4716298431158066,
|
|
"reward_before_mean": 0.18680993653833866,
|
|
"reward_before_std": 0.4280705275014043,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3302964009344578,
|
|
"reward_change_min": -0.5444469898939133,
|
|
"reward_change_std": 0.20489494875073433,
|
|
"reward_std": 0.4716298636049032,
|
|
"rewards/accuracy_reward": 0.2291666753590107,
|
|
"rewards/cosine_scaled_reward": -0.04235674999654293,
|
|
"step": 286
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2068.4375343322754,
|
|
"epoch": 0.328,
|
|
"grad_norm": 0.11217836290597916,
|
|
"kl": 3.282725811004639e-05,
|
|
"lambda_div_used": 0.576636016368866,
|
|
"learning_rate": 5.154764373429315e-07,
|
|
"loss": -0.0127,
|
|
"reward": -0.015125550329685211,
|
|
"reward_after_mean": -0.015125550329685211,
|
|
"reward_after_std": 0.44022079929709435,
|
|
"reward_before_mean": 0.4338516741991043,
|
|
"reward_before_std": 0.349846001714468,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4489772208034992,
|
|
"reward_change_min": -0.6400703266263008,
|
|
"reward_change_std": 0.25500839948654175,
|
|
"reward_std": 0.44022080302238464,
|
|
"rewards/accuracy_reward": 0.3125000074505806,
|
|
"rewards/cosine_scaled_reward": 0.12135165929794312,
|
|
"step": 287
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3021.1666870117188,
|
|
"epoch": 0.3291428571428571,
|
|
"grad_norm": 0.05326732248067856,
|
|
"kl": 2.1582163753919303e-05,
|
|
"lambda_div_used": 0.5814583152532578,
|
|
"learning_rate": 5.123449705004581e-07,
|
|
"loss": 0.0215,
|
|
"reward": -0.23482287488877773,
|
|
"reward_after_mean": -0.23482287488877773,
|
|
"reward_after_std": 0.4797380156815052,
|
|
"reward_before_mean": 0.10618079453706741,
|
|
"reward_before_std": 0.37177785113453865,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34100368432700634,
|
|
"reward_change_min": -0.5246531553566456,
|
|
"reward_change_std": 0.190623770467937,
|
|
"reward_std": 0.47973802499473095,
|
|
"rewards/accuracy_reward": 0.18750000186264515,
|
|
"rewards/cosine_scaled_reward": -0.08131920825690031,
|
|
"step": 288
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2210.0208435058594,
|
|
"epoch": 0.3302857142857143,
|
|
"grad_norm": 0.0918356254696846,
|
|
"kl": 3.3445656299591064e-05,
|
|
"lambda_div_used": 0.5579611882567406,
|
|
"learning_rate": 5.09215338910999e-07,
|
|
"loss": 0.0308,
|
|
"reward": -0.1890019178390503,
|
|
"reward_after_mean": -0.1890019178390503,
|
|
"reward_after_std": 0.38851393200457096,
|
|
"reward_before_mean": 0.2269407268613577,
|
|
"reward_before_std": 0.2580757178366184,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4159426633268595,
|
|
"reward_change_min": -0.5701811872422695,
|
|
"reward_change_std": 0.21641669981181622,
|
|
"reward_std": 0.38851393945515156,
|
|
"rewards/accuracy_reward": 0.27083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.04389259871095419,
|
|
"step": 289
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1820.6666870117188,
|
|
"epoch": 0.3314285714285714,
|
|
"grad_norm": 0.10334110260009766,
|
|
"kl": 1.7639249563217163e-05,
|
|
"lambda_div_used": 0.649879202246666,
|
|
"learning_rate": 5.060876951083828e-07,
|
|
"loss": -0.0389,
|
|
"reward": 0.04819735325872898,
|
|
"reward_after_mean": 0.04819735325872898,
|
|
"reward_after_std": 0.752808591350913,
|
|
"reward_before_mean": 0.39133079699240625,
|
|
"reward_before_std": 0.7000208692625165,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3431334514170885,
|
|
"reward_change_min": -0.5988858677446842,
|
|
"reward_change_std": 0.2255254928022623,
|
|
"reward_std": 0.7528086155653,
|
|
"rewards/accuracy_reward": 0.3333333358168602,
|
|
"rewards/cosine_scaled_reward": 0.05799746699631214,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2717.166732788086,
|
|
"epoch": 0.3325714285714286,
|
|
"grad_norm": 0.07149659842252731,
|
|
"kl": 9.842216968536377e-06,
|
|
"lambda_div_used": 0.6230586618185043,
|
|
"learning_rate": 5.02962191529556e-07,
|
|
"loss": 0.0634,
|
|
"reward": 0.04544975981116295,
|
|
"reward_after_mean": 0.04544975981116295,
|
|
"reward_after_std": 0.6337935384362936,
|
|
"reward_before_mean": 0.4299341347068548,
|
|
"reward_before_std": 0.5707205794751644,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38448438234627247,
|
|
"reward_change_min": -0.6367709413170815,
|
|
"reward_change_std": 0.2398422248661518,
|
|
"reward_std": 0.6337935607880354,
|
|
"rewards/accuracy_reward": 0.33333333767950535,
|
|
"rewards/cosine_scaled_reward": 0.09660080214962363,
|
|
"step": 291
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3176.979202270508,
|
|
"epoch": 0.33371428571428574,
|
|
"grad_norm": 0.05150657147169113,
|
|
"kl": 2.0240433514118195e-05,
|
|
"lambda_div_used": 0.5560602247714996,
|
|
"learning_rate": 4.998389805071536e-07,
|
|
"loss": 0.062,
|
|
"reward": -0.4294139966368675,
|
|
"reward_after_mean": -0.4294139966368675,
|
|
"reward_after_std": 0.3452510107308626,
|
|
"reward_before_mean": -0.15922172274440527,
|
|
"reward_before_std": 0.2578093442134559,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27019229158759117,
|
|
"reward_change_min": -0.42562897875905037,
|
|
"reward_change_std": 0.15438843425363302,
|
|
"reward_std": 0.3452510181814432,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.20088838692754507,
|
|
"step": 292
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2630.9791946411133,
|
|
"epoch": 0.33485714285714285,
|
|
"grad_norm": 0.05954331159591675,
|
|
"kl": 1.2833625078201294e-05,
|
|
"lambda_div_used": 0.5555919855833054,
|
|
"learning_rate": 4.967182142620745e-07,
|
|
"loss": 0.0009,
|
|
"reward": -0.19852645695209503,
|
|
"reward_after_mean": -0.19852645695209503,
|
|
"reward_after_std": 0.3515181578695774,
|
|
"reward_before_mean": 0.20652466267347336,
|
|
"reward_before_std": 0.24862384609878063,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.40505112148821354,
|
|
"reward_change_min": -0.5747470110654831,
|
|
"reward_change_std": 0.22049889154732227,
|
|
"reward_std": 0.35151816345751286,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.022642023861408234,
|
|
"step": 293
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3005.5,
|
|
"epoch": 0.336,
|
|
"grad_norm": 0.06993379443883896,
|
|
"kl": 2.8274953365325928e-05,
|
|
"lambda_div_used": 0.5608418732881546,
|
|
"learning_rate": 4.93600044896063e-07,
|
|
"loss": -0.0248,
|
|
"reward": -0.3667301833629608,
|
|
"reward_after_mean": -0.3667301833629608,
|
|
"reward_after_std": 0.32350156269967556,
|
|
"reward_before_mean": -0.07280110754072666,
|
|
"reward_before_std": 0.2737845163792372,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29392906464636326,
|
|
"reward_change_min": -0.4593726359307766,
|
|
"reward_change_std": 0.17168805841356516,
|
|
"reward_std": 0.3235015720129013,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.15613443590700626,
|
|
"step": 294
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3221.2291717529297,
|
|
"epoch": 0.33714285714285713,
|
|
"grad_norm": 0.06364311277866364,
|
|
"kl": 1.817569136619568e-05,
|
|
"lambda_div_used": 0.5979067236185074,
|
|
"learning_rate": 4.904846243842949e-07,
|
|
"loss": 0.0107,
|
|
"reward": -0.259306401014328,
|
|
"reward_after_mean": -0.259306401014328,
|
|
"reward_after_std": 0.5024331342428923,
|
|
"reward_before_mean": 0.018260781886056066,
|
|
"reward_before_std": 0.45087322127074003,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.277567183598876,
|
|
"reward_change_min": -0.41658853366971016,
|
|
"reward_change_std": 0.1643626783043146,
|
|
"reward_std": 0.5024331398308277,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.10673921927809715,
|
|
"step": 295
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3282.8750610351562,
|
|
"epoch": 0.3382857142857143,
|
|
"grad_norm": 0.0698896273970604,
|
|
"kl": 2.3877248167991638e-05,
|
|
"lambda_div_used": 0.5625706240534782,
|
|
"learning_rate": 4.873721045679706e-07,
|
|
"loss": 0.0079,
|
|
"reward": -0.39721857756376266,
|
|
"reward_after_mean": -0.39721857756376266,
|
|
"reward_after_std": 0.34284412302076817,
|
|
"reward_before_mean": -0.1091517936438322,
|
|
"reward_before_std": 0.28544116113334894,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28806679882109165,
|
|
"reward_change_min": -0.4639837518334389,
|
|
"reward_change_std": 0.1706175785511732,
|
|
"reward_std": 0.3428441286087036,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.15081845596432686,
|
|
"step": 296
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3569.8541870117188,
|
|
"epoch": 0.3394285714285714,
|
|
"grad_norm": 0.04541197791695595,
|
|
"kl": 2.245139330625534e-05,
|
|
"lambda_div_used": 0.5649159774184227,
|
|
"learning_rate": 4.842626371469149e-07,
|
|
"loss": 0.0071,
|
|
"reward": -0.43140796944499016,
|
|
"reward_after_mean": -0.43140796944499016,
|
|
"reward_after_std": 0.3433863054960966,
|
|
"reward_before_mean": -0.17089181207120419,
|
|
"reward_before_std": 0.29400468710809946,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26051616482436657,
|
|
"reward_change_min": -0.4413977116346359,
|
|
"reward_change_std": 0.15714262332767248,
|
|
"reward_std": 0.3433863129466772,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.2125584715977311,
|
|
"step": 297
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2834.479202270508,
|
|
"epoch": 0.3405714285714286,
|
|
"grad_norm": 0.07559877634048462,
|
|
"kl": 1.9371509552001953e-05,
|
|
"lambda_div_used": 0.5950812250375748,
|
|
"learning_rate": 4.811563736721829e-07,
|
|
"loss": -0.0217,
|
|
"reward": -0.13096300419420004,
|
|
"reward_after_mean": -0.13096300419420004,
|
|
"reward_after_std": 0.505608232691884,
|
|
"reward_before_mean": 0.22305661533027887,
|
|
"reward_before_std": 0.43375879526138306,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.354019645601511,
|
|
"reward_change_min": -0.5659407489001751,
|
|
"reward_change_std": 0.20670694950968027,
|
|
"reward_std": 0.5056082457304001,
|
|
"rewards/accuracy_reward": 0.29166667349636555,
|
|
"rewards/cosine_scaled_reward": -0.0686100497841835,
|
|
"step": 298
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3288.5833587646484,
|
|
"epoch": 0.3417142857142857,
|
|
"grad_norm": 0.05182049795985222,
|
|
"kl": 3.013480454683304e-05,
|
|
"lambda_div_used": 0.5971293970942497,
|
|
"learning_rate": 4.780534655386743e-07,
|
|
"loss": -0.0035,
|
|
"reward": -0.1649590004235506,
|
|
"reward_after_mean": -0.1649590004235506,
|
|
"reward_after_std": 0.476910138502717,
|
|
"reward_before_mean": 0.15552489459514618,
|
|
"reward_before_std": 0.44229450821876526,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3204838838428259,
|
|
"reward_change_min": -0.5031619034707546,
|
|
"reward_change_std": 0.19439208041876554,
|
|
"reward_std": 0.47691015154123306,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.052808452397584915,
|
|
"step": 299
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3455.3958435058594,
|
|
"epoch": 0.34285714285714286,
|
|
"grad_norm": 0.059302717447280884,
|
|
"kl": 3.039836883544922e-05,
|
|
"lambda_div_used": 0.5659952610731125,
|
|
"learning_rate": 4.749540639777539e-07,
|
|
"loss": 0.0631,
|
|
"reward": -0.4138263203203678,
|
|
"reward_after_mean": -0.4138263203203678,
|
|
"reward_after_std": 0.35950295627117157,
|
|
"reward_before_mean": -0.14833886176347733,
|
|
"reward_before_std": 0.30170741491019726,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26548744924366474,
|
|
"reward_change_min": -0.45029690116643906,
|
|
"reward_change_std": 0.16043098457157612,
|
|
"reward_std": 0.35950295627117157,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.19000552780926228,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2581.2917404174805,
|
|
"epoch": 0.344,
|
|
"grad_norm": 0.0829557478427887,
|
|
"kl": 1.7702579498291016e-05,
|
|
"lambda_div_used": 0.6020706444978714,
|
|
"learning_rate": 4.7185832004988133e-07,
|
|
"loss": 0.0033,
|
|
"reward": -0.2905968725681305,
|
|
"reward_after_mean": -0.2905968725681305,
|
|
"reward_after_std": 0.522302333265543,
|
|
"reward_before_mean": -0.03232934419065714,
|
|
"reward_before_std": 0.47291796933859587,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2582675274461508,
|
|
"reward_change_min": -0.4681916609406471,
|
|
"reward_change_std": 0.16694430727511644,
|
|
"reward_std": 0.5223023407161236,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.13649601582437754,
|
|
"step": 301
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2299.250026702881,
|
|
"epoch": 0.34514285714285714,
|
|
"grad_norm": 0.09259702265262604,
|
|
"kl": 2.473965287208557e-05,
|
|
"lambda_div_used": 0.6064160838723183,
|
|
"learning_rate": 4.68766384637248e-07,
|
|
"loss": 0.0049,
|
|
"reward": 0.1023632986471057,
|
|
"reward_after_mean": 0.1023632986471057,
|
|
"reward_after_std": 0.5881610047072172,
|
|
"reward_before_mean": 0.5643375236541033,
|
|
"reward_before_std": 0.48729276517406106,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.46197423338890076,
|
|
"reward_change_min": -0.6793596595525742,
|
|
"reward_change_std": 0.2643775464966893,
|
|
"reward_std": 0.588161014020443,
|
|
"rewards/accuracy_reward": 0.39583334513008595,
|
|
"rewards/cosine_scaled_reward": 0.16850417526438832,
|
|
"step": 302
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2496.604202270508,
|
|
"epoch": 0.3462857142857143,
|
|
"grad_norm": 0.10342813283205032,
|
|
"kl": 2.822279930114746e-05,
|
|
"lambda_div_used": 0.6173663139343262,
|
|
"learning_rate": 4.656784084364238e-07,
|
|
"loss": -0.0508,
|
|
"reward": -0.21569269057363272,
|
|
"reward_after_mean": -0.21569269057363272,
|
|
"reward_after_std": 0.5887450613081455,
|
|
"reward_before_mean": 0.048040480352938175,
|
|
"reward_before_std": 0.5433634500950575,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2637331634759903,
|
|
"reward_change_min": -0.4554433934390545,
|
|
"reward_change_std": 0.17205023765563965,
|
|
"reward_std": 0.5887450724840164,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.09779285965487361,
|
|
"step": 303
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2828.375030517578,
|
|
"epoch": 0.3474285714285714,
|
|
"grad_norm": 0.06273287534713745,
|
|
"kl": 3.515370190143585e-05,
|
|
"lambda_div_used": 0.5850896239280701,
|
|
"learning_rate": 4.6259454195101267e-07,
|
|
"loss": 0.0395,
|
|
"reward": -0.351226020604372,
|
|
"reward_after_mean": -0.351226020604372,
|
|
"reward_after_std": 0.4407376032322645,
|
|
"reward_before_mean": -0.091004628688097,
|
|
"reward_before_std": 0.39247662480920553,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2602214030921459,
|
|
"reward_change_min": -0.4692201763391495,
|
|
"reward_change_std": 0.16711712814867496,
|
|
"reward_std": 0.4407376106828451,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.1743379645049572,
|
|
"step": 304
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2963.270835876465,
|
|
"epoch": 0.3485714285714286,
|
|
"grad_norm": 0.07068406045436859,
|
|
"kl": 3.521144390106201e-05,
|
|
"lambda_div_used": 0.5555498078465462,
|
|
"learning_rate": 4.59514935484316e-07,
|
|
"loss": 0.0077,
|
|
"reward": -0.44899775832891464,
|
|
"reward_after_mean": -0.44899775832891464,
|
|
"reward_after_std": 0.32930242642760277,
|
|
"reward_before_mean": -0.17746215965598822,
|
|
"reward_before_std": 0.2488851365633309,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27153559774160385,
|
|
"reward_change_min": -0.41173194721341133,
|
|
"reward_change_std": 0.14773181919008493,
|
|
"reward_std": 0.32930243387818336,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.19829548941925168,
|
|
"step": 305
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2542.8750228881836,
|
|
"epoch": 0.3497142857142857,
|
|
"grad_norm": 0.09160702675580978,
|
|
"kl": 2.777576446533203e-05,
|
|
"lambda_div_used": 0.5710241869091988,
|
|
"learning_rate": 4.5643973913200837e-07,
|
|
"loss": 0.001,
|
|
"reward": -0.28438286297023296,
|
|
"reward_after_mean": -0.28438286297023296,
|
|
"reward_after_std": 0.45469350554049015,
|
|
"reward_before_mean": 0.05048683221684769,
|
|
"reward_before_std": 0.3197702756151557,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.33486970886588097,
|
|
"reward_change_min": -0.46306542679667473,
|
|
"reward_change_std": 0.1733616916462779,
|
|
"reward_std": 0.45469350926578045,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.11617984622716904,
|
|
"step": 306
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2358.9167098999023,
|
|
"epoch": 0.35085714285714287,
|
|
"grad_norm": 0.08308543264865875,
|
|
"kl": 2.5270506739616394e-05,
|
|
"lambda_div_used": 0.6283661872148514,
|
|
"learning_rate": 4.5336910277482155e-07,
|
|
"loss": 0.0233,
|
|
"reward": -0.023870151489973068,
|
|
"reward_after_mean": -0.023870151489973068,
|
|
"reward_after_std": 0.5968872811645269,
|
|
"reward_before_mean": 0.2986760139465332,
|
|
"reward_before_std": 0.602552474476397,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3225461672991514,
|
|
"reward_change_min": -0.5788566246628761,
|
|
"reward_change_std": 0.22817020770162344,
|
|
"reward_std": 0.5968873165547848,
|
|
"rewards/accuracy_reward": 0.2916666716337204,
|
|
"rewards/cosine_scaled_reward": 0.007009351626038551,
|
|
"step": 307
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3364.187530517578,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.04729427769780159,
|
|
"kl": 1.2226402759552002e-05,
|
|
"lambda_div_used": 0.5575956255197525,
|
|
"learning_rate": 4.503031760712397e-07,
|
|
"loss": -0.0147,
|
|
"reward": -0.364587739109993,
|
|
"reward_after_mean": -0.364587739109993,
|
|
"reward_after_std": 0.3172164801508188,
|
|
"reward_before_mean": -0.06630371138453484,
|
|
"reward_before_std": 0.25998193118721247,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29828402772545815,
|
|
"reward_change_min": -0.4606154337525368,
|
|
"reward_change_std": 0.17100436985492706,
|
|
"reward_std": 0.31721648946404457,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.14963705092668533,
|
|
"step": 308
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3246.750030517578,
|
|
"epoch": 0.35314285714285715,
|
|
"grad_norm": 0.054204076528549194,
|
|
"kl": 1.4858320355415344e-05,
|
|
"lambda_div_used": 0.6364770829677582,
|
|
"learning_rate": 4.4724210845020494e-07,
|
|
"loss": 0.0936,
|
|
"reward": -0.09930397570133209,
|
|
"reward_after_mean": -0.09930397570133209,
|
|
"reward_after_std": 0.6563027147203684,
|
|
"reward_before_mean": 0.18065341375768185,
|
|
"reward_before_std": 0.6305609010159969,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27995740808546543,
|
|
"reward_change_min": -0.5127242244780064,
|
|
"reward_change_std": 0.19019456766545773,
|
|
"reward_std": 0.6563027296215296,
|
|
"rewards/accuracy_reward": 0.2083333358168602,
|
|
"rewards/cosine_scaled_reward": -0.027679930441081524,
|
|
"step": 309
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2454.750015258789,
|
|
"epoch": 0.35428571428571426,
|
|
"grad_norm": 0.10188445448875427,
|
|
"kl": 3.9263395592570305e-05,
|
|
"lambda_div_used": 0.5863905549049377,
|
|
"learning_rate": 4.441860491038345e-07,
|
|
"loss": -0.0091,
|
|
"reward": -0.358197920024395,
|
|
"reward_after_mean": -0.358197920024395,
|
|
"reward_after_std": 0.4752412661910057,
|
|
"reward_before_mean": -0.10783447185531259,
|
|
"reward_before_std": 0.39106855262070894,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25036344304680824,
|
|
"reward_change_min": -0.36833222955465317,
|
|
"reward_change_std": 0.13517011515796185,
|
|
"reward_std": 0.47524126805365086,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.17033447965513915,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2549.5000228881836,
|
|
"epoch": 0.3554285714285714,
|
|
"grad_norm": 0.07070475816726685,
|
|
"kl": 1.1476688086986542e-05,
|
|
"lambda_div_used": 0.6028245091438293,
|
|
"learning_rate": 4.4113514698014953e-07,
|
|
"loss": -0.0234,
|
|
"reward": -0.07257996033877134,
|
|
"reward_after_mean": -0.07257996033877134,
|
|
"reward_after_std": 0.5427698846906424,
|
|
"reward_before_mean": 0.32260693423449993,
|
|
"reward_before_std": 0.4735551681369543,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3951868824660778,
|
|
"reward_change_min": -0.6274620294570923,
|
|
"reward_change_std": 0.2447280865162611,
|
|
"reward_std": 0.5427699014544487,
|
|
"rewards/accuracy_reward": 0.3125000037252903,
|
|
"rewards/cosine_scaled_reward": 0.010106915608048439,
|
|
"step": 311
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2062.020851135254,
|
|
"epoch": 0.3565714285714286,
|
|
"grad_norm": 0.0864594504237175,
|
|
"kl": 2.2163614630699158e-05,
|
|
"lambda_div_used": 0.5457132831215858,
|
|
"learning_rate": 4.3808955077581546e-07,
|
|
"loss": 0.0122,
|
|
"reward": -0.03406492620706558,
|
|
"reward_after_mean": -0.03406492620706558,
|
|
"reward_after_std": 0.3851391561329365,
|
|
"reward_before_mean": 0.5170513242483139,
|
|
"reward_before_std": 0.20092127658426762,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5511162541806698,
|
|
"reward_change_min": -0.7572819888591766,
|
|
"reward_change_std": 0.2855915669351816,
|
|
"reward_std": 0.38513917103409767,
|
|
"rewards/accuracy_reward": 0.375,
|
|
"rewards/cosine_scaled_reward": 0.1420513167977333,
|
|
"step": 312
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2939.958351135254,
|
|
"epoch": 0.3577142857142857,
|
|
"grad_norm": 0.06731264293193817,
|
|
"kl": 2.4283304810523987e-05,
|
|
"lambda_div_used": 0.5877447426319122,
|
|
"learning_rate": 4.350494089288943e-07,
|
|
"loss": -0.0439,
|
|
"reward": -0.15467195864766836,
|
|
"reward_after_mean": -0.15467195864766836,
|
|
"reward_after_std": 0.522945849224925,
|
|
"reward_before_mean": 0.20540180057287216,
|
|
"reward_before_std": 0.40051606576889753,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3600737862288952,
|
|
"reward_change_min": -0.5124829597771168,
|
|
"reward_change_std": 0.19857864920049906,
|
|
"reward_std": 0.5229458846151829,
|
|
"rewards/accuracy_reward": 0.2291666679084301,
|
|
"rewards/cosine_scaled_reward": -0.02376485476270318,
|
|
"step": 313
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2541.18754196167,
|
|
"epoch": 0.3588571428571429,
|
|
"grad_norm": 0.08012691885232925,
|
|
"kl": 3.474205732345581e-05,
|
|
"lambda_div_used": 0.607881672680378,
|
|
"learning_rate": 4.3201486961161093e-07,
|
|
"loss": 0.0168,
|
|
"reward": -0.20861644856631756,
|
|
"reward_after_mean": -0.20861644856631756,
|
|
"reward_after_std": 0.5089061111211777,
|
|
"reward_before_mean": 0.08002243563532829,
|
|
"reward_before_std": 0.49900877848267555,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28863888792693615,
|
|
"reward_change_min": -0.548176895827055,
|
|
"reward_change_std": 0.2009973768144846,
|
|
"reward_std": 0.5089061167091131,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.08664422971196473,
|
|
"step": 314
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3124.3333740234375,
|
|
"epoch": 0.36,
|
|
"grad_norm": 0.0558871328830719,
|
|
"kl": 1.9735191017389297e-05,
|
|
"lambda_div_used": 0.5734824016690254,
|
|
"learning_rate": 4.2898608072313045e-07,
|
|
"loss": 0.0494,
|
|
"reward": -0.1729298224672675,
|
|
"reward_after_mean": -0.1729298224672675,
|
|
"reward_after_std": 0.45045475475490093,
|
|
"reward_before_mean": 0.20758753083646297,
|
|
"reward_before_std": 0.33522335812449455,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3805173486471176,
|
|
"reward_change_min": -0.5440769977867603,
|
|
"reward_change_std": 0.20876472163945436,
|
|
"reward_std": 0.4504547640681267,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.02157914894632995,
|
|
"step": 315
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3509.5416870117188,
|
|
"epoch": 0.36114285714285715,
|
|
"grad_norm": 0.05331319198012352,
|
|
"kl": 2.981536090373993e-05,
|
|
"lambda_div_used": 0.5893311724066734,
|
|
"learning_rate": 4.2596318988235037e-07,
|
|
"loss": 0.0135,
|
|
"reward": -0.22396450489759445,
|
|
"reward_after_mean": -0.22396450489759445,
|
|
"reward_after_std": 0.43868801929056644,
|
|
"reward_before_mean": 0.07456053979694843,
|
|
"reward_before_std": 0.414919788017869,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29852502048015594,
|
|
"reward_change_min": -0.4823254942893982,
|
|
"reward_change_std": 0.1912739286199212,
|
|
"reward_std": 0.43868803791701794,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.09210613742470741,
|
|
"step": 316
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3099.5833435058594,
|
|
"epoch": 0.36228571428571427,
|
|
"grad_norm": 0.06200157478451729,
|
|
"kl": 3.091990947723389e-05,
|
|
"lambda_div_used": 0.5825675800442696,
|
|
"learning_rate": 4.2294634442070553e-07,
|
|
"loss": 0.0437,
|
|
"reward": -0.35854507237672806,
|
|
"reward_after_mean": -0.35854507237672806,
|
|
"reward_after_std": 0.43180895783007145,
|
|
"reward_before_mean": -0.102005859836936,
|
|
"reward_before_std": 0.3769839182496071,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25653921999037266,
|
|
"reward_change_min": -0.4170425795018673,
|
|
"reward_change_std": 0.15367608424276114,
|
|
"reward_std": 0.4318089634180069,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.18533920124173164,
|
|
"step": 317
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2281.000068664551,
|
|
"epoch": 0.36342857142857143,
|
|
"grad_norm": 0.0932127833366394,
|
|
"kl": 3.602728247642517e-05,
|
|
"lambda_div_used": 0.6223095878958702,
|
|
"learning_rate": 4.1993569137498776e-07,
|
|
"loss": 0.0631,
|
|
"reward": -0.07363937655463815,
|
|
"reward_after_mean": -0.07363937655463815,
|
|
"reward_after_std": 0.5691333152353764,
|
|
"reward_before_mean": 0.24711408838629723,
|
|
"reward_before_std": 0.5656038168817759,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3207534924149513,
|
|
"reward_change_min": -0.5814453661441803,
|
|
"reward_change_std": 0.2219030074775219,
|
|
"reward_std": 0.569133322685957,
|
|
"rewards/accuracy_reward": 0.2500000074505806,
|
|
"rewards/cosine_scaled_reward": -0.0028859074227511883,
|
|
"step": 318
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2854.5625076293945,
|
|
"epoch": 0.36457142857142855,
|
|
"grad_norm": 0.08494079858064651,
|
|
"kl": 2.7507543563842773e-05,
|
|
"lambda_div_used": 0.5380512997508049,
|
|
"learning_rate": 4.1693137748017915e-07,
|
|
"loss": -0.0389,
|
|
"reward": -0.4965253435075283,
|
|
"reward_after_mean": -0.4965253435075283,
|
|
"reward_after_std": 0.2333353590220213,
|
|
"reward_before_mean": -0.2120634987950325,
|
|
"reward_before_std": 0.16711975168436766,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28446184657514095,
|
|
"reward_change_min": -0.4124374948441982,
|
|
"reward_change_std": 0.15511877462267876,
|
|
"reward_std": 0.2333353627473116,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2120634950697422,
|
|
"step": 319
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1964.8958740234375,
|
|
"epoch": 0.3657142857142857,
|
|
"grad_norm": 0.09068436175584793,
|
|
"kl": 3.33394855260849e-05,
|
|
"lambda_div_used": 0.6160280704498291,
|
|
"learning_rate": 4.1393354916230005e-07,
|
|
"loss": 0.0117,
|
|
"reward": -0.19740011962130666,
|
|
"reward_after_mean": -0.19740011962130666,
|
|
"reward_after_std": 0.5963947810232639,
|
|
"reward_before_mean": 0.0693174353800714,
|
|
"reward_before_std": 0.5336382519453764,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26671755872666836,
|
|
"reward_change_min": -0.4143567681312561,
|
|
"reward_change_std": 0.15817437414079905,
|
|
"reward_std": 0.5963947977870703,
|
|
"rewards/accuracy_reward": 0.12500000186264515,
|
|
"rewards/cosine_scaled_reward": -0.05568256159313023,
|
|
"step": 320
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1974.8958778381348,
|
|
"epoch": 0.3668571428571429,
|
|
"grad_norm": 0.0693729817867279,
|
|
"kl": 1.574307680130005e-05,
|
|
"lambda_div_used": 0.6290735602378845,
|
|
"learning_rate": 4.1094235253127374e-07,
|
|
"loss": 0.0387,
|
|
"reward": 0.11023577488958836,
|
|
"reward_after_mean": 0.11023577488958836,
|
|
"reward_after_std": 0.6442763805389404,
|
|
"reward_before_mean": 0.5218545235693455,
|
|
"reward_before_std": 0.6013733670115471,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4116187300533056,
|
|
"reward_change_min": -0.6836260408163071,
|
|
"reward_change_std": 0.2705372450873256,
|
|
"reward_std": 0.6442763898521662,
|
|
"rewards/accuracy_reward": 0.3958333395421505,
|
|
"rewards/cosine_scaled_reward": 0.12602117005735636,
|
|
"step": 321
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2837.833381652832,
|
|
"epoch": 0.368,
|
|
"grad_norm": 0.10218925029039383,
|
|
"kl": 4.9501657485961914e-05,
|
|
"lambda_div_used": 0.6059064492583275,
|
|
"learning_rate": 4.079579333738039e-07,
|
|
"loss": -0.0178,
|
|
"reward": -0.2954628551378846,
|
|
"reward_after_mean": -0.2954628551378846,
|
|
"reward_after_std": 0.5665331184864044,
|
|
"reward_before_mean": -0.04816420469433069,
|
|
"reward_before_std": 0.48093545995652676,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24729863554239273,
|
|
"reward_change_min": -0.3516420107334852,
|
|
"reward_change_std": 0.13184416200965643,
|
|
"reward_std": 0.5665331222116947,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.13149754563346505,
|
|
"step": 322
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3015.666717529297,
|
|
"epoch": 0.36914285714285716,
|
|
"grad_norm": 0.0596209317445755,
|
|
"kl": 1.7877668142318726e-05,
|
|
"lambda_div_used": 0.5565166473388672,
|
|
"learning_rate": 4.0498043714627006e-07,
|
|
"loss": 0.0465,
|
|
"reward": -0.16177266091108322,
|
|
"reward_after_mean": -0.16177266091108322,
|
|
"reward_after_std": 0.42436218820512295,
|
|
"reward_before_mean": 0.2816486116498709,
|
|
"reward_before_std": 0.2578566027805209,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4434212874621153,
|
|
"reward_change_min": -0.6335049495100975,
|
|
"reward_change_std": 0.23841418512165546,
|
|
"reward_std": 0.4243622049689293,
|
|
"rewards/accuracy_reward": 0.2916666679084301,
|
|
"rewards/cosine_scaled_reward": -0.010018057189881802,
|
|
"step": 323
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2751.6458435058594,
|
|
"epoch": 0.3702857142857143,
|
|
"grad_norm": 0.10376841574907303,
|
|
"kl": 2.1306797862052917e-05,
|
|
"lambda_div_used": 0.5670785158872604,
|
|
"learning_rate": 4.020100089676376e-07,
|
|
"loss": -0.0038,
|
|
"reward": -0.10797288408502936,
|
|
"reward_after_mean": -0.10797288408502936,
|
|
"reward_after_std": 0.4300071895122528,
|
|
"reward_before_mean": 0.3295288155786693,
|
|
"reward_before_std": 0.3011532872915268,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.43750171549618244,
|
|
"reward_change_min": -0.6185614429414272,
|
|
"reward_change_std": 0.2333526872098446,
|
|
"reward_std": 0.43000719882547855,
|
|
"rewards/accuracy_reward": 0.25000000558793545,
|
|
"rewards/cosine_scaled_reward": 0.07952881557866931,
|
|
"step": 324
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2943.8958587646484,
|
|
"epoch": 0.37142857142857144,
|
|
"grad_norm": 0.07239633053541183,
|
|
"kl": 9.275972843170166e-06,
|
|
"lambda_div_used": 0.6356078162789345,
|
|
"learning_rate": 3.9904679361238526e-07,
|
|
"loss": 0.0667,
|
|
"reward": -0.050798285752534866,
|
|
"reward_after_mean": -0.050798285752534866,
|
|
"reward_after_std": 0.6440786644816399,
|
|
"reward_before_mean": 0.24087253957986832,
|
|
"reward_before_std": 0.6331603992730379,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2916708290576935,
|
|
"reward_change_min": -0.4932599440217018,
|
|
"reward_change_std": 0.20057004038244486,
|
|
"reward_std": 0.6440786886960268,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": -0.02996080555021763,
|
|
"step": 325
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2375.645851135254,
|
|
"epoch": 0.37257142857142855,
|
|
"grad_norm": 0.09299691766500473,
|
|
"kl": 2.9705464839935303e-05,
|
|
"lambda_div_used": 0.5330315083265305,
|
|
"learning_rate": 3.9609093550344907e-07,
|
|
"loss": -0.0979,
|
|
"reward": -0.20862240344285965,
|
|
"reward_after_mean": -0.20862240344285965,
|
|
"reward_after_std": 0.3296848703175783,
|
|
"reward_before_mean": 0.2517512815538794,
|
|
"reward_before_std": 0.1450797226279974,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.460373692214489,
|
|
"reward_change_min": -0.6264396719634533,
|
|
"reward_change_std": 0.23280893173068762,
|
|
"reward_std": 0.3296848740428686,
|
|
"rewards/accuracy_reward": 0.25,
|
|
"rewards/cosine_scaled_reward": 0.001751287141814828,
|
|
"step": 326
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2810.187511444092,
|
|
"epoch": 0.3737142857142857,
|
|
"grad_norm": 0.06210591271519661,
|
|
"kl": 8.400529623031616e-06,
|
|
"lambda_div_used": 0.5783005133271217,
|
|
"learning_rate": 3.931425787051832e-07,
|
|
"loss": -0.0483,
|
|
"reward": -0.11793380603194237,
|
|
"reward_after_mean": -0.11793380603194237,
|
|
"reward_after_std": 0.45353105291724205,
|
|
"reward_before_mean": 0.2935123089700937,
|
|
"reward_before_std": 0.3566841436550021,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41144610941410065,
|
|
"reward_change_min": -0.6089312434196472,
|
|
"reward_change_std": 0.23675687983632088,
|
|
"reward_std": 0.4535310585051775,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": 0.022678975947201252,
|
|
"step": 327
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3518.8958435058594,
|
|
"epoch": 0.37485714285714283,
|
|
"grad_norm": 0.05192619562149048,
|
|
"kl": 2.485513687133789e-05,
|
|
"lambda_div_used": 0.5715365409851074,
|
|
"learning_rate": 3.902018669163384e-07,
|
|
"loss": 0.0155,
|
|
"reward": -0.4497411046177149,
|
|
"reward_after_mean": -0.4497411046177149,
|
|
"reward_after_std": 0.40177739411592484,
|
|
"reward_before_mean": -0.21567542850971222,
|
|
"reward_before_std": 0.32284149527549744,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23406569100916386,
|
|
"reward_change_min": -0.33814629539847374,
|
|
"reward_change_std": 0.12489900179207325,
|
|
"reward_std": 0.40177739597857,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.2573420889675617,
|
|
"step": 328
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2243.0416946411133,
|
|
"epoch": 0.376,
|
|
"grad_norm": 0.10950610786676407,
|
|
"kl": 2.469681203365326e-05,
|
|
"lambda_div_used": 0.6176783889532089,
|
|
"learning_rate": 3.872689434630585e-07,
|
|
"loss": -0.0562,
|
|
"reward": -0.013319691643118858,
|
|
"reward_after_mean": -0.013319691643118858,
|
|
"reward_after_std": 0.5674807205796242,
|
|
"reward_before_mean": 0.3346351385116577,
|
|
"reward_before_std": 0.5435493532568216,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34795483760535717,
|
|
"reward_change_min": -0.5437430925667286,
|
|
"reward_change_std": 0.22110824659466743,
|
|
"reward_std": 0.5674807410687208,
|
|
"rewards/accuracy_reward": 0.27083334140479565,
|
|
"rewards/cosine_scaled_reward": 0.06380179291591048,
|
|
"step": 329
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2141.479190826416,
|
|
"epoch": 0.37714285714285717,
|
|
"grad_norm": 0.10051169991493225,
|
|
"kl": 4.89354133605957e-05,
|
|
"lambda_div_used": 0.5594945177435875,
|
|
"learning_rate": 3.843439512918949e-07,
|
|
"loss": -0.0064,
|
|
"reward": -0.43674849811941385,
|
|
"reward_after_mean": -0.43674849811941385,
|
|
"reward_after_std": 0.32430145144462585,
|
|
"reward_before_mean": -0.17029727809131145,
|
|
"reward_before_std": 0.2681279256939888,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.266451220959425,
|
|
"reward_change_min": -0.44791119545698166,
|
|
"reward_change_std": 0.1575082140043378,
|
|
"reward_std": 0.32430145516991615,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.2119639493757859,
|
|
"step": 330
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2564.604202270508,
|
|
"epoch": 0.3782857142857143,
|
|
"grad_norm": 0.12382425367832184,
|
|
"kl": 4.225596785545349e-05,
|
|
"lambda_div_used": 0.5574543103575706,
|
|
"learning_rate": 3.8142703296283953e-07,
|
|
"loss": 0.0413,
|
|
"reward": -0.41342686861753464,
|
|
"reward_after_mean": -0.41342686861753464,
|
|
"reward_after_std": 0.33145094849169254,
|
|
"reward_before_mean": -0.13134576752781868,
|
|
"reward_before_std": 0.25930391903966665,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2820810880511999,
|
|
"reward_change_min": -0.4496995583176613,
|
|
"reward_change_std": 0.16215670108795166,
|
|
"reward_std": 0.33145095966756344,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.17301243357360363,
|
|
"step": 331
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2420.6666870117188,
|
|
"epoch": 0.37942857142857145,
|
|
"grad_norm": 0.07938601076602936,
|
|
"kl": 2.1241605281829834e-05,
|
|
"lambda_div_used": 0.5416555106639862,
|
|
"learning_rate": 3.785183306423767e-07,
|
|
"loss": 0.0287,
|
|
"reward": -0.21761326864361763,
|
|
"reward_after_mean": -0.21761326864361763,
|
|
"reward_after_std": 0.34953613951802254,
|
|
"reward_before_mean": 0.2352729644626379,
|
|
"reward_before_std": 0.18298226408660412,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.45288625732064247,
|
|
"reward_change_min": -0.6215734221041203,
|
|
"reward_change_std": 0.2345103258267045,
|
|
"reward_std": 0.34953614324331284,
|
|
"rewards/accuracy_reward": 0.25,
|
|
"rewards/cosine_scaled_reward": -0.014727018773555756,
|
|
"step": 332
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2350.6875381469727,
|
|
"epoch": 0.38057142857142856,
|
|
"grad_norm": 0.08657004684209824,
|
|
"kl": 2.1725893020629883e-05,
|
|
"lambda_div_used": 0.6729468256235123,
|
|
"learning_rate": 3.7561798609655373e-07,
|
|
"loss": 0.0369,
|
|
"reward": -0.01155824027955532,
|
|
"reward_after_mean": -0.01155824027955532,
|
|
"reward_after_std": 0.8104779217392206,
|
|
"reward_before_mean": 0.23989354074001312,
|
|
"reward_before_std": 0.8117707390338182,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2514517717063427,
|
|
"reward_change_min": -0.5040576457977295,
|
|
"reward_change_std": 0.19348772894591093,
|
|
"reward_std": 0.810477938503027,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": -0.030939804390072823,
|
|
"step": 333
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3491.6041870117188,
|
|
"epoch": 0.38171428571428573,
|
|
"grad_norm": 0.05208956450223923,
|
|
"kl": 1.1973665095865726e-05,
|
|
"lambda_div_used": 0.5869207382202148,
|
|
"learning_rate": 3.72726140684072e-07,
|
|
"loss": -0.009,
|
|
"reward": -0.2944636158645153,
|
|
"reward_after_mean": -0.2944636158645153,
|
|
"reward_after_std": 0.43844909220933914,
|
|
"reward_before_mean": -0.013160821050405502,
|
|
"reward_before_std": 0.40495526185259223,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2813027612864971,
|
|
"reward_change_min": -0.4717176593840122,
|
|
"reward_change_std": 0.1835445323958993,
|
|
"reward_std": 0.4384490940719843,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.13816083781421185,
|
|
"step": 334
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2589.0833892822266,
|
|
"epoch": 0.38285714285714284,
|
|
"grad_norm": 0.0683097094297409,
|
|
"kl": 1.9595026969909668e-05,
|
|
"lambda_div_used": 0.6027273386716843,
|
|
"learning_rate": 3.6984293534939737e-07,
|
|
"loss": -0.0488,
|
|
"reward": 0.005996011197566986,
|
|
"reward_after_mean": 0.005996011197566986,
|
|
"reward_after_std": 0.5422599408775568,
|
|
"reward_before_mean": 0.41999348998069763,
|
|
"reward_before_std": 0.4701922629028559,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41399746760725975,
|
|
"reward_change_min": -0.623339332640171,
|
|
"reward_change_std": 0.24341293703764677,
|
|
"reward_std": 0.5422599650919437,
|
|
"rewards/accuracy_reward": 0.33333334140479565,
|
|
"rewards/cosine_scaled_reward": 0.08666014112532139,
|
|
"step": 335
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3162.1458740234375,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.067026786506176,
|
|
"kl": 2.0024715922772884e-05,
|
|
"lambda_div_used": 0.5950900241732597,
|
|
"learning_rate": 3.6696851061588994e-07,
|
|
"loss": 0.0138,
|
|
"reward": -0.16571878641843796,
|
|
"reward_after_mean": -0.16571878641843796,
|
|
"reward_after_std": 0.4802239239215851,
|
|
"reward_before_mean": 0.14471458829939365,
|
|
"reward_before_std": 0.4368621027097106,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31043340265750885,
|
|
"reward_change_min": -0.45523249730467796,
|
|
"reward_change_std": 0.18632046319544315,
|
|
"reward_std": 0.48022393323481083,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.06361875729635358,
|
|
"step": 336
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3228.916717529297,
|
|
"epoch": 0.3851428571428571,
|
|
"grad_norm": 0.05528295040130615,
|
|
"kl": 1.9825994968414307e-05,
|
|
"lambda_div_used": 0.6006625667214394,
|
|
"learning_rate": 3.641030065789562e-07,
|
|
"loss": 0.0552,
|
|
"reward": -0.2433483125641942,
|
|
"reward_after_mean": -0.2433483125641942,
|
|
"reward_after_std": 0.5013067033141851,
|
|
"reward_before_mean": 0.03825543075799942,
|
|
"reward_before_std": 0.4624515902251005,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28160375356674194,
|
|
"reward_change_min": -0.4847262054681778,
|
|
"reward_change_std": 0.17836903873831034,
|
|
"reward_std": 0.501306714490056,
|
|
"rewards/accuracy_reward": 0.14583333767950535,
|
|
"rewards/cosine_scaled_reward": -0.10757789574563503,
|
|
"step": 337
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2393.645866394043,
|
|
"epoch": 0.3862857142857143,
|
|
"grad_norm": 0.10292567312717438,
|
|
"kl": 3.5159289836883545e-05,
|
|
"lambda_div_used": 0.6184682846069336,
|
|
"learning_rate": 3.612465628992203e-07,
|
|
"loss": 0.1082,
|
|
"reward": -0.010080082342028618,
|
|
"reward_after_mean": -0.010080082342028618,
|
|
"reward_after_std": 0.6298389285802841,
|
|
"reward_before_mean": 0.3685674872249365,
|
|
"reward_before_std": 0.546540604904294,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3786476030945778,
|
|
"reward_change_min": -0.5825354494154453,
|
|
"reward_change_std": 0.2281006295233965,
|
|
"reward_std": 0.6298389509320259,
|
|
"rewards/accuracy_reward": 0.3125000037252903,
|
|
"rewards/cosine_scaled_reward": 0.05606749979779124,
|
|
"step": 338
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3026.9583892822266,
|
|
"epoch": 0.38742857142857146,
|
|
"grad_norm": 0.0688738226890564,
|
|
"kl": 2.4460256099700928e-05,
|
|
"lambda_div_used": 0.5621443763375282,
|
|
"learning_rate": 3.5839931879571725e-07,
|
|
"loss": -0.0178,
|
|
"reward": -0.32898029685020447,
|
|
"reward_after_mean": -0.32898029685020447,
|
|
"reward_after_std": 0.3227359801530838,
|
|
"reward_before_mean": -0.01589403674006462,
|
|
"reward_before_std": 0.2781273443251848,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31308628246188164,
|
|
"reward_change_min": -0.48185204714536667,
|
|
"reward_change_std": 0.1813461184501648,
|
|
"reward_std": 0.3227359913289547,
|
|
"rewards/accuracy_reward": 0.1041666716337204,
|
|
"rewards/cosine_scaled_reward": -0.12006070092320442,
|
|
"step": 339
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2447.937515258789,
|
|
"epoch": 0.38857142857142857,
|
|
"grad_norm": 0.10290948301553726,
|
|
"kl": 2.363821113249287e-05,
|
|
"lambda_div_used": 0.5592405423521996,
|
|
"learning_rate": 3.555614130391079e-07,
|
|
"loss": -0.0046,
|
|
"reward": -0.42997122183442116,
|
|
"reward_after_mean": -0.42997122183442116,
|
|
"reward_after_std": 0.33643546886742115,
|
|
"reward_before_mean": -0.1512418081983924,
|
|
"reward_before_std": 0.2663228642195463,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2787294201552868,
|
|
"reward_change_min": -0.4129678010940552,
|
|
"reward_change_std": 0.1523351836949587,
|
|
"reward_std": 0.3364354781806469,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.17207514215260744,
|
|
"step": 340
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2488.9583587646484,
|
|
"epoch": 0.38971428571428574,
|
|
"grad_norm": 0.061042170971632004,
|
|
"kl": 1.909211277961731e-05,
|
|
"lambda_div_used": 0.5834595933556557,
|
|
"learning_rate": 3.5273298394491515e-07,
|
|
"loss": -0.0023,
|
|
"reward": 0.009830114664509892,
|
|
"reward_after_mean": 0.009830114664509892,
|
|
"reward_after_std": 0.5199873205274343,
|
|
"reward_before_mean": 0.4715769328176975,
|
|
"reward_before_std": 0.38039534725248814,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.46174679696559906,
|
|
"reward_change_min": -0.6357490979135036,
|
|
"reward_change_std": 0.2529716519638896,
|
|
"reward_std": 0.5199873335659504,
|
|
"rewards/accuracy_reward": 0.33333334140479565,
|
|
"rewards/cosine_scaled_reward": 0.13824361143633723,
|
|
"step": 341
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3020.520854949951,
|
|
"epoch": 0.39085714285714285,
|
|
"grad_norm": 0.07325860857963562,
|
|
"kl": 2.4762004613876343e-05,
|
|
"lambda_div_used": 0.5985761880874634,
|
|
"learning_rate": 3.4991416936678276e-07,
|
|
"loss": 0.0007,
|
|
"reward": -0.06264204811304808,
|
|
"reward_after_mean": -0.06264204811304808,
|
|
"reward_after_std": 0.5387034490704536,
|
|
"reward_before_mean": 0.33080895524472,
|
|
"reward_before_std": 0.45346040464937687,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3934510052204132,
|
|
"reward_change_min": -0.5952773988246918,
|
|
"reward_change_std": 0.23232300952076912,
|
|
"reward_std": 0.5387034583836794,
|
|
"rewards/accuracy_reward": 0.29166666977107525,
|
|
"rewards/cosine_scaled_reward": 0.03914228640496731,
|
|
"step": 342
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3405.1875610351562,
|
|
"epoch": 0.392,
|
|
"grad_norm": 0.04665439575910568,
|
|
"kl": 2.7433037757873535e-05,
|
|
"lambda_div_used": 0.6150833070278168,
|
|
"learning_rate": 3.471051066897562e-07,
|
|
"loss": 0.0509,
|
|
"reward": -0.1582264108583331,
|
|
"reward_after_mean": -0.1582264108583331,
|
|
"reward_after_std": 0.5790915079414845,
|
|
"reward_before_mean": 0.13680532574653625,
|
|
"reward_before_std": 0.5264813583344221,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29503174126148224,
|
|
"reward_change_min": -0.4708465002477169,
|
|
"reward_change_std": 0.1767220702022314,
|
|
"reward_std": 0.5790915191173553,
|
|
"rewards/accuracy_reward": 0.1875000074505806,
|
|
"rewards/cosine_scaled_reward": -0.050694676116108894,
|
|
"step": 343
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2360.875045776367,
|
|
"epoch": 0.3931428571428571,
|
|
"grad_norm": 0.07271434366703033,
|
|
"kl": 1.041218638420105e-05,
|
|
"lambda_div_used": 0.5770246461033821,
|
|
"learning_rate": 3.4430593282358777e-07,
|
|
"loss": -0.0008,
|
|
"reward": 0.04454847797751427,
|
|
"reward_after_mean": 0.04454847797751427,
|
|
"reward_after_std": 0.5018230397254229,
|
|
"reward_before_mean": 0.5492894258350134,
|
|
"reward_before_std": 0.35145203582942486,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5047409404069185,
|
|
"reward_change_min": -0.7455123476684093,
|
|
"reward_change_std": 0.2790954224765301,
|
|
"reward_std": 0.5018230620771646,
|
|
"rewards/accuracy_reward": 0.4375,
|
|
"rewards/cosine_scaled_reward": 0.1117894072085619,
|
|
"step": 344
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3030.0833740234375,
|
|
"epoch": 0.3942857142857143,
|
|
"grad_norm": 0.0703606829047203,
|
|
"kl": 3.1925737857818604e-05,
|
|
"lambda_div_used": 0.5804503262042999,
|
|
"learning_rate": 3.4151678419606233e-07,
|
|
"loss": 0.0646,
|
|
"reward": -0.22463034093379974,
|
|
"reward_after_mean": -0.22463034093379974,
|
|
"reward_after_std": 0.41369511373341084,
|
|
"reward_before_mean": 0.09700941108167171,
|
|
"reward_before_std": 0.37086532823741436,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.32163975574076176,
|
|
"reward_change_min": -0.5076877251267433,
|
|
"reward_change_std": 0.1956562791019678,
|
|
"reward_std": 0.4136951379477978,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.06965726800262928,
|
|
"step": 345
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3186.6458740234375,
|
|
"epoch": 0.3954285714285714,
|
|
"grad_norm": 0.05292130261659622,
|
|
"kl": 1.4215707778930664e-05,
|
|
"lambda_div_used": 0.5958857163786888,
|
|
"learning_rate": 3.387377967463493e-07,
|
|
"loss": 0.0331,
|
|
"reward": -0.24454469978809357,
|
|
"reward_after_mean": -0.24454469978809357,
|
|
"reward_after_std": 0.48920249938964844,
|
|
"reward_before_mean": 0.04633236164227128,
|
|
"reward_before_std": 0.44041235372424126,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29087705351412296,
|
|
"reward_change_min": -0.4697858430445194,
|
|
"reward_change_std": 0.179568306542933,
|
|
"reward_std": 0.4892025087028742,
|
|
"rewards/accuracy_reward": 0.16666667349636555,
|
|
"rewards/cosine_scaled_reward": -0.12033431546296924,
|
|
"step": 346
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3055.4583587646484,
|
|
"epoch": 0.3965714285714286,
|
|
"grad_norm": 0.07354738563299179,
|
|
"kl": 6.0675665736198425e-06,
|
|
"lambda_div_used": 0.5582710355520248,
|
|
"learning_rate": 3.359691059183761e-07,
|
|
"loss": 0.0132,
|
|
"reward": -0.42621116526424885,
|
|
"reward_after_mean": -0.42621116526424885,
|
|
"reward_after_std": 0.3322943150997162,
|
|
"reward_before_mean": -0.15516536496579647,
|
|
"reward_before_std": 0.2633455842733383,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2710457965731621,
|
|
"reward_change_min": -0.4307188205420971,
|
|
"reward_change_std": 0.15532648842781782,
|
|
"reward_std": 0.33229432441294193,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.19683203659951687,
|
|
"step": 347
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2852.4583435058594,
|
|
"epoch": 0.3977142857142857,
|
|
"grad_norm": 0.09605983644723892,
|
|
"kl": 3.167241811752319e-05,
|
|
"lambda_div_used": 0.5788434967398643,
|
|
"learning_rate": 3.3321084665422803e-07,
|
|
"loss": -0.0017,
|
|
"reward": -0.2087371125817299,
|
|
"reward_after_mean": -0.2087371125817299,
|
|
"reward_after_std": 0.40125221759080887,
|
|
"reward_before_mean": 0.12491290923207998,
|
|
"reward_before_std": 0.36025606002658606,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3336500097066164,
|
|
"reward_change_min": -0.5146553069353104,
|
|
"reward_change_std": 0.20004253275692463,
|
|
"reward_std": 0.40125224366784096,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.04175376985222101,
|
|
"step": 348
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2964.875015258789,
|
|
"epoch": 0.39885714285714285,
|
|
"grad_norm": 0.06869405508041382,
|
|
"kl": 1.731887459754944e-05,
|
|
"lambda_div_used": 0.5606642514467239,
|
|
"learning_rate": 3.3046315338757026e-07,
|
|
"loss": 0.0031,
|
|
"reward": -0.2807905152440071,
|
|
"reward_after_mean": -0.2807905152440071,
|
|
"reward_after_std": 0.39816635474562645,
|
|
"reward_before_mean": 0.08202904835343361,
|
|
"reward_before_std": 0.2763519547879696,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36281956918537617,
|
|
"reward_change_min": -0.54488330706954,
|
|
"reward_change_std": 0.20150058157742023,
|
|
"reward_std": 0.39816636219620705,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.0846376121044159,
|
|
"step": 349
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2521.9791870117188,
|
|
"epoch": 0.4,
|
|
"grad_norm": 0.10303740203380585,
|
|
"kl": 2.6671215891838074e-05,
|
|
"lambda_div_used": 0.6121688261628151,
|
|
"learning_rate": 3.2772616003709616e-07,
|
|
"loss": -0.0711,
|
|
"reward": -0.04820730444043875,
|
|
"reward_after_mean": -0.04820730444043875,
|
|
"reward_after_std": 0.5604431573301554,
|
|
"reward_before_mean": 0.287683189380914,
|
|
"reward_before_std": 0.5214535812847316,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3358904607594013,
|
|
"reward_change_min": -0.5252225995063782,
|
|
"reward_change_std": 0.21241459622979164,
|
|
"reward_std": 0.5604431666433811,
|
|
"rewards/accuracy_reward": 0.27083334140479565,
|
|
"rewards/cosine_scaled_reward": 0.016849845182150602,
|
|
"step": 350
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3268.250030517578,
|
|
"epoch": 0.40114285714285713,
|
|
"grad_norm": 0.04833042621612549,
|
|
"kl": 2.4568289518356323e-05,
|
|
"lambda_div_used": 0.5601532310247421,
|
|
"learning_rate": 3.250000000000001e-07,
|
|
"loss": 0.0519,
|
|
"reward": -0.250777292996645,
|
|
"reward_after_mean": -0.250777292996645,
|
|
"reward_after_std": 0.38171103224158287,
|
|
"reward_before_mean": 0.1175131555646658,
|
|
"reward_before_std": 0.2718004435300827,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36829047463834286,
|
|
"reward_change_min": -0.5422449931502342,
|
|
"reward_change_std": 0.20415859669446945,
|
|
"reward_std": 0.3817110415548086,
|
|
"rewards/accuracy_reward": 0.1875,
|
|
"rewards/cosine_scaled_reward": -0.06998682580888271,
|
|
"step": 351
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2589.333366394043,
|
|
"epoch": 0.4022857142857143,
|
|
"grad_norm": 0.06552080065011978,
|
|
"kl": 1.905485987663269e-05,
|
|
"lambda_div_used": 0.6104472130537033,
|
|
"learning_rate": 3.222848061454764e-07,
|
|
"loss": 0.0025,
|
|
"reward": -0.09347447147592902,
|
|
"reward_after_mean": -0.09347447147592902,
|
|
"reward_after_std": 0.6156985405832529,
|
|
"reward_before_mean": 0.26158976616716245,
|
|
"reward_before_std": 0.5061299707740545,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3550642393529415,
|
|
"reward_change_min": -0.5232127867639065,
|
|
"reward_change_std": 0.1957317991182208,
|
|
"reward_std": 0.6156985703855753,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": 0.01158975763246417,
|
|
"step": 352
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2533.7916946411133,
|
|
"epoch": 0.4034285714285714,
|
|
"grad_norm": 0.07949981093406677,
|
|
"kl": 4.393863491714001e-05,
|
|
"lambda_div_used": 0.6204324588179588,
|
|
"learning_rate": 3.195807108082429e-07,
|
|
"loss": 0.0456,
|
|
"reward": -0.12379613053053617,
|
|
"reward_after_mean": -0.12379613053053617,
|
|
"reward_after_std": 0.6454745382070541,
|
|
"reward_before_mean": 0.203694608528167,
|
|
"reward_before_std": 0.5591385969892144,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.327490733936429,
|
|
"reward_change_min": -0.5519858598709106,
|
|
"reward_change_std": 0.1984806014224887,
|
|
"reward_std": 0.6454745400696993,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": -0.0463053984567523,
|
|
"step": 353
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1914.562515258789,
|
|
"epoch": 0.4045714285714286,
|
|
"grad_norm": 0.07853464037179947,
|
|
"kl": 2.3346394300460815e-05,
|
|
"lambda_div_used": 0.5937526449561119,
|
|
"learning_rate": 3.168878457820915e-07,
|
|
"loss": -0.0345,
|
|
"reward": 0.036306386813521385,
|
|
"reward_after_mean": 0.036306386813521385,
|
|
"reward_after_std": 0.5187021996825933,
|
|
"reward_before_mean": 0.4854283039458096,
|
|
"reward_before_std": 0.42800275422632694,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4491219110786915,
|
|
"reward_change_min": -0.637521255761385,
|
|
"reward_change_std": 0.252533134073019,
|
|
"reward_std": 0.5187022183090448,
|
|
"rewards/accuracy_reward": 0.33333334140479565,
|
|
"rewards/cosine_scaled_reward": 0.15209495089948177,
|
|
"step": 354
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2335.8958740234375,
|
|
"epoch": 0.4057142857142857,
|
|
"grad_norm": 0.09039249271154404,
|
|
"kl": 2.7535483241081238e-05,
|
|
"lambda_div_used": 0.6392721608281136,
|
|
"learning_rate": 3.142063423134644e-07,
|
|
"loss": 0.0158,
|
|
"reward": 0.017912205308675766,
|
|
"reward_after_mean": 0.017912205308675766,
|
|
"reward_after_std": 0.6705970745533705,
|
|
"reward_before_mean": 0.355070261284709,
|
|
"reward_before_std": 0.6396346259862185,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.33715808019042015,
|
|
"reward_change_min": -0.5581717267632484,
|
|
"reward_change_std": 0.21758130192756653,
|
|
"reward_std": 0.6705970987677574,
|
|
"rewards/accuracy_reward": 0.354166679084301,
|
|
"rewards/cosine_scaled_reward": 0.0009035973343998194,
|
|
"step": 355
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2585.4791870117188,
|
|
"epoch": 0.40685714285714286,
|
|
"grad_norm": 0.06332956254482269,
|
|
"kl": 1.6089528799057007e-05,
|
|
"lambda_div_used": 0.6472217217087746,
|
|
"learning_rate": 3.115363310950578e-07,
|
|
"loss": -0.0329,
|
|
"reward": -0.00402236171066761,
|
|
"reward_after_mean": -0.00402236171066761,
|
|
"reward_after_std": 0.6937260664999485,
|
|
"reward_before_mean": 0.2978215580806136,
|
|
"reward_before_std": 0.6849680617451668,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3018439058214426,
|
|
"reward_change_min": -0.4991762898862362,
|
|
"reward_change_std": 0.2024542335420847,
|
|
"reward_std": 0.6937260907143354,
|
|
"rewards/accuracy_reward": 0.2916666753590107,
|
|
"rewards/cosine_scaled_reward": 0.00615486316382885,
|
|
"step": 356
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3264.8125610351562,
|
|
"epoch": 0.408,
|
|
"grad_norm": 0.057182345539331436,
|
|
"kl": 2.91336327791214e-05,
|
|
"lambda_div_used": 0.5862653851509094,
|
|
"learning_rate": 3.0887794225945143e-07,
|
|
"loss": 0.0104,
|
|
"reward": -0.31888771802186966,
|
|
"reward_after_mean": -0.31888771802186966,
|
|
"reward_after_std": 0.43322005309164524,
|
|
"reward_before_mean": -0.049007685855031013,
|
|
"reward_before_std": 0.39840223640203476,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2698800265789032,
|
|
"reward_change_min": -0.48054200410842896,
|
|
"reward_change_std": 0.1758509548380971,
|
|
"reward_std": 0.43322005309164524,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1531743509694934,
|
|
"step": 357
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3050.2500534057617,
|
|
"epoch": 0.40914285714285714,
|
|
"grad_norm": 0.05330098420381546,
|
|
"kl": 2.4370267055928707e-05,
|
|
"lambda_div_used": 0.6431510671973228,
|
|
"learning_rate": 3.062313053727671e-07,
|
|
"loss": 0.003,
|
|
"reward": -0.07201657444238663,
|
|
"reward_after_mean": -0.07201657444238663,
|
|
"reward_after_std": 0.688179362565279,
|
|
"reward_before_mean": 0.2027557131368667,
|
|
"reward_before_std": 0.6673171781003475,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27477228827774525,
|
|
"reward_change_min": -0.4944618083536625,
|
|
"reward_change_std": 0.18934499006718397,
|
|
"reward_std": 0.6881793700158596,
|
|
"rewards/accuracy_reward": 0.2291666716337204,
|
|
"rewards/cosine_scaled_reward": -0.026410957798361778,
|
|
"step": 358
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2671.937530517578,
|
|
"epoch": 0.4102857142857143,
|
|
"grad_norm": 0.07592163234949112,
|
|
"kl": 3.884732723236084e-05,
|
|
"lambda_div_used": 0.5795001536607742,
|
|
"learning_rate": 3.0359654942835247e-07,
|
|
"loss": 0.0464,
|
|
"reward": -0.2641681991517544,
|
|
"reward_after_mean": -0.2641681991517544,
|
|
"reward_after_std": 0.4112328961491585,
|
|
"reward_before_mean": 0.03885669261217117,
|
|
"reward_before_std": 0.36149344593286514,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30302487686276436,
|
|
"reward_change_min": -0.4962599165737629,
|
|
"reward_change_std": 0.18252001702785492,
|
|
"reward_std": 0.4112329035997391,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.10697665251791477,
|
|
"step": 359
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2798.937530517578,
|
|
"epoch": 0.4114285714285714,
|
|
"grad_norm": 0.06665439158678055,
|
|
"kl": 3.1637027859687805e-05,
|
|
"lambda_div_used": 0.6062901616096497,
|
|
"learning_rate": 3.0097380284049523e-07,
|
|
"loss": -0.0029,
|
|
"reward": -0.0033570118248462677,
|
|
"reward_after_mean": -0.0033570118248462677,
|
|
"reward_after_std": 0.5593161657452583,
|
|
"reward_before_mean": 0.4031880460679531,
|
|
"reward_before_std": 0.4958424214273691,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.40654509887099266,
|
|
"reward_change_min": -0.6473257802426815,
|
|
"reward_change_std": 0.25588439870625734,
|
|
"reward_std": 0.5593161787837744,
|
|
"rewards/accuracy_reward": 0.3333333358168602,
|
|
"rewards/cosine_scaled_reward": 0.0698547288775444,
|
|
"step": 360
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3150.625030517578,
|
|
"epoch": 0.4125714285714286,
|
|
"grad_norm": 0.06971946358680725,
|
|
"kl": 2.9014074243605137e-05,
|
|
"lambda_div_used": 0.6219506114721298,
|
|
"learning_rate": 2.9836319343816397e-07,
|
|
"loss": 0.0048,
|
|
"reward": -0.11981392558664083,
|
|
"reward_after_mean": -0.11981392558664083,
|
|
"reward_after_std": 0.5868565142154694,
|
|
"reward_before_mean": 0.17182117886841297,
|
|
"reward_before_std": 0.564973471686244,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29163510724902153,
|
|
"reward_change_min": -0.5129627995193005,
|
|
"reward_change_std": 0.19524423126131296,
|
|
"reward_std": 0.5868565402925014,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.057345494627952576,
|
|
"step": 361
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1471.916706085205,
|
|
"epoch": 0.4137142857142857,
|
|
"grad_norm": 0.12450961768627167,
|
|
"kl": 2.4568289518356323e-05,
|
|
"lambda_div_used": 0.571638435125351,
|
|
"learning_rate": 2.9576484845877793e-07,
|
|
"loss": -0.0303,
|
|
"reward": 0.013313630130141973,
|
|
"reward_after_mean": 0.013313630130141973,
|
|
"reward_after_std": 0.4897063076496124,
|
|
"reward_before_mean": 0.505017813295126,
|
|
"reward_before_std": 0.32288316125050187,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4917041715234518,
|
|
"reward_change_min": -0.6615025624632835,
|
|
"reward_change_std": 0.25815817900002,
|
|
"reward_std": 0.4897063188254833,
|
|
"rewards/accuracy_reward": 0.37500000558793545,
|
|
"rewards/cosine_scaled_reward": 0.1300177900120616,
|
|
"step": 362
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2174.270881652832,
|
|
"epoch": 0.41485714285714287,
|
|
"grad_norm": 0.08419760316610336,
|
|
"kl": 4.357472062110901e-05,
|
|
"lambda_div_used": 0.6037088930606842,
|
|
"learning_rate": 2.931788945420058e-07,
|
|
"loss": 0.0176,
|
|
"reward": -0.09517315030097961,
|
|
"reward_after_mean": -0.09517315030097961,
|
|
"reward_after_std": 0.57970448769629,
|
|
"reward_before_mean": 0.282845395617187,
|
|
"reward_before_std": 0.4830307289958,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.37801856361329556,
|
|
"reward_change_min": -0.5875305943191051,
|
|
"reward_change_std": 0.22446841653436422,
|
|
"reward_std": 0.57970448769629,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": 0.03284538397565484,
|
|
"step": 363
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2614.9167098999023,
|
|
"epoch": 0.416,
|
|
"grad_norm": 0.07822614908218384,
|
|
"kl": 2.289540134370327e-05,
|
|
"lambda_div_used": 0.5602610111236572,
|
|
"learning_rate": 2.9060545772359305e-07,
|
|
"loss": 0.0079,
|
|
"reward": -0.4077296704053879,
|
|
"reward_after_mean": -0.4077296704053879,
|
|
"reward_after_std": 0.3345623780041933,
|
|
"reward_before_mean": -0.1330121699720621,
|
|
"reward_before_std": 0.27498441375792027,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2747174873948097,
|
|
"reward_change_min": -0.4260003827512264,
|
|
"reward_change_std": 0.16245513781905174,
|
|
"reward_std": 0.3345623817294836,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.195512181147933,
|
|
"step": 364
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3003.604202270508,
|
|
"epoch": 0.41714285714285715,
|
|
"grad_norm": 0.04828161001205444,
|
|
"kl": 6.87967985868454e-06,
|
|
"lambda_div_used": 0.6036554351449013,
|
|
"learning_rate": 2.8804466342921987e-07,
|
|
"loss": 0.0555,
|
|
"reward": -0.31679424038156867,
|
|
"reward_after_mean": -0.31679424038156867,
|
|
"reward_after_std": 0.5247047282755375,
|
|
"reward_before_mean": -0.07733823172748089,
|
|
"reward_before_std": 0.4768796032294631,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23945600911974907,
|
|
"reward_change_min": -0.42952996119856834,
|
|
"reward_change_std": 0.15284609980881214,
|
|
"reward_std": 0.5247047450393438,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.18150490219704807,
|
|
"step": 365
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1893.00004196167,
|
|
"epoch": 0.41828571428571426,
|
|
"grad_norm": 0.08456390351057053,
|
|
"kl": 1.6473233699798584e-05,
|
|
"lambda_div_used": 0.6091032102704048,
|
|
"learning_rate": 2.854966364683872e-07,
|
|
"loss": -0.0195,
|
|
"reward": 0.005720527842640877,
|
|
"reward_after_mean": 0.005720527842640877,
|
|
"reward_after_std": 0.6514943428337574,
|
|
"reward_before_mean": 0.43899719044566154,
|
|
"reward_before_std": 0.49832448456436396,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4332766607403755,
|
|
"reward_change_min": -0.6058122254908085,
|
|
"reward_change_std": 0.23482245951890945,
|
|
"reward_std": 0.6514943763613701,
|
|
"rewards/accuracy_reward": 0.39583333767950535,
|
|
"rewards/cosine_scaled_reward": 0.043163834139704704,
|
|
"step": 366
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2780.937515258789,
|
|
"epoch": 0.41942857142857143,
|
|
"grad_norm": 0.06994574517011642,
|
|
"kl": 2.946704626083374e-05,
|
|
"lambda_div_used": 0.6079575195908546,
|
|
"learning_rate": 2.829615010283344e-07,
|
|
"loss": 0.0087,
|
|
"reward": -0.16347008850425482,
|
|
"reward_after_mean": -0.16347008850425482,
|
|
"reward_after_std": 0.5466162711381912,
|
|
"reward_before_mean": 0.1403359491378069,
|
|
"reward_before_std": 0.49341694079339504,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30380604416131973,
|
|
"reward_change_min": -0.4883100688457489,
|
|
"reward_change_std": 0.18378268275409937,
|
|
"reward_std": 0.5466162823140621,
|
|
"rewards/accuracy_reward": 0.1875000074505806,
|
|
"rewards/cosine_scaled_reward": -0.0471640374744311,
|
|
"step": 367
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2967.041679382324,
|
|
"epoch": 0.4205714285714286,
|
|
"grad_norm": 0.07414139062166214,
|
|
"kl": 4.155188798904419e-05,
|
|
"lambda_div_used": 0.5803077146410942,
|
|
"learning_rate": 2.8043938066798645e-07,
|
|
"loss": 0.0504,
|
|
"reward": -0.27402403950691223,
|
|
"reward_after_mean": -0.27402403950691223,
|
|
"reward_after_std": 0.4059589561074972,
|
|
"reward_before_mean": 0.008191026747226715,
|
|
"reward_before_std": 0.371637674048543,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28221505135297775,
|
|
"reward_change_min": -0.44202784821391106,
|
|
"reward_change_std": 0.17603088915348053,
|
|
"reward_std": 0.40595896542072296,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.15847563743591309,
|
|
"step": 368
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2863.1667404174805,
|
|
"epoch": 0.4217142857142857,
|
|
"grad_norm": 0.07171155512332916,
|
|
"kl": 2.0891427993774414e-05,
|
|
"lambda_div_used": 0.6472674459218979,
|
|
"learning_rate": 2.7793039831193133e-07,
|
|
"loss": -0.0044,
|
|
"reward": -0.04304695222526789,
|
|
"reward_after_mean": -0.04304695222526789,
|
|
"reward_after_std": 0.687617726624012,
|
|
"reward_before_mean": 0.25146727729588747,
|
|
"reward_before_std": 0.6888436311855912,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29451421834528446,
|
|
"reward_change_min": -0.6169496960937977,
|
|
"reward_change_std": 0.22693553566932678,
|
|
"reward_std": 0.6876177359372377,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": 0.0014672745019197464,
|
|
"step": 369
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3261.2291870117188,
|
|
"epoch": 0.4228571428571429,
|
|
"grad_norm": 0.058094967156648636,
|
|
"kl": 2.1554529666900635e-05,
|
|
"lambda_div_used": 0.5657742545008659,
|
|
"learning_rate": 2.7543467624442956e-07,
|
|
"loss": 0.0031,
|
|
"reward": -0.2483972143381834,
|
|
"reward_after_mean": -0.2483972143381834,
|
|
"reward_after_std": 0.4499538466334343,
|
|
"reward_before_mean": 0.13386818021535873,
|
|
"reward_before_std": 0.29791492875665426,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38226539455354214,
|
|
"reward_change_min": -0.51585279032588,
|
|
"reward_change_std": 0.19617348536849022,
|
|
"reward_std": 0.4499538540840149,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.03279848676174879,
|
|
"step": 370
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1885.645866394043,
|
|
"epoch": 0.424,
|
|
"grad_norm": 0.12381042540073395,
|
|
"kl": 1.5240773791447282e-05,
|
|
"lambda_div_used": 0.5325883999466896,
|
|
"learning_rate": 2.729523361034538e-07,
|
|
"loss": -0.0304,
|
|
"reward": -0.14332129061222076,
|
|
"reward_after_mean": -0.14332129061222076,
|
|
"reward_after_std": 0.3713335506618023,
|
|
"reward_before_mean": 0.3775896169245243,
|
|
"reward_before_std": 0.14313361048698425,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5209108907729387,
|
|
"reward_change_min": -0.6799871250987053,
|
|
"reward_change_std": 0.26304140500724316,
|
|
"reward_std": 0.3713335543870926,
|
|
"rewards/accuracy_reward": 0.375,
|
|
"rewards/cosine_scaled_reward": 0.0025896020233631134,
|
|
"step": 371
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3061.6250228881836,
|
|
"epoch": 0.42514285714285716,
|
|
"grad_norm": 0.0565616711974144,
|
|
"kl": 1.3771001249551773e-05,
|
|
"lambda_div_used": 0.6153379678726196,
|
|
"learning_rate": 2.7048349887476037e-07,
|
|
"loss": 0.0019,
|
|
"reward": 0.08842738252133131,
|
|
"reward_after_mean": 0.08842738252133131,
|
|
"reward_after_std": 0.6899000462144613,
|
|
"reward_before_mean": 0.5513297475408763,
|
|
"reward_before_std": 0.5352799613028765,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.46290238574147224,
|
|
"reward_change_min": -0.6905807815492153,
|
|
"reward_change_std": 0.26435263454914093,
|
|
"reward_std": 0.6899000480771065,
|
|
"rewards/accuracy_reward": 0.3958333358168602,
|
|
"rewards/cosine_scaled_reward": 0.15549640776589513,
|
|
"step": 372
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1999.791690826416,
|
|
"epoch": 0.42628571428571427,
|
|
"grad_norm": 0.11289634555578232,
|
|
"kl": 2.6244670152664185e-05,
|
|
"lambda_div_used": 0.5901408270001411,
|
|
"learning_rate": 2.6802828488599294e-07,
|
|
"loss": -0.0028,
|
|
"reward": -0.25084885116666555,
|
|
"reward_after_mean": -0.25084885116666555,
|
|
"reward_after_std": 0.47954990342259407,
|
|
"reward_before_mean": 0.03629123326390982,
|
|
"reward_before_std": 0.41134614683687687,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28714008443057537,
|
|
"reward_change_min": -0.4200817756354809,
|
|
"reward_change_std": 0.1626526527106762,
|
|
"reward_std": 0.47954992204904556,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.08870877418667078,
|
|
"step": 373
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2700.791717529297,
|
|
"epoch": 0.42742857142857144,
|
|
"grad_norm": 0.0688440129160881,
|
|
"kl": 2.674013376235962e-05,
|
|
"lambda_div_used": 0.6036395579576492,
|
|
"learning_rate": 2.655868138008171e-07,
|
|
"loss": -0.0525,
|
|
"reward": -0.17511339485645294,
|
|
"reward_after_mean": -0.17511339485645294,
|
|
"reward_after_std": 0.5073369853198528,
|
|
"reward_before_mean": 0.13348117470741272,
|
|
"reward_before_std": 0.48259105905890465,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30859458073973656,
|
|
"reward_change_min": -0.5309739634394646,
|
|
"reward_change_std": 0.20630291104316711,
|
|
"reward_std": 0.5073370076715946,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.07485215552151203,
|
|
"step": 374
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3033.791702270508,
|
|
"epoch": 0.42857142857142855,
|
|
"grad_norm": 0.05866050720214844,
|
|
"kl": 1.8252991139888763e-05,
|
|
"lambda_div_used": 0.6017400398850441,
|
|
"learning_rate": 2.631592046130896e-07,
|
|
"loss": 0.0477,
|
|
"reward": -0.18571221362799406,
|
|
"reward_after_mean": -0.18571221362799406,
|
|
"reward_after_std": 0.5023255608975887,
|
|
"reward_before_mean": 0.11062880232930183,
|
|
"reward_before_std": 0.46861019916832447,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2963410019874573,
|
|
"reward_change_min": -0.45132842287421227,
|
|
"reward_change_std": 0.1817708509042859,
|
|
"reward_std": 0.5023255832493305,
|
|
"rewards/accuracy_reward": 0.20833334140479565,
|
|
"rewards/cosine_scaled_reward": -0.097704553976655,
|
|
"step": 375
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2246.770851135254,
|
|
"epoch": 0.4297142857142857,
|
|
"grad_norm": 0.08337263762950897,
|
|
"kl": 8.605420589447021e-06,
|
|
"lambda_div_used": 0.5853807479143143,
|
|
"learning_rate": 2.6074557564105724e-07,
|
|
"loss": 0.0128,
|
|
"reward": -0.3334618601948023,
|
|
"reward_after_mean": -0.3334618601948023,
|
|
"reward_after_std": 0.4486316703259945,
|
|
"reward_before_mean": -0.05933803477091715,
|
|
"reward_before_std": 0.3892780668102205,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2741238009184599,
|
|
"reward_change_min": -0.46821001917123795,
|
|
"reward_change_std": 0.1663337228819728,
|
|
"reward_std": 0.44863167591392994,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.163504708558321,
|
|
"step": 376
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3488.3541870117188,
|
|
"epoch": 0.4308571428571429,
|
|
"grad_norm": 0.055966660380363464,
|
|
"kl": 2.5756657123565674e-05,
|
|
"lambda_div_used": 0.5742340236902237,
|
|
"learning_rate": 2.583460445215911e-07,
|
|
"loss": -0.0032,
|
|
"reward": -0.3738445993512869,
|
|
"reward_after_mean": -0.3738445993512869,
|
|
"reward_after_std": 0.39967909548431635,
|
|
"reward_before_mean": -0.10715527582215145,
|
|
"reward_before_std": 0.33878533728420734,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26668932288885117,
|
|
"reward_change_min": -0.4109174869954586,
|
|
"reward_change_std": 0.15686567965894938,
|
|
"reward_std": 0.39967911317944527,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.19048861227929592,
|
|
"step": 377
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2134.3333740234375,
|
|
"epoch": 0.432,
|
|
"grad_norm": 0.13181807100772858,
|
|
"kl": 2.8654932975769043e-05,
|
|
"lambda_div_used": 0.6004306748509407,
|
|
"learning_rate": 2.5596072820445254e-07,
|
|
"loss": -0.0331,
|
|
"reward": -0.06989812850952148,
|
|
"reward_after_mean": -0.06989812850952148,
|
|
"reward_after_std": 0.5459046512842178,
|
|
"reward_before_mean": 0.31881395215168595,
|
|
"reward_before_std": 0.46263560838997364,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3887120857834816,
|
|
"reward_change_min": -0.5917952992022038,
|
|
"reward_change_std": 0.23005317710340023,
|
|
"reward_std": 0.5459046605974436,
|
|
"rewards/accuracy_reward": 0.29166666977107525,
|
|
"rewards/cosine_scaled_reward": 0.027147281914949417,
|
|
"step": 378
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3121.0416870117188,
|
|
"epoch": 0.43314285714285716,
|
|
"grad_norm": 0.07427296042442322,
|
|
"kl": 1.5733763575553894e-05,
|
|
"lambda_div_used": 0.5833418369293213,
|
|
"learning_rate": 2.5358974294659373e-07,
|
|
"loss": -0.0048,
|
|
"reward": -0.29101302847266197,
|
|
"reward_after_mean": -0.29101302847266197,
|
|
"reward_after_std": 0.4290795102715492,
|
|
"reward_before_mean": -0.009382706135511398,
|
|
"reward_before_std": 0.38597164303064346,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2816303465515375,
|
|
"reward_change_min": -0.47839244455099106,
|
|
"reward_change_std": 0.17911072820425034,
|
|
"reward_std": 0.4290795363485813,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.13438269309699535,
|
|
"step": 379
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2823.416702270508,
|
|
"epoch": 0.4342857142857143,
|
|
"grad_norm": 0.06611248105764389,
|
|
"kl": 2.2601336240768433e-05,
|
|
"lambda_div_used": 0.6066281795501709,
|
|
"learning_rate": 2.512332043064913e-07,
|
|
"loss": 0.0533,
|
|
"reward": -0.17613293696194887,
|
|
"reward_after_mean": -0.17613293696194887,
|
|
"reward_after_std": 0.495351817458868,
|
|
"reward_before_mean": 0.1346770692616701,
|
|
"reward_before_std": 0.48867712169885635,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31081000342965126,
|
|
"reward_change_min": -0.5689719989895821,
|
|
"reward_change_std": 0.21053248085081577,
|
|
"reward_std": 0.4953518286347389,
|
|
"rewards/accuracy_reward": 0.1875000074505806,
|
|
"rewards/cosine_scaled_reward": -0.05282294252538122,
|
|
"step": 380
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2983.7916984558105,
|
|
"epoch": 0.43542857142857144,
|
|
"grad_norm": 0.08519254624843597,
|
|
"kl": 3.2504089176654816e-05,
|
|
"lambda_div_used": 0.6109358817338943,
|
|
"learning_rate": 2.488912271385139e-07,
|
|
"loss": 0.0165,
|
|
"reward": -0.2218069350346923,
|
|
"reward_after_mean": -0.2218069350346923,
|
|
"reward_after_std": 0.5642109382897615,
|
|
"reward_before_mean": 0.040106164291501045,
|
|
"reward_before_std": 0.5099438140168786,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.261913089081645,
|
|
"reward_change_min": -0.45296039059758186,
|
|
"reward_change_std": 0.1642473293468356,
|
|
"reward_std": 0.5642109606415033,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.10572716826573014,
|
|
"step": 381
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2500.437545776367,
|
|
"epoch": 0.43657142857142855,
|
|
"grad_norm": 0.07325682789087296,
|
|
"kl": 3.65767627954483e-05,
|
|
"lambda_div_used": 0.5600408837199211,
|
|
"learning_rate": 2.465639255873246e-07,
|
|
"loss": -0.0096,
|
|
"reward": -0.4078812226653099,
|
|
"reward_after_mean": -0.4078812226653099,
|
|
"reward_after_std": 0.3340430334210396,
|
|
"reward_before_mean": -0.13890772312879562,
|
|
"reward_before_std": 0.27443209011107683,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26897350139915943,
|
|
"reward_change_min": -0.42554087564349174,
|
|
"reward_change_std": 0.15754834469407797,
|
|
"reward_std": 0.3340430427342653,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.20140772499144077,
|
|
"step": 382
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2824.291717529297,
|
|
"epoch": 0.4377142857142857,
|
|
"grad_norm": 0.08796168118715286,
|
|
"kl": 4.8138201236724854e-05,
|
|
"lambda_div_used": 0.5781937688589096,
|
|
"learning_rate": 2.4425141308231765e-07,
|
|
"loss": -0.0086,
|
|
"reward": -0.16291768848896027,
|
|
"reward_after_mean": -0.16291768848896027,
|
|
"reward_after_std": 0.387674568220973,
|
|
"reward_before_mean": 0.1924248207360506,
|
|
"reward_before_std": 0.35597400926053524,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3553424943238497,
|
|
"reward_change_min": -0.5255323797464371,
|
|
"reward_change_std": 0.21135967783629894,
|
|
"reward_std": 0.3876745719462633,
|
|
"rewards/accuracy_reward": 0.2083333432674408,
|
|
"rewards/cosine_scaled_reward": -0.01590852066874504,
|
|
"step": 383
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2347.6042404174805,
|
|
"epoch": 0.43885714285714283,
|
|
"grad_norm": 0.09212023764848709,
|
|
"kl": 2.508983016014099e-05,
|
|
"lambda_div_used": 0.6615950018167496,
|
|
"learning_rate": 2.4195380233209006e-07,
|
|
"loss": 0.0408,
|
|
"reward": 0.27819300815463066,
|
|
"reward_after_mean": 0.27819300815463066,
|
|
"reward_after_std": 0.7268820777535439,
|
|
"reward_before_mean": 0.6785084716975689,
|
|
"reward_before_std": 0.7540020374581218,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4003154691308737,
|
|
"reward_change_min": -0.6902611702680588,
|
|
"reward_change_std": 0.283959056250751,
|
|
"reward_std": 0.7268821019679308,
|
|
"rewards/accuracy_reward": 0.4583333432674408,
|
|
"rewards/cosine_scaled_reward": 0.22017512656748295,
|
|
"step": 384
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2711.958396911621,
|
|
"epoch": 0.44,
|
|
"grad_norm": 0.07038652151823044,
|
|
"kl": 1.942366361618042e-05,
|
|
"lambda_div_used": 0.6165113672614098,
|
|
"learning_rate": 2.3967120531894857e-07,
|
|
"loss": 0.012,
|
|
"reward": -0.25626325886696577,
|
|
"reward_after_mean": -0.25626325886696577,
|
|
"reward_after_std": 0.5892766248434782,
|
|
"reward_before_mean": -0.0074533987790346146,
|
|
"reward_before_std": 0.5360177559778094,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24880987033247948,
|
|
"reward_change_min": -0.44854912906885147,
|
|
"reward_change_std": 0.1595793990418315,
|
|
"reward_std": 0.5892766322940588,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.13245339877903461,
|
|
"step": 385
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2889.1042404174805,
|
|
"epoch": 0.44114285714285717,
|
|
"grad_norm": 0.060970280319452286,
|
|
"kl": 1.8231570720672607e-05,
|
|
"lambda_div_used": 0.5940364003181458,
|
|
"learning_rate": 2.374037332934512e-07,
|
|
"loss": -0.0287,
|
|
"reward": -0.24576714914292097,
|
|
"reward_after_mean": -0.24576714914292097,
|
|
"reward_after_std": 0.4883785657584667,
|
|
"reward_before_mean": 0.04634229093790054,
|
|
"reward_before_std": 0.4279829766601324,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29210945032536983,
|
|
"reward_change_min": -0.4432514049112797,
|
|
"reward_change_std": 0.17009661067277193,
|
|
"reward_std": 0.48837856762111187,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.07865770626813173,
|
|
"step": 386
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2997.3125,
|
|
"epoch": 0.4422857142857143,
|
|
"grad_norm": 0.07781542837619781,
|
|
"kl": 1.3821758329868317e-05,
|
|
"lambda_div_used": 0.5551474094390869,
|
|
"learning_rate": 2.3515149676898552e-07,
|
|
"loss": -0.0529,
|
|
"reward": -0.38486793637275696,
|
|
"reward_after_mean": -0.38486793637275696,
|
|
"reward_after_std": 0.31045267172157764,
|
|
"reward_before_mean": -0.10442159324884415,
|
|
"reward_before_std": 0.24840335873886943,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28044634498655796,
|
|
"reward_change_min": -0.4241391494870186,
|
|
"reward_change_std": 0.15805031638592482,
|
|
"reward_std": 0.31045267917215824,
|
|
"rewards/accuracy_reward": 0.1041666716337204,
|
|
"rewards/cosine_scaled_reward": -0.20858826488256454,
|
|
"step": 387
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2572.2708587646484,
|
|
"epoch": 0.44342857142857145,
|
|
"grad_norm": 0.06563723832368851,
|
|
"kl": 2.9848888516426086e-06,
|
|
"lambda_div_used": 0.5865650475025177,
|
|
"learning_rate": 2.3291460551638237e-07,
|
|
"loss": 0.0407,
|
|
"reward": -0.15434425324201584,
|
|
"reward_after_mean": -0.15434425324201584,
|
|
"reward_after_std": 0.47601727209985256,
|
|
"reward_before_mean": 0.20021704956889153,
|
|
"reward_before_std": 0.39916726760566235,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.354561323300004,
|
|
"reward_change_min": -0.5467559210956097,
|
|
"reward_change_std": 0.21171134896576405,
|
|
"reward_std": 0.47601727209985256,
|
|
"rewards/accuracy_reward": 0.2708333358168602,
|
|
"rewards/cosine_scaled_reward": -0.07061627879738808,
|
|
"step": 388
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2650.937545776367,
|
|
"epoch": 0.44457142857142856,
|
|
"grad_norm": 0.07840488851070404,
|
|
"kl": 1.73286534845829e-05,
|
|
"lambda_div_used": 0.6045578718185425,
|
|
"learning_rate": 2.306931685585657e-07,
|
|
"loss": -0.0394,
|
|
"reward": -0.26122746989130974,
|
|
"reward_after_mean": -0.26122746989130974,
|
|
"reward_after_std": 0.5210577324032784,
|
|
"reward_before_mean": 0.002998221665620804,
|
|
"reward_before_std": 0.486898148432374,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26422569528222084,
|
|
"reward_change_min": -0.4951511509716511,
|
|
"reward_change_std": 0.18348107766360044,
|
|
"reward_std": 0.5210577566176653,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.1428351059439592,
|
|
"step": 389
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3007.666679382324,
|
|
"epoch": 0.44571428571428573,
|
|
"grad_norm": 0.07837636768817902,
|
|
"kl": 1.6082078218460083e-05,
|
|
"lambda_div_used": 0.61600511521101,
|
|
"learning_rate": 2.2848729416523859e-07,
|
|
"loss": 0.0713,
|
|
"reward": -0.18647840432822704,
|
|
"reward_after_mean": -0.18647840432822704,
|
|
"reward_after_std": 0.5758322961628437,
|
|
"reward_before_mean": 0.08954079262912273,
|
|
"reward_before_std": 0.5347468825057149,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27601918019354343,
|
|
"reward_change_min": -0.47189946845173836,
|
|
"reward_change_std": 0.17452176753431559,
|
|
"reward_std": 0.5758323054760695,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.07712590089067817,
|
|
"step": 390
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2716.708351135254,
|
|
"epoch": 0.44685714285714284,
|
|
"grad_norm": 0.09803462773561478,
|
|
"kl": 2.5559216737747192e-05,
|
|
"lambda_div_used": 0.6362191960215569,
|
|
"learning_rate": 2.2629708984760706e-07,
|
|
"loss": 0.0166,
|
|
"reward": -0.013867436617147177,
|
|
"reward_after_mean": -0.013867436617147177,
|
|
"reward_after_std": 0.6719868443906307,
|
|
"reward_before_mean": 0.30256712157279253,
|
|
"reward_before_std": 0.6324740117415786,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3164345696568489,
|
|
"reward_change_min": -0.5261501334607601,
|
|
"reward_change_std": 0.20623697619885206,
|
|
"reward_std": 0.6719868592917919,
|
|
"rewards/accuracy_reward": 0.2708333432674408,
|
|
"rewards/cosine_scaled_reward": 0.03173377411440015,
|
|
"step": 391
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2130.9791984558105,
|
|
"epoch": 0.448,
|
|
"grad_norm": 0.11540649086236954,
|
|
"kl": 1.8610619008541107e-05,
|
|
"lambda_div_used": 0.6027960926294327,
|
|
"learning_rate": 2.2412266235313973e-07,
|
|
"loss": -0.0919,
|
|
"reward": -0.16835473664104939,
|
|
"reward_after_mean": -0.16835473664104939,
|
|
"reward_after_std": 0.5122340489178896,
|
|
"reward_before_mean": 0.1393747702240944,
|
|
"reward_before_std": 0.4794600326567888,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3077295087277889,
|
|
"reward_change_min": -0.4898943528532982,
|
|
"reward_change_std": 0.1954572731629014,
|
|
"reward_std": 0.5122340768575668,
|
|
"rewards/accuracy_reward": 0.16666666977107525,
|
|
"rewards/cosine_scaled_reward": -0.027291906299069524,
|
|
"step": 392
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2649.562530517578,
|
|
"epoch": 0.4491428571428571,
|
|
"grad_norm": 0.07226411253213882,
|
|
"kl": 2.648681402206421e-05,
|
|
"lambda_div_used": 0.5953627824783325,
|
|
"learning_rate": 2.2196411766036487e-07,
|
|
"loss": -0.067,
|
|
"reward": -0.20140517689287663,
|
|
"reward_after_mean": -0.20140517689287663,
|
|
"reward_after_std": 0.4766168761998415,
|
|
"reward_before_mean": 0.09705937840044498,
|
|
"reward_before_std": 0.4375506564974785,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2984645199030638,
|
|
"reward_change_min": -0.47758448868989944,
|
|
"reward_change_std": 0.18242334388196468,
|
|
"reward_std": 0.47661688551306725,
|
|
"rewards/accuracy_reward": 0.18750000558793545,
|
|
"rewards/cosine_scaled_reward": -0.09044064581394196,
|
|
"step": 393
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3104.1458740234375,
|
|
"epoch": 0.4502857142857143,
|
|
"grad_norm": 0.07446404546499252,
|
|
"kl": 2.555176615715027e-05,
|
|
"lambda_div_used": 0.5502220839262009,
|
|
"learning_rate": 2.1982156097370557e-07,
|
|
"loss": -0.1089,
|
|
"reward": -0.46874738670885563,
|
|
"reward_after_mean": -0.46874738670885563,
|
|
"reward_after_std": 0.30342659167945385,
|
|
"reward_before_mean": -0.20392105542123318,
|
|
"reward_before_std": 0.22309745661914349,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2648263294249773,
|
|
"reward_change_min": -0.3818593733012676,
|
|
"reward_change_std": 0.14172559697180986,
|
|
"reward_std": 0.30342659167945385,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.22475438751280308,
|
|
"step": 394
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2284.6458435058594,
|
|
"epoch": 0.4514285714285714,
|
|
"grad_norm": 0.12902304530143738,
|
|
"kl": 4.9501657485961914e-05,
|
|
"lambda_div_used": 0.5602920204401016,
|
|
"learning_rate": 2.1769509671835223e-07,
|
|
"loss": -0.1675,
|
|
"reward": -0.3165151756256819,
|
|
"reward_after_mean": -0.3165151756256819,
|
|
"reward_after_std": 0.3793674483895302,
|
|
"reward_before_mean": 0.0216768067330122,
|
|
"reward_before_std": 0.27139727398753166,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3381919823586941,
|
|
"reward_change_min": -0.48955149203538895,
|
|
"reward_change_std": 0.17990652937442064,
|
|
"reward_std": 0.3793674521148205,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.12415653187781572,
|
|
"step": 395
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3048.1458587646484,
|
|
"epoch": 0.45257142857142857,
|
|
"grad_norm": 0.054839495569467545,
|
|
"kl": 7.04331323504448e-05,
|
|
"lambda_div_used": 0.6417915895581245,
|
|
"learning_rate": 2.1558482853517253e-07,
|
|
"loss": 0.0192,
|
|
"reward": -0.09411877207458019,
|
|
"reward_after_mean": -0.09411877207458019,
|
|
"reward_after_std": 0.6742421016097069,
|
|
"reward_before_mean": 0.1871709941624431,
|
|
"reward_before_std": 0.659186695702374,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2812897562980652,
|
|
"reward_change_min": -0.5664796940982342,
|
|
"reward_change_std": 0.20405743923038244,
|
|
"reward_std": 0.6742421071976423,
|
|
"rewards/accuracy_reward": 0.20833333767950535,
|
|
"rewards/cosine_scaled_reward": -0.021162351593375206,
|
|
"step": 396
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2982.750030517578,
|
|
"epoch": 0.45371428571428574,
|
|
"grad_norm": 0.05798732861876488,
|
|
"kl": 2.0850449800491333e-05,
|
|
"lambda_div_used": 0.5765915215015411,
|
|
"learning_rate": 2.134908592756607e-07,
|
|
"loss": 0.0391,
|
|
"reward": -0.36006474308669567,
|
|
"reward_after_mean": -0.36006474308669567,
|
|
"reward_after_std": 0.40458301082253456,
|
|
"reward_before_mean": -0.08499279711395502,
|
|
"reward_before_std": 0.3492864612489939,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2750719413161278,
|
|
"reward_change_min": -0.4323217496275902,
|
|
"reward_change_std": 0.16262990981340408,
|
|
"reward_std": 0.40458302199840546,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.16832613572478294,
|
|
"step": 397
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2822.5209045410156,
|
|
"epoch": 0.45485714285714285,
|
|
"grad_norm": 0.06360434740781784,
|
|
"kl": 2.2810418158769608e-05,
|
|
"lambda_div_used": 0.6480221897363663,
|
|
"learning_rate": 2.1141329099692406e-07,
|
|
"loss": 0.0465,
|
|
"reward": -0.09229415841400623,
|
|
"reward_after_mean": -0.09229415841400623,
|
|
"reward_after_std": 0.7079674638807774,
|
|
"reward_before_mean": 0.1752314588520676,
|
|
"reward_before_std": 0.6917341919615865,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26752561889588833,
|
|
"reward_change_min": -0.5181693434715271,
|
|
"reward_change_std": 0.19102454278618097,
|
|
"reward_std": 0.7079674787819386,
|
|
"rewards/accuracy_reward": 0.2083333358168602,
|
|
"rewards/cosine_scaled_reward": -0.03310186788439751,
|
|
"step": 398
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2191.583351135254,
|
|
"epoch": 0.456,
|
|
"grad_norm": 0.07585066556930542,
|
|
"kl": 3.449246287345886e-05,
|
|
"lambda_div_used": 0.6071069464087486,
|
|
"learning_rate": 2.0935222495670968e-07,
|
|
"loss": -0.0428,
|
|
"reward": -0.06968086212873459,
|
|
"reward_after_mean": -0.06968086212873459,
|
|
"reward_after_std": 0.5623702071607113,
|
|
"reward_before_mean": 0.29000685061328113,
|
|
"reward_before_std": 0.4939764440059662,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35968772508203983,
|
|
"reward_change_min": -0.5781176537275314,
|
|
"reward_change_std": 0.2176226656883955,
|
|
"reward_std": 0.5623702295124531,
|
|
"rewards/accuracy_reward": 0.27083333767950535,
|
|
"rewards/cosine_scaled_reward": 0.01917351223528385,
|
|
"step": 399
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1814.4375305175781,
|
|
"epoch": 0.45714285714285713,
|
|
"grad_norm": 0.09906096011400223,
|
|
"kl": 3.718771040439606e-05,
|
|
"lambda_div_used": 0.615766242146492,
|
|
"learning_rate": 2.0730776160846853e-07,
|
|
"loss": -0.0203,
|
|
"reward": 0.023047026246786118,
|
|
"reward_after_mean": 0.023047026246786118,
|
|
"reward_after_std": 0.6236102003604174,
|
|
"reward_before_mean": 0.43256790889427066,
|
|
"reward_before_std": 0.5322064086794853,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.40952087566256523,
|
|
"reward_change_min": -0.6220400035381317,
|
|
"reward_change_std": 0.2354581467807293,
|
|
"reward_std": 0.62361023388803,
|
|
"rewards/accuracy_reward": 0.3125000074505806,
|
|
"rewards/cosine_scaled_reward": 0.12006791215389967,
|
|
"step": 400
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3091.7916870117188,
|
|
"epoch": 0.4582857142857143,
|
|
"grad_norm": 0.05475914105772972,
|
|
"kl": 3.5446137189865112e-06,
|
|
"lambda_div_used": 0.562153548002243,
|
|
"learning_rate": 2.0528000059645995e-07,
|
|
"loss": 0.0508,
|
|
"reward": -0.29909564182162285,
|
|
"reward_after_mean": -0.29909564182162285,
|
|
"reward_after_std": 0.3837758805602789,
|
|
"reward_before_mean": 0.04575100168585777,
|
|
"reward_before_std": 0.28164876997470856,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3448466807603836,
|
|
"reward_change_min": -0.5300594680011272,
|
|
"reward_change_std": 0.19381076097488403,
|
|
"reward_std": 0.38377588987350464,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.12091565364971757,
|
|
"step": 401
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2440.1458740234375,
|
|
"epoch": 0.4594285714285714,
|
|
"grad_norm": 0.11770489811897278,
|
|
"kl": 1.8846243619918823e-05,
|
|
"lambda_div_used": 0.5605455562472343,
|
|
"learning_rate": 2.032690407508949e-07,
|
|
"loss": -0.0261,
|
|
"reward": -0.2383374497294426,
|
|
"reward_after_mean": -0.2383374497294426,
|
|
"reward_after_std": 0.3794025480747223,
|
|
"reward_before_mean": 0.14294641837477684,
|
|
"reward_before_std": 0.27562023140490055,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.38128384202718735,
|
|
"reward_change_min": -0.5627436973154545,
|
|
"reward_change_std": 0.2136234436184168,
|
|
"reward_std": 0.3794025518000126,
|
|
"rewards/accuracy_reward": 0.1875,
|
|
"rewards/cosine_scaled_reward": -0.044553620740771294,
|
|
"step": 402
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1985.6875343322754,
|
|
"epoch": 0.4605714285714286,
|
|
"grad_norm": 0.10363903641700745,
|
|
"kl": 2.3838132619857788e-05,
|
|
"lambda_div_used": 0.5793934315443039,
|
|
"learning_rate": 2.0127498008311922e-07,
|
|
"loss": 0.003,
|
|
"reward": -0.17537187691777945,
|
|
"reward_after_mean": -0.17537187691777945,
|
|
"reward_after_std": 0.46652381494641304,
|
|
"reward_before_mean": 0.20323466695845127,
|
|
"reward_before_std": 0.3647587588056922,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.37860656157135963,
|
|
"reward_change_min": -0.5506669841706753,
|
|
"reward_change_std": 0.21334033645689487,
|
|
"reward_std": 0.4665238317102194,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.025931986048817635,
|
|
"step": 403
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2137.375,
|
|
"epoch": 0.4617142857142857,
|
|
"grad_norm": 0.10717064887285233,
|
|
"kl": 3.7202611565589905e-05,
|
|
"lambda_div_used": 0.5993036478757858,
|
|
"learning_rate": 1.9929791578083655e-07,
|
|
"loss": -0.0043,
|
|
"reward": -0.053446926176548004,
|
|
"reward_after_mean": -0.053446926176548004,
|
|
"reward_after_std": 0.4612566214054823,
|
|
"reward_before_mean": 0.3198554217815399,
|
|
"reward_before_std": 0.4542539082467556,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3733023554086685,
|
|
"reward_change_min": -0.5774630047380924,
|
|
"reward_change_std": 0.2339334823191166,
|
|
"reward_std": 0.4612566400319338,
|
|
"rewards/accuracy_reward": 0.2708333432674408,
|
|
"rewards/cosine_scaled_reward": 0.04902207478880882,
|
|
"step": 404
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2319.5000228881836,
|
|
"epoch": 0.46285714285714286,
|
|
"grad_norm": 0.13891565799713135,
|
|
"kl": 2.9403716325759888e-05,
|
|
"lambda_div_used": 0.6237343773245811,
|
|
"learning_rate": 1.9733794420337213e-07,
|
|
"loss": -0.0602,
|
|
"reward": 0.1285000964999199,
|
|
"reward_after_mean": 0.1285000964999199,
|
|
"reward_after_std": 0.7067493386566639,
|
|
"reward_before_mean": 0.5776062086224556,
|
|
"reward_before_std": 0.5746421907097101,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.449106115847826,
|
|
"reward_change_min": -0.6593505516648293,
|
|
"reward_change_std": 0.263070298358798,
|
|
"reward_std": 0.7067493461072445,
|
|
"rewards/accuracy_reward": 0.4166666679084301,
|
|
"rewards/cosine_scaled_reward": 0.1609395444393158,
|
|
"step": 405
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2442.1458892822266,
|
|
"epoch": 0.464,
|
|
"grad_norm": 0.07689624279737473,
|
|
"kl": 8.527189493179321e-06,
|
|
"lambda_div_used": 0.6336105018854141,
|
|
"learning_rate": 1.9539516087697517e-07,
|
|
"loss": -0.0,
|
|
"reward": -0.11170937749557197,
|
|
"reward_after_mean": -0.11170937749557197,
|
|
"reward_after_std": 0.6453428398817778,
|
|
"reward_before_mean": 0.17541324836201966,
|
|
"reward_before_std": 0.6142212487757206,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2871226370334625,
|
|
"reward_change_min": -0.5107162520289421,
|
|
"reward_change_std": 0.19029081240296364,
|
|
"reward_std": 0.645342854782939,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.032920089550316334,
|
|
"step": 406
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2516.6041946411133,
|
|
"epoch": 0.46514285714285714,
|
|
"grad_norm": 0.09361904859542847,
|
|
"kl": 2.995133399963379e-05,
|
|
"lambda_div_used": 0.5926980003714561,
|
|
"learning_rate": 1.934696604901642e-07,
|
|
"loss": 0.011,
|
|
"reward": -0.16482560662552714,
|
|
"reward_after_mean": -0.16482560662552714,
|
|
"reward_after_std": 0.5236028637737036,
|
|
"reward_before_mean": 0.18611273169517517,
|
|
"reward_before_std": 0.4258856289088726,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35093834809958935,
|
|
"reward_change_min": -0.5517234578728676,
|
|
"reward_change_std": 0.20658957865089178,
|
|
"reward_std": 0.5236028656363487,
|
|
"rewards/accuracy_reward": 0.25000000186264515,
|
|
"rewards/cosine_scaled_reward": -0.06388726737350225,
|
|
"step": 407
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2709.4791984558105,
|
|
"epoch": 0.4662857142857143,
|
|
"grad_norm": 0.09306048601865768,
|
|
"kl": 3.203703090548515e-05,
|
|
"lambda_div_used": 0.5583930537104607,
|
|
"learning_rate": 1.915615368891117e-07,
|
|
"loss": -0.111,
|
|
"reward": -0.17684245621785522,
|
|
"reward_after_mean": -0.17684245621785522,
|
|
"reward_after_std": 0.42921141162514687,
|
|
"reward_before_mean": 0.2615569829940796,
|
|
"reward_before_std": 0.26226984336972237,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.43839943781495094,
|
|
"reward_change_min": -0.6072936318814754,
|
|
"reward_change_std": 0.22694199439138174,
|
|
"reward_std": 0.42921141907572746,
|
|
"rewards/accuracy_reward": 0.27083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.009276359807699919,
|
|
"step": 408
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3529.3541870117188,
|
|
"epoch": 0.4674285714285714,
|
|
"grad_norm": 0.048929836601018906,
|
|
"kl": 7.6089054346084595e-06,
|
|
"lambda_div_used": 0.6116252392530441,
|
|
"learning_rate": 1.8967088307307e-07,
|
|
"loss": 0.001,
|
|
"reward": -0.17062923312187195,
|
|
"reward_after_mean": -0.17062923312187195,
|
|
"reward_after_std": 0.5682655684649944,
|
|
"reward_before_mean": 0.12847105879336596,
|
|
"reward_before_std": 0.5097848381847143,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2991002984344959,
|
|
"reward_change_min": -0.43997345492243767,
|
|
"reward_change_std": 0.17078326642513275,
|
|
"reward_std": 0.5682655889540911,
|
|
"rewards/accuracy_reward": 0.16666667349636555,
|
|
"rewards/cosine_scaled_reward": -0.038195611676201224,
|
|
"step": 409
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2435.625045776367,
|
|
"epoch": 0.4685714285714286,
|
|
"grad_norm": 0.08304905891418457,
|
|
"kl": 1.5234574675559998e-05,
|
|
"lambda_div_used": 0.6268336623907089,
|
|
"learning_rate": 1.8779779118983867e-07,
|
|
"loss": 0.0499,
|
|
"reward": -0.03433734131976962,
|
|
"reward_after_mean": -0.03433734131976962,
|
|
"reward_after_std": 0.6416803412139416,
|
|
"reward_before_mean": 0.2960522407665849,
|
|
"reward_before_std": 0.5826444877311587,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3303895927965641,
|
|
"reward_change_min": -0.4872638136148453,
|
|
"reward_change_std": 0.19508948456496,
|
|
"reward_std": 0.6416803747415543,
|
|
"rewards/accuracy_reward": 0.25000000931322575,
|
|
"rewards/cosine_scaled_reward": 0.04605222793179564,
|
|
"step": 410
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3163.1041717529297,
|
|
"epoch": 0.4697142857142857,
|
|
"grad_norm": 0.06846357136964798,
|
|
"kl": 1.529604196548462e-05,
|
|
"lambda_div_used": 0.5874563306570053,
|
|
"learning_rate": 1.8594235253127372e-07,
|
|
"loss": 0.0214,
|
|
"reward": -0.22584301605820656,
|
|
"reward_after_mean": -0.22584301605820656,
|
|
"reward_after_std": 0.42311797849833965,
|
|
"reward_before_mean": 0.0774743240326643,
|
|
"reward_before_std": 0.4020446836948395,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3033173345029354,
|
|
"reward_change_min": -0.48705917969346046,
|
|
"reward_change_std": 0.1920422399416566,
|
|
"reward_std": 0.42311798594892025,
|
|
"rewards/accuracy_reward": 0.1666666716337204,
|
|
"rewards/cosine_scaled_reward": -0.08919236063957214,
|
|
"step": 411
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2888.125015258789,
|
|
"epoch": 0.47085714285714286,
|
|
"grad_norm": 0.06694348156452179,
|
|
"kl": 2.148747444152832e-05,
|
|
"lambda_div_used": 0.6005539819598198,
|
|
"learning_rate": 1.8410465752883758e-07,
|
|
"loss": 0.049,
|
|
"reward": -0.21726901549845934,
|
|
"reward_after_mean": -0.21726901549845934,
|
|
"reward_after_std": 0.48868509009480476,
|
|
"reward_before_mean": 0.08159955404698849,
|
|
"reward_before_std": 0.463420107960701,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2988685816526413,
|
|
"reward_change_min": -0.554147582501173,
|
|
"reward_change_std": 0.19932363275438547,
|
|
"reward_std": 0.48868510872125626,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.06423377990722656,
|
|
"step": 412
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2530.2708702087402,
|
|
"epoch": 0.472,
|
|
"grad_norm": 0.08418448269367218,
|
|
"kl": 2.208724617958069e-05,
|
|
"lambda_div_used": 0.6184473037719727,
|
|
"learning_rate": 1.822847957491922e-07,
|
|
"loss": 0.0462,
|
|
"reward": 0.05549921467900276,
|
|
"reward_after_mean": 0.05549921467900276,
|
|
"reward_after_std": 0.6010984163731337,
|
|
"reward_before_mean": 0.4671566132456064,
|
|
"reward_before_std": 0.546179112046957,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4116574004292488,
|
|
"reward_change_min": -0.6464647352695465,
|
|
"reward_change_std": 0.253952544182539,
|
|
"reward_std": 0.6010984499007463,
|
|
"rewards/accuracy_reward": 0.37500000931322575,
|
|
"rewards/cosine_scaled_reward": 0.09215660532936454,
|
|
"step": 413
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2948.562530517578,
|
|
"epoch": 0.47314285714285714,
|
|
"grad_norm": 0.06618716567754745,
|
|
"kl": 1.0458752512931824e-05,
|
|
"lambda_div_used": 0.6192215830087662,
|
|
"learning_rate": 1.804828558898332e-07,
|
|
"loss": -0.0483,
|
|
"reward": -0.21263186633586884,
|
|
"reward_after_mean": -0.21263186633586884,
|
|
"reward_after_std": 0.5998369809240103,
|
|
"reward_before_mean": 0.04798411298543215,
|
|
"reward_before_std": 0.555210480466485,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2606159746646881,
|
|
"reward_change_min": -0.4564817361533642,
|
|
"reward_change_std": 0.16899613942950964,
|
|
"reward_std": 0.5998369976878166,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.0978492172434926,
|
|
"step": 414
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3234.8125,
|
|
"epoch": 0.4742857142857143,
|
|
"grad_norm": 0.06085793301463127,
|
|
"kl": 2.8954818844795227e-05,
|
|
"lambda_div_used": 0.5570018589496613,
|
|
"learning_rate": 1.7869892577476722e-07,
|
|
"loss": 0.0073,
|
|
"reward": -0.46066740388050675,
|
|
"reward_after_mean": -0.46066740388050675,
|
|
"reward_after_std": 0.3419057931751013,
|
|
"reward_before_mean": -0.2106443401426077,
|
|
"reward_before_std": 0.2546057654544711,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2500230632722378,
|
|
"reward_change_min": -0.36453111097216606,
|
|
"reward_change_std": 0.1324605904519558,
|
|
"reward_std": 0.3419058118015528,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.23147768154740334,
|
|
"step": 415
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1748.9583892822266,
|
|
"epoch": 0.4754285714285714,
|
|
"grad_norm": 0.09120064228773117,
|
|
"kl": 8.981674909591675e-06,
|
|
"lambda_div_used": 0.6067373231053352,
|
|
"learning_rate": 1.7693309235023127e-07,
|
|
"loss": -0.0771,
|
|
"reward": -0.13937147613614798,
|
|
"reward_after_mean": -0.13937147613614798,
|
|
"reward_after_std": 0.6240962240844965,
|
|
"reward_before_mean": 0.20647221896797419,
|
|
"reward_before_std": 0.48884215706493706,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34584368020296097,
|
|
"reward_change_min": -0.5141387544572353,
|
|
"reward_change_std": 0.18731264490634203,
|
|
"reward_std": 0.6240962333977222,
|
|
"rewards/accuracy_reward": 0.2500000037252903,
|
|
"rewards/cosine_scaled_reward": -0.04352780181216076,
|
|
"step": 416
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3307.250015258789,
|
|
"epoch": 0.4765714285714286,
|
|
"grad_norm": 0.05766940861940384,
|
|
"kl": 3.1495699658989906e-05,
|
|
"lambda_div_used": 0.5544050261378288,
|
|
"learning_rate": 1.7518544168045524e-07,
|
|
"loss": -0.0019,
|
|
"reward": -0.4980150870978832,
|
|
"reward_after_mean": -0.4980150870978832,
|
|
"reward_after_std": 0.3123048096895218,
|
|
"reward_before_mean": -0.2511206082999706,
|
|
"reward_before_std": 0.24326619878411293,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24689447320997715,
|
|
"reward_change_min": -0.3710709176957607,
|
|
"reward_change_std": 0.13346257898956537,
|
|
"reward_std": 0.3123048171401024,
|
|
"rewards/accuracy_reward": 0.02083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.2719539441168308,
|
|
"step": 417
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2171.625015258789,
|
|
"epoch": 0.4777142857142857,
|
|
"grad_norm": 0.0854194164276123,
|
|
"kl": 2.8558075428009033e-05,
|
|
"lambda_div_used": 0.6033797562122345,
|
|
"learning_rate": 1.7345605894346726e-07,
|
|
"loss": 0.0073,
|
|
"reward": -0.0949536501429975,
|
|
"reward_after_mean": -0.0949536501429975,
|
|
"reward_after_std": 0.540143633261323,
|
|
"reward_before_mean": 0.25413690507411957,
|
|
"reward_before_std": 0.4725718079134822,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34909053333103657,
|
|
"reward_change_min": -0.5411153584718704,
|
|
"reward_change_std": 0.2063078135251999,
|
|
"reward_std": 0.5401436407119036,
|
|
"rewards/accuracy_reward": 0.29166667349636555,
|
|
"rewards/cosine_scaled_reward": -0.03752976981922984,
|
|
"step": 418
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2464.3750228881836,
|
|
"epoch": 0.47885714285714287,
|
|
"grad_norm": 0.08315848559141159,
|
|
"kl": 3.0182301998138428e-05,
|
|
"lambda_div_used": 0.5937136337161064,
|
|
"learning_rate": 1.7174502842694212e-07,
|
|
"loss": 0.03,
|
|
"reward": -0.27206650376319885,
|
|
"reward_after_mean": -0.27206650376319885,
|
|
"reward_after_std": 0.49108788557350636,
|
|
"reward_before_mean": 0.0038498505018651485,
|
|
"reward_before_std": 0.4298001816496253,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27591635659337044,
|
|
"reward_change_min": -0.4215012192726135,
|
|
"reward_change_std": 0.16396540496498346,
|
|
"reward_std": 0.49108790047466755,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.12115013878792524,
|
|
"step": 419
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1972.7083625793457,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.11145550012588501,
|
|
"kl": 3.493949770927429e-05,
|
|
"lambda_div_used": 0.5797194987535477,
|
|
"learning_rate": 1.7005243352409333e-07,
|
|
"loss": -0.0607,
|
|
"reward": -0.1652970388531685,
|
|
"reward_after_mean": -0.1652970388531685,
|
|
"reward_after_std": 0.4477591011673212,
|
|
"reward_before_mean": 0.18980162939988077,
|
|
"reward_before_std": 0.36517443507909775,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.35509867407381535,
|
|
"reward_change_min": -0.5067496970295906,
|
|
"reward_change_std": 0.19984195847064257,
|
|
"reward_std": 0.4477591197937727,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.03936504106968641,
|
|
"step": 420
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3191.937515258789,
|
|
"epoch": 0.48114285714285715,
|
|
"grad_norm": 0.07170204073190689,
|
|
"kl": 3.11434268951416e-05,
|
|
"lambda_div_used": 0.5753046199679375,
|
|
"learning_rate": 1.6837835672960831e-07,
|
|
"loss": 0.0088,
|
|
"reward": -0.38796948781237006,
|
|
"reward_after_mean": -0.38796948781237006,
|
|
"reward_after_std": 0.4225973077118397,
|
|
"reward_before_mean": -0.12759371474385262,
|
|
"reward_before_std": 0.3405588921159506,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2603757604956627,
|
|
"reward_change_min": -0.37436668202281,
|
|
"reward_change_std": 0.13996374886482954,
|
|
"reward_std": 0.4225973132997751,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.16926039289683104,
|
|
"step": 421
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2776.437545776367,
|
|
"epoch": 0.48228571428571426,
|
|
"grad_norm": 0.06681375205516815,
|
|
"kl": 1.9788742065429688e-05,
|
|
"lambda_div_used": 0.5904401987791061,
|
|
"learning_rate": 1.6672287963562852e-07,
|
|
"loss": 0.0293,
|
|
"reward": -0.25396816432476044,
|
|
"reward_after_mean": -0.25396816432476044,
|
|
"reward_after_std": 0.4787884410470724,
|
|
"reward_before_mean": 0.03486193250864744,
|
|
"reward_before_std": 0.4093271289020777,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2888300847262144,
|
|
"reward_change_min": -0.43693453073501587,
|
|
"reward_change_std": 0.16317449882626534,
|
|
"reward_std": 0.4787884559482336,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.11097141215577722,
|
|
"step": 422
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2945.9583740234375,
|
|
"epoch": 0.48342857142857143,
|
|
"grad_norm": 0.06767871230840683,
|
|
"kl": 2.205371856689453e-05,
|
|
"lambda_div_used": 0.611551009118557,
|
|
"learning_rate": 1.6508608292777203e-07,
|
|
"loss": 0.0153,
|
|
"reward": -0.16482537053525448,
|
|
"reward_after_mean": -0.16482537053525448,
|
|
"reward_after_std": 0.5363675616681576,
|
|
"reward_before_mean": 0.12604539189487696,
|
|
"reward_before_std": 0.518328445032239,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2908707782626152,
|
|
"reward_change_min": -0.4775175042450428,
|
|
"reward_change_std": 0.18889948446303606,
|
|
"reward_std": 0.5363675802946091,
|
|
"rewards/accuracy_reward": 0.16666666977107525,
|
|
"rewards/cosine_scaled_reward": -0.040621266700327396,
|
|
"step": 423
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3215.0208740234375,
|
|
"epoch": 0.4845714285714286,
|
|
"grad_norm": 0.061716482043266296,
|
|
"kl": 1.1460855603218079e-05,
|
|
"lambda_div_used": 0.5937408357858658,
|
|
"learning_rate": 1.6346804638120098e-07,
|
|
"loss": 0.0327,
|
|
"reward": -0.3415182586759329,
|
|
"reward_after_mean": -0.3415182586759329,
|
|
"reward_after_std": 0.48476750776171684,
|
|
"reward_before_mean": -0.08947536488994956,
|
|
"reward_before_std": 0.4272408355027437,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2520429063588381,
|
|
"reward_change_min": -0.42084335163235664,
|
|
"reward_change_std": 0.15120856929570436,
|
|
"reward_std": 0.4847675133496523,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.17280870315153152,
|
|
"step": 424
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2154.9375610351562,
|
|
"epoch": 0.4857142857142857,
|
|
"grad_norm": 0.09371069073677063,
|
|
"kl": 3.830809146165848e-05,
|
|
"lambda_div_used": 0.6315138638019562,
|
|
"learning_rate": 1.6186884885673413e-07,
|
|
"loss": 0.0058,
|
|
"reward": 0.34828917868435383,
|
|
"reward_after_mean": 0.34828917868435383,
|
|
"reward_after_std": 0.7491090279072523,
|
|
"reward_before_mean": 0.91605463065207,
|
|
"reward_before_std": 0.6105400957167149,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5677654705941677,
|
|
"reward_change_min": -0.9185826852917671,
|
|
"reward_change_std": 0.34253316558897495,
|
|
"reward_std": 0.7491090428084135,
|
|
"rewards/accuracy_reward": 0.5833333358168602,
|
|
"rewards/cosine_scaled_reward": 0.3327212668955326,
|
|
"step": 425
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2205.3125190734863,
|
|
"epoch": 0.4868571428571429,
|
|
"grad_norm": 0.09298980236053467,
|
|
"kl": 2.3175030946731567e-05,
|
|
"lambda_div_used": 0.6215192526578903,
|
|
"learning_rate": 1.6028856829700258e-07,
|
|
"loss": -0.002,
|
|
"reward": -0.029936233535408974,
|
|
"reward_after_mean": -0.029936233535408974,
|
|
"reward_after_std": 0.5604026541113853,
|
|
"reward_before_mean": 0.3139693345874548,
|
|
"reward_before_std": 0.5635973755270243,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34390556812286377,
|
|
"reward_change_min": -0.5907084755599499,
|
|
"reward_change_std": 0.23498705588281155,
|
|
"reward_std": 0.5604026671499014,
|
|
"rewards/accuracy_reward": 0.3125000074505806,
|
|
"rewards/cosine_scaled_reward": 0.0014693308621644974,
|
|
"step": 426
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3295.7708740234375,
|
|
"epoch": 0.488,
|
|
"grad_norm": 0.0548052154481411,
|
|
"kl": 2.4527311325073242e-05,
|
|
"lambda_div_used": 0.5990054532885551,
|
|
"learning_rate": 1.5872728172265146e-07,
|
|
"loss": 0.0088,
|
|
"reward": -0.2071160115301609,
|
|
"reward_after_mean": -0.2071160115301609,
|
|
"reward_after_std": 0.49672279693186283,
|
|
"reward_before_mean": 0.08893106225878,
|
|
"reward_before_std": 0.454012256115675,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2960470784455538,
|
|
"reward_change_min": -0.4723533205688,
|
|
"reward_change_std": 0.18442231137305498,
|
|
"reward_std": 0.4967228155583143,
|
|
"rewards/accuracy_reward": 0.18750000558793545,
|
|
"rewards/cosine_scaled_reward": -0.09856893587857485,
|
|
"step": 427
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2302.520881652832,
|
|
"epoch": 0.48914285714285716,
|
|
"grad_norm": 0.11483462899923325,
|
|
"kl": 0.000337366946041584,
|
|
"lambda_div_used": 0.5547136962413788,
|
|
"learning_rate": 1.5718506522858572e-07,
|
|
"loss": 0.0311,
|
|
"reward": -0.42663199454545975,
|
|
"reward_after_mean": -0.42663199454545975,
|
|
"reward_after_std": 0.29241302236914635,
|
|
"reward_before_mean": -0.1547260768711567,
|
|
"reward_before_std": 0.24498768709599972,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27190591767430305,
|
|
"reward_change_min": -0.42438486963510513,
|
|
"reward_change_std": 0.1579811777919531,
|
|
"reward_std": 0.2924130354076624,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.2380594089627266,
|
|
"step": 428
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2431.520881652832,
|
|
"epoch": 0.49028571428571427,
|
|
"grad_norm": 0.0878632664680481,
|
|
"kl": 2.4685636162757874e-05,
|
|
"lambda_div_used": 0.6025609895586967,
|
|
"learning_rate": 1.5566199398026147e-07,
|
|
"loss": 0.0378,
|
|
"reward": -0.057155030546709895,
|
|
"reward_after_mean": -0.057155030546709895,
|
|
"reward_after_std": 0.5567535478621721,
|
|
"reward_before_mean": 0.3208682704716921,
|
|
"reward_before_std": 0.47341752983629704,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3780232574790716,
|
|
"reward_change_min": -0.5763550102710724,
|
|
"reward_change_std": 0.22364642471075058,
|
|
"reward_std": 0.5567535553127527,
|
|
"rewards/accuracy_reward": 0.29166666977107525,
|
|
"rewards/cosine_scaled_reward": 0.029201554832980037,
|
|
"step": 429
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2577.3125610351562,
|
|
"epoch": 0.49142857142857144,
|
|
"grad_norm": 0.08103640377521515,
|
|
"kl": 1.4763325452804565e-05,
|
|
"lambda_div_used": 0.6012379750609398,
|
|
"learning_rate": 1.5415814221002265e-07,
|
|
"loss": -0.0049,
|
|
"reward": 0.031602535396814346,
|
|
"reward_after_mean": 0.031602535396814346,
|
|
"reward_after_std": 0.5440461356192827,
|
|
"reward_before_mean": 0.4564003311097622,
|
|
"reward_before_std": 0.46839726250618696,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.42479780688881874,
|
|
"reward_change_min": -0.6648109555244446,
|
|
"reward_change_std": 0.2579617351293564,
|
|
"reward_std": 0.5440461542457342,
|
|
"rewards/accuracy_reward": 0.3541666753590107,
|
|
"rewards/cosine_scaled_reward": 0.1022336557507515,
|
|
"step": 430
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2690.6458740234375,
|
|
"epoch": 0.49257142857142855,
|
|
"grad_norm": 0.06538030505180359,
|
|
"kl": 2.8170645236968994e-05,
|
|
"lambda_div_used": 0.5918548330664635,
|
|
"learning_rate": 1.5267358321348285e-07,
|
|
"loss": -0.0328,
|
|
"reward": -0.24143310775980353,
|
|
"reward_after_mean": -0.24143310775980353,
|
|
"reward_after_std": 0.481786971911788,
|
|
"reward_before_mean": 0.05766227189451456,
|
|
"reward_before_std": 0.41715206764638424,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29909538850188255,
|
|
"reward_change_min": -0.4450513869524002,
|
|
"reward_change_std": 0.1692206682637334,
|
|
"reward_std": 0.4817869979888201,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.08817105047637597,
|
|
"step": 431
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2843.750045776367,
|
|
"epoch": 0.4937142857142857,
|
|
"grad_norm": 0.05780716612935066,
|
|
"kl": 3.104284405708313e-05,
|
|
"lambda_div_used": 0.576793298125267,
|
|
"learning_rate": 1.5120838934595337e-07,
|
|
"loss": 0.0053,
|
|
"reward": -0.42155745439231396,
|
|
"reward_after_mean": -0.42155745439231396,
|
|
"reward_after_std": 0.4230448566377163,
|
|
"reward_before_mean": -0.19034346495755017,
|
|
"reward_before_std": 0.35024640895426273,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23121398873627186,
|
|
"reward_change_min": -0.3715337961912155,
|
|
"reward_change_std": 0.1338915005326271,
|
|
"reward_std": 0.4230448678135872,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.2528434665873647,
|
|
"step": 432
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2992.062530517578,
|
|
"epoch": 0.4948571428571429,
|
|
"grad_norm": 0.06562553346157074,
|
|
"kl": 3.0972063541412354e-05,
|
|
"lambda_div_used": 0.6079665347933769,
|
|
"learning_rate": 1.4976263201891613e-07,
|
|
"loss": 0.1201,
|
|
"reward": -0.26266857516020536,
|
|
"reward_after_mean": -0.26266857516020536,
|
|
"reward_after_std": 0.5306479204446077,
|
|
"reward_before_mean": -0.0102414321154356,
|
|
"reward_before_std": 0.5030816271901131,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25242713652551174,
|
|
"reward_change_min": -0.4955142140388489,
|
|
"reward_change_std": 0.1781205264851451,
|
|
"reward_std": 0.5306479260325432,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.15607476886361837,
|
|
"step": 433
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2915.625030517578,
|
|
"epoch": 0.496,
|
|
"grad_norm": 0.06934478878974915,
|
|
"kl": 3.8422178477048874e-05,
|
|
"lambda_div_used": 0.5604980885982513,
|
|
"learning_rate": 1.483363816965435e-07,
|
|
"loss": -0.004,
|
|
"reward": -0.4280230412259698,
|
|
"reward_after_mean": -0.4280230412259698,
|
|
"reward_after_std": 0.32627478428184986,
|
|
"reward_before_mean": -0.15262745507061481,
|
|
"reward_before_std": 0.2731653768569231,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27539557591080666,
|
|
"reward_change_min": -0.44822341948747635,
|
|
"reward_change_std": 0.16339552495628595,
|
|
"reward_std": 0.32627478800714016,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1942941201850772,
|
|
"step": 434
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2320.750026702881,
|
|
"epoch": 0.49714285714285716,
|
|
"grad_norm": 0.13675029575824738,
|
|
"kl": 4.8140063881874084e-05,
|
|
"lambda_div_used": 0.5811168253421783,
|
|
"learning_rate": 1.469297078922642e-07,
|
|
"loss": -0.0374,
|
|
"reward": -0.17623315937817097,
|
|
"reward_after_mean": -0.17623315937817097,
|
|
"reward_after_std": 0.483237961307168,
|
|
"reward_before_mean": 0.19585114251822233,
|
|
"reward_before_std": 0.3759047882631421,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.37208431400358677,
|
|
"reward_change_min": -0.5468379594385624,
|
|
"reward_change_std": 0.20962903555482626,
|
|
"reward_std": 0.4832379762083292,
|
|
"rewards/accuracy_reward": 0.20833333395421505,
|
|
"rewards/cosine_scaled_reward": -0.01248218398541212,
|
|
"step": 435
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2316.3125381469727,
|
|
"epoch": 0.4982857142857143,
|
|
"grad_norm": 0.09783437103033066,
|
|
"kl": 2.839416265487671e-05,
|
|
"lambda_div_used": 0.6289880573749542,
|
|
"learning_rate": 1.4554267916537495e-07,
|
|
"loss": 0.0826,
|
|
"reward": 0.12041758373379707,
|
|
"reward_after_mean": 0.12041758373379707,
|
|
"reward_after_std": 0.6426001656800508,
|
|
"reward_before_mean": 0.5348608233034611,
|
|
"reward_before_std": 0.6021685730665922,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.41444322280585766,
|
|
"reward_change_min": -0.6993351243436337,
|
|
"reward_change_std": 0.271469890139997,
|
|
"reward_std": 0.6426001694053411,
|
|
"rewards/accuracy_reward": 0.3958333395421505,
|
|
"rewards/cosine_scaled_reward": 0.13902746886014938,
|
|
"step": 436
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2954.0416717529297,
|
|
"epoch": 0.49942857142857144,
|
|
"grad_norm": 0.05496148392558098,
|
|
"kl": 0.00023943744599819183,
|
|
"lambda_div_used": 0.5626013651490211,
|
|
"learning_rate": 1.4417536311769885e-07,
|
|
"loss": 0.05,
|
|
"reward": -0.42826351523399353,
|
|
"reward_after_mean": -0.42826351523399353,
|
|
"reward_after_std": 0.33650987036526203,
|
|
"reward_before_mean": -0.16868688352406025,
|
|
"reward_before_std": 0.2832178361713886,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25957662612199783,
|
|
"reward_change_min": -0.42033713683485985,
|
|
"reward_change_std": 0.1560918828472495,
|
|
"reward_std": 0.3365098759531975,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.2311868779361248,
|
|
"step": 437
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3007.500015258789,
|
|
"epoch": 0.5005714285714286,
|
|
"grad_norm": 0.05682806298136711,
|
|
"kl": 4.358415026217699e-06,
|
|
"lambda_div_used": 0.6039799600839615,
|
|
"learning_rate": 1.4282782639029128e-07,
|
|
"loss": 0.0289,
|
|
"reward": -0.29300207551568747,
|
|
"reward_after_mean": -0.29300207551568747,
|
|
"reward_after_std": 0.5204536523669958,
|
|
"reward_before_mean": -0.03094739466905594,
|
|
"reward_before_std": 0.4829921592026949,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2620546855032444,
|
|
"reward_change_min": -0.5145607963204384,
|
|
"reward_change_std": 0.18371508549898863,
|
|
"reward_std": 0.5204536579549313,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.15594739385414869,
|
|
"step": 438
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2249.5000076293945,
|
|
"epoch": 0.5017142857142857,
|
|
"grad_norm": 0.0828532725572586,
|
|
"kl": 3.217160701751709e-05,
|
|
"lambda_div_used": 0.5708463862538338,
|
|
"learning_rate": 1.4150013466019114e-07,
|
|
"loss": 0.021,
|
|
"reward": -0.40206424333155155,
|
|
"reward_after_mean": -0.40206424333155155,
|
|
"reward_after_std": 0.40689974650740623,
|
|
"reward_before_mean": -0.14063972979784012,
|
|
"reward_before_std": 0.32079045102000237,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.261424507945776,
|
|
"reward_change_min": -0.38026969507336617,
|
|
"reward_change_std": 0.14067976083606482,
|
|
"reward_std": 0.406899768859148,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1823064020718448,
|
|
"step": 439
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2964.000011444092,
|
|
"epoch": 0.5028571428571429,
|
|
"grad_norm": 0.07350306212902069,
|
|
"kl": 2.7701258659362793e-05,
|
|
"lambda_div_used": 0.5365751013159752,
|
|
"learning_rate": 1.4019235263722034e-07,
|
|
"loss": 0.0201,
|
|
"reward": -0.5367953963577747,
|
|
"reward_after_mean": -0.5367953963577747,
|
|
"reward_after_std": 0.231884878128767,
|
|
"reward_before_mean": -0.28081973269581795,
|
|
"reward_before_std": 0.16059848852455616,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25597566179931164,
|
|
"reward_change_min": -0.38940757140517235,
|
|
"reward_change_std": 0.13862022012472153,
|
|
"reward_std": 0.2318848893046379,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.28081973642110825,
|
|
"step": 440
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3007.770896911621,
|
|
"epoch": 0.504,
|
|
"grad_norm": 0.07498018443584442,
|
|
"kl": 2.944841980934143e-05,
|
|
"lambda_div_used": 0.6011143997311592,
|
|
"learning_rate": 1.3890454406082956e-07,
|
|
"loss": -0.0116,
|
|
"reward": -0.13889071717858315,
|
|
"reward_after_mean": -0.13889071717858315,
|
|
"reward_after_std": 0.4864419028162956,
|
|
"reward_before_mean": 0.17084867507219315,
|
|
"reward_before_std": 0.4689741334877908,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3097394183278084,
|
|
"reward_change_min": -0.514279019087553,
|
|
"reward_change_std": 0.20097953081130981,
|
|
"reward_std": 0.48644191212952137,
|
|
"rewards/accuracy_reward": 0.2291666753590107,
|
|
"rewards/cosine_scaled_reward": -0.05831799004226923,
|
|
"step": 441
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2837.4583435058594,
|
|
"epoch": 0.5051428571428571,
|
|
"grad_norm": 0.0673719272017479,
|
|
"kl": 3.160163760185242e-05,
|
|
"lambda_div_used": 0.5689455196261406,
|
|
"learning_rate": 1.3763677169699217e-07,
|
|
"loss": 0.0084,
|
|
"reward": -0.31079866108484566,
|
|
"reward_after_mean": -0.31079866108484566,
|
|
"reward_after_std": 0.3982803635299206,
|
|
"reward_before_mean": -0.010625829687342048,
|
|
"reward_before_std": 0.31661996035836637,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.30017283000051975,
|
|
"reward_change_min": -0.4261031448841095,
|
|
"reward_change_std": 0.16800945159047842,
|
|
"reward_std": 0.3982803765684366,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.1147925069089979,
|
|
"step": 442
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3194.270854949951,
|
|
"epoch": 0.5062857142857143,
|
|
"grad_norm": 0.05984271690249443,
|
|
"kl": 1.745857298374176e-05,
|
|
"lambda_div_used": 0.5607094466686249,
|
|
"learning_rate": 1.3638909733514452e-07,
|
|
"loss": 0.0053,
|
|
"reward": -0.38279488682746887,
|
|
"reward_after_mean": -0.38279488682746887,
|
|
"reward_after_std": 0.33906700275838375,
|
|
"reward_before_mean": -0.09817820321768522,
|
|
"reward_before_std": 0.2764374865218997,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2846166864037514,
|
|
"reward_change_min": -0.4438001811504364,
|
|
"reward_change_std": 0.16594033408910036,
|
|
"reward_std": 0.3390670083463192,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.16067820694297552,
|
|
"step": 443
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2902.2291717529297,
|
|
"epoch": 0.5074285714285715,
|
|
"grad_norm": 0.0638304203748703,
|
|
"kl": 1.5033408999443054e-05,
|
|
"lambda_div_used": 0.5774018093943596,
|
|
"learning_rate": 1.351615817851748e-07,
|
|
"loss": -0.0355,
|
|
"reward": -0.3352237604558468,
|
|
"reward_after_mean": -0.3352237604558468,
|
|
"reward_after_std": 0.4039857666939497,
|
|
"reward_before_mean": -0.05285666696727276,
|
|
"reward_before_std": 0.35096561443060637,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2823671065270901,
|
|
"reward_change_min": -0.42594215646386147,
|
|
"reward_change_std": 0.16656992863863707,
|
|
"reward_std": 0.40398577041924,
|
|
"rewards/accuracy_reward": 0.10416666977107525,
|
|
"rewards/cosine_scaled_reward": -0.15702332742512226,
|
|
"step": 444
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3006.916717529297,
|
|
"epoch": 0.5085714285714286,
|
|
"grad_norm": 0.06397882103919983,
|
|
"kl": 2.842303365468979e-05,
|
|
"lambda_div_used": 0.6194909662008286,
|
|
"learning_rate": 1.3395428487445914e-07,
|
|
"loss": 0.0715,
|
|
"reward": -0.05263347551226616,
|
|
"reward_after_mean": -0.05263347551226616,
|
|
"reward_after_std": 0.5679135732352734,
|
|
"reward_before_mean": 0.2764017879962921,
|
|
"reward_before_std": 0.5510673765093088,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3290352523326874,
|
|
"reward_change_min": -0.5197134874761105,
|
|
"reward_change_std": 0.21186745446175337,
|
|
"reward_std": 0.5679135844111443,
|
|
"rewards/accuracy_reward": 0.27083334140479565,
|
|
"rewards/cosine_scaled_reward": 0.0055684298276901245,
|
|
"step": 445
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3028.8333892822266,
|
|
"epoch": 0.5097142857142857,
|
|
"grad_norm": 0.06140498071908951,
|
|
"kl": 3.2648444175720215e-05,
|
|
"lambda_div_used": 0.606703408062458,
|
|
"learning_rate": 1.3276726544494571e-07,
|
|
"loss": -0.0241,
|
|
"reward": -0.2073356769979,
|
|
"reward_after_mean": -0.2073356769979,
|
|
"reward_after_std": 0.5240648984909058,
|
|
"reward_before_mean": 0.0723939798772335,
|
|
"reward_before_std": 0.4952498711645603,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27972964756190777,
|
|
"reward_change_min": -0.5379137881100178,
|
|
"reward_change_std": 0.1909602265805006,
|
|
"reward_std": 0.5240649078041315,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.07343936339020729,
|
|
"step": 446
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2571.166721343994,
|
|
"epoch": 0.5108571428571429,
|
|
"grad_norm": 0.08778514713048935,
|
|
"kl": 9.072478860616684e-06,
|
|
"lambda_div_used": 0.5970501974225044,
|
|
"learning_rate": 1.316005813502869e-07,
|
|
"loss": 0.0018,
|
|
"reward": -0.19514422863721848,
|
|
"reward_after_mean": -0.19514422863721848,
|
|
"reward_after_std": 0.4898754768073559,
|
|
"reward_before_mean": 0.10967571474611759,
|
|
"reward_before_std": 0.4461768325418234,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3048199340701103,
|
|
"reward_change_min": -0.48957687243819237,
|
|
"reward_change_std": 0.18721044715493917,
|
|
"reward_std": 0.4898754861205816,
|
|
"rewards/accuracy_reward": 0.18750000558793545,
|
|
"rewards/cosine_scaled_reward": -0.07782429456710815,
|
|
"step": 447
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2039.6042022705078,
|
|
"epoch": 0.512,
|
|
"grad_norm": 0.12122303992509842,
|
|
"kl": 3.530830144882202e-05,
|
|
"lambda_div_used": 0.5591797679662704,
|
|
"learning_rate": 1.3045428945301953e-07,
|
|
"loss": 0.0057,
|
|
"reward": -0.3564775697886944,
|
|
"reward_after_mean": -0.3564775697886944,
|
|
"reward_after_std": 0.33722602762281895,
|
|
"reward_before_mean": -0.06660962291061878,
|
|
"reward_before_std": 0.27086107339709997,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.28986795619130135,
|
|
"reward_change_min": -0.4345347508788109,
|
|
"reward_change_std": 0.1656003799289465,
|
|
"reward_std": 0.33722603134810925,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.14994295500218868,
|
|
"step": 448
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2680.0208435058594,
|
|
"epoch": 0.5131428571428571,
|
|
"grad_norm": 0.07745695859193802,
|
|
"kl": 3.194063901901245e-05,
|
|
"lambda_div_used": 0.5827131420373917,
|
|
"learning_rate": 1.2932844562179352e-07,
|
|
"loss": -0.0247,
|
|
"reward": -0.278532937169075,
|
|
"reward_after_mean": -0.278532937169075,
|
|
"reward_after_std": 0.44566163793206215,
|
|
"reward_before_mean": 0.01611769199371338,
|
|
"reward_before_std": 0.3775772461667657,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29465061984956264,
|
|
"reward_change_min": -0.4545559212565422,
|
|
"reward_change_std": 0.17489958554506302,
|
|
"reward_std": 0.44566163793206215,
|
|
"rewards/accuracy_reward": 0.18750000558793545,
|
|
"rewards/cosine_scaled_reward": -0.17138232477009296,
|
|
"step": 449
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2405.0833702087402,
|
|
"epoch": 0.5142857142857142,
|
|
"grad_norm": 0.07359456270933151,
|
|
"kl": 4.271417856216431e-05,
|
|
"lambda_div_used": 0.5313882529735565,
|
|
"learning_rate": 1.2822310472864885e-07,
|
|
"loss": 0.0047,
|
|
"reward": -0.40455422177910805,
|
|
"reward_after_mean": -0.40455422177910805,
|
|
"reward_after_std": 0.27055008336901665,
|
|
"reward_before_mean": -0.05470774322748184,
|
|
"reward_before_std": 0.13779542688280344,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34984651021659374,
|
|
"reward_change_min": -0.482446551322937,
|
|
"reward_change_std": 0.18013106007128954,
|
|
"reward_std": 0.2705500964075327,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.17970772832632065,
|
|
"step": 450
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2726.7916870117188,
|
|
"epoch": 0.5154285714285715,
|
|
"grad_norm": 0.10456772893667221,
|
|
"kl": 5.123019218444824e-05,
|
|
"lambda_div_used": 0.5659012496471405,
|
|
"learning_rate": 1.2713832064634125e-07,
|
|
"loss": 0.0437,
|
|
"reward": -0.233762688934803,
|
|
"reward_after_mean": -0.233762688934803,
|
|
"reward_after_std": 0.3937326893210411,
|
|
"reward_before_mean": 0.14449233608320355,
|
|
"reward_before_std": 0.30359079129993916,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3782550562173128,
|
|
"reward_change_min": -0.5748258531093597,
|
|
"reward_change_std": 0.21745321340858936,
|
|
"reward_std": 0.3937326930463314,
|
|
"rewards/accuracy_reward": 0.1875,
|
|
"rewards/cosine_scaled_reward": -0.0430076620541513,
|
|
"step": 451
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3173.812530517578,
|
|
"epoch": 0.5165714285714286,
|
|
"grad_norm": 0.05382152646780014,
|
|
"kl": 1.621060073375702e-05,
|
|
"lambda_div_used": 0.628274716436863,
|
|
"learning_rate": 1.260741462457165e-07,
|
|
"loss": 0.0152,
|
|
"reward": 0.08829959481954575,
|
|
"reward_after_mean": 0.08829959481954575,
|
|
"reward_after_std": 0.6491915434598923,
|
|
"reward_before_mean": 0.49067158019170165,
|
|
"reward_before_std": 0.5948440972715616,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4023719858378172,
|
|
"reward_change_min": -0.6429549157619476,
|
|
"reward_change_std": 0.2526876190677285,
|
|
"reward_std": 0.6491915658116341,
|
|
"rewards/accuracy_reward": 0.37500000558793545,
|
|
"rewards/cosine_scaled_reward": 0.11567158252000809,
|
|
"step": 452
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2601.1666870117188,
|
|
"epoch": 0.5177142857142857,
|
|
"grad_norm": 0.10710335522890091,
|
|
"kl": 4.57763671875e-05,
|
|
"lambda_div_used": 0.5769856572151184,
|
|
"learning_rate": 1.2503063339313356e-07,
|
|
"loss": -0.0233,
|
|
"reward": -0.4120060931891203,
|
|
"reward_after_mean": -0.4120060931891203,
|
|
"reward_after_std": 0.4247955661267042,
|
|
"reward_before_mean": -0.17395742796361446,
|
|
"reward_before_std": 0.3489131908863783,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23804865032434464,
|
|
"reward_change_min": -0.3744778670370579,
|
|
"reward_change_std": 0.12989939749240875,
|
|
"reward_std": 0.4247955847531557,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.21562409959733486,
|
|
"step": 453
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2794.8958740234375,
|
|
"epoch": 0.5188571428571429,
|
|
"grad_norm": 0.059399981051683426,
|
|
"kl": 1.829490065574646e-05,
|
|
"lambda_div_used": 0.604412779211998,
|
|
"learning_rate": 1.2400783294793668e-07,
|
|
"loss": 0.0084,
|
|
"reward": -0.23108407109975815,
|
|
"reward_after_mean": -0.23108407109975815,
|
|
"reward_after_std": 0.5207228269428015,
|
|
"reward_before_mean": 0.05918463226407766,
|
|
"reward_before_std": 0.4853199487552047,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29026869870722294,
|
|
"reward_change_min": -0.5276229903101921,
|
|
"reward_change_std": 0.1960765514522791,
|
|
"reward_std": 0.5207228306680918,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.08664871653309092,
|
|
"step": 454
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3120.0416717529297,
|
|
"epoch": 0.52,
|
|
"grad_norm": 0.07823975384235382,
|
|
"kl": 1.1070631444454193e-05,
|
|
"lambda_div_used": 0.5381916239857674,
|
|
"learning_rate": 1.2300579475997657e-07,
|
|
"loss": 0.0035,
|
|
"reward": -0.5376988351345062,
|
|
"reward_after_mean": -0.5376988351345062,
|
|
"reward_after_std": 0.2399882897734642,
|
|
"reward_before_mean": -0.28268107399344444,
|
|
"reward_before_std": 0.16770172398537397,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25501775927841663,
|
|
"reward_change_min": -0.38403724879026413,
|
|
"reward_change_std": 0.1382020702585578,
|
|
"reward_std": 0.23998829536139965,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.28268107026815414,
|
|
"step": 455
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3183.125,
|
|
"epoch": 0.5211428571428571,
|
|
"grad_norm": 0.07441714406013489,
|
|
"kl": 1.7248094081878662e-05,
|
|
"lambda_div_used": 0.535825714468956,
|
|
"learning_rate": 1.220245676671809e-07,
|
|
"loss": -0.0061,
|
|
"reward": -0.5242688432335854,
|
|
"reward_after_mean": -0.5242688432335854,
|
|
"reward_after_std": 0.2355129700154066,
|
|
"reward_before_mean": -0.2603513076901436,
|
|
"reward_before_std": 0.15733365854248405,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2639175299555063,
|
|
"reward_change_min": -0.3936190530657768,
|
|
"reward_change_std": 0.14126356784254313,
|
|
"reward_std": 0.23551297932863235,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2603513114154339,
|
|
"step": 456
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3090.9166717529297,
|
|
"epoch": 0.5222857142857142,
|
|
"grad_norm": 0.06740739941596985,
|
|
"kl": 1.6771256923675537e-05,
|
|
"lambda_div_used": 0.5661213397979736,
|
|
"learning_rate": 1.2106419949317388e-07,
|
|
"loss": 0.0183,
|
|
"reward": -0.2666686773300171,
|
|
"reward_after_mean": -0.2666686773300171,
|
|
"reward_after_std": 0.3630063198506832,
|
|
"reward_before_mean": 0.06939902156591415,
|
|
"reward_before_std": 0.3003841144964099,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3360677044838667,
|
|
"reward_change_min": -0.5109980814158916,
|
|
"reward_change_std": 0.19570716377347708,
|
|
"reward_std": 0.36300632916390896,
|
|
"rewards/accuracy_reward": 0.1458333395421505,
|
|
"rewards/cosine_scaled_reward": -0.0764343123883009,
|
|
"step": 457
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2337.0416717529297,
|
|
"epoch": 0.5234285714285715,
|
|
"grad_norm": 0.10038017481565475,
|
|
"kl": 1.5580561012029648e-05,
|
|
"lambda_div_used": 0.587464913725853,
|
|
"learning_rate": 1.2012473704494537e-07,
|
|
"loss": 0.0337,
|
|
"reward": -0.2921891398727894,
|
|
"reward_after_mean": -0.2921891398727894,
|
|
"reward_after_std": 0.46863692067563534,
|
|
"reward_before_mean": -0.017078701872378588,
|
|
"reward_before_std": 0.3978575337678194,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27511043660342693,
|
|
"reward_change_min": -0.4165792725980282,
|
|
"reward_change_std": 0.15801922790706158,
|
|
"reward_std": 0.4686369299888611,
|
|
"rewards/accuracy_reward": 0.1250000037252903,
|
|
"rewards/cosine_scaled_reward": -0.14207870024256408,
|
|
"step": 458
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1897.9791870117188,
|
|
"epoch": 0.5245714285714286,
|
|
"grad_norm": 0.10566994547843933,
|
|
"kl": 3.1791627407073975e-05,
|
|
"lambda_div_used": 0.6483638733625412,
|
|
"learning_rate": 1.1920622611056974e-07,
|
|
"loss": -0.0327,
|
|
"reward": 0.04763873480260372,
|
|
"reward_after_mean": 0.04763873480260372,
|
|
"reward_after_std": 0.6893943976610899,
|
|
"reward_before_mean": 0.37104589492082596,
|
|
"reward_before_std": 0.6905025038868189,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3234071359038353,
|
|
"reward_change_min": -0.573150984942913,
|
|
"reward_change_std": 0.22597133833914995,
|
|
"reward_std": 0.6893944274634123,
|
|
"rewards/accuracy_reward": 0.29166667349636555,
|
|
"rewards/cosine_scaled_reward": 0.07937921397387981,
|
|
"step": 459
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3152.9583587646484,
|
|
"epoch": 0.5257142857142857,
|
|
"grad_norm": 0.0553775392472744,
|
|
"kl": 1.6976147890090942e-05,
|
|
"lambda_div_used": 0.6212490126490593,
|
|
"learning_rate": 1.1830871145697412e-07,
|
|
"loss": -0.0403,
|
|
"reward": -0.2310712798498571,
|
|
"reward_after_mean": -0.2310712798498571,
|
|
"reward_after_std": 0.5967046339064837,
|
|
"reward_before_mean": 0.018506756518036127,
|
|
"reward_before_std": 0.5584379080682993,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24957803264260292,
|
|
"reward_change_min": -0.45073715783655643,
|
|
"reward_change_std": 0.16405375488102436,
|
|
"reward_std": 0.5967046469449997,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.1273265864001587,
|
|
"step": 460
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2965.625015258789,
|
|
"epoch": 0.5268571428571428,
|
|
"grad_norm": 0.0683179497718811,
|
|
"kl": 3.898143768310547e-05,
|
|
"lambda_div_used": 0.5838751494884491,
|
|
"learning_rate": 1.1743223682775649e-07,
|
|
"loss": 0.0201,
|
|
"reward": -0.36440329253673553,
|
|
"reward_after_mean": -0.36440329253673553,
|
|
"reward_after_std": 0.44049660488963127,
|
|
"reward_before_mean": -0.10854836623184383,
|
|
"reward_before_std": 0.3863721750676632,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25585492700338364,
|
|
"reward_change_min": -0.41783962957561016,
|
|
"reward_change_std": 0.15460436698049307,
|
|
"reward_std": 0.44049660861492157,
|
|
"rewards/accuracy_reward": 0.08333333395421505,
|
|
"rewards/cosine_scaled_reward": -0.19188169576227665,
|
|
"step": 461
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2874.3125076293945,
|
|
"epoch": 0.528,
|
|
"grad_norm": 0.07832667976617813,
|
|
"kl": 2.7470290660858154e-05,
|
|
"lambda_div_used": 0.5419389680027962,
|
|
"learning_rate": 1.1657684494105386e-07,
|
|
"loss": -0.0381,
|
|
"reward": -0.5123195722699165,
|
|
"reward_after_mean": -0.5123195722699165,
|
|
"reward_after_std": 0.2568210382014513,
|
|
"reward_before_mean": -0.2502432279288769,
|
|
"reward_before_std": 0.1842196974903345,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2620763499289751,
|
|
"reward_change_min": -0.4005816727876663,
|
|
"reward_change_std": 0.14368234388530254,
|
|
"reward_std": 0.2568210456520319,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.2502432242035866,
|
|
"step": 462
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2986.8333740234375,
|
|
"epoch": 0.5291428571428571,
|
|
"grad_norm": 0.053950026631355286,
|
|
"kl": 8.771196007728577e-06,
|
|
"lambda_div_used": 0.622698076069355,
|
|
"learning_rate": 1.1574257748745986e-07,
|
|
"loss": 0.0106,
|
|
"reward": -0.2041953857988119,
|
|
"reward_after_mean": -0.2041953857988119,
|
|
"reward_after_std": 0.6069091446697712,
|
|
"reward_before_mean": 0.05050618201494217,
|
|
"reward_before_std": 0.5684067364782095,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25470156222581863,
|
|
"reward_change_min": -0.4568561315536499,
|
|
"reward_change_std": 0.16765617858618498,
|
|
"reward_std": 0.6069091446697712,
|
|
"rewards/accuracy_reward": 0.1458333358168602,
|
|
"rewards/cosine_scaled_reward": -0.09532715613022447,
|
|
"step": 463
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1749.0416717529297,
|
|
"epoch": 0.5302857142857142,
|
|
"grad_norm": 0.11516103893518448,
|
|
"kl": 3.466010093688965e-05,
|
|
"lambda_div_used": 0.5341480076313019,
|
|
"learning_rate": 1.1492947512799328e-07,
|
|
"loss": -0.0088,
|
|
"reward": -0.040389321744441986,
|
|
"reward_after_mean": -0.040389321744441986,
|
|
"reward_after_std": 0.37696985714137554,
|
|
"reward_before_mean": 0.5386080192402005,
|
|
"reward_before_std": 0.14989514648914337,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.5789973307400942,
|
|
"reward_change_min": -0.7674467526376247,
|
|
"reward_change_std": 0.2917799688875675,
|
|
"reward_std": 0.37696986459195614,
|
|
"rewards/accuracy_reward": 0.375,
|
|
"rewards/cosine_scaled_reward": 0.16360801365226507,
|
|
"step": 464
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3036.854179382324,
|
|
"epoch": 0.5314285714285715,
|
|
"grad_norm": 0.08283663541078568,
|
|
"kl": 2.4955719709396362e-05,
|
|
"lambda_div_used": 0.572467640042305,
|
|
"learning_rate": 1.1413757749211602e-07,
|
|
"loss": 0.0373,
|
|
"reward": -0.40219551732297987,
|
|
"reward_after_mean": -0.40219551732297987,
|
|
"reward_after_std": 0.40797613374888897,
|
|
"reward_before_mean": -0.1455877646803856,
|
|
"reward_before_std": 0.32708599977195263,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.256607748568058,
|
|
"reward_change_min": -0.37336331233382225,
|
|
"reward_change_std": 0.1377662243321538,
|
|
"reward_std": 0.40797614119946957,
|
|
"rewards/accuracy_reward": 0.0416666679084301,
|
|
"rewards/cosine_scaled_reward": -0.18725443072617054,
|
|
"step": 465
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3122.9583435058594,
|
|
"epoch": 0.5325714285714286,
|
|
"grad_norm": 0.07085831463336945,
|
|
"kl": 2.0582228899002075e-05,
|
|
"lambda_div_used": 0.5319224968552589,
|
|
"learning_rate": 1.1336692317580158e-07,
|
|
"loss": -0.0563,
|
|
"reward": -0.36368466913700104,
|
|
"reward_after_mean": -0.36368466913700104,
|
|
"reward_after_std": 0.27606455981731415,
|
|
"reward_before_mean": 0.005439521744847298,
|
|
"reward_before_std": 0.14014938473701477,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.36912417598068714,
|
|
"reward_change_min": -0.5061142668128014,
|
|
"reward_change_std": 0.1899967910721898,
|
|
"reward_std": 0.2760645691305399,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.11956048291176558,
|
|
"step": 466
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3113.2708854675293,
|
|
"epoch": 0.5337142857142857,
|
|
"grad_norm": 0.07966674119234085,
|
|
"kl": 3.802310675382614e-05,
|
|
"lambda_div_used": 0.5612561926245689,
|
|
"learning_rate": 1.1261754973965422e-07,
|
|
"loss": -0.0067,
|
|
"reward": -0.3137255348265171,
|
|
"reward_after_mean": -0.3137255348265171,
|
|
"reward_after_std": 0.39207486622035503,
|
|
"reward_before_mean": 0.03320633992552757,
|
|
"reward_before_std": 0.2764817178249359,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3469318952411413,
|
|
"reward_change_min": -0.4828169047832489,
|
|
"reward_change_std": 0.18068420328199863,
|
|
"reward_std": 0.3920748811215162,
|
|
"rewards/accuracy_reward": 0.14583333395421505,
|
|
"rewards/cosine_scaled_reward": -0.11262698657810688,
|
|
"step": 467
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2898.770854949951,
|
|
"epoch": 0.5348571428571428,
|
|
"grad_norm": 0.07577298581600189,
|
|
"kl": 4.957616329193115e-05,
|
|
"lambda_div_used": 0.5363663136959076,
|
|
"learning_rate": 1.1188949370707787e-07,
|
|
"loss": 0.0338,
|
|
"reward": -0.5352834053337574,
|
|
"reward_after_mean": -0.5352834053337574,
|
|
"reward_after_std": 0.22454985231161118,
|
|
"reward_before_mean": -0.2775777019560337,
|
|
"reward_before_std": 0.15978037798777223,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2577057033777237,
|
|
"reward_change_min": -0.3789172098040581,
|
|
"reward_change_std": 0.14186344109475613,
|
|
"reward_std": 0.22454985417425632,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/cosine_scaled_reward": -0.27757770381867886,
|
|
"step": 468
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2854.8750228881836,
|
|
"epoch": 0.536,
|
|
"grad_norm": 0.09182782471179962,
|
|
"kl": 2.0341947674751282e-05,
|
|
"lambda_div_used": 0.5550094842910767,
|
|
"learning_rate": 1.1118279056249653e-07,
|
|
"loss": 0.0301,
|
|
"reward": -0.35969678312540054,
|
|
"reward_after_mean": -0.35969678312540054,
|
|
"reward_after_std": 0.30523936823010445,
|
|
"reward_before_mean": -0.04356633126735687,
|
|
"reward_before_std": 0.2476841462776065,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31613045185804367,
|
|
"reward_change_min": -0.45744797959923744,
|
|
"reward_change_std": 0.17962745483964682,
|
|
"reward_std": 0.3052393738180399,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.12689966335892677,
|
|
"step": 469
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3333.562530517578,
|
|
"epoch": 0.5371428571428571,
|
|
"grad_norm": 0.05610484257340431,
|
|
"kl": 2.9811635613441467e-05,
|
|
"lambda_div_used": 0.5911690816283226,
|
|
"learning_rate": 1.1049747474962444e-07,
|
|
"loss": 0.0307,
|
|
"reward": -0.37843912467360497,
|
|
"reward_after_mean": -0.37843912467360497,
|
|
"reward_after_std": 0.49228920973837376,
|
|
"reward_before_mean": -0.14763083557772916,
|
|
"reward_before_std": 0.4128831513226032,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.230808287858963,
|
|
"reward_change_min": -0.3457215949892998,
|
|
"reward_change_std": 0.12627399526536465,
|
|
"reward_std": 0.49228921718895435,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.21013083728030324,
|
|
"step": 470
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3032.312515258789,
|
|
"epoch": 0.5382857142857143,
|
|
"grad_norm": 0.053547170013189316,
|
|
"kl": 2.0138919353485107e-05,
|
|
"lambda_div_used": 0.5520770102739334,
|
|
"learning_rate": 1.0983357966978745e-07,
|
|
"loss": -0.0036,
|
|
"reward": -0.32707202807068825,
|
|
"reward_after_mean": -0.32707202807068825,
|
|
"reward_after_std": 0.30774626694619656,
|
|
"reward_before_mean": 0.006061417981982231,
|
|
"reward_before_std": 0.23228682670742273,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3331334535032511,
|
|
"reward_change_min": -0.47613633051514626,
|
|
"reward_change_std": 0.181988756172359,
|
|
"reward_std": 0.3077462762594223,
|
|
"rewards/accuracy_reward": 0.1041666716337204,
|
|
"rewards/cosine_scaled_reward": -0.09810524806380272,
|
|
"step": 471
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3115.8333435058594,
|
|
"epoch": 0.5394285714285715,
|
|
"grad_norm": 0.05183921754360199,
|
|
"kl": 1.757591962814331e-05,
|
|
"lambda_div_used": 0.6081600487232208,
|
|
"learning_rate": 1.0919113768029517e-07,
|
|
"loss": 0.0248,
|
|
"reward": -0.22974545462056994,
|
|
"reward_after_mean": -0.22974545462056994,
|
|
"reward_after_std": 0.5618578754365444,
|
|
"reward_before_mean": 0.03380160592496395,
|
|
"reward_before_std": 0.4960998175665736,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2635470647364855,
|
|
"reward_change_min": -0.4026007801294327,
|
|
"reward_change_std": 0.1525269951671362,
|
|
"reward_std": 0.5618579015135765,
|
|
"rewards/accuracy_reward": 0.14583333767950535,
|
|
"rewards/cosine_scaled_reward": -0.1120317269815132,
|
|
"step": 472
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3389.7291870117188,
|
|
"epoch": 0.5405714285714286,
|
|
"grad_norm": 0.05399390682578087,
|
|
"kl": 2.3730099201202393e-05,
|
|
"lambda_div_used": 0.6023471653461456,
|
|
"learning_rate": 1.0857018009286381e-07,
|
|
"loss": -0.0081,
|
|
"reward": -0.16527403378859162,
|
|
"reward_after_mean": -0.16527403378859162,
|
|
"reward_after_std": 0.4859522972255945,
|
|
"reward_before_mean": 0.15660450607538223,
|
|
"reward_before_std": 0.469901567324996,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3218785412609577,
|
|
"reward_change_min": -0.5562809407711029,
|
|
"reward_change_std": 0.21282578073441982,
|
|
"reward_std": 0.48595230281352997,
|
|
"rewards/accuracy_reward": 0.2083333395421505,
|
|
"rewards/cosine_scaled_reward": -0.051728841848671436,
|
|
"step": 473
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2609.3541946411133,
|
|
"epoch": 0.5417142857142857,
|
|
"grad_norm": 0.07730484753847122,
|
|
"kl": 1.3811513781547546e-05,
|
|
"lambda_div_used": 0.5951687097549438,
|
|
"learning_rate": 1.0797073717209013e-07,
|
|
"loss": 0.085,
|
|
"reward": 0.0976928174495697,
|
|
"reward_after_mean": 0.0976928174495697,
|
|
"reward_after_std": 0.5153173375874758,
|
|
"reward_before_mean": 0.5706379320472479,
|
|
"reward_before_std": 0.4373670984059572,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.47294510155916214,
|
|
"reward_change_min": -0.6950589865446091,
|
|
"reward_change_std": 0.27640796452760696,
|
|
"reward_std": 0.515317365527153,
|
|
"rewards/accuracy_reward": 0.3958333432674408,
|
|
"rewards/cosine_scaled_reward": 0.1748045664280653,
|
|
"step": 474
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2361.020851135254,
|
|
"epoch": 0.5428571428571428,
|
|
"grad_norm": 0.08214546740055084,
|
|
"kl": 9.525567293167114e-06,
|
|
"lambda_div_used": 0.6207183972001076,
|
|
"learning_rate": 1.0739283813397639e-07,
|
|
"loss": 0.0204,
|
|
"reward": 0.0007177861407399178,
|
|
"reward_after_mean": 0.0007177861407399178,
|
|
"reward_after_std": 0.5817896965891123,
|
|
"reward_before_mean": 0.34289272502064705,
|
|
"reward_before_std": 0.5596933793276548,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3421749472618103,
|
|
"reward_change_min": -0.5431834943592548,
|
|
"reward_change_std": 0.21874003671109676,
|
|
"reward_std": 0.581789730116725,
|
|
"rewards/accuracy_reward": 0.29166667722165585,
|
|
"rewards/cosine_scaled_reward": 0.05122605012729764,
|
|
"step": 475
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2583.8333740234375,
|
|
"epoch": 0.544,
|
|
"grad_norm": 0.08052952587604523,
|
|
"kl": 3.0007213354110718e-05,
|
|
"lambda_div_used": 0.6784488782286644,
|
|
"learning_rate": 1.068365111445064e-07,
|
|
"loss": 0.0154,
|
|
"reward": -0.031517775263637304,
|
|
"reward_after_mean": -0.031517775263637304,
|
|
"reward_after_std": 0.8641770519316196,
|
|
"reward_before_mean": 0.2038349723443389,
|
|
"reward_before_std": 0.8286011293530464,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.23535274900496006,
|
|
"reward_change_min": -0.41804002970457077,
|
|
"reward_change_std": 0.16118939872831106,
|
|
"reward_std": 0.8641770891845226,
|
|
"rewards/accuracy_reward": 0.22916666977107525,
|
|
"rewards/cosine_scaled_reward": -0.025331700686365366,
|
|
"step": 476
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2308.291732788086,
|
|
"epoch": 0.5451428571428572,
|
|
"grad_norm": 0.06839341670274734,
|
|
"kl": 8.73953104019165e-06,
|
|
"lambda_div_used": 0.6096622571349144,
|
|
"learning_rate": 1.063017833182728e-07,
|
|
"loss": 0.1111,
|
|
"reward": 0.19456686079502106,
|
|
"reward_after_mean": 0.19456686079502106,
|
|
"reward_after_std": 0.5847878716886044,
|
|
"reward_before_mean": 0.678333050571382,
|
|
"reward_before_std": 0.5006472393870354,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.48376619443297386,
|
|
"reward_change_min": -0.6835142783820629,
|
|
"reward_change_std": 0.2774948738515377,
|
|
"reward_std": 0.5847878959029913,
|
|
"rewards/accuracy_reward": 0.45833334885537624,
|
|
"rewards/cosine_scaled_reward": 0.21999971382319927,
|
|
"step": 477
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3169.3125610351562,
|
|
"epoch": 0.5462857142857143,
|
|
"grad_norm": 0.052864328026771545,
|
|
"kl": 6.990041583776474e-06,
|
|
"lambda_div_used": 0.5758631080389023,
|
|
"learning_rate": 1.0578868071715544e-07,
|
|
"loss": 0.0349,
|
|
"reward": -0.23655124567449093,
|
|
"reward_after_mean": -0.23655124567449093,
|
|
"reward_after_std": 0.4532997701317072,
|
|
"reward_before_mean": 0.11144567281007767,
|
|
"reward_before_std": 0.3459607223048806,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3479969333857298,
|
|
"reward_change_min": -0.5187661461532116,
|
|
"reward_change_std": 0.19546091184020042,
|
|
"reward_std": 0.4532997887581587,
|
|
"rewards/accuracy_reward": 0.20833333395421505,
|
|
"rewards/cosine_scaled_reward": -0.09688764810562134,
|
|
"step": 478
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2825.916702270508,
|
|
"epoch": 0.5474285714285714,
|
|
"grad_norm": 0.08355361968278885,
|
|
"kl": 3.739655949175358e-05,
|
|
"lambda_div_used": 0.5902442187070847,
|
|
"learning_rate": 1.0529722834905125e-07,
|
|
"loss": 0.033,
|
|
"reward": -0.37585191056132317,
|
|
"reward_after_mean": -0.37585191056132317,
|
|
"reward_after_std": 0.45466959848999977,
|
|
"reward_before_mean": -0.12638765759766102,
|
|
"reward_before_std": 0.4168459586799145,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2494642548263073,
|
|
"reward_change_min": -0.4229847304522991,
|
|
"reward_change_std": 0.15689124166965485,
|
|
"reward_std": 0.4546696189790964,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.18888765573501587,
|
|
"step": 479
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2415.895866394043,
|
|
"epoch": 0.5485714285714286,
|
|
"grad_norm": 0.08123134076595306,
|
|
"kl": 1.4044344425201416e-05,
|
|
"lambda_div_used": 0.5890126600861549,
|
|
"learning_rate": 1.0482745016665526e-07,
|
|
"loss": 0.0739,
|
|
"reward": -0.36289478465914726,
|
|
"reward_after_mean": -0.36289478465914726,
|
|
"reward_after_std": 0.4815117195248604,
|
|
"reward_before_mean": -0.11086839716881514,
|
|
"reward_before_std": 0.4026770405471325,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.25202639400959015,
|
|
"reward_change_min": -0.37619682028889656,
|
|
"reward_change_std": 0.13623447716236115,
|
|
"reward_std": 0.4815117232501507,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.17336839414201677,
|
|
"step": 480
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3057.625030517578,
|
|
"epoch": 0.5497142857142857,
|
|
"grad_norm": 0.05632660165429115,
|
|
"kl": 2.8233975172042847e-05,
|
|
"lambda_div_used": 0.5841230228543282,
|
|
"learning_rate": 1.0437936906629334e-07,
|
|
"loss": 0.0198,
|
|
"reward": -0.386552257463336,
|
|
"reward_after_mean": -0.386552257463336,
|
|
"reward_after_std": 0.4468122199177742,
|
|
"reward_before_mean": -0.14256569184362888,
|
|
"reward_before_std": 0.38717135414481163,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.24398657120764256,
|
|
"reward_change_min": -0.42186837270855904,
|
|
"reward_change_std": 0.14736179821193218,
|
|
"reward_std": 0.4468122236430645,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.20506569370627403,
|
|
"step": 481
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2824.5833587646484,
|
|
"epoch": 0.5508571428571428,
|
|
"grad_norm": 0.0865793228149414,
|
|
"kl": 2.4417764507234097e-05,
|
|
"lambda_div_used": 0.622517004609108,
|
|
"learning_rate": 1.0395300688680625e-07,
|
|
"loss": -0.0313,
|
|
"reward": -0.05876433290541172,
|
|
"reward_after_mean": -0.05876433290541172,
|
|
"reward_after_std": 0.5989817604422569,
|
|
"reward_before_mean": 0.259432727470994,
|
|
"reward_before_std": 0.5737235806882381,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.31819707341492176,
|
|
"reward_change_min": -0.5269695781171322,
|
|
"reward_change_std": 0.21138717606663704,
|
|
"reward_std": 0.5989817790687084,
|
|
"rewards/accuracy_reward": 0.22916667349636555,
|
|
"rewards/cosine_scaled_reward": 0.030266055837273598,
|
|
"step": 482
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2869.250015258789,
|
|
"epoch": 0.552,
|
|
"grad_norm": 0.08511485904455185,
|
|
"kl": 3.474205732345581e-05,
|
|
"lambda_div_used": 0.6013401597738266,
|
|
"learning_rate": 1.0354838440848501e-07,
|
|
"loss": -0.0837,
|
|
"reward": -0.2791392467916012,
|
|
"reward_after_mean": -0.2791392467916012,
|
|
"reward_after_std": 0.5272991992533207,
|
|
"reward_before_mean": -0.02371177263557911,
|
|
"reward_before_std": 0.4689871799200773,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2554274704307318,
|
|
"reward_change_min": -0.40494704991579056,
|
|
"reward_change_std": 0.15485912468284369,
|
|
"reward_std": 0.5272992141544819,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.1278784405440092,
|
|
"step": 483
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2591.7500228881836,
|
|
"epoch": 0.5531428571428572,
|
|
"grad_norm": 0.08745326101779938,
|
|
"kl": 3.6368146538734436e-07,
|
|
"lambda_div_used": 0.6466837078332901,
|
|
"learning_rate": 1.0316552135205837e-07,
|
|
"loss": 0.0082,
|
|
"reward": -0.04187892563641071,
|
|
"reward_after_mean": -0.04187892563641071,
|
|
"reward_after_std": 0.6843612100929022,
|
|
"reward_before_mean": 0.242530676885508,
|
|
"reward_before_std": 0.6844684220850468,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.284409599378705,
|
|
"reward_change_min": -0.49204646795988083,
|
|
"reward_change_std": 0.1993796620517969,
|
|
"reward_std": 0.6843612212687731,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": -0.0283026653341949,
|
|
"step": 484
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2046.2500381469727,
|
|
"epoch": 0.5542857142857143,
|
|
"grad_norm": 0.09076947718858719,
|
|
"kl": 2.752244472503662e-05,
|
|
"lambda_div_used": 0.5719796344637871,
|
|
"learning_rate": 1.0280443637773163e-07,
|
|
"loss": 0.0035,
|
|
"reward": -0.40644849208183587,
|
|
"reward_after_mean": -0.40644849208183587,
|
|
"reward_after_std": 0.398155614733696,
|
|
"reward_before_mean": -0.14403727410535794,
|
|
"reward_before_std": 0.3282261691056192,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2624112106859684,
|
|
"reward_change_min": -0.43003450334072113,
|
|
"reward_change_std": 0.15388297475874424,
|
|
"reward_std": 0.39815562404692173,
|
|
"rewards/accuracy_reward": 0.06250000186264515,
|
|
"rewards/cosine_scaled_reward": -0.20653727487660944,
|
|
"step": 485
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1967.7916946411133,
|
|
"epoch": 0.5554285714285714,
|
|
"grad_norm": 0.09958811849355698,
|
|
"kl": 3.91155481338501e-05,
|
|
"lambda_div_used": 0.5703976079821587,
|
|
"learning_rate": 1.0246514708427701e-07,
|
|
"loss": 0.0966,
|
|
"reward": -0.26609380822628736,
|
|
"reward_after_mean": -0.26609380822628736,
|
|
"reward_after_std": 0.4386676363646984,
|
|
"reward_before_mean": 0.07946969009935856,
|
|
"reward_before_std": 0.31633214373141527,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.34556350111961365,
|
|
"reward_change_min": -0.48576217144727707,
|
|
"reward_change_std": 0.1828850321471691,
|
|
"reward_std": 0.4386676475405693,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.08719698037020862,
|
|
"step": 486
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1967.0625228881836,
|
|
"epoch": 0.5565714285714286,
|
|
"grad_norm": 0.10718576610088348,
|
|
"kl": 3.7085264921188354e-06,
|
|
"lambda_div_used": 0.6087248548865318,
|
|
"learning_rate": 1.0214767000817596e-07,
|
|
"loss": 0.0461,
|
|
"reward": 0.08780635055154562,
|
|
"reward_after_mean": 0.08780635055154562,
|
|
"reward_after_std": 0.6550032701343298,
|
|
"reward_before_mean": 0.5625860100844875,
|
|
"reward_before_std": 0.5011934200301766,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.47477963753044605,
|
|
"reward_change_min": -0.7034505233168602,
|
|
"reward_change_std": 0.26853484753519297,
|
|
"reward_std": 0.6550032943487167,
|
|
"rewards/accuracy_reward": 0.3958333358168602,
|
|
"rewards/cosine_scaled_reward": 0.16675265738740563,
|
|
"step": 487
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2356.8125228881836,
|
|
"epoch": 0.5577142857142857,
|
|
"grad_norm": 0.08583448082208633,
|
|
"kl": 2.1670013666152954e-05,
|
|
"lambda_div_used": 0.5598724335432053,
|
|
"learning_rate": 1.0185202062281336e-07,
|
|
"loss": -0.0446,
|
|
"reward": -0.39712974801659584,
|
|
"reward_after_mean": -0.39712974801659584,
|
|
"reward_after_std": 0.3282298669219017,
|
|
"reward_before_mean": -0.11880321707576513,
|
|
"reward_before_std": 0.2731999019160867,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27832652255892754,
|
|
"reward_change_min": -0.42709672823548317,
|
|
"reward_change_std": 0.16301770228892565,
|
|
"reward_std": 0.3282298743724823,
|
|
"rewards/accuracy_reward": 0.0833333358168602,
|
|
"rewards/cosine_scaled_reward": -0.20213654916733503,
|
|
"step": 488
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3039.2916870117188,
|
|
"epoch": 0.5588571428571428,
|
|
"grad_norm": 0.08493036031723022,
|
|
"kl": 4.955753684043884e-05,
|
|
"lambda_div_used": 0.5549413114786148,
|
|
"learning_rate": 1.0157821333772304e-07,
|
|
"loss": -0.0127,
|
|
"reward": -0.4395306259393692,
|
|
"reward_after_mean": -0.4395306259393692,
|
|
"reward_after_std": 0.30519232526421547,
|
|
"reward_before_mean": -0.16155868768692017,
|
|
"reward_before_std": 0.24803727120161057,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.27797193080186844,
|
|
"reward_change_min": -0.44194458797574043,
|
|
"reward_change_std": 0.1623495165258646,
|
|
"reward_std": 0.3051923308521509,
|
|
"rewards/accuracy_reward": 0.0625,
|
|
"rewards/cosine_scaled_reward": -0.22405867651104927,
|
|
"step": 489
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2838.0833892822266,
|
|
"epoch": 0.56,
|
|
"grad_norm": 0.06069381162524223,
|
|
"kl": 1.2032687664031982e-05,
|
|
"lambda_div_used": 0.583185076713562,
|
|
"learning_rate": 1.013262614978859e-07,
|
|
"loss": 0.0139,
|
|
"reward": -0.1338807214051485,
|
|
"reward_after_mean": -0.1338807214051485,
|
|
"reward_after_std": 0.4649778436869383,
|
|
"reward_before_mean": 0.25663699954748154,
|
|
"reward_before_std": 0.3789667785167694,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.39051773957908154,
|
|
"reward_change_min": -0.6341840960085392,
|
|
"reward_change_std": 0.2300328817218542,
|
|
"reward_std": 0.46497784554958344,
|
|
"rewards/accuracy_reward": 0.2708333395421505,
|
|
"rewards/cosine_scaled_reward": -0.014196328818798065,
|
|
"step": 490
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2593.7291870117188,
|
|
"epoch": 0.5611428571428572,
|
|
"grad_norm": 0.07340901345014572,
|
|
"kl": 3.531062975525856e-05,
|
|
"lambda_div_used": 0.5965912714600563,
|
|
"learning_rate": 1.0109617738307911e-07,
|
|
"loss": 0.0268,
|
|
"reward": 0.01746355928480625,
|
|
"reward_after_mean": 0.01746355928480625,
|
|
"reward_after_std": 0.5940692499279976,
|
|
"reward_before_mean": 0.47233812790364027,
|
|
"reward_before_std": 0.4421461224555969,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4548745583742857,
|
|
"reward_change_min": -0.6400704979896545,
|
|
"reward_change_std": 0.24487750884145498,
|
|
"reward_std": 0.5940692741423845,
|
|
"rewards/accuracy_reward": 0.3541666679084301,
|
|
"rewards/cosine_scaled_reward": 0.11817142926156521,
|
|
"step": 491
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2583.5416679382324,
|
|
"epoch": 0.5622857142857143,
|
|
"grad_norm": 0.08282249420881271,
|
|
"kl": 2.4866312742233276e-05,
|
|
"lambda_div_used": 0.5367531701922417,
|
|
"learning_rate": 1.0088797220727779e-07,
|
|
"loss": 0.0376,
|
|
"reward": -0.3908469006419182,
|
|
"reward_after_mean": -0.3908469006419182,
|
|
"reward_after_std": 0.27869052439928055,
|
|
"reward_before_mean": -0.03699151985347271,
|
|
"reward_before_std": 0.16138391755521297,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3538553789258003,
|
|
"reward_change_min": -0.5079905577003956,
|
|
"reward_change_std": 0.18725439626723528,
|
|
"reward_std": 0.27869053184986115,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/cosine_scaled_reward": -0.1619915273040533,
|
|
"step": 492
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2176.8333740234375,
|
|
"epoch": 0.5634285714285714,
|
|
"grad_norm": 0.09936831891536713,
|
|
"kl": 6.021931767463684e-06,
|
|
"lambda_div_used": 0.596942737698555,
|
|
"learning_rate": 1.0070165611810855e-07,
|
|
"loss": 0.0246,
|
|
"reward": -0.037120603024959564,
|
|
"reward_after_mean": -0.037120603024959564,
|
|
"reward_after_std": 0.46812310442328453,
|
|
"reward_before_mean": 0.34137603268027306,
|
|
"reward_before_std": 0.44823691714555025,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.3784966245293617,
|
|
"reward_change_min": -0.5661123469471931,
|
|
"reward_change_std": 0.234741548076272,
|
|
"reward_std": 0.4681231305003166,
|
|
"rewards/accuracy_reward": 0.2708333432674408,
|
|
"rewards/cosine_scaled_reward": 0.07054269965738058,
|
|
"step": 493
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2352.583427429199,
|
|
"epoch": 0.5645714285714286,
|
|
"grad_norm": 0.07574566453695297,
|
|
"kl": 1.6648322343826294e-05,
|
|
"lambda_div_used": 0.6796721071004868,
|
|
"learning_rate": 1.005372381963547e-07,
|
|
"loss": 0.092,
|
|
"reward": 0.10963849350810051,
|
|
"reward_after_mean": 0.10963849350810051,
|
|
"reward_after_std": 0.8239130303263664,
|
|
"reward_before_mean": 0.4074633736163378,
|
|
"reward_before_std": 0.848267612978816,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2978248745203018,
|
|
"reward_change_min": -0.6511921100318432,
|
|
"reward_change_std": 0.2445714958012104,
|
|
"reward_std": 0.8239130582660437,
|
|
"rewards/accuracy_reward": 0.3125000037252903,
|
|
"rewards/cosine_scaled_reward": 0.09496336756274104,
|
|
"step": 494
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 3203.8959045410156,
|
|
"epoch": 0.5657142857142857,
|
|
"grad_norm": 0.057527996599674225,
|
|
"kl": 2.3846514523029327e-05,
|
|
"lambda_div_used": 0.5994853675365448,
|
|
"learning_rate": 1.0039472645551372e-07,
|
|
"loss": 0.0637,
|
|
"reward": -0.21300244145095348,
|
|
"reward_after_mean": -0.21300244145095348,
|
|
"reward_after_std": 0.48377062380313873,
|
|
"reward_before_mean": 0.08397414721548557,
|
|
"reward_before_std": 0.4531490486115217,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.29697658494114876,
|
|
"reward_change_min": -0.49157723411917686,
|
|
"reward_change_std": 0.1875515878200531,
|
|
"reward_std": 0.4837706368416548,
|
|
"rewards/accuracy_reward": 0.16666667349636555,
|
|
"rewards/cosine_scaled_reward": -0.08269252139143646,
|
|
"step": 495
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 1895.9375457763672,
|
|
"epoch": 0.5668571428571428,
|
|
"grad_norm": 0.10791665315628052,
|
|
"kl": 1.890142448246479e-05,
|
|
"lambda_div_used": 0.6198414713144302,
|
|
"learning_rate": 1.002741278414069e-07,
|
|
"loss": 0.0183,
|
|
"reward": 0.09977892541792244,
|
|
"reward_after_mean": 0.09977892541792244,
|
|
"reward_after_std": 0.6343465056270361,
|
|
"reward_before_mean": 0.5255357641726732,
|
|
"reward_before_std": 0.555769513361156,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.4257568307220936,
|
|
"reward_change_min": -0.6588771641254425,
|
|
"reward_change_std": 0.25742017664015293,
|
|
"reward_std": 0.634346516802907,
|
|
"rewards/accuracy_reward": 0.35416667349636555,
|
|
"rewards/cosine_scaled_reward": 0.17136909160763025,
|
|
"step": 496
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2577.645851135254,
|
|
"epoch": 0.568,
|
|
"grad_norm": 0.06887742131948471,
|
|
"kl": 2.493336796760559e-05,
|
|
"lambda_div_used": 0.6232517957687378,
|
|
"learning_rate": 1.0017544823184055e-07,
|
|
"loss": 0.0143,
|
|
"reward": 0.039356768131256104,
|
|
"reward_after_mean": 0.039356768131256104,
|
|
"reward_after_std": 0.5658366903662682,
|
|
"reward_before_mean": 0.401943476870656,
|
|
"reward_before_std": 0.5692849718034267,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.362586697563529,
|
|
"reward_change_min": -0.6212925836443901,
|
|
"reward_change_std": 0.24119753576815128,
|
|
"reward_std": 0.5658366959542036,
|
|
"rewards/accuracy_reward": 0.3333333469927311,
|
|
"rewards/cosine_scaled_reward": 0.06861014291644096,
|
|
"step": 497
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2857.2916946411133,
|
|
"epoch": 0.5691428571428572,
|
|
"grad_norm": 0.06479962170124054,
|
|
"kl": 9.963754564523697e-06,
|
|
"lambda_div_used": 0.6236685812473297,
|
|
"learning_rate": 1.0009869243631952e-07,
|
|
"loss": 0.0008,
|
|
"reward": -0.18094617873430252,
|
|
"reward_after_mean": -0.18094617873430252,
|
|
"reward_after_std": 0.6155845355242491,
|
|
"reward_before_mean": 0.07468715589493513,
|
|
"reward_before_std": 0.5768492119386792,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2556333262473345,
|
|
"reward_change_min": -0.4431779384613037,
|
|
"reward_change_std": 0.16846577636897564,
|
|
"reward_std": 0.6155845616012812,
|
|
"rewards/accuracy_reward": 0.1666666679084301,
|
|
"rewards/cosine_scaled_reward": -0.0919795110821724,
|
|
"step": 498
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2707.791702270508,
|
|
"epoch": 0.5702857142857143,
|
|
"grad_norm": 0.06317302584648132,
|
|
"kl": 1.5532365068793297e-05,
|
|
"lambda_div_used": 0.6250721588730812,
|
|
"learning_rate": 1.000438641958131e-07,
|
|
"loss": -0.0298,
|
|
"reward": -0.12677906453609467,
|
|
"reward_after_mean": -0.12677906453609467,
|
|
"reward_after_std": 0.6044176463037729,
|
|
"reward_before_mean": 0.1698420336470008,
|
|
"reward_before_std": 0.5829241154715419,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.2966211009770632,
|
|
"reward_change_min": -0.5905468240380287,
|
|
"reward_change_std": 0.2108100038021803,
|
|
"reward_std": 0.6044176481664181,
|
|
"rewards/accuracy_reward": 0.2083333358168602,
|
|
"rewards/cosine_scaled_reward": -0.03849130589514971,
|
|
"step": 499
|
|
},
|
|
{
|
|
"clip_fraction": 0.0,
|
|
"completion_length": 2980.9167098999023,
|
|
"epoch": 0.5714285714285714,
|
|
"grad_norm": 0.0621151477098465,
|
|
"kl": 1.033581793308258e-05,
|
|
"lambda_div_used": 0.6023986041545868,
|
|
"learning_rate": 1.0001096618257236e-07,
|
|
"loss": 0.0314,
|
|
"reward": -0.26747864981007297,
|
|
"reward_after_mean": -0.26747864981007297,
|
|
"reward_after_std": 0.5251844674348831,
|
|
"reward_before_mean": -0.0025623496621847153,
|
|
"reward_before_std": 0.4696931503713131,
|
|
"reward_change_max": 0.0,
|
|
"reward_change_mean": -0.26491627655923367,
|
|
"reward_change_min": -0.43205036222934723,
|
|
"reward_change_std": 0.15884541906416416,
|
|
"reward_std": 0.5251844730228186,
|
|
"rewards/accuracy_reward": 0.1041666679084301,
|
|
"rewards/cosine_scaled_reward": -0.10672901570796967,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.5714285714285714,
|
|
"step": 500,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.005314271912367986,
|
|
"train_runtime": 106074.3369,
|
|
"train_samples_per_second": 0.226,
|
|
"train_steps_per_second": 0.005
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 500,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 10,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 6,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|