1635 lines
65 KiB
JSON
1635 lines
65 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.3888888888888888,
|
|
"eval_steps": 500,
|
|
"global_step": 500,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3453125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2711.7,
|
|
"completions/mean_length": 3878.6890625,
|
|
"completions/mean_terminated_length": 1619.041455078125,
|
|
"completions/min_length": 609.9,
|
|
"completions/min_terminated_length": 609.9,
|
|
"entropy": 0.5342199512757361,
|
|
"epoch": 0.027777777777777776,
|
|
"frac_reward_zero_std": 0.275,
|
|
"grad_norm": 0.2434006631374359,
|
|
"learning_rate": 3e-07,
|
|
"loss": -0.0252,
|
|
"num_tokens": 2530705.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.4778271645307541,
|
|
"rewards/reward_correct/mean": 0.3828125,
|
|
"rewards/reward_correct/std": 0.47782717645168304,
|
|
"sampling/importance_sampling_ratio/max": 2.229939651489258,
|
|
"sampling/importance_sampling_ratio/mean": 0.28947882503271105,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 20.731588554382324,
|
|
"sampling/sampling_logp_difference/mean": 0.042482714727520944,
|
|
"step": 10,
|
|
"step_time": 97.8341444697231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.296875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2827.0,
|
|
"completions/mean_length": 3604.1640625,
|
|
"completions/mean_terminated_length": 1672.880029296875,
|
|
"completions/min_length": 578.1,
|
|
"completions/min_terminated_length": 578.1,
|
|
"entropy": 0.5231569674331695,
|
|
"epoch": 0.05555555555555555,
|
|
"frac_reward_zero_std": 0.175,
|
|
"grad_norm": 0.1266385316848755,
|
|
"learning_rate": 6.333333333333332e-07,
|
|
"loss": -0.0326,
|
|
"num_tokens": 4884674.0,
|
|
"reward": 0.4109375,
|
|
"reward_std": 0.47415409684181214,
|
|
"rewards/reward_correct/mean": 0.4109375,
|
|
"rewards/reward_correct/std": 0.4741541087627411,
|
|
"sampling/importance_sampling_ratio/max": 2.4683921217918394,
|
|
"sampling/importance_sampling_ratio/mean": 0.3206924065947533,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 19.872397041320802,
|
|
"sampling/sampling_logp_difference/mean": 0.04122307561337948,
|
|
"step": 20,
|
|
"step_time": 91.14647748004646
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.284375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2853.0,
|
|
"completions/mean_length": 3462.8375,
|
|
"completions/mean_terminated_length": 1606.2551391601562,
|
|
"completions/min_length": 553.3,
|
|
"completions/min_terminated_length": 553.3,
|
|
"entropy": 0.489674887387082,
|
|
"epoch": 0.08333333333333333,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.10141309350728989,
|
|
"learning_rate": 9.666666666666666e-07,
|
|
"loss": -0.0178,
|
|
"num_tokens": 7152210.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.4725195735692978,
|
|
"rewards/reward_correct/mean": 0.45,
|
|
"rewards/reward_correct/std": 0.47251958549022677,
|
|
"sampling/importance_sampling_ratio/max": 2.1169467091560366,
|
|
"sampling/importance_sampling_ratio/mean": 0.3329242721199989,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.969456291198732,
|
|
"sampling/sampling_logp_difference/mean": 0.03882257957011461,
|
|
"step": 30,
|
|
"step_time": 89.79702748470008
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2707.2,
|
|
"completions/mean_length": 4290.6703125,
|
|
"completions/mean_terminated_length": 1551.9008666992188,
|
|
"completions/min_length": 537.6,
|
|
"completions/min_terminated_length": 537.6,
|
|
"entropy": 0.5838452530559153,
|
|
"epoch": 0.1111111111111111,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.11795759201049805,
|
|
"learning_rate": 9.997876019358083e-07,
|
|
"loss": -0.0185,
|
|
"num_tokens": 9945431.0,
|
|
"reward": 0.3640625,
|
|
"reward_std": 0.4712675392627716,
|
|
"rewards/reward_correct/mean": 0.3640625,
|
|
"rewards/reward_correct/std": 0.471267557144165,
|
|
"sampling/importance_sampling_ratio/max": 2.536797857284546,
|
|
"sampling/importance_sampling_ratio/mean": 0.29656108319759367,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.390139389038087,
|
|
"sampling/sampling_logp_difference/mean": 0.04604039713740349,
|
|
"step": 40,
|
|
"step_time": 104.82241893839091
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3453125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2767.5,
|
|
"completions/mean_length": 3832.1796875,
|
|
"completions/mean_terminated_length": 1559.965576171875,
|
|
"completions/min_length": 624.6,
|
|
"completions/min_terminated_length": 624.6,
|
|
"entropy": 0.5342930512968451,
|
|
"epoch": 0.1388888888888889,
|
|
"frac_reward_zero_std": 0.3125,
|
|
"grad_norm": 1.948204517364502,
|
|
"learning_rate": 9.990536180750723e-07,
|
|
"loss": -0.0307,
|
|
"num_tokens": 12443042.0,
|
|
"reward": 0.403125,
|
|
"reward_std": 0.4718716889619827,
|
|
"rewards/reward_correct/mean": 0.403125,
|
|
"rewards/reward_correct/std": 0.4718716889619827,
|
|
"sampling/importance_sampling_ratio/max": 2.631464684009552,
|
|
"sampling/importance_sampling_ratio/mean": 0.3282940939068794,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.956416893005372,
|
|
"sampling/sampling_logp_difference/mean": 0.045470394939184186,
|
|
"step": 50,
|
|
"step_time": 96.40531954467296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.353125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2789.8,
|
|
"completions/mean_length": 3885.1921875,
|
|
"completions/mean_terminated_length": 1538.73271484375,
|
|
"completions/min_length": 554.3,
|
|
"completions/min_terminated_length": 554.3,
|
|
"entropy": 0.5527781342156232,
|
|
"epoch": 0.16666666666666666,
|
|
"frac_reward_zero_std": 0.2125,
|
|
"grad_norm": 0.2698710858821869,
|
|
"learning_rate": 9.97796195883804e-07,
|
|
"loss": -0.0329,
|
|
"num_tokens": 14976877.0,
|
|
"reward": 0.4359375,
|
|
"reward_std": 0.4925714761018753,
|
|
"rewards/reward_correct/mean": 0.4359375,
|
|
"rewards/reward_correct/std": 0.49257148802280426,
|
|
"sampling/importance_sampling_ratio/max": 2.5233256936073305,
|
|
"sampling/importance_sampling_ratio/mean": 0.32925176024436953,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.101856422424316,
|
|
"sampling/sampling_logp_difference/mean": 0.04616186544299126,
|
|
"step": 60,
|
|
"step_time": 97.68703336380422
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.346875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2801.9,
|
|
"completions/mean_length": 3842.059375,
|
|
"completions/mean_terminated_length": 1533.8720336914062,
|
|
"completions/min_length": 606.3,
|
|
"completions/min_terminated_length": 606.3,
|
|
"entropy": 0.5389381644781679,
|
|
"epoch": 0.19444444444444445,
|
|
"frac_reward_zero_std": 0.275,
|
|
"grad_norm": 0.1625521332025528,
|
|
"learning_rate": 9.960166542242428e-07,
|
|
"loss": -0.0281,
|
|
"num_tokens": 17480451.0,
|
|
"reward": 0.4265625,
|
|
"reward_std": 0.4911981523036957,
|
|
"rewards/reward_correct/mean": 0.4265625,
|
|
"rewards/reward_correct/std": 0.49119815826416013,
|
|
"sampling/importance_sampling_ratio/max": 2.4373825907707216,
|
|
"sampling/importance_sampling_ratio/mean": 0.32550778836011884,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.449700927734376,
|
|
"sampling/sampling_logp_difference/mean": 0.04671826101839542,
|
|
"step": 70,
|
|
"step_time": 96.85919274576008
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2702.0,
|
|
"completions/mean_length": 3792.390625,
|
|
"completions/mean_terminated_length": 1554.6982788085938,
|
|
"completions/min_length": 523.0,
|
|
"completions/min_terminated_length": 523.0,
|
|
"entropy": 0.5755266681313514,
|
|
"epoch": 0.2222222222222222,
|
|
"frac_reward_zero_std": 0.275,
|
|
"grad_norm": 0.2741508483886719,
|
|
"learning_rate": 9.937168595898508e-07,
|
|
"loss": -0.0216,
|
|
"num_tokens": 19956309.0,
|
|
"reward": 0.396875,
|
|
"reward_std": 0.48693000376224516,
|
|
"rewards/reward_correct/mean": 0.396875,
|
|
"rewards/reward_correct/std": 0.48693000674247744,
|
|
"sampling/importance_sampling_ratio/max": 2.2106072545051574,
|
|
"sampling/importance_sampling_ratio/mean": 0.30533475875854493,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.59271354675293,
|
|
"sampling/sampling_logp_difference/mean": 0.04881819523870945,
|
|
"step": 80,
|
|
"step_time": 95.62034683078528
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3796875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2839.8,
|
|
"completions/mean_length": 4084.5625,
|
|
"completions/mean_terminated_length": 1582.0239868164062,
|
|
"completions/min_length": 598.2,
|
|
"completions/min_terminated_length": 598.2,
|
|
"entropy": 0.5755758906248957,
|
|
"epoch": 0.25,
|
|
"frac_reward_zero_std": 0.2375,
|
|
"grad_norm": 0.11705684661865234,
|
|
"learning_rate": 9.908992241476186e-07,
|
|
"loss": -0.0328,
|
|
"num_tokens": 22618117.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.46752808690071107,
|
|
"rewards/reward_correct/mean": 0.3671875,
|
|
"rewards/reward_correct/std": 0.4675280928611755,
|
|
"sampling/importance_sampling_ratio/max": 2.1488356232643127,
|
|
"sampling/importance_sampling_ratio/mean": 0.282461379468441,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.50088768005371,
|
|
"sampling/sampling_logp_difference/mean": 0.04865880124270916,
|
|
"step": 90,
|
|
"step_time": 101.01973776686937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4203125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2645.3,
|
|
"completions/mean_length": 4314.771875,
|
|
"completions/mean_terminated_length": 1509.4629516601562,
|
|
"completions/min_length": 569.4,
|
|
"completions/min_terminated_length": 569.4,
|
|
"entropy": 0.5954360837582499,
|
|
"epoch": 0.2777777777777778,
|
|
"frac_reward_zero_std": 0.275,
|
|
"grad_norm": 0.09120581299066544,
|
|
"learning_rate": 9.875667032080352e-07,
|
|
"loss": -0.0301,
|
|
"num_tokens": 25430539.0,
|
|
"reward": 0.353125,
|
|
"reward_std": 0.4764838844537735,
|
|
"rewards/reward_correct/mean": 0.353125,
|
|
"rewards/reward_correct/std": 0.4764839023351669,
|
|
"sampling/importance_sampling_ratio/max": 2.3067823886871337,
|
|
"sampling/importance_sampling_ratio/mean": 0.28874450623989106,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.806869316101075,
|
|
"sampling/sampling_logp_difference/mean": 0.050743330270051956,
|
|
"step": 100,
|
|
"step_time": 106.27121401969343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2765625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2885.7,
|
|
"completions/mean_length": 3313.5390625,
|
|
"completions/mean_terminated_length": 1451.3173950195312,
|
|
"completions/min_length": 524.1,
|
|
"completions/min_terminated_length": 524.1,
|
|
"entropy": 0.5162243159022182,
|
|
"epoch": 0.3055555555555556,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.20105834305286407,
|
|
"learning_rate": 9.837227921253745e-07,
|
|
"loss": -0.0222,
|
|
"num_tokens": 27594124.0,
|
|
"reward": 0.4359375,
|
|
"reward_std": 0.48942558765411376,
|
|
"rewards/reward_correct/mean": 0.4359375,
|
|
"rewards/reward_correct/std": 0.4894255995750427,
|
|
"sampling/importance_sampling_ratio/max": 2.4590456604957582,
|
|
"sampling/importance_sampling_ratio/mean": 0.36281110197305677,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.5187047958374,
|
|
"sampling/sampling_logp_difference/mean": 0.04329596869647503,
|
|
"step": 110,
|
|
"step_time": 88.54633442051708
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3703125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2659.6,
|
|
"completions/mean_length": 3929.0671875,
|
|
"completions/mean_terminated_length": 1439.959716796875,
|
|
"completions/min_length": 555.3,
|
|
"completions/min_terminated_length": 555.3,
|
|
"entropy": 0.5905495826154947,
|
|
"epoch": 0.3333333333333333,
|
|
"frac_reward_zero_std": 0.2375,
|
|
"grad_norm": 0.08803282678127289,
|
|
"learning_rate": 9.793715226315528e-07,
|
|
"loss": -0.0261,
|
|
"num_tokens": 30159399.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.47909272611141207,
|
|
"rewards/reward_correct/mean": 0.421875,
|
|
"rewards/reward_correct/std": 0.47909273505210875,
|
|
"sampling/importance_sampling_ratio/max": 2.126016139984131,
|
|
"sampling/importance_sampling_ratio/mean": 0.27617028504610064,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.618224716186525,
|
|
"sampling/sampling_logp_difference/mean": 0.05163493566215038,
|
|
"step": 120,
|
|
"step_time": 99.54315063767135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.378125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2735.7,
|
|
"completions/mean_length": 4040.6890625,
|
|
"completions/mean_terminated_length": 1536.42490234375,
|
|
"completions/min_length": 645.0,
|
|
"completions/min_terminated_length": 645.0,
|
|
"entropy": 0.5655143281910568,
|
|
"epoch": 0.3611111111111111,
|
|
"frac_reward_zero_std": 0.2125,
|
|
"grad_norm": 0.1966542899608612,
|
|
"learning_rate": 9.745174586073982e-07,
|
|
"loss": -0.0311,
|
|
"num_tokens": 32795344.0,
|
|
"reward": 0.384375,
|
|
"reward_std": 0.47214375138282777,
|
|
"rewards/reward_correct/mean": 0.384375,
|
|
"rewards/reward_correct/std": 0.4721437692642212,
|
|
"sampling/importance_sampling_ratio/max": 2.5283960938453673,
|
|
"sampling/importance_sampling_ratio/mean": 0.3151987254619598,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.23869686126709,
|
|
"sampling/sampling_logp_difference/mean": 0.049242925643920896,
|
|
"step": 130,
|
|
"step_time": 100.30694740768521
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2765625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2714.3,
|
|
"completions/mean_length": 3301.6859375,
|
|
"completions/mean_terminated_length": 1435.6876953125,
|
|
"completions/min_length": 568.2,
|
|
"completions/min_terminated_length": 568.2,
|
|
"entropy": 0.5224350173491985,
|
|
"epoch": 0.3888888888888889,
|
|
"frac_reward_zero_std": 0.225,
|
|
"grad_norm": 0.266464501619339,
|
|
"learning_rate": 9.691656912957684e-07,
|
|
"loss": -0.0196,
|
|
"num_tokens": 34952999.0,
|
|
"reward": 0.478125,
|
|
"reward_std": 0.5006843686103821,
|
|
"rewards/reward_correct/mean": 0.478125,
|
|
"rewards/reward_correct/std": 0.500684380531311,
|
|
"sampling/importance_sampling_ratio/max": 2.5264899134635925,
|
|
"sampling/importance_sampling_ratio/mean": 0.4062116891145706,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.331268119812012,
|
|
"sampling/sampling_logp_difference/mean": 0.0450795441865921,
|
|
"step": 140,
|
|
"step_time": 88.51336960084737
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.321875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2729.3,
|
|
"completions/mean_length": 3610.175,
|
|
"completions/mean_terminated_length": 1446.8697387695313,
|
|
"completions/min_length": 536.4,
|
|
"completions/min_terminated_length": 536.4,
|
|
"entropy": 0.5646064455155283,
|
|
"epoch": 0.4166666666666667,
|
|
"frac_reward_zero_std": 0.225,
|
|
"grad_norm": 0.2582959830760956,
|
|
"learning_rate": 9.633218339615432e-07,
|
|
"loss": -0.0341,
|
|
"num_tokens": 37314103.0,
|
|
"reward": 0.4046875,
|
|
"reward_std": 0.47060964107513426,
|
|
"rewards/reward_correct/mean": 0.4046875,
|
|
"rewards/reward_correct/std": 0.470609650015831,
|
|
"sampling/importance_sampling_ratio/max": 2.4045764088630674,
|
|
"sampling/importance_sampling_ratio/mean": 0.3732295721769333,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.39520835876465,
|
|
"sampling/sampling_logp_difference/mean": 0.04807058311998844,
|
|
"step": 150,
|
|
"step_time": 93.85687965862454
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2705.6,
|
|
"completions/mean_length": 3746.8171875,
|
|
"completions/mean_terminated_length": 1490.0490234375,
|
|
"completions/min_length": 519.2,
|
|
"completions/min_terminated_length": 519.2,
|
|
"entropy": 0.5504673863761127,
|
|
"epoch": 0.4444444444444444,
|
|
"frac_reward_zero_std": 0.1625,
|
|
"grad_norm": 0.20146092772483826,
|
|
"learning_rate": 9.569920160040814e-07,
|
|
"loss": -0.0387,
|
|
"num_tokens": 39761090.0,
|
|
"reward": 0.4515625,
|
|
"reward_std": 0.48955525159835817,
|
|
"rewards/reward_correct/mean": 0.4515625,
|
|
"rewards/reward_correct/std": 0.48955525159835817,
|
|
"sampling/importance_sampling_ratio/max": 2.39810996055603,
|
|
"sampling/importance_sampling_ratio/mean": 0.35599239766597746,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.339636421203615,
|
|
"sampling/sampling_logp_difference/mean": 0.04856264889240265,
|
|
"step": 160,
|
|
"step_time": 95.43752448409796
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3578125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2787.9,
|
|
"completions/mean_length": 3883.9484375,
|
|
"completions/mean_terminated_length": 1488.986962890625,
|
|
"completions/min_length": 600.0,
|
|
"completions/min_terminated_length": 600.0,
|
|
"entropy": 0.5517090468667447,
|
|
"epoch": 0.4722222222222222,
|
|
"frac_reward_zero_std": 0.3625,
|
|
"grad_norm": 0.23341695964336395,
|
|
"learning_rate": 9.501828765283294e-07,
|
|
"loss": -0.0221,
|
|
"num_tokens": 42296545.0,
|
|
"reward": 0.3953125,
|
|
"reward_std": 0.47710200548172,
|
|
"rewards/reward_correct/mean": 0.3953125,
|
|
"rewards/reward_correct/std": 0.4771020084619522,
|
|
"sampling/importance_sampling_ratio/max": 2.4312403798103333,
|
|
"sampling/importance_sampling_ratio/mean": 0.3166383683681488,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.572778701782227,
|
|
"sampling/sampling_logp_difference/mean": 0.04768793135881424,
|
|
"step": 170,
|
|
"step_time": 98.4165175858885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.340625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2983.3,
|
|
"completions/mean_length": 3747.296875,
|
|
"completions/mean_terminated_length": 1446.30537109375,
|
|
"completions/min_length": 559.2,
|
|
"completions/min_terminated_length": 559.2,
|
|
"entropy": 0.5527338188607246,
|
|
"epoch": 0.5,
|
|
"frac_reward_zero_std": 0.2375,
|
|
"grad_norm": 0.13981030881404877,
|
|
"learning_rate": 9.429015573813162e-07,
|
|
"loss": -0.0258,
|
|
"num_tokens": 44746079.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.48312163949012754,
|
|
"rewards/reward_correct/mean": 0.390625,
|
|
"rewards/reward_correct/std": 0.4831216514110565,
|
|
"sampling/importance_sampling_ratio/max": 2.236375367641449,
|
|
"sampling/importance_sampling_ratio/mean": 0.35382367074489596,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.376666259765624,
|
|
"sampling/sampling_logp_difference/mean": 0.04669986329972744,
|
|
"step": 180,
|
|
"step_time": 95.65556847061961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.31875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2660.9,
|
|
"completions/mean_length": 3635.1734375,
|
|
"completions/mean_terminated_length": 1505.3306030273438,
|
|
"completions/min_length": 580.8,
|
|
"completions/min_terminated_length": 580.8,
|
|
"entropy": 0.536462911311537,
|
|
"epoch": 0.5277777777777778,
|
|
"frac_reward_zero_std": 0.1875,
|
|
"grad_norm": 0.16308102011680603,
|
|
"learning_rate": 9.351556956613422e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 47120230.0,
|
|
"reward": 0.41875,
|
|
"reward_std": 0.48690163493156435,
|
|
"rewards/reward_correct/mean": 0.41875,
|
|
"rewards/reward_correct/std": 0.48690164685249326,
|
|
"sampling/importance_sampling_ratio/max": 2.3551482439041136,
|
|
"sampling/importance_sampling_ratio/mean": 0.3526595965027809,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.034694099426268,
|
|
"sampling/sampling_logp_difference/mean": 0.04380165822803974,
|
|
"step": 190,
|
|
"step_time": 92.7884238248691
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3421875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2682.7,
|
|
"completions/mean_length": 3784.634375,
|
|
"completions/mean_terminated_length": 1501.1128051757812,
|
|
"completions/min_length": 506.2,
|
|
"completions/min_terminated_length": 506.2,
|
|
"entropy": 0.5288248831871897,
|
|
"epoch": 0.5555555555555556,
|
|
"frac_reward_zero_std": 0.225,
|
|
"grad_norm": 0.3080485165119171,
|
|
"learning_rate": 9.269534157077176e-07,
|
|
"loss": -0.0415,
|
|
"num_tokens": 49589284.0,
|
|
"reward": 0.4328125,
|
|
"reward_std": 0.49167430102825166,
|
|
"rewards/reward_correct/mean": 0.4328125,
|
|
"rewards/reward_correct/std": 0.4916743189096451,
|
|
"sampling/importance_sampling_ratio/max": 2.2188847064971924,
|
|
"sampling/importance_sampling_ratio/mean": 0.3379298195242882,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.151332664489747,
|
|
"sampling/sampling_logp_difference/mean": 0.04422352984547615,
|
|
"step": 200,
|
|
"step_time": 96.42145730126649
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3015625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2692.1,
|
|
"completions/mean_length": 3486.7875,
|
|
"completions/mean_terminated_length": 1456.8675537109375,
|
|
"completions/min_length": 424.7,
|
|
"completions/min_terminated_length": 424.7,
|
|
"entropy": 0.5138872295152396,
|
|
"epoch": 0.5833333333333334,
|
|
"frac_reward_zero_std": 0.2375,
|
|
"grad_norm": 0.14368104934692383,
|
|
"learning_rate": 9.183033205794524e-07,
|
|
"loss": -0.0271,
|
|
"num_tokens": 51866052.0,
|
|
"reward": 0.4359375,
|
|
"reward_std": 0.4865183919668198,
|
|
"rewards/reward_correct/mean": 0.4359375,
|
|
"rewards/reward_correct/std": 0.48651840090751647,
|
|
"sampling/importance_sampling_ratio/max": 2.529734969139099,
|
|
"sampling/importance_sampling_ratio/mean": 0.3310353755950928,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.605073738098145,
|
|
"sampling/sampling_logp_difference/mean": 0.04353258796036243,
|
|
"step": 210,
|
|
"step_time": 91.41761814020575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3109375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2680.2,
|
|
"completions/mean_length": 3544.5046875,
|
|
"completions/mean_terminated_length": 1457.0344970703125,
|
|
"completions/min_length": 569.1,
|
|
"completions/min_terminated_length": 569.1,
|
|
"entropy": 0.5135231144260615,
|
|
"epoch": 0.6111111111111112,
|
|
"frac_reward_zero_std": 0.225,
|
|
"grad_norm": 0.17278236150741577,
|
|
"learning_rate": 9.092144830318357e-07,
|
|
"loss": -0.0254,
|
|
"num_tokens": 54182551.0,
|
|
"reward": 0.43125,
|
|
"reward_std": 0.4832146823406219,
|
|
"rewards/reward_correct/mean": 0.43125,
|
|
"rewards/reward_correct/std": 0.4832146853208542,
|
|
"sampling/importance_sampling_ratio/max": 2.305103623867035,
|
|
"sampling/importance_sampling_ratio/mean": 0.35843545794487,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.62167282104492,
|
|
"sampling/sampling_logp_difference/mean": 0.04431740902364254,
|
|
"step": 220,
|
|
"step_time": 92.90139632392675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3234375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2705.8,
|
|
"completions/mean_length": 3632.15,
|
|
"completions/mean_terminated_length": 1470.5746459960938,
|
|
"completions/min_length": 552.3,
|
|
"completions/min_terminated_length": 552.3,
|
|
"entropy": 0.5150754484347999,
|
|
"epoch": 0.6388888888888888,
|
|
"frac_reward_zero_std": 0.1875,
|
|
"grad_norm": 0.4685378968715668,
|
|
"learning_rate": 8.99696436000368e-07,
|
|
"loss": -0.0234,
|
|
"num_tokens": 56552135.0,
|
|
"reward": 0.4265625,
|
|
"reward_std": 0.48878895938396455,
|
|
"rewards/reward_correct/mean": 0.4265625,
|
|
"rewards/reward_correct/std": 0.48878897428512574,
|
|
"sampling/importance_sampling_ratio/max": 2.552997076511383,
|
|
"sampling/importance_sampling_ratio/mean": 0.3578918635845184,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.531307792663576,
|
|
"sampling/sampling_logp_difference/mean": 0.04508708380162716,
|
|
"step": 230,
|
|
"step_time": 93.72880325838923
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3234375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 3232.5,
|
|
"completions/mean_length": 3705.865625,
|
|
"completions/mean_terminated_length": 1559.2656982421875,
|
|
"completions/min_length": 649.0,
|
|
"completions/min_terminated_length": 649.0,
|
|
"entropy": 0.5137905349489301,
|
|
"epoch": 0.6666666666666666,
|
|
"frac_reward_zero_std": 0.2875,
|
|
"grad_norm": 0.1393118053674698,
|
|
"learning_rate": 8.897591626020284e-07,
|
|
"loss": -0.0271,
|
|
"num_tokens": 58972081.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.4902578145265579,
|
|
"rewards/reward_correct/mean": 0.4296875,
|
|
"rewards/reward_correct/std": 0.49025782346725466,
|
|
"sampling/importance_sampling_ratio/max": 2.492326486110687,
|
|
"sampling/importance_sampling_ratio/mean": 0.33982470631599426,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 19.714303398132323,
|
|
"sampling/sampling_logp_difference/mean": 0.04368722774088383,
|
|
"step": 240,
|
|
"step_time": 94.55036701764911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3140625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2707.3,
|
|
"completions/mean_length": 3529.0140625,
|
|
"completions/mean_terminated_length": 1392.2738159179687,
|
|
"completions/min_length": 495.1,
|
|
"completions/min_terminated_length": 495.1,
|
|
"entropy": 0.5081628792919218,
|
|
"epoch": 0.6944444444444444,
|
|
"frac_reward_zero_std": 0.175,
|
|
"grad_norm": 0.08988256007432938,
|
|
"learning_rate": 8.794130856643633e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 61275938.0,
|
|
"reward": 0.4484375,
|
|
"reward_std": 0.49697442054748536,
|
|
"rewards/reward_correct/mean": 0.4484375,
|
|
"rewards/reward_correct/std": 0.4969744235277176,
|
|
"sampling/importance_sampling_ratio/max": 2.635455274581909,
|
|
"sampling/importance_sampling_ratio/mean": 0.35613458752632143,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.6952974319458,
|
|
"sampling/sampling_logp_difference/mean": 0.044385163486003874,
|
|
"step": 250,
|
|
"step_time": 92.77678105384112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.275,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 3083.6,
|
|
"completions/mean_length": 3301.2484375,
|
|
"completions/mean_terminated_length": 1451.84169921875,
|
|
"completions/min_length": 537.7,
|
|
"completions/min_terminated_length": 537.7,
|
|
"entropy": 0.48980009178631007,
|
|
"epoch": 0.7222222222222222,
|
|
"frac_reward_zero_std": 0.175,
|
|
"grad_norm": 0.13209204375743866,
|
|
"learning_rate": 8.686690567933801e-07,
|
|
"loss": -0.0239,
|
|
"num_tokens": 63433217.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.5000784158706665,
|
|
"rewards/reward_correct/mean": 0.4625,
|
|
"rewards/reward_correct/std": 0.5000784218311309,
|
|
"sampling/importance_sampling_ratio/max": 2.4186819911003115,
|
|
"sampling/importance_sampling_ratio/mean": 0.3675165086984634,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.942532539367676,
|
|
"sampling/sampling_logp_difference/mean": 0.04302049204707146,
|
|
"step": 260,
|
|
"step_time": 88.44859280735254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3078125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2721.3,
|
|
"completions/mean_length": 3572.4546875,
|
|
"completions/mean_terminated_length": 1534.3489501953125,
|
|
"completions/min_length": 564.8,
|
|
"completions/min_terminated_length": 564.8,
|
|
"entropy": 0.5065572742838412,
|
|
"epoch": 0.75,
|
|
"frac_reward_zero_std": 0.2625,
|
|
"grad_norm": 0.3504384756088257,
|
|
"learning_rate": 8.575383449917102e-07,
|
|
"loss": -0.0168,
|
|
"num_tokens": 65767596.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.4748325616121292,
|
|
"rewards/reward_correct/mean": 0.453125,
|
|
"rewards/reward_correct/std": 0.4748325675725937,
|
|
"sampling/importance_sampling_ratio/max": 2.2907237648963927,
|
|
"sampling/importance_sampling_ratio/mean": 0.3160614877939224,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 25.577795219421386,
|
|
"sampling/sampling_logp_difference/mean": 0.043240266665816304,
|
|
"step": 270,
|
|
"step_time": 92.66369731239975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2734375,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2857.0,
|
|
"completions/mean_length": 3278.6984375,
|
|
"completions/mean_terminated_length": 1444.8076782226562,
|
|
"completions/min_length": 553.0,
|
|
"completions/min_terminated_length": 553.0,
|
|
"entropy": 0.4928085923660547,
|
|
"epoch": 0.7777777777777778,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.11743981391191483,
|
|
"learning_rate": 8.460326248389824e-07,
|
|
"loss": -0.0328,
|
|
"num_tokens": 67909899.0,
|
|
"reward": 0.4796875,
|
|
"reward_std": 0.4822675257921219,
|
|
"rewards/reward_correct/mean": 0.4796875,
|
|
"rewards/reward_correct/std": 0.4822675287723541,
|
|
"sampling/importance_sampling_ratio/max": 2.5242951631546022,
|
|
"sampling/importance_sampling_ratio/mean": 0.38842698484659194,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.473637580871582,
|
|
"sampling/sampling_logp_difference/mean": 0.04052827805280686,
|
|
"step": 280,
|
|
"step_time": 87.84348612818867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2953125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2763.8,
|
|
"completions/mean_length": 3433.0734375,
|
|
"completions/mean_terminated_length": 1448.4840942382812,
|
|
"completions/min_length": 560.3,
|
|
"completions/min_terminated_length": 560.3,
|
|
"entropy": 0.5100364525802433,
|
|
"epoch": 0.8055555555555556,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.2548004686832428,
|
|
"learning_rate": 8.341639642468001e-07,
|
|
"loss": -0.0231,
|
|
"num_tokens": 70154122.0,
|
|
"reward": 0.39375,
|
|
"reward_std": 0.4714914709329605,
|
|
"rewards/reward_correct/mean": 0.39375,
|
|
"rewards/reward_correct/std": 0.4714914858341217,
|
|
"sampling/importance_sampling_ratio/max": 2.33950617313385,
|
|
"sampling/importance_sampling_ratio/mean": 0.3633014544844627,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.3909854888916,
|
|
"sampling/sampling_logp_difference/mean": 0.04575198851525784,
|
|
"step": 290,
|
|
"step_time": 90.32039845213293
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.265625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2642.8,
|
|
"completions/mean_length": 3212.525,
|
|
"completions/mean_terminated_length": 1411.774169921875,
|
|
"completions/min_length": 572.2,
|
|
"completions/min_terminated_length": 572.2,
|
|
"entropy": 0.47054290105588736,
|
|
"epoch": 0.8333333333333334,
|
|
"frac_reward_zero_std": 0.2125,
|
|
"grad_norm": 0.27349135279655457,
|
|
"learning_rate": 8.219448118011687e-07,
|
|
"loss": -0.0334,
|
|
"num_tokens": 72256530.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.4876495271921158,
|
|
"rewards/reward_correct/mean": 0.421875,
|
|
"rewards/reward_correct/std": 0.48764953911304476,
|
|
"sampling/importance_sampling_ratio/max": 2.2759945511817934,
|
|
"sampling/importance_sampling_ratio/mean": 0.3820392429828644,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.031887817382813,
|
|
"sampling/sampling_logp_difference/mean": 0.041293776035308837,
|
|
"step": 300,
|
|
"step_time": 86.86856737490743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2796875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2730.5,
|
|
"completions/mean_length": 3313.04375,
|
|
"completions/mean_terminated_length": 1431.5855224609375,
|
|
"completions/min_length": 443.0,
|
|
"completions/min_terminated_length": 443.0,
|
|
"entropy": 0.500835029873997,
|
|
"epoch": 0.8611111111111112,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.460864394903183,
|
|
"learning_rate": 8.093879837056485e-07,
|
|
"loss": -0.0167,
|
|
"num_tokens": 74422630.0,
|
|
"reward": 0.4078125,
|
|
"reward_std": 0.4829194873571396,
|
|
"rewards/reward_correct/mean": 0.4078125,
|
|
"rewards/reward_correct/std": 0.4829194962978363,
|
|
"sampling/importance_sampling_ratio/max": 2.56204674243927,
|
|
"sampling/importance_sampling_ratio/mean": 0.3848576098680496,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.089742851257324,
|
|
"sampling/sampling_logp_difference/mean": 0.04200447015464306,
|
|
"step": 310,
|
|
"step_time": 88.47893992438912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.303125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2956.5,
|
|
"completions/mean_length": 3513.2265625,
|
|
"completions/mean_terminated_length": 1483.2241455078124,
|
|
"completions/min_length": 610.5,
|
|
"completions/min_terminated_length": 610.5,
|
|
"entropy": 0.514991466421634,
|
|
"epoch": 0.8888888888888888,
|
|
"frac_reward_zero_std": 0.2625,
|
|
"grad_norm": 0.179782897233963,
|
|
"learning_rate": 7.965066503389264e-07,
|
|
"loss": -0.0309,
|
|
"num_tokens": 76716127.0,
|
|
"reward": 0.365625,
|
|
"reward_std": 0.47816264927387236,
|
|
"rewards/reward_correct/mean": 0.365625,
|
|
"rewards/reward_correct/std": 0.4781626582145691,
|
|
"sampling/importance_sampling_ratio/max": 2.519623303413391,
|
|
"sampling/importance_sampling_ratio/mean": 0.3532961577177048,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.083291053771973,
|
|
"sampling/sampling_logp_difference/mean": 0.04559468738734722,
|
|
"step": 320,
|
|
"step_time": 91.2618560783565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.253125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2747.5,
|
|
"completions/mean_length": 3135.4203125,
|
|
"completions/mean_terminated_length": 1434.197314453125,
|
|
"completions/min_length": 520.0,
|
|
"completions/min_terminated_length": 520.0,
|
|
"entropy": 0.4833090817555785,
|
|
"epoch": 0.9166666666666666,
|
|
"frac_reward_zero_std": 0.1875,
|
|
"grad_norm": 0.13914607465267181,
|
|
"learning_rate": 7.833143224409075e-07,
|
|
"loss": -0.0312,
|
|
"num_tokens": 78771436.0,
|
|
"reward": 0.4390625,
|
|
"reward_std": 0.4904475510120392,
|
|
"rewards/reward_correct/mean": 0.4390625,
|
|
"rewards/reward_correct/std": 0.49044756293296815,
|
|
"sampling/importance_sampling_ratio/max": 2.455558753013611,
|
|
"sampling/importance_sampling_ratio/mean": 0.38001332730054854,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.64868927001953,
|
|
"sampling/sampling_logp_difference/mean": 0.04199961461126804,
|
|
"step": 330,
|
|
"step_time": 85.40838945042341
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.290625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2742.6,
|
|
"completions/mean_length": 3391.753125,
|
|
"completions/mean_terminated_length": 1421.10107421875,
|
|
"completions/min_length": 449.9,
|
|
"completions/min_terminated_length": 449.9,
|
|
"entropy": 0.4844976777676493,
|
|
"epoch": 0.9444444444444444,
|
|
"frac_reward_zero_std": 0.225,
|
|
"grad_norm": 0.16815045475959778,
|
|
"learning_rate": 7.698248369418146e-07,
|
|
"loss": -0.0224,
|
|
"num_tokens": 80986918.0,
|
|
"reward": 0.4578125,
|
|
"reward_std": 0.49601399302482607,
|
|
"rewards/reward_correct/mean": 0.4578125,
|
|
"rewards/reward_correct/std": 0.496014004945755,
|
|
"sampling/importance_sampling_ratio/max": 2.3630632042884825,
|
|
"sampling/importance_sampling_ratio/mean": 0.3652906835079193,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.45225372314453,
|
|
"sampling/sampling_logp_difference/mean": 0.0402510404586792,
|
|
"step": 340,
|
|
"step_time": 89.63123638182878
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2764.0,
|
|
"completions/mean_length": 3510.5140625,
|
|
"completions/mean_terminated_length": 1515.0855346679687,
|
|
"completions/min_length": 589.2,
|
|
"completions/min_terminated_length": 589.2,
|
|
"entropy": 0.48650123639963566,
|
|
"epoch": 0.9722222222222222,
|
|
"frac_reward_zero_std": 0.2125,
|
|
"grad_norm": 0.34350690245628357,
|
|
"learning_rate": 7.560523424491594e-07,
|
|
"loss": -0.0327,
|
|
"num_tokens": 83279343.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.48524410724639894,
|
|
"rewards/reward_correct/mean": 0.40625,
|
|
"rewards/reward_correct/std": 0.4852441310882568,
|
|
"sampling/importance_sampling_ratio/max": 2.621637427806854,
|
|
"sampling/importance_sampling_ratio/mean": 0.3800614088773727,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.8350399017334,
|
|
"sampling/sampling_logp_difference/mean": 0.04253811165690422,
|
|
"step": 350,
|
|
"step_time": 90.70822816956789
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.315625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2796.9,
|
|
"completions/mean_length": 3580.165625,
|
|
"completions/mean_terminated_length": 1469.3232055664062,
|
|
"completions/min_length": 481.0,
|
|
"completions/min_terminated_length": 481.0,
|
|
"entropy": 0.5141544749494642,
|
|
"epoch": 1.0,
|
|
"frac_reward_zero_std": 0.2375,
|
|
"grad_norm": 0.09957528114318848,
|
|
"learning_rate": 7.420112844078065e-07,
|
|
"loss": -0.0264,
|
|
"num_tokens": 85620353.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.48938204646110534,
|
|
"rewards/reward_correct/mean": 0.45,
|
|
"rewards/reward_correct/std": 0.48938204944133756,
|
|
"sampling/importance_sampling_ratio/max": 2.5152801275253296,
|
|
"sampling/importance_sampling_ratio/mean": 0.3644652560353279,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.44343662261963,
|
|
"sampling/sampling_logp_difference/mean": 0.043626537173986436,
|
|
"step": 360,
|
|
"step_time": 92.83569850176573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.2875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2694.5,
|
|
"completions/mean_length": 3396.1671875,
|
|
"completions/mean_terminated_length": 1472.170263671875,
|
|
"completions/min_length": 472.1,
|
|
"completions/min_terminated_length": 472.1,
|
|
"entropy": 0.5021241253707558,
|
|
"epoch": 1.0277777777777777,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.20708823204040527,
|
|
"learning_rate": 7.277163899486974e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 87839972.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.4824483096599579,
|
|
"rewards/reward_correct/mean": 0.4609375,
|
|
"rewards/reward_correct/std": 0.4824483096599579,
|
|
"sampling/importance_sampling_ratio/max": 2.517948567867279,
|
|
"sampling/importance_sampling_ratio/mean": 0.4134147673845291,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.941598892211914,
|
|
"sampling/sampling_logp_difference/mean": 0.04388279486447573,
|
|
"step": 370,
|
|
"step_time": 89.34413652997463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3171875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2708.5,
|
|
"completions/mean_length": 3588.9296875,
|
|
"completions/mean_terminated_length": 1454.8683471679688,
|
|
"completions/min_length": 524.1,
|
|
"completions/min_terminated_length": 524.1,
|
|
"entropy": 0.502693377295509,
|
|
"epoch": 1.0555555555555556,
|
|
"frac_reward_zero_std": 0.275,
|
|
"grad_norm": 0.1858949214220047,
|
|
"learning_rate": 7.131826524421229e-07,
|
|
"loss": -0.0318,
|
|
"num_tokens": 90182887.0,
|
|
"reward": 0.440625,
|
|
"reward_std": 0.4875584036111832,
|
|
"rewards/reward_correct/mean": 0.440625,
|
|
"rewards/reward_correct/std": 0.4875584065914154,
|
|
"sampling/importance_sampling_ratio/max": 2.4215506672859193,
|
|
"sampling/importance_sampling_ratio/mean": 0.36463034301996233,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.958362197875978,
|
|
"sampling/sampling_logp_difference/mean": 0.04441062249243259,
|
|
"step": 380,
|
|
"step_time": 92.67752550859004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3890625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2740.1,
|
|
"completions/mean_length": 4121.328125,
|
|
"completions/mean_terminated_length": 1532.1543334960938,
|
|
"completions/min_length": 603.0,
|
|
"completions/min_terminated_length": 603.0,
|
|
"entropy": 0.5237304116133601,
|
|
"epoch": 1.0833333333333333,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.09304118901491165,
|
|
"learning_rate": 6.984253157717485e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 92870865.0,
|
|
"reward": 0.4078125,
|
|
"reward_std": 0.49269229769706724,
|
|
"rewards/reward_correct/mean": 0.4078125,
|
|
"rewards/reward_correct/std": 0.492692306637764,
|
|
"sampling/importance_sampling_ratio/max": 2.14275484085083,
|
|
"sampling/importance_sampling_ratio/mean": 0.29739712178707123,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.217792129516603,
|
|
"sampling/sampling_logp_difference/mean": 0.04599142037332058,
|
|
"step": 390,
|
|
"step_time": 102.05327154491097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3640625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2618.8,
|
|
"completions/mean_length": 3917.4375,
|
|
"completions/mean_terminated_length": 1481.8446899414062,
|
|
"completions/min_length": 622.0,
|
|
"completions/min_terminated_length": 622.0,
|
|
"entropy": 0.5371272402349859,
|
|
"epoch": 1.1111111111111112,
|
|
"frac_reward_zero_std": 0.2125,
|
|
"grad_norm": 0.14799313247203827,
|
|
"learning_rate": 6.834598583458861e-07,
|
|
"loss": -0.0227,
|
|
"num_tokens": 95421425.0,
|
|
"reward": 0.428125,
|
|
"reward_std": 0.482084384560585,
|
|
"rewards/reward_correct/mean": 0.428125,
|
|
"rewards/reward_correct/std": 0.4820843905210495,
|
|
"sampling/importance_sampling_ratio/max": 2.424322748184204,
|
|
"sampling/importance_sampling_ratio/mean": 0.32391665428876876,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.81523914337158,
|
|
"sampling/sampling_logp_difference/mean": 0.04645326770842075,
|
|
"step": 400,
|
|
"step_time": 99.04672075342387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.371875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2735.0,
|
|
"completions/mean_length": 3984.6125,
|
|
"completions/mean_terminated_length": 1514.8505249023438,
|
|
"completions/min_length": 551.9,
|
|
"completions/min_terminated_length": 551.9,
|
|
"entropy": 0.5294327767100185,
|
|
"epoch": 1.1388888888888888,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.2487545609474182,
|
|
"learning_rate": 6.683019768627794e-07,
|
|
"loss": -0.0202,
|
|
"num_tokens": 98020353.0,
|
|
"reward": 0.4203125,
|
|
"reward_std": 0.4853229820728302,
|
|
"rewards/reward_correct/mean": 0.4203125,
|
|
"rewards/reward_correct/std": 0.48532298803329466,
|
|
"sampling/importance_sampling_ratio/max": 2.425787329673767,
|
|
"sampling/importance_sampling_ratio/mean": 0.34082319438457487,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.107694625854492,
|
|
"sampling/sampling_logp_difference/mean": 0.04505102261900902,
|
|
"step": 410,
|
|
"step_time": 100.24638866372406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4046875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 3061.0,
|
|
"completions/mean_length": 4253.1953125,
|
|
"completions/mean_terminated_length": 1604.5595947265624,
|
|
"completions/min_length": 520.9,
|
|
"completions/min_terminated_length": 520.9,
|
|
"entropy": 0.5594399727415293,
|
|
"epoch": 1.1666666666666667,
|
|
"frac_reward_zero_std": 0.3125,
|
|
"grad_norm": 0.09589551389217377,
|
|
"learning_rate": 6.529675698469369e-07,
|
|
"loss": -0.0194,
|
|
"num_tokens": 100791518.0,
|
|
"reward": 0.340625,
|
|
"reward_std": 0.4659861117601395,
|
|
"rewards/reward_correct/mean": 0.340625,
|
|
"rewards/reward_correct/std": 0.4659861266613007,
|
|
"sampling/importance_sampling_ratio/max": 2.2143738508224486,
|
|
"sampling/importance_sampling_ratio/mean": 0.2883808270096779,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 24.680903816223143,
|
|
"sampling/sampling_logp_difference/mean": 0.045639732107520106,
|
|
"step": 420,
|
|
"step_time": 103.95541290417313
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3640625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2721.9,
|
|
"completions/mean_length": 3928.7765625,
|
|
"completions/mean_terminated_length": 1515.8645141601562,
|
|
"completions/min_length": 658.3,
|
|
"completions/min_terminated_length": 658.3,
|
|
"entropy": 0.5309726831037551,
|
|
"epoch": 1.1944444444444444,
|
|
"frac_reward_zero_std": 0.2125,
|
|
"grad_norm": 0.11377198994159698,
|
|
"learning_rate": 6.374727209737742e-07,
|
|
"loss": -0.0375,
|
|
"num_tokens": 103356455.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.4833220809698105,
|
|
"rewards/reward_correct/mean": 0.3875,
|
|
"rewards/reward_correct/std": 0.48332209289073946,
|
|
"sampling/importance_sampling_ratio/max": 2.697994363307953,
|
|
"sampling/importance_sampling_ratio/mean": 0.3253849983215332,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.634774589538573,
|
|
"sampling/sampling_logp_difference/mean": 0.04580756314098835,
|
|
"step": 430,
|
|
"step_time": 98.64194328989834
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3828125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2776.0,
|
|
"completions/mean_length": 4030.6875,
|
|
"completions/mean_terminated_length": 1456.2660522460938,
|
|
"completions/min_length": 534.9,
|
|
"completions/min_terminated_length": 534.9,
|
|
"entropy": 0.5253222627099603,
|
|
"epoch": 1.2222222222222223,
|
|
"frac_reward_zero_std": 0.2875,
|
|
"grad_norm": 0.1834605634212494,
|
|
"learning_rate": 6.218336822000597e-07,
|
|
"loss": -0.0311,
|
|
"num_tokens": 105985199.0,
|
|
"reward": 0.371875,
|
|
"reward_std": 0.4680081695318222,
|
|
"rewards/reward_correct/mean": 0.371875,
|
|
"rewards/reward_correct/std": 0.46800817251205445,
|
|
"sampling/importance_sampling_ratio/max": 2.4830429673194887,
|
|
"sampling/importance_sampling_ratio/mean": 0.3344820261001587,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 25.512220191955567,
|
|
"sampling/sampling_logp_difference/mean": 0.04608744233846664,
|
|
"step": 440,
|
|
"step_time": 101.21402945108711
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.4046875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2800.7,
|
|
"completions/mean_length": 4262.6609375,
|
|
"completions/mean_terminated_length": 1580.7835083007812,
|
|
"completions/min_length": 585.1,
|
|
"completions/min_terminated_length": 585.1,
|
|
"entropy": 0.5569739128928631,
|
|
"epoch": 1.25,
|
|
"frac_reward_zero_std": 0.175,
|
|
"grad_norm": 0.12380044907331467,
|
|
"learning_rate": 6.060668567178559e-07,
|
|
"loss": -0.0332,
|
|
"num_tokens": 108762526.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.4926897346973419,
|
|
"rewards/reward_correct/mean": 0.4375,
|
|
"rewards/reward_correct/std": 0.4926897376775742,
|
|
"sampling/importance_sampling_ratio/max": 2.0959500551223753,
|
|
"sampling/importance_sampling_ratio/mean": 0.28630392253398895,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.07707347869873,
|
|
"sampling/sampling_logp_difference/mean": 0.048703621700406076,
|
|
"step": 450,
|
|
"step_time": 104.36413167230785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3203125,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2636.1,
|
|
"completions/mean_length": 3645.103125,
|
|
"completions/mean_terminated_length": 1502.7676513671875,
|
|
"completions/min_length": 578.3,
|
|
"completions/min_terminated_length": 578.3,
|
|
"entropy": 0.5052359893452376,
|
|
"epoch": 1.2777777777777777,
|
|
"frac_reward_zero_std": 0.1375,
|
|
"grad_norm": 0.3007439970970154,
|
|
"learning_rate": 5.901887817498367e-07,
|
|
"loss": -0.0368,
|
|
"num_tokens": 111138808.0,
|
|
"reward": 0.41875,
|
|
"reward_std": 0.49026038646698,
|
|
"rewards/reward_correct/mean": 0.41875,
|
|
"rewards/reward_correct/std": 0.49026039242744446,
|
|
"sampling/importance_sampling_ratio/max": 2.2202050924301147,
|
|
"sampling/importance_sampling_ratio/mean": 0.3308444514870644,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.17190055847168,
|
|
"sampling/sampling_logp_difference/mean": 0.04389919862151146,
|
|
"step": 460,
|
|
"step_time": 93.38349273055792
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3265625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2857.9,
|
|
"completions/mean_length": 3689.3828125,
|
|
"completions/mean_terminated_length": 1499.214501953125,
|
|
"completions/min_length": 502.0,
|
|
"completions/min_terminated_length": 502.0,
|
|
"entropy": 0.5224037424195558,
|
|
"epoch": 1.3055555555555556,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.2663036286830902,
|
|
"learning_rate": 5.742161112040236e-07,
|
|
"loss": -0.0214,
|
|
"num_tokens": 113549213.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.4806476831436157,
|
|
"rewards/reward_correct/mean": 0.375,
|
|
"rewards/reward_correct/std": 0.48064770102500914,
|
|
"sampling/importance_sampling_ratio/max": 2.4689416170120237,
|
|
"sampling/importance_sampling_ratio/mean": 0.3618432849645615,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 20.738582611083984,
|
|
"sampling/sampling_logp_difference/mean": 0.046205943822860716,
|
|
"step": 470,
|
|
"step_time": 94.04668763801456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.35,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 3171.3,
|
|
"completions/mean_length": 3906.228125,
|
|
"completions/mean_terminated_length": 1615.4124145507812,
|
|
"completions/min_length": 649.1,
|
|
"completions/min_terminated_length": 649.1,
|
|
"entropy": 0.5468584059271961,
|
|
"epoch": 1.3333333333333333,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1444406360387802,
|
|
"learning_rate": 5.581655982061366e-07,
|
|
"loss": -0.0381,
|
|
"num_tokens": 116095783.0,
|
|
"reward": 0.4109375,
|
|
"reward_std": 0.48344514071941375,
|
|
"rewards/reward_correct/mean": 0.4109375,
|
|
"rewards/reward_correct/std": 0.48344514071941375,
|
|
"sampling/importance_sampling_ratio/max": 2.212104618549347,
|
|
"sampling/importance_sampling_ratio/mean": 0.3308627545833588,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.928424453735353,
|
|
"sampling/sampling_logp_difference/mean": 0.04624480046331882,
|
|
"step": 480,
|
|
"step_time": 97.89495770912617
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.3640625,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2847.0,
|
|
"completions/mean_length": 3933.9484375,
|
|
"completions/mean_terminated_length": 1534.0626586914063,
|
|
"completions/min_length": 625.0,
|
|
"completions/min_terminated_length": 625.0,
|
|
"entropy": 0.5143518089316785,
|
|
"epoch": 1.3611111111111112,
|
|
"frac_reward_zero_std": 0.1625,
|
|
"grad_norm": 0.14443330466747284,
|
|
"learning_rate": 5.420540775278788e-07,
|
|
"loss": -0.0338,
|
|
"num_tokens": 118660662.0,
|
|
"reward": 0.453125,
|
|
"reward_std": 0.490170481801033,
|
|
"rewards/reward_correct/mean": 0.453125,
|
|
"rewards/reward_correct/std": 0.49017049074172975,
|
|
"sampling/importance_sampling_ratio/max": 2.4240808367729185,
|
|
"sampling/importance_sampling_ratio/mean": 0.32956405729055405,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 25.39155864715576,
|
|
"sampling/sampling_logp_difference/mean": 0.044797389209270476,
|
|
"step": 490,
|
|
"step_time": 99.29671751111746
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.31875,
|
|
"completions/max_length": 8192.0,
|
|
"completions/max_terminated_length": 2790.6,
|
|
"completions/mean_length": 3592.6796875,
|
|
"completions/mean_terminated_length": 1455.6937377929687,
|
|
"completions/min_length": 574.1,
|
|
"completions/min_terminated_length": 574.1,
|
|
"entropy": 0.5269695220049471,
|
|
"epoch": 1.3888888888888888,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.15532448887825012,
|
|
"learning_rate": 5.258984479295852e-07,
|
|
"loss": -0.028,
|
|
"num_tokens": 121006129.0,
|
|
"reward": 0.4578125,
|
|
"reward_std": 0.4924586594104767,
|
|
"rewards/reward_correct/mean": 0.4578125,
|
|
"rewards/reward_correct/std": 0.49245867431163787,
|
|
"sampling/importance_sampling_ratio/max": 2.5489956021308897,
|
|
"sampling/importance_sampling_ratio/mean": 0.37822050750255587,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.463522529602052,
|
|
"sampling/sampling_logp_difference/mean": 0.046779343485832216,
|
|
"step": 500,
|
|
"step_time": 92.61194766778499
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1000,
|
|
"num_input_tokens_seen": 121006129,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 100,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|