25613 lines
937 KiB
JSON
25613 lines
937 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.5,
|
|
"eval_steps": 500,
|
|
"global_step": 882,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 787.265625,
|
|
"completions/mean_terminated_length": 787.265625,
|
|
"completions/min_length": 415.0,
|
|
"completions/min_terminated_length": 415.0,
|
|
"entropy": 0.7912661358714104,
|
|
"epoch": 0.0005668934240362812,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0145,
|
|
"num_tokens": 58849.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 1,
|
|
"step_time": 53.208744254894555
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/max_terminated_length": 1396.0,
|
|
"completions/mean_length": 720.953125,
|
|
"completions/mean_terminated_length": 720.953125,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 0.8267659060657024,
|
|
"epoch": 0.0011337868480725624,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.314453125,
|
|
"learning_rate": 9.994331065759637e-07,
|
|
"loss": 0.0152,
|
|
"num_tokens": 116702.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 2,
|
|
"step_time": 52.46114222612232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1339.0,
|
|
"completions/max_terminated_length": 1339.0,
|
|
"completions/mean_length": 722.0625,
|
|
"completions/mean_terminated_length": 722.0625,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 0.9556892551481724,
|
|
"epoch": 0.0017006802721088435,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 9.988662131519275e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 172418.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 3,
|
|
"step_time": 51.01601474173367
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1369.0,
|
|
"completions/max_terminated_length": 1369.0,
|
|
"completions/mean_length": 726.9375,
|
|
"completions/mean_terminated_length": 726.9375,
|
|
"completions/min_length": 33.0,
|
|
"completions/min_terminated_length": 33.0,
|
|
"entropy": 0.8995401579886675,
|
|
"epoch": 0.0022675736961451248,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.232421875,
|
|
"learning_rate": 9.982993197278912e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 229150.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 4,
|
|
"step_time": 50.94503266736865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2224.0,
|
|
"completions/max_terminated_length": 2224.0,
|
|
"completions/mean_length": 798.359375,
|
|
"completions/mean_terminated_length": 798.359375,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.870071342214942,
|
|
"epoch": 0.002834467120181406,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.24609375,
|
|
"learning_rate": 9.977324263038547e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 289981.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 5,
|
|
"step_time": 84.87245248258114
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 777.734375,
|
|
"completions/mean_terminated_length": 777.734375,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.5162014495581388,
|
|
"epoch": 0.003401360544217687,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.971655328798185e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 350580.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 6,
|
|
"step_time": 57.13563923817128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1465.0,
|
|
"completions/max_terminated_length": 1465.0,
|
|
"completions/mean_length": 745.953125,
|
|
"completions/mean_terminated_length": 745.953125,
|
|
"completions/min_length": 350.0,
|
|
"completions/min_terminated_length": 350.0,
|
|
"entropy": 0.9106043875217438,
|
|
"epoch": 0.003968253968253968,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 9.965986394557822e-07,
|
|
"loss": 0.027,
|
|
"num_tokens": 411089.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 7,
|
|
"step_time": 56.46439360547811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1174.0,
|
|
"completions/max_terminated_length": 1174.0,
|
|
"completions/mean_length": 663.9375,
|
|
"completions/mean_terminated_length": 663.9375,
|
|
"completions/min_length": 210.0,
|
|
"completions/min_terminated_length": 210.0,
|
|
"entropy": 0.7852413542568684,
|
|
"epoch": 0.0045351473922902496,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2275390625,
|
|
"learning_rate": 9.96031746031746e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 465445.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 8,
|
|
"step_time": 44.557856001891196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1411.0,
|
|
"completions/max_terminated_length": 1411.0,
|
|
"completions/mean_length": 754.640625,
|
|
"completions/mean_terminated_length": 754.640625,
|
|
"completions/min_length": 415.0,
|
|
"completions/min_terminated_length": 415.0,
|
|
"entropy": 0.6974279470741749,
|
|
"epoch": 0.00510204081632653,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2021484375,
|
|
"learning_rate": 9.954648526077097e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 525142.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 9,
|
|
"step_time": 53.639586839824915
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1870.0,
|
|
"completions/max_terminated_length": 1870.0,
|
|
"completions/mean_length": 809.296875,
|
|
"completions/mean_terminated_length": 809.296875,
|
|
"completions/min_length": 454.0,
|
|
"completions/min_terminated_length": 454.0,
|
|
"entropy": 0.8046941738575697,
|
|
"epoch": 0.005668934240362812,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.271484375,
|
|
"learning_rate": 9.948979591836735e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 586313.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 10,
|
|
"step_time": 70.25523345544934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1355.0,
|
|
"completions/max_terminated_length": 1355.0,
|
|
"completions/mean_length": 843.921875,
|
|
"completions/mean_terminated_length": 843.921875,
|
|
"completions/min_length": 379.0,
|
|
"completions/min_terminated_length": 379.0,
|
|
"entropy": 0.7391964737325907,
|
|
"epoch": 0.006235827664399093,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 9.94331065759637e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 650068.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 11,
|
|
"step_time": 50.96021834015846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1932.0,
|
|
"completions/mean_length": 940.75,
|
|
"completions/mean_terminated_length": 890.666748046875,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.6950870044529438,
|
|
"epoch": 0.006802721088435374,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2255859375,
|
|
"learning_rate": 9.937641723356008e-07,
|
|
"loss": 0.0054,
|
|
"num_tokens": 720268.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 12,
|
|
"step_time": 175.690794384107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1704.0,
|
|
"completions/max_terminated_length": 1704.0,
|
|
"completions/mean_length": 896.03125,
|
|
"completions/mean_terminated_length": 896.03125,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7128410097211599,
|
|
"epoch": 0.007369614512471655,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.314453125,
|
|
"learning_rate": 9.931972789115645e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 788798.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 13,
|
|
"step_time": 65.53362340852618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1462.0,
|
|
"completions/max_terminated_length": 1462.0,
|
|
"completions/mean_length": 774.09375,
|
|
"completions/mean_terminated_length": 774.09375,
|
|
"completions/min_length": 406.0,
|
|
"completions/min_terminated_length": 406.0,
|
|
"entropy": 0.8867389839142561,
|
|
"epoch": 0.007936507936507936,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 9.926303854875283e-07,
|
|
"loss": -0.0287,
|
|
"num_tokens": 849668.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 14,
|
|
"step_time": 56.14234718307853
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1392.0,
|
|
"completions/max_terminated_length": 1392.0,
|
|
"completions/mean_length": 806.171875,
|
|
"completions/mean_terminated_length": 806.171875,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.680386021733284,
|
|
"epoch": 0.008503401360544218,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 9.92063492063492e-07,
|
|
"loss": -0.007,
|
|
"num_tokens": 912015.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 15,
|
|
"step_time": 52.486787100322545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1232.0,
|
|
"completions/max_terminated_length": 1232.0,
|
|
"completions/mean_length": 737.03125,
|
|
"completions/mean_terminated_length": 737.03125,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.897417239844799,
|
|
"epoch": 0.009070294784580499,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.914965986394558e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 969129.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 16,
|
|
"step_time": 46.14659155998379
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1403.0,
|
|
"completions/max_terminated_length": 1403.0,
|
|
"completions/mean_length": 765.453125,
|
|
"completions/mean_terminated_length": 765.453125,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.9181688856333494,
|
|
"epoch": 0.00963718820861678,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 9.909297052154193e-07,
|
|
"loss": -0.0073,
|
|
"num_tokens": 1029094.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 17,
|
|
"step_time": 52.30169053096324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1291.0,
|
|
"completions/max_terminated_length": 1291.0,
|
|
"completions/mean_length": 795.53125,
|
|
"completions/mean_terminated_length": 795.53125,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.7358187921345234,
|
|
"epoch": 0.01020408163265306,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.234375,
|
|
"learning_rate": 9.90362811791383e-07,
|
|
"loss": -0.0018,
|
|
"num_tokens": 1089720.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 18,
|
|
"step_time": 48.606017048470676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1462.0,
|
|
"completions/max_terminated_length": 1462.0,
|
|
"completions/mean_length": 711.921875,
|
|
"completions/mean_terminated_length": 711.921875,
|
|
"completions/min_length": 405.0,
|
|
"completions/min_terminated_length": 405.0,
|
|
"entropy": 0.9461786858737469,
|
|
"epoch": 0.010770975056689343,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28515625,
|
|
"learning_rate": 9.897959183673468e-07,
|
|
"loss": -0.0098,
|
|
"num_tokens": 1144827.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 19,
|
|
"step_time": 54.73584616929293
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1962.0,
|
|
"completions/max_terminated_length": 1962.0,
|
|
"completions/mean_length": 717.609375,
|
|
"completions/mean_terminated_length": 717.609375,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.7700510062277317,
|
|
"epoch": 0.011337868480725623,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 9.892290249433106e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 1200890.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 20,
|
|
"step_time": 74.84192624688148
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1629.0,
|
|
"completions/max_terminated_length": 1629.0,
|
|
"completions/mean_length": 862.453125,
|
|
"completions/mean_terminated_length": 862.453125,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.7410835660994053,
|
|
"epoch": 0.011904761904761904,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.205078125,
|
|
"learning_rate": 9.886621315192743e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 1269087.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 21,
|
|
"step_time": 62.02429956756532
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1385.0,
|
|
"completions/max_terminated_length": 1385.0,
|
|
"completions/mean_length": 655.640625,
|
|
"completions/mean_terminated_length": 655.640625,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 1.062008362263441,
|
|
"epoch": 0.012471655328798186,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 9.88095238095238e-07,
|
|
"loss": 0.0065,
|
|
"num_tokens": 1329856.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 22,
|
|
"step_time": 56.07742853090167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1237.0,
|
|
"completions/max_terminated_length": 1237.0,
|
|
"completions/mean_length": 741.921875,
|
|
"completions/mean_terminated_length": 741.921875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.8006270676851273,
|
|
"epoch": 0.013038548752834467,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 9.875283446712018e-07,
|
|
"loss": -0.0079,
|
|
"num_tokens": 1388403.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 23,
|
|
"step_time": 46.51893087476492
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 685.40625,
|
|
"completions/mean_terminated_length": 685.40625,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"entropy": 0.9855949841439724,
|
|
"epoch": 0.013605442176870748,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41796875,
|
|
"learning_rate": 9.869614512471656e-07,
|
|
"loss": 0.0031,
|
|
"num_tokens": 1442821.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.2567298710346222,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 24,
|
|
"step_time": 51.394197618588805
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1163.0,
|
|
"completions/max_terminated_length": 1163.0,
|
|
"completions/mean_length": 753.390625,
|
|
"completions/mean_terminated_length": 753.390625,
|
|
"completions/min_length": 31.0,
|
|
"completions/min_terminated_length": 31.0,
|
|
"entropy": 0.8061298858374357,
|
|
"epoch": 0.01417233560090703,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 9.86394557823129e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 1501582.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 25,
|
|
"step_time": 44.023181112483144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 733.625,
|
|
"completions/mean_terminated_length": 733.625,
|
|
"completions/min_length": 20.0,
|
|
"completions/min_terminated_length": 20.0,
|
|
"entropy": 0.8445104025304317,
|
|
"epoch": 0.01473922902494331,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.365234375,
|
|
"learning_rate": 9.858276643990928e-07,
|
|
"loss": -0.0117,
|
|
"num_tokens": 1557806.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.39708447456359863,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 26,
|
|
"step_time": 53.340097087435424
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1720.0,
|
|
"completions/max_terminated_length": 1720.0,
|
|
"completions/mean_length": 724.765625,
|
|
"completions/mean_terminated_length": 724.765625,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 1.036143023520708,
|
|
"epoch": 0.015306122448979591,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 9.852607709750566e-07,
|
|
"loss": 0.0017,
|
|
"num_tokens": 1614767.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 27,
|
|
"step_time": 64.8979874541983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 730.71875,
|
|
"completions/mean_terminated_length": 730.71875,
|
|
"completions/min_length": 177.0,
|
|
"completions/min_terminated_length": 177.0,
|
|
"entropy": 0.7145545389503241,
|
|
"epoch": 0.015873015873015872,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.392578125,
|
|
"learning_rate": 9.846938775510203e-07,
|
|
"loss": 0.0183,
|
|
"num_tokens": 1671429.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 28,
|
|
"step_time": 52.964277925901115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1306.0,
|
|
"completions/max_terminated_length": 1306.0,
|
|
"completions/mean_length": 806.96875,
|
|
"completions/mean_terminated_length": 806.96875,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.9442988224327564,
|
|
"epoch": 0.016439909297052153,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 9.84126984126984e-07,
|
|
"loss": 0.0258,
|
|
"num_tokens": 1736139.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 29,
|
|
"step_time": 50.33796348888427
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1497.0,
|
|
"completions/max_terminated_length": 1497.0,
|
|
"completions/mean_length": 751.125,
|
|
"completions/mean_terminated_length": 751.125,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 0.9300839081406593,
|
|
"epoch": 0.017006802721088437,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 9.835600907029478e-07,
|
|
"loss": -0.0215,
|
|
"num_tokens": 1795699.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 30,
|
|
"step_time": 58.92226161248982
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1455.0,
|
|
"completions/max_terminated_length": 1455.0,
|
|
"completions/mean_length": 748.625,
|
|
"completions/mean_terminated_length": 748.625,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 0.9305003769695759,
|
|
"epoch": 0.017573696145124718,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.829931972789116e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1853307.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 31,
|
|
"step_time": 55.984347719699144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 756.6875,
|
|
"completions/mean_terminated_length": 756.6875,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 1.0353158302605152,
|
|
"epoch": 0.018140589569160998,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 9.824263038548753e-07,
|
|
"loss": -0.028,
|
|
"num_tokens": 1914359.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 32,
|
|
"step_time": 48.36243443284184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1262.0,
|
|
"completions/max_terminated_length": 1262.0,
|
|
"completions/mean_length": 752.609375,
|
|
"completions/mean_terminated_length": 752.609375,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 0.7963434420526028,
|
|
"epoch": 0.01870748299319728,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 9.818594104308389e-07,
|
|
"loss": -0.0104,
|
|
"num_tokens": 1971198.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 33,
|
|
"step_time": 47.883975078351796
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2255.0,
|
|
"completions/max_terminated_length": 2255.0,
|
|
"completions/mean_length": 762.859375,
|
|
"completions/mean_terminated_length": 762.859375,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.7279575727880001,
|
|
"epoch": 0.01927437641723356,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2431640625,
|
|
"learning_rate": 9.812925170068026e-07,
|
|
"loss": 0.0052,
|
|
"num_tokens": 2030189.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 34,
|
|
"step_time": 87.02857953030616
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1432.0,
|
|
"completions/max_terminated_length": 1432.0,
|
|
"completions/mean_length": 823.03125,
|
|
"completions/mean_terminated_length": 823.03125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.8386581428349018,
|
|
"epoch": 0.01984126984126984,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 9.807256235827664e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 2094439.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 35,
|
|
"step_time": 54.508007938973606
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1454.0,
|
|
"completions/max_terminated_length": 1454.0,
|
|
"completions/mean_length": 729.65625,
|
|
"completions/mean_terminated_length": 729.65625,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.9026074595749378,
|
|
"epoch": 0.02040816326530612,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.220703125,
|
|
"learning_rate": 9.801587301587301e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 2152001.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 36,
|
|
"step_time": 55.1510893618688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1198.0,
|
|
"completions/max_terminated_length": 1198.0,
|
|
"completions/mean_length": 730.28125,
|
|
"completions/mean_terminated_length": 730.28125,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.7190846279263496,
|
|
"epoch": 0.020975056689342405,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.380859375,
|
|
"learning_rate": 9.795918367346939e-07,
|
|
"loss": 0.0085,
|
|
"num_tokens": 2210139.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 37,
|
|
"step_time": 45.36145204398781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1166.0,
|
|
"completions/max_terminated_length": 1166.0,
|
|
"completions/mean_length": 751.625,
|
|
"completions/mean_terminated_length": 751.625,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 0.7631574179977179,
|
|
"epoch": 0.021541950113378686,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 9.790249433106576e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 2267779.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6666666865348816,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 38,
|
|
"step_time": 43.88726707175374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1659.0,
|
|
"completions/max_terminated_length": 1659.0,
|
|
"completions/mean_length": 777.15625,
|
|
"completions/mean_terminated_length": 777.15625,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 1.039644118398428,
|
|
"epoch": 0.022108843537414966,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"learning_rate": 9.784580498866214e-07,
|
|
"loss": -0.0216,
|
|
"num_tokens": 2328453.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 39,
|
|
"step_time": 63.13692790642381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1534.0,
|
|
"completions/max_terminated_length": 1534.0,
|
|
"completions/mean_length": 730.8125,
|
|
"completions/mean_terminated_length": 730.8125,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 1.0110173597931862,
|
|
"epoch": 0.022675736961451247,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 9.778911564625851e-07,
|
|
"loss": 0.0069,
|
|
"num_tokens": 2389921.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 40,
|
|
"step_time": 60.27846311684698
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1052.0,
|
|
"completions/mean_length": 690.328125,
|
|
"completions/mean_terminated_length": 636.2698974609375,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 1.0578433759510517,
|
|
"epoch": 0.023242630385487528,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 9.773242630385487e-07,
|
|
"loss": -0.0046,
|
|
"num_tokens": 2444566.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 41,
|
|
"step_time": 177.39915514178574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 838.0,
|
|
"completions/mean_terminated_length": 838.0,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8202644884586334,
|
|
"epoch": 0.023809523809523808,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 9.767573696145124e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 2509566.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 42,
|
|
"step_time": 56.90364085417241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 734.171875,
|
|
"completions/mean_terminated_length": 734.171875,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7877535559237003,
|
|
"epoch": 0.024376417233560092,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"learning_rate": 9.761904761904762e-07,
|
|
"loss": 0.008,
|
|
"num_tokens": 2566745.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 43,
|
|
"step_time": 55.869060398079455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1434.0,
|
|
"completions/max_terminated_length": 1434.0,
|
|
"completions/mean_length": 814.359375,
|
|
"completions/mean_terminated_length": 814.359375,
|
|
"completions/min_length": 6.0,
|
|
"completions/min_terminated_length": 6.0,
|
|
"entropy": 0.7830752562731504,
|
|
"epoch": 0.024943310657596373,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 9.7562358276644e-07,
|
|
"loss": -0.0146,
|
|
"num_tokens": 2628928.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 44,
|
|
"step_time": 55.32074639480561
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1261.0,
|
|
"completions/max_terminated_length": 1261.0,
|
|
"completions/mean_length": 732.40625,
|
|
"completions/mean_terminated_length": 732.40625,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 0.9837802685797215,
|
|
"epoch": 0.025510204081632654,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3984375,
|
|
"learning_rate": 9.750566893424036e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 2687210.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 45,
|
|
"step_time": 48.296295342966914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1630.0,
|
|
"completions/max_terminated_length": 1630.0,
|
|
"completions/mean_length": 826.90625,
|
|
"completions/mean_terminated_length": 826.90625,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.8597951680421829,
|
|
"epoch": 0.026077097505668934,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 9.744897959183674e-07,
|
|
"loss": -0.0273,
|
|
"num_tokens": 2750828.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 46,
|
|
"step_time": 62.49716377258301
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1509.0,
|
|
"completions/max_terminated_length": 1509.0,
|
|
"completions/mean_length": 766.0625,
|
|
"completions/mean_terminated_length": 766.0625,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 0.8254429344087839,
|
|
"epoch": 0.026643990929705215,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"learning_rate": 9.739229024943311e-07,
|
|
"loss": 0.0219,
|
|
"num_tokens": 2813176.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 47,
|
|
"step_time": 58.94816019106656
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1348.0,
|
|
"completions/max_terminated_length": 1348.0,
|
|
"completions/mean_length": 796.171875,
|
|
"completions/mean_terminated_length": 796.171875,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 0.70322042144835,
|
|
"epoch": 0.027210884353741496,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 9.733560090702947e-07,
|
|
"loss": -0.0156,
|
|
"num_tokens": 2873451.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 48,
|
|
"step_time": 51.40357466414571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1043.0,
|
|
"completions/max_terminated_length": 1043.0,
|
|
"completions/mean_length": 642.03125,
|
|
"completions/mean_terminated_length": 642.03125,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 1.0623097755014896,
|
|
"epoch": 0.027777777777777776,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 9.727891156462584e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 2923909.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 49,
|
|
"step_time": 39.97872541937977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1270.0,
|
|
"completions/max_terminated_length": 1270.0,
|
|
"completions/mean_length": 765.9375,
|
|
"completions/mean_terminated_length": 765.9375,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.8148674815893173,
|
|
"epoch": 0.02834467120181406,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.38671875,
|
|
"learning_rate": 9.722222222222222e-07,
|
|
"loss": -0.0207,
|
|
"num_tokens": 2983305.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 50,
|
|
"step_time": 48.24801494460553
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 821.859375,
|
|
"completions/mean_terminated_length": 769.888916015625,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.7434265501797199,
|
|
"epoch": 0.02891156462585034,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 9.71655328798186e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 3049592.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 51,
|
|
"step_time": 188.78284354228526
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1639.0,
|
|
"completions/max_terminated_length": 1639.0,
|
|
"completions/mean_length": 812.09375,
|
|
"completions/mean_terminated_length": 812.09375,
|
|
"completions/min_length": 421.0,
|
|
"completions/min_terminated_length": 421.0,
|
|
"entropy": 0.9160140417516232,
|
|
"epoch": 0.02947845804988662,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.710884353741497e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 3110910.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 52,
|
|
"step_time": 61.9008500315249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1485.0,
|
|
"completions/max_terminated_length": 1485.0,
|
|
"completions/mean_length": 694.109375,
|
|
"completions/mean_terminated_length": 694.109375,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 1.017433162778616,
|
|
"epoch": 0.030045351473922902,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.31640625,
|
|
"learning_rate": 9.705215419501134e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 3165373.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 53,
|
|
"step_time": 56.43010967876762
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1607.0,
|
|
"completions/max_terminated_length": 1607.0,
|
|
"completions/mean_length": 828.953125,
|
|
"completions/mean_terminated_length": 828.953125,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.7901779320091009,
|
|
"epoch": 0.030612244897959183,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1689453125,
|
|
"learning_rate": 9.69954648526077e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 3229466.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 54,
|
|
"step_time": 61.095109866932034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 707.84375,
|
|
"completions/mean_terminated_length": 707.84375,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 0.9682547487318516,
|
|
"epoch": 0.031179138321995464,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"learning_rate": 9.693877551020407e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 3286064.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 55,
|
|
"step_time": 49.53000052366406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1407.0,
|
|
"completions/max_terminated_length": 1407.0,
|
|
"completions/mean_length": 747.03125,
|
|
"completions/mean_terminated_length": 747.03125,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.8757524751126766,
|
|
"epoch": 0.031746031746031744,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 9.688208616780045e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 3345554.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 56,
|
|
"step_time": 54.49297194555402
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1591.0,
|
|
"completions/max_terminated_length": 1591.0,
|
|
"completions/mean_length": 735.1875,
|
|
"completions/mean_terminated_length": 735.1875,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7640372887253761,
|
|
"epoch": 0.03231292517006803,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.63671875,
|
|
"learning_rate": 9.682539682539682e-07,
|
|
"loss": 0.0149,
|
|
"num_tokens": 3405638.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 57,
|
|
"step_time": 63.35214420687407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 664.578125,
|
|
"completions/mean_terminated_length": 664.578125,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.7871280889958143,
|
|
"epoch": 0.032879818594104306,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 9.67687074829932e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 3457443.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 58,
|
|
"step_time": 53.59946721140295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1491.0,
|
|
"completions/mean_length": 756.1875,
|
|
"completions/mean_terminated_length": 703.1746215820312,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.9718666542321444,
|
|
"epoch": 0.03344671201814059,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 9.671201814058957e-07,
|
|
"loss": -0.0036,
|
|
"num_tokens": 3516543.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 59,
|
|
"step_time": 176.96932354010642
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1990.0,
|
|
"completions/max_terminated_length": 1990.0,
|
|
"completions/mean_length": 785.265625,
|
|
"completions/mean_terminated_length": 785.265625,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 0.9386273510754108,
|
|
"epoch": 0.034013605442176874,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.224609375,
|
|
"learning_rate": 9.665532879818595e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 3577824.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 60,
|
|
"step_time": 76.28884133696556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2057.0,
|
|
"completions/max_terminated_length": 2057.0,
|
|
"completions/mean_length": 800.65625,
|
|
"completions/mean_terminated_length": 800.65625,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.9068919755518436,
|
|
"epoch": 0.03458049886621315,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 9.65986394557823e-07,
|
|
"loss": 0.0151,
|
|
"num_tokens": 3640170.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 61,
|
|
"step_time": 79.0729680787772
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 678.296875,
|
|
"completions/mean_terminated_length": 678.296875,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.9080477468669415,
|
|
"epoch": 0.035147392290249435,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4375,
|
|
"learning_rate": 9.654195011337868e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 3692229.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 62,
|
|
"step_time": 49.03559826966375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1295.0,
|
|
"completions/max_terminated_length": 1295.0,
|
|
"completions/mean_length": 733.890625,
|
|
"completions/mean_terminated_length": 733.890625,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 1.0118759498000145,
|
|
"epoch": 0.03571428571428571,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"learning_rate": 9.648526077097505e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 3749190.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 63,
|
|
"step_time": 48.98392139002681
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1745.0,
|
|
"completions/max_terminated_length": 1745.0,
|
|
"completions/mean_length": 793.625,
|
|
"completions/mean_terminated_length": 793.625,
|
|
"completions/min_length": 204.0,
|
|
"completions/min_terminated_length": 204.0,
|
|
"entropy": 0.8415250703692436,
|
|
"epoch": 0.036281179138321996,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1875,
|
|
"learning_rate": 9.642857142857142e-07,
|
|
"loss": -0.0139,
|
|
"num_tokens": 3812422.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 64,
|
|
"step_time": 66.7228564498946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1802.0,
|
|
"completions/mean_length": 813.0,
|
|
"completions/mean_terminated_length": 760.888916015625,
|
|
"completions/min_length": 253.0,
|
|
"completions/min_terminated_length": 253.0,
|
|
"entropy": 0.9139939080923796,
|
|
"epoch": 0.036848072562358274,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 9.63718820861678e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 3876014.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 65,
|
|
"step_time": 178.41644801665097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1971.0,
|
|
"completions/max_terminated_length": 1971.0,
|
|
"completions/mean_length": 887.59375,
|
|
"completions/mean_terminated_length": 887.59375,
|
|
"completions/min_length": 386.0,
|
|
"completions/min_terminated_length": 386.0,
|
|
"entropy": 0.7073164060711861,
|
|
"epoch": 0.03741496598639456,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 9.631519274376417e-07,
|
|
"loss": -0.0054,
|
|
"num_tokens": 3945324.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 66,
|
|
"step_time": 75.84505661018193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1218.0,
|
|
"completions/max_terminated_length": 1218.0,
|
|
"completions/mean_length": 760.796875,
|
|
"completions/mean_terminated_length": 760.796875,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7621981650590897,
|
|
"epoch": 0.03798185941043084,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 9.625850340136053e-07,
|
|
"loss": -0.0338,
|
|
"num_tokens": 4004503.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 67,
|
|
"step_time": 46.655542940832675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1647.0,
|
|
"completions/max_terminated_length": 1647.0,
|
|
"completions/mean_length": 870.921875,
|
|
"completions/mean_terminated_length": 870.921875,
|
|
"completions/min_length": 335.0,
|
|
"completions/min_terminated_length": 335.0,
|
|
"entropy": 0.6609465070068836,
|
|
"epoch": 0.03854875283446712,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 9.62018140589569e-07,
|
|
"loss": 0.0241,
|
|
"num_tokens": 4070570.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 68,
|
|
"step_time": 64.57767817284912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 810.359375,
|
|
"completions/mean_terminated_length": 810.359375,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.7866592034697533,
|
|
"epoch": 0.0391156462585034,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.19921875,
|
|
"learning_rate": 9.614512471655328e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 4134009.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 69,
|
|
"step_time": 54.660805217921734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1522.0,
|
|
"completions/max_terminated_length": 1522.0,
|
|
"completions/mean_length": 793.765625,
|
|
"completions/mean_terminated_length": 793.765625,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.8732692003250122,
|
|
"epoch": 0.03968253968253968,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"learning_rate": 9.608843537414965e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 4195066.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 70,
|
|
"step_time": 58.68067215476185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/max_terminated_length": 1202.0,
|
|
"completions/mean_length": 666.203125,
|
|
"completions/mean_terminated_length": 666.203125,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.7925896048545837,
|
|
"epoch": 0.040249433106575964,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 9.603174603174603e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 4247335.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 71,
|
|
"step_time": 45.4902665251866
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1203.0,
|
|
"completions/max_terminated_length": 1203.0,
|
|
"completions/mean_length": 583.375,
|
|
"completions/mean_terminated_length": 583.375,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.8753242213279009,
|
|
"epoch": 0.04081632653061224,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 9.59750566893424e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 4292759.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 72,
|
|
"step_time": 44.927397458814085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1914.0,
|
|
"completions/max_terminated_length": 1914.0,
|
|
"completions/mean_length": 742.28125,
|
|
"completions/mean_terminated_length": 742.28125,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.8473658263683319,
|
|
"epoch": 0.041383219954648526,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2734375,
|
|
"learning_rate": 9.591836734693876e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 4358281.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 73,
|
|
"step_time": 81.2505574543029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1081.0,
|
|
"completions/max_terminated_length": 1081.0,
|
|
"completions/mean_length": 655.671875,
|
|
"completions/mean_terminated_length": 655.671875,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 1.105690035969019,
|
|
"epoch": 0.04195011337868481,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 9.586167800453513e-07,
|
|
"loss": -0.0107,
|
|
"num_tokens": 4409628.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 74,
|
|
"step_time": 43.225347391329706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1425.0,
|
|
"completions/max_terminated_length": 1425.0,
|
|
"completions/mean_length": 799.859375,
|
|
"completions/mean_terminated_length": 799.859375,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.7495364528149366,
|
|
"epoch": 0.04251700680272109,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 9.58049886621315e-07,
|
|
"loss": -0.0215,
|
|
"num_tokens": 4471059.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 75,
|
|
"step_time": 54.34133177995682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1372.0,
|
|
"completions/max_terminated_length": 1372.0,
|
|
"completions/mean_length": 701.703125,
|
|
"completions/mean_terminated_length": 701.703125,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.7809962090104818,
|
|
"epoch": 0.04308390022675737,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"learning_rate": 9.574829931972788e-07,
|
|
"loss": 0.0035,
|
|
"num_tokens": 4524760.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 76,
|
|
"step_time": 51.31494827289134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1443.0,
|
|
"completions/max_terminated_length": 1443.0,
|
|
"completions/mean_length": 782.890625,
|
|
"completions/mean_terminated_length": 782.890625,
|
|
"completions/min_length": 416.0,
|
|
"completions/min_terminated_length": 416.0,
|
|
"entropy": 0.9791093207895756,
|
|
"epoch": 0.04365079365079365,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 9.569160997732426e-07,
|
|
"loss": -0.0071,
|
|
"num_tokens": 4584777.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 77,
|
|
"step_time": 54.10253896191716
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1675.0,
|
|
"completions/max_terminated_length": 1675.0,
|
|
"completions/mean_length": 823.84375,
|
|
"completions/mean_terminated_length": 823.84375,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.8013289663940668,
|
|
"epoch": 0.04421768707482993,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34765625,
|
|
"learning_rate": 9.563492063492063e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 4649015.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 78,
|
|
"step_time": 63.31329367030412
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1449.0,
|
|
"completions/max_terminated_length": 1449.0,
|
|
"completions/mean_length": 789.515625,
|
|
"completions/mean_terminated_length": 789.515625,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.7154162228107452,
|
|
"epoch": 0.04478458049886622,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"learning_rate": 9.5578231292517e-07,
|
|
"loss": 0.0182,
|
|
"num_tokens": 4710208.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 79,
|
|
"step_time": 54.76761769596487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1659.0,
|
|
"completions/max_terminated_length": 1659.0,
|
|
"completions/mean_length": 806.765625,
|
|
"completions/mean_terminated_length": 806.765625,
|
|
"completions/min_length": 8.0,
|
|
"completions/min_terminated_length": 8.0,
|
|
"entropy": 0.7990427520126104,
|
|
"epoch": 0.045351473922902494,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34765625,
|
|
"learning_rate": 9.552154195011338e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 4772081.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 80,
|
|
"step_time": 62.50960550829768
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1240.0,
|
|
"completions/max_terminated_length": 1240.0,
|
|
"completions/mean_length": 713.703125,
|
|
"completions/mean_terminated_length": 713.703125,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.8844226524233818,
|
|
"epoch": 0.04591836734693878,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.291015625,
|
|
"learning_rate": 9.546485260770974e-07,
|
|
"loss": 0.0076,
|
|
"num_tokens": 4827382.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 81,
|
|
"step_time": 46.83812944497913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1190.0,
|
|
"completions/max_terminated_length": 1190.0,
|
|
"completions/mean_length": 640.75,
|
|
"completions/mean_terminated_length": 640.75,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"entropy": 0.9863478187471628,
|
|
"epoch": 0.046485260770975055,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 9.54081632653061e-07,
|
|
"loss": 0.0056,
|
|
"num_tokens": 4882950.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 82,
|
|
"step_time": 47.364373445510864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1294.0,
|
|
"completions/mean_length": 836.703125,
|
|
"completions/mean_terminated_length": 784.9683227539062,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.7088115140795708,
|
|
"epoch": 0.04705215419501134,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 9.53514739229025e-07,
|
|
"loss": 0.0159,
|
|
"num_tokens": 4945459.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 83,
|
|
"step_time": 177.49385809525847
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1561.0,
|
|
"completions/max_terminated_length": 1561.0,
|
|
"completions/mean_length": 784.484375,
|
|
"completions/mean_terminated_length": 784.484375,
|
|
"completions/min_length": 450.0,
|
|
"completions/min_terminated_length": 450.0,
|
|
"entropy": 0.9638878479599953,
|
|
"epoch": 0.047619047619047616,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 9.529478458049887e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 5006394.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 84,
|
|
"step_time": 58.921785313636065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1333.0,
|
|
"completions/max_terminated_length": 1333.0,
|
|
"completions/mean_length": 925.96875,
|
|
"completions/mean_terminated_length": 925.96875,
|
|
"completions/min_length": 464.0,
|
|
"completions/min_terminated_length": 464.0,
|
|
"entropy": 0.7016284838318825,
|
|
"epoch": 0.0481859410430839,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2373046875,
|
|
"learning_rate": 9.523809523809522e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 5077376.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 85,
|
|
"step_time": 50.29745448753238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1323.0,
|
|
"completions/max_terminated_length": 1323.0,
|
|
"completions/mean_length": 808.703125,
|
|
"completions/mean_terminated_length": 808.703125,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 0.8223045617341995,
|
|
"epoch": 0.048752834467120185,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.51814058956916e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 5140029.0,
|
|
"reward": 0.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 86,
|
|
"step_time": 50.399625142104924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1543.0,
|
|
"completions/max_terminated_length": 1543.0,
|
|
"completions/mean_length": 835.5625,
|
|
"completions/mean_terminated_length": 835.5625,
|
|
"completions/min_length": 342.0,
|
|
"completions/min_terminated_length": 342.0,
|
|
"entropy": 0.9934357516467571,
|
|
"epoch": 0.04931972789115646,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 9.512471655328797e-07,
|
|
"loss": 0.0201,
|
|
"num_tokens": 5206257.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 87,
|
|
"step_time": 58.83743468020111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1540.0,
|
|
"completions/max_terminated_length": 1540.0,
|
|
"completions/mean_length": 807.65625,
|
|
"completions/mean_terminated_length": 807.65625,
|
|
"completions/min_length": 425.0,
|
|
"completions/min_terminated_length": 425.0,
|
|
"entropy": 0.9231362417340279,
|
|
"epoch": 0.049886621315192746,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.306640625,
|
|
"learning_rate": 9.506802721088435e-07,
|
|
"loss": 0.0085,
|
|
"num_tokens": 5277547.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 88,
|
|
"step_time": 64.1513889292255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1456.0,
|
|
"completions/max_terminated_length": 1456.0,
|
|
"completions/mean_length": 725.40625,
|
|
"completions/mean_terminated_length": 725.40625,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.8447514325380325,
|
|
"epoch": 0.05045351473922902,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"learning_rate": 9.501133786848072e-07,
|
|
"loss": 0.0131,
|
|
"num_tokens": 5334709.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 89,
|
|
"step_time": 55.60451160091907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1420.0,
|
|
"completions/max_terminated_length": 1420.0,
|
|
"completions/mean_length": 801.5625,
|
|
"completions/mean_terminated_length": 801.5625,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"entropy": 0.9044834561645985,
|
|
"epoch": 0.05102040816326531,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2216796875,
|
|
"learning_rate": 9.49546485260771e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 5397441.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 90,
|
|
"step_time": 53.2394202882424
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 795.109375,
|
|
"completions/mean_terminated_length": 795.109375,
|
|
"completions/min_length": 200.0,
|
|
"completions/min_terminated_length": 200.0,
|
|
"entropy": 0.9016128778457642,
|
|
"epoch": 0.051587301587301584,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 9.489795918367347e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 5460848.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 91,
|
|
"step_time": 57.886049394495785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1684.0,
|
|
"completions/max_terminated_length": 1684.0,
|
|
"completions/mean_length": 761.984375,
|
|
"completions/mean_terminated_length": 761.984375,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.7569958660751581,
|
|
"epoch": 0.05215419501133787,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 9.484126984126984e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 5519727.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 92,
|
|
"step_time": 63.76254240516573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2312.0,
|
|
"completions/max_terminated_length": 2312.0,
|
|
"completions/mean_length": 824.375,
|
|
"completions/mean_terminated_length": 824.375,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.8328082952648401,
|
|
"epoch": 0.05272108843537415,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 9.47845804988662e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 5584447.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 93,
|
|
"step_time": 89.38122331723571
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1060.0,
|
|
"completions/max_terminated_length": 1060.0,
|
|
"completions/mean_length": 682.65625,
|
|
"completions/mean_terminated_length": 682.65625,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 1.166295189410448,
|
|
"epoch": 0.05328798185941043,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 9.472789115646258e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 5639625.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 94,
|
|
"step_time": 40.6492495816201
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1426.0,
|
|
"completions/max_terminated_length": 1426.0,
|
|
"completions/mean_length": 710.375,
|
|
"completions/mean_terminated_length": 710.375,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.9604226835072041,
|
|
"epoch": 0.053854875283446714,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.279296875,
|
|
"learning_rate": 9.467120181405895e-07,
|
|
"loss": 0.0113,
|
|
"num_tokens": 5695033.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 95,
|
|
"step_time": 53.77150951512158
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1531.0,
|
|
"completions/max_terminated_length": 1531.0,
|
|
"completions/mean_length": 730.59375,
|
|
"completions/mean_terminated_length": 730.59375,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7991134524345398,
|
|
"epoch": 0.05442176870748299,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4140625,
|
|
"learning_rate": 9.461451247165533e-07,
|
|
"loss": 0.0271,
|
|
"num_tokens": 5752639.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 96,
|
|
"step_time": 59.56408981606364
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1226.0,
|
|
"completions/max_terminated_length": 1226.0,
|
|
"completions/mean_length": 776.53125,
|
|
"completions/mean_terminated_length": 776.53125,
|
|
"completions/min_length": 402.0,
|
|
"completions/min_terminated_length": 402.0,
|
|
"entropy": 0.9329401925206184,
|
|
"epoch": 0.054988662131519275,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40625,
|
|
"learning_rate": 9.45578231292517e-07,
|
|
"loss": -0.0129,
|
|
"num_tokens": 5813473.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 97,
|
|
"step_time": 47.72492705844343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1445.0,
|
|
"completions/max_terminated_length": 1445.0,
|
|
"completions/mean_length": 850.984375,
|
|
"completions/mean_terminated_length": 850.984375,
|
|
"completions/min_length": 438.0,
|
|
"completions/min_terminated_length": 438.0,
|
|
"entropy": 0.7094423845410347,
|
|
"epoch": 0.05555555555555555,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 9.450113378684807e-07,
|
|
"loss": 0.0181,
|
|
"num_tokens": 5878904.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 98,
|
|
"step_time": 56.325513327494264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1597.0,
|
|
"completions/max_terminated_length": 1597.0,
|
|
"completions/mean_length": 879.5,
|
|
"completions/mean_terminated_length": 879.5,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.7695587165653706,
|
|
"epoch": 0.05612244897959184,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2265625,
|
|
"learning_rate": 9.444444444444444e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 5948408.0,
|
|
"reward": 0.0078125,
|
|
"reward_std": 0.0625,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 99,
|
|
"step_time": 63.71368716843426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1517.0,
|
|
"completions/max_terminated_length": 1517.0,
|
|
"completions/mean_length": 794.984375,
|
|
"completions/mean_terminated_length": 794.984375,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.7346187010407448,
|
|
"epoch": 0.05668934240362812,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 9.438775510204082e-07,
|
|
"loss": 0.0258,
|
|
"num_tokens": 6010095.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5890451669692993,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 100,
|
|
"step_time": 57.80585550609976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1724.0,
|
|
"completions/max_terminated_length": 1724.0,
|
|
"completions/mean_length": 755.90625,
|
|
"completions/mean_terminated_length": 755.90625,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.7995297256857157,
|
|
"epoch": 0.0572562358276644,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 9.433106575963718e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 6068169.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.48795002698898315,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 101,
|
|
"step_time": 65.5606642998755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1146.0,
|
|
"completions/max_terminated_length": 1146.0,
|
|
"completions/mean_length": 799.046875,
|
|
"completions/mean_terminated_length": 799.046875,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.7355476096272469,
|
|
"epoch": 0.05782312925170068,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 9.427437641723356e-07,
|
|
"loss": 0.0018,
|
|
"num_tokens": 6129364.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.35486623644828796,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 102,
|
|
"step_time": 43.53606962505728
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1734.0,
|
|
"completions/max_terminated_length": 1734.0,
|
|
"completions/mean_length": 830.765625,
|
|
"completions/mean_terminated_length": 830.765625,
|
|
"completions/min_length": 460.0,
|
|
"completions/min_terminated_length": 460.0,
|
|
"entropy": 0.8551389183849096,
|
|
"epoch": 0.05839002267573696,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"learning_rate": 9.421768707482993e-07,
|
|
"loss": 0.0269,
|
|
"num_tokens": 6192565.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5883605480194092,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 103,
|
|
"step_time": 66.09313382022083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1485.0,
|
|
"completions/max_terminated_length": 1485.0,
|
|
"completions/mean_length": 813.265625,
|
|
"completions/mean_terminated_length": 813.265625,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.7450036723166704,
|
|
"epoch": 0.05895691609977324,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 9.41609977324263e-07,
|
|
"loss": 0.0198,
|
|
"num_tokens": 6257174.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 104,
|
|
"step_time": 57.193663199432194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1614.0,
|
|
"completions/max_terminated_length": 1614.0,
|
|
"completions/mean_length": 689.921875,
|
|
"completions/mean_terminated_length": 689.921875,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.7431341577321291,
|
|
"epoch": 0.05952380952380952,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.416015625,
|
|
"learning_rate": 9.410430839002267e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 6310881.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5875696539878845,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 105,
|
|
"step_time": 61.51257424522191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2372.0,
|
|
"completions/max_terminated_length": 2372.0,
|
|
"completions/mean_length": 755.953125,
|
|
"completions/mean_terminated_length": 755.953125,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"entropy": 0.7027264926582575,
|
|
"epoch": 0.060090702947845805,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34375,
|
|
"learning_rate": 9.404761904761904e-07,
|
|
"loss": -0.0087,
|
|
"num_tokens": 6368918.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 106,
|
|
"step_time": 92.04900511726737
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/max_terminated_length": 1276.0,
|
|
"completions/mean_length": 800.71875,
|
|
"completions/mean_terminated_length": 800.71875,
|
|
"completions/min_length": 453.0,
|
|
"completions/min_terminated_length": 453.0,
|
|
"entropy": 0.7530606593936682,
|
|
"epoch": 0.06065759637188209,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.302734375,
|
|
"learning_rate": 9.399092970521542e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 6432204.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 107,
|
|
"step_time": 49.60997568629682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1581.0,
|
|
"completions/max_terminated_length": 1581.0,
|
|
"completions/mean_length": 834.015625,
|
|
"completions/mean_terminated_length": 834.015625,
|
|
"completions/min_length": 472.0,
|
|
"completions/min_terminated_length": 472.0,
|
|
"entropy": 0.6710873208940029,
|
|
"epoch": 0.061224489795918366,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"learning_rate": 9.393424036281179e-07,
|
|
"loss": -0.0135,
|
|
"num_tokens": 6494893.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 108,
|
|
"step_time": 60.02176415827125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1426.0,
|
|
"completions/max_terminated_length": 1426.0,
|
|
"completions/mean_length": 681.40625,
|
|
"completions/mean_terminated_length": 681.40625,
|
|
"completions/min_length": 245.0,
|
|
"completions/min_terminated_length": 245.0,
|
|
"entropy": 1.0327333752065897,
|
|
"epoch": 0.06179138321995465,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 9.387755102040816e-07,
|
|
"loss": 0.0027,
|
|
"num_tokens": 6547679.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 109,
|
|
"step_time": 54.22360837832093
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 715.171875,
|
|
"completions/mean_terminated_length": 715.171875,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.7466475442051888,
|
|
"epoch": 0.06235827664399093,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 9.382086167800452e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 6603386.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 110,
|
|
"step_time": 51.49598786607385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1622.0,
|
|
"completions/max_terminated_length": 1622.0,
|
|
"completions/mean_length": 757.90625,
|
|
"completions/mean_terminated_length": 757.90625,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.8256755955517292,
|
|
"epoch": 0.06292517006802721,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.287109375,
|
|
"learning_rate": 9.37641723356009e-07,
|
|
"loss": -0.0077,
|
|
"num_tokens": 6663844.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 111,
|
|
"step_time": 62.764745325781405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1264.0,
|
|
"completions/max_terminated_length": 1264.0,
|
|
"completions/mean_length": 717.5625,
|
|
"completions/mean_terminated_length": 717.5625,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8293892573565245,
|
|
"epoch": 0.06349206349206349,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 9.370748299319727e-07,
|
|
"loss": -0.0144,
|
|
"num_tokens": 6720208.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 112,
|
|
"step_time": 48.27981117833406
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 743.015625,
|
|
"completions/mean_terminated_length": 743.015625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8873293474316597,
|
|
"epoch": 0.06405895691609978,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.205078125,
|
|
"learning_rate": 9.365079365079365e-07,
|
|
"loss": 0.0021,
|
|
"num_tokens": 6779377.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 113,
|
|
"step_time": 58.38554639369249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 751.15625,
|
|
"completions/mean_terminated_length": 751.15625,
|
|
"completions/min_length": 395.0,
|
|
"completions/min_terminated_length": 395.0,
|
|
"entropy": 0.9458110071718693,
|
|
"epoch": 0.06462585034013606,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 9.359410430839002e-07,
|
|
"loss": -0.0463,
|
|
"num_tokens": 6839931.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 114,
|
|
"step_time": 61.09925704449415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 601.734375,
|
|
"completions/mean_terminated_length": 601.734375,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.9978123791515827,
|
|
"epoch": 0.06519274376417233,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 9.35374149659864e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 6888666.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 115,
|
|
"step_time": 49.341413654387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1531.0,
|
|
"completions/max_terminated_length": 1531.0,
|
|
"completions/mean_length": 724.390625,
|
|
"completions/mean_terminated_length": 724.390625,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.6574038378894329,
|
|
"epoch": 0.06575963718820861,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2412109375,
|
|
"learning_rate": 9.348072562358276e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 6951011.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 116,
|
|
"step_time": 61.11546309199184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1904.0,
|
|
"completions/max_terminated_length": 1904.0,
|
|
"completions/mean_length": 814.578125,
|
|
"completions/mean_terminated_length": 814.578125,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.8695726897567511,
|
|
"epoch": 0.0663265306122449,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.494140625,
|
|
"learning_rate": 9.342403628117913e-07,
|
|
"loss": -0.0349,
|
|
"num_tokens": 7016536.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 117,
|
|
"step_time": 74.26174595579505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1279.0,
|
|
"completions/max_terminated_length": 1279.0,
|
|
"completions/mean_length": 706.703125,
|
|
"completions/mean_terminated_length": 706.703125,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.785661468282342,
|
|
"epoch": 0.06689342403628118,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"learning_rate": 9.33673469387755e-07,
|
|
"loss": -0.0256,
|
|
"num_tokens": 7073613.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 118,
|
|
"step_time": 48.82230156194419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1785.0,
|
|
"completions/max_terminated_length": 1785.0,
|
|
"completions/mean_length": 718.65625,
|
|
"completions/mean_terminated_length": 718.65625,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.8869774080812931,
|
|
"epoch": 0.06746031746031746,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 9.331065759637188e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 7128311.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 119,
|
|
"step_time": 66.68152226600796
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1645.0,
|
|
"completions/max_terminated_length": 1645.0,
|
|
"completions/mean_length": 865.1875,
|
|
"completions/mean_terminated_length": 865.1875,
|
|
"completions/min_length": 466.0,
|
|
"completions/min_terminated_length": 466.0,
|
|
"entropy": 0.7705853059887886,
|
|
"epoch": 0.06802721088435375,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 9.325396825396825e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 7194195.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 120,
|
|
"step_time": 62.07781745027751
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1929.0,
|
|
"completions/max_terminated_length": 1929.0,
|
|
"completions/mean_length": 821.390625,
|
|
"completions/mean_terminated_length": 821.390625,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.6111497618257999,
|
|
"epoch": 0.06859410430839002,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.310546875,
|
|
"learning_rate": 9.319727891156463e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 7258436.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 121,
|
|
"step_time": 74.65006504673511
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1487.0,
|
|
"completions/mean_length": 881.765625,
|
|
"completions/mean_terminated_length": 830.74609375,
|
|
"completions/min_length": 424.0,
|
|
"completions/min_terminated_length": 424.0,
|
|
"entropy": 0.6640894301235676,
|
|
"epoch": 0.0691609977324263,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3984375,
|
|
"learning_rate": 9.3140589569161e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 7326005.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 122,
|
|
"step_time": 178.2570448340848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1314.0,
|
|
"completions/max_terminated_length": 1314.0,
|
|
"completions/mean_length": 593.46875,
|
|
"completions/mean_terminated_length": 593.46875,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.9700460098683834,
|
|
"epoch": 0.06972789115646258,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 9.308390022675737e-07,
|
|
"loss": 0.0146,
|
|
"num_tokens": 7377635.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 123,
|
|
"step_time": 51.13704787194729
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 988.0,
|
|
"completions/max_terminated_length": 988.0,
|
|
"completions/mean_length": 579.09375,
|
|
"completions/mean_terminated_length": 579.09375,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 1.065275426954031,
|
|
"epoch": 0.07029478458049887,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.318359375,
|
|
"learning_rate": 9.302721088435374e-07,
|
|
"loss": -0.0053,
|
|
"num_tokens": 7424409.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 124,
|
|
"step_time": 37.35036712232977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 777.375,
|
|
"completions/mean_terminated_length": 777.375,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.9211017340421677,
|
|
"epoch": 0.07086167800453515,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 9.29705215419501e-07,
|
|
"loss": 0.0199,
|
|
"num_tokens": 7486521.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 125,
|
|
"step_time": 57.95168686658144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1432.0,
|
|
"completions/max_terminated_length": 1432.0,
|
|
"completions/mean_length": 770.90625,
|
|
"completions/mean_terminated_length": 770.90625,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 0.8487519323825836,
|
|
"epoch": 0.07142857142857142,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28125,
|
|
"learning_rate": 9.291383219954648e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 7546539.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.40999117493629456,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 126,
|
|
"step_time": 54.54803218692541
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 780.203125,
|
|
"completions/mean_terminated_length": 780.203125,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8813410773873329,
|
|
"epoch": 0.07199546485260772,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"learning_rate": 9.285714285714285e-07,
|
|
"loss": -0.0022,
|
|
"num_tokens": 7607624.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 127,
|
|
"step_time": 51.01048209145665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2171.0,
|
|
"completions/max_terminated_length": 2171.0,
|
|
"completions/mean_length": 727.984375,
|
|
"completions/mean_terminated_length": 727.984375,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.8846711702644825,
|
|
"epoch": 0.07256235827664399,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.306640625,
|
|
"learning_rate": 9.280045351473923e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 7664759.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.35073620080947876,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 128,
|
|
"step_time": 82.85143149085343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1223.0,
|
|
"completions/max_terminated_length": 1223.0,
|
|
"completions/mean_length": 785.359375,
|
|
"completions/mean_terminated_length": 785.359375,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.6674748081713915,
|
|
"epoch": 0.07312925170068027,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 9.274376417233559e-07,
|
|
"loss": -0.0248,
|
|
"num_tokens": 7724710.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 129,
|
|
"step_time": 46.273179126903415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 840.625,
|
|
"completions/mean_terminated_length": 840.625,
|
|
"completions/min_length": 305.0,
|
|
"completions/min_terminated_length": 305.0,
|
|
"entropy": 0.777132011950016,
|
|
"epoch": 0.07369614512471655,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 9.268707482993197e-07,
|
|
"loss": 0.0132,
|
|
"num_tokens": 7788878.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 130,
|
|
"step_time": 57.40823499020189
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1504.0,
|
|
"completions/max_terminated_length": 1504.0,
|
|
"completions/mean_length": 703.9375,
|
|
"completions/mean_terminated_length": 703.9375,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.9815900027751923,
|
|
"epoch": 0.07426303854875284,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 9.263038548752834e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 7844554.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 131,
|
|
"step_time": 57.04882840067148
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1490.0,
|
|
"completions/max_terminated_length": 1490.0,
|
|
"completions/mean_length": 837.015625,
|
|
"completions/mean_terminated_length": 837.015625,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.7387430816888809,
|
|
"epoch": 0.07482993197278912,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 9.257369614512472e-07,
|
|
"loss": 0.0338,
|
|
"num_tokens": 7908603.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.6147472858428955,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 132,
|
|
"step_time": 56.945903515443206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 692.265625,
|
|
"completions/mean_terminated_length": 692.265625,
|
|
"completions/min_length": 333.0,
|
|
"completions/min_terminated_length": 333.0,
|
|
"entropy": 0.7826277390122414,
|
|
"epoch": 0.07539682539682539,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 9.251700680272108e-07,
|
|
"loss": 0.0395,
|
|
"num_tokens": 7963396.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 133,
|
|
"step_time": 48.388916741125286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1395.0,
|
|
"completions/max_terminated_length": 1395.0,
|
|
"completions/mean_length": 789.28125,
|
|
"completions/mean_terminated_length": 789.28125,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.7802317626774311,
|
|
"epoch": 0.07596371882086168,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 9.246031746031746e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 8025022.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 134,
|
|
"step_time": 53.4332234794274
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1510.0,
|
|
"completions/max_terminated_length": 1510.0,
|
|
"completions/mean_length": 618.0,
|
|
"completions/mean_terminated_length": 618.0,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"entropy": 0.8391514271497726,
|
|
"epoch": 0.07653061224489796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 9.240362811791382e-07,
|
|
"loss": 0.0519,
|
|
"num_tokens": 8073766.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7622999548912048,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 135,
|
|
"step_time": 56.804534021764994
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1985.0,
|
|
"completions/max_terminated_length": 1985.0,
|
|
"completions/mean_length": 829.875,
|
|
"completions/mean_terminated_length": 829.875,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 0.7645440567284822,
|
|
"epoch": 0.07709750566893424,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.43359375,
|
|
"learning_rate": 9.23469387755102e-07,
|
|
"loss": 0.0181,
|
|
"num_tokens": 8136686.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 136,
|
|
"step_time": 75.18006173055619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1155.0,
|
|
"completions/max_terminated_length": 1155.0,
|
|
"completions/mean_length": 691.046875,
|
|
"completions/mean_terminated_length": 691.046875,
|
|
"completions/min_length": 335.0,
|
|
"completions/min_terminated_length": 335.0,
|
|
"entropy": 1.0325789265334606,
|
|
"epoch": 0.07766439909297052,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 9.229024943310657e-07,
|
|
"loss": 0.021,
|
|
"num_tokens": 8191625.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.46691107749938965,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 137,
|
|
"step_time": 44.112369927577674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 788.390625,
|
|
"completions/mean_terminated_length": 788.390625,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.8837102428078651,
|
|
"epoch": 0.0782312925170068,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 9.223356009070295e-07,
|
|
"loss": 0.0164,
|
|
"num_tokens": 8255322.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 138,
|
|
"step_time": 50.744255722500384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1479.0,
|
|
"completions/max_terminated_length": 1479.0,
|
|
"completions/mean_length": 714.609375,
|
|
"completions/mean_terminated_length": 714.609375,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.803769901394844,
|
|
"epoch": 0.07879818594104308,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.462890625,
|
|
"learning_rate": 9.217687074829932e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 8312857.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 139,
|
|
"step_time": 56.574633402749896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1645.0,
|
|
"completions/mean_length": 887.4375,
|
|
"completions/mean_terminated_length": 836.5079956054688,
|
|
"completions/min_length": 397.0,
|
|
"completions/min_terminated_length": 397.0,
|
|
"entropy": 0.6532765198498964,
|
|
"epoch": 0.07936507936507936,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 9.21201814058957e-07,
|
|
"loss": -0.0169,
|
|
"num_tokens": 8378405.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 140,
|
|
"step_time": 176.8985596196726
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1290.0,
|
|
"completions/max_terminated_length": 1290.0,
|
|
"completions/mean_length": 686.421875,
|
|
"completions/mean_terminated_length": 686.421875,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 0.8225036654621363,
|
|
"epoch": 0.07993197278911565,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4765625,
|
|
"learning_rate": 9.206349206349205e-07,
|
|
"loss": -0.0061,
|
|
"num_tokens": 8431888.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5409793853759766,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 141,
|
|
"step_time": 49.527410535141826
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1527.0,
|
|
"completions/max_terminated_length": 1527.0,
|
|
"completions/mean_length": 781.078125,
|
|
"completions/mean_terminated_length": 781.078125,
|
|
"completions/min_length": 436.0,
|
|
"completions/min_terminated_length": 436.0,
|
|
"entropy": 0.8449603244662285,
|
|
"epoch": 0.08049886621315193,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 9.200680272108843e-07,
|
|
"loss": -0.0182,
|
|
"num_tokens": 8491925.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.3177144229412079,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 142,
|
|
"step_time": 58.418819515034556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1393.0,
|
|
"completions/max_terminated_length": 1393.0,
|
|
"completions/mean_length": 795.625,
|
|
"completions/mean_terminated_length": 795.625,
|
|
"completions/min_length": 431.0,
|
|
"completions/min_terminated_length": 431.0,
|
|
"entropy": 0.8637427650392056,
|
|
"epoch": 0.0810657596371882,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 9.19501133786848e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 8553165.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 143,
|
|
"step_time": 52.9595802705735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1339.0,
|
|
"completions/max_terminated_length": 1339.0,
|
|
"completions/mean_length": 659.859375,
|
|
"completions/mean_terminated_length": 659.859375,
|
|
"completions/min_length": 183.0,
|
|
"completions/min_terminated_length": 183.0,
|
|
"entropy": 0.8931311331689358,
|
|
"epoch": 0.08163265306122448,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"learning_rate": 9.189342403628118e-07,
|
|
"loss": -0.0258,
|
|
"num_tokens": 8605972.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 144,
|
|
"step_time": 51.218512947671115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1310.0,
|
|
"completions/max_terminated_length": 1310.0,
|
|
"completions/mean_length": 780.859375,
|
|
"completions/mean_terminated_length": 780.859375,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.8530476838350296,
|
|
"epoch": 0.08219954648526077,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.388671875,
|
|
"learning_rate": 9.183673469387755e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 8667731.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 145,
|
|
"step_time": 49.99847659096122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1645.0,
|
|
"completions/max_terminated_length": 1645.0,
|
|
"completions/mean_length": 727.046875,
|
|
"completions/mean_terminated_length": 727.046875,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7862106617540121,
|
|
"epoch": 0.08276643990929705,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 9.178004535147393e-07,
|
|
"loss": 0.02,
|
|
"num_tokens": 8727598.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5890451669692993,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 146,
|
|
"step_time": 63.81167842168361
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 816.78125,
|
|
"completions/mean_terminated_length": 816.78125,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.8034506104886532,
|
|
"epoch": 0.08333333333333333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 9.172335600907029e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 8790232.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 147,
|
|
"step_time": 52.338583000935614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1086.0,
|
|
"completions/max_terminated_length": 1086.0,
|
|
"completions/mean_length": 726.625,
|
|
"completions/mean_terminated_length": 726.625,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 0.8533828165382147,
|
|
"epoch": 0.08390022675736962,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"learning_rate": 9.166666666666665e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 8846584.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.4315783977508545,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 148,
|
|
"step_time": 41.05783475842327
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1764.0,
|
|
"completions/max_terminated_length": 1764.0,
|
|
"completions/mean_length": 781.515625,
|
|
"completions/mean_terminated_length": 781.515625,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.7435462810099125,
|
|
"epoch": 0.0844671201814059,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 9.160997732426303e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 8907089.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 149,
|
|
"step_time": 67.09477553144097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1922.0,
|
|
"completions/max_terminated_length": 1922.0,
|
|
"completions/mean_length": 794.375,
|
|
"completions/mean_terminated_length": 794.375,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.5769841112196445,
|
|
"epoch": 0.08503401360544217,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 9.15532879818594e-07,
|
|
"loss": 0.0269,
|
|
"num_tokens": 8968057.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 150,
|
|
"step_time": 73.29869169741869
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1612.0,
|
|
"completions/max_terminated_length": 1612.0,
|
|
"completions/mean_length": 775.984375,
|
|
"completions/mean_terminated_length": 775.984375,
|
|
"completions/min_length": 395.0,
|
|
"completions/min_terminated_length": 395.0,
|
|
"entropy": 0.9547706730663776,
|
|
"epoch": 0.08560090702947845,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 9.149659863945578e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 9028016.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 151,
|
|
"step_time": 61.82334432005882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 736.53125,
|
|
"completions/mean_terminated_length": 736.53125,
|
|
"completions/min_length": 286.0,
|
|
"completions/min_terminated_length": 286.0,
|
|
"entropy": 0.8605308867990971,
|
|
"epoch": 0.08616780045351474,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 9.143990929705215e-07,
|
|
"loss": -0.0206,
|
|
"num_tokens": 9086130.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 152,
|
|
"step_time": 52.93216622900218
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1617.0,
|
|
"completions/max_terminated_length": 1617.0,
|
|
"completions/mean_length": 731.5,
|
|
"completions/mean_terminated_length": 731.5,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.8619019202888012,
|
|
"epoch": 0.08673469387755102,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 9.138321995464853e-07,
|
|
"loss": -0.0132,
|
|
"num_tokens": 9143674.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 153,
|
|
"step_time": 61.16917031444609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1289.0,
|
|
"completions/max_terminated_length": 1289.0,
|
|
"completions/mean_length": 795.3125,
|
|
"completions/mean_terminated_length": 795.3125,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.8963804133236408,
|
|
"epoch": 0.0873015873015873,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 9.132653061224489e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 9204638.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 154,
|
|
"step_time": 48.77144674025476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1480.0,
|
|
"completions/max_terminated_length": 1480.0,
|
|
"completions/mean_length": 802.421875,
|
|
"completions/mean_terminated_length": 802.421875,
|
|
"completions/min_length": 360.0,
|
|
"completions/min_terminated_length": 360.0,
|
|
"entropy": 0.8584241755306721,
|
|
"epoch": 0.08786848072562359,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 9.126984126984127e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 9270737.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 155,
|
|
"step_time": 58.38633061386645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1231.0,
|
|
"completions/max_terminated_length": 1231.0,
|
|
"completions/mean_length": 786.234375,
|
|
"completions/mean_terminated_length": 786.234375,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.7147355619817972,
|
|
"epoch": 0.08843537414965986,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 9.121315192743763e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 9332240.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 156,
|
|
"step_time": 47.72322576586157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1425.0,
|
|
"completions/max_terminated_length": 1425.0,
|
|
"completions/mean_length": 698.53125,
|
|
"completions/mean_terminated_length": 698.53125,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.9679968729615211,
|
|
"epoch": 0.08900226757369614,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4765625,
|
|
"learning_rate": 9.115646258503401e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 9387138.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 157,
|
|
"step_time": 54.63682753685862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 686.03125,
|
|
"completions/mean_terminated_length": 686.03125,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8306704685091972,
|
|
"epoch": 0.08956916099773243,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 9.109977324263038e-07,
|
|
"loss": -0.0268,
|
|
"num_tokens": 9439876.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 158,
|
|
"step_time": 50.167413502931595
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1665.0,
|
|
"completions/max_terminated_length": 1665.0,
|
|
"completions/mean_length": 804.484375,
|
|
"completions/mean_terminated_length": 804.484375,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"entropy": 0.7235718593001366,
|
|
"epoch": 0.09013605442176871,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4296875,
|
|
"learning_rate": 9.104308390022676e-07,
|
|
"loss": 0.0075,
|
|
"num_tokens": 9502635.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.42891231179237366,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 159,
|
|
"step_time": 63.26405255496502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1866.0,
|
|
"completions/max_terminated_length": 1866.0,
|
|
"completions/mean_length": 707.109375,
|
|
"completions/mean_terminated_length": 707.109375,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.8616630770266056,
|
|
"epoch": 0.09070294784580499,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 9.098639455782312e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 9557034.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 160,
|
|
"step_time": 70.56565687526017
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/max_terminated_length": 1344.0,
|
|
"completions/mean_length": 768.59375,
|
|
"completions/mean_terminated_length": 768.59375,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.9379465486854315,
|
|
"epoch": 0.09126984126984126,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 9.09297052154195e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 9617640.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6546536684036255,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 161,
|
|
"step_time": 50.923309312202036
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 716.359375,
|
|
"completions/mean_terminated_length": 716.359375,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.589280379936099,
|
|
"epoch": 0.09183673469387756,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 9.087301587301587e-07,
|
|
"loss": 0.0246,
|
|
"num_tokens": 9672767.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 162,
|
|
"step_time": 53.908720002509654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1277.0,
|
|
"completions/max_terminated_length": 1277.0,
|
|
"completions/mean_length": 778.640625,
|
|
"completions/mean_terminated_length": 778.640625,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.9540521949529648,
|
|
"epoch": 0.09240362811791383,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 9.081632653061225e-07,
|
|
"loss": 0.009,
|
|
"num_tokens": 9733720.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 163,
|
|
"step_time": 48.228524887003005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1618.0,
|
|
"completions/max_terminated_length": 1618.0,
|
|
"completions/mean_length": 798.796875,
|
|
"completions/mean_terminated_length": 798.796875,
|
|
"completions/min_length": 430.0,
|
|
"completions/min_terminated_length": 430.0,
|
|
"entropy": 0.9369715973734856,
|
|
"epoch": 0.09297052154195011,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.458984375,
|
|
"learning_rate": 9.075963718820861e-07,
|
|
"loss": -0.0221,
|
|
"num_tokens": 9796915.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 164,
|
|
"step_time": 61.795095819979906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1539.0,
|
|
"completions/max_terminated_length": 1539.0,
|
|
"completions/mean_length": 717.359375,
|
|
"completions/mean_terminated_length": 717.359375,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.8061159290373325,
|
|
"epoch": 0.0935374149659864,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 9.070294784580499e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 9851794.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.41277948021888733,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 165,
|
|
"step_time": 58.08895343821496
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1349.0,
|
|
"completions/max_terminated_length": 1349.0,
|
|
"completions/mean_length": 686.625,
|
|
"completions/mean_terminated_length": 686.625,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.9296346679329872,
|
|
"epoch": 0.09410430839002268,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3125,
|
|
"learning_rate": 9.064625850340135e-07,
|
|
"loss": 0.0125,
|
|
"num_tokens": 9906666.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 166,
|
|
"step_time": 52.012140300124884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1470.0,
|
|
"completions/max_terminated_length": 1470.0,
|
|
"completions/mean_length": 749.0,
|
|
"completions/mean_terminated_length": 749.0,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 0.6455147042870522,
|
|
"epoch": 0.09467120181405896,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 9.058956916099772e-07,
|
|
"loss": 0.0119,
|
|
"num_tokens": 9964658.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.8344857692718506,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 167,
|
|
"step_time": 56.627189056016505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1420.0,
|
|
"completions/max_terminated_length": 1420.0,
|
|
"completions/mean_length": 728.671875,
|
|
"completions/mean_terminated_length": 728.671875,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.8471074942499399,
|
|
"epoch": 0.09523809523809523,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 9.05328798185941e-07,
|
|
"loss": -0.0208,
|
|
"num_tokens": 10022517.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 168,
|
|
"step_time": 55.34592215809971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 789.671875,
|
|
"completions/mean_terminated_length": 789.671875,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.9063250534236431,
|
|
"epoch": 0.09580498866213152,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28515625,
|
|
"learning_rate": 9.047619047619047e-07,
|
|
"loss": 0.0261,
|
|
"num_tokens": 10084376.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 169,
|
|
"step_time": 56.44608405418694
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1144.0,
|
|
"completions/max_terminated_length": 1144.0,
|
|
"completions/mean_length": 740.6875,
|
|
"completions/mean_terminated_length": 740.6875,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.9391395784914494,
|
|
"epoch": 0.0963718820861678,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 9.041950113378685e-07,
|
|
"loss": -0.0212,
|
|
"num_tokens": 10145532.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 170,
|
|
"step_time": 45.221647115424275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1706.0,
|
|
"completions/max_terminated_length": 1706.0,
|
|
"completions/mean_length": 722.953125,
|
|
"completions/mean_terminated_length": 722.953125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.8834142237901688,
|
|
"epoch": 0.09693877551020408,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 9.036281179138322e-07,
|
|
"loss": 0.009,
|
|
"num_tokens": 10200745.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 171,
|
|
"step_time": 66.33940491080284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1845.0,
|
|
"completions/max_terminated_length": 1845.0,
|
|
"completions/mean_length": 715.828125,
|
|
"completions/mean_terminated_length": 715.828125,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.9365953505039215,
|
|
"epoch": 0.09750566893424037,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.275390625,
|
|
"learning_rate": 9.030612244897958e-07,
|
|
"loss": 0.0068,
|
|
"num_tokens": 10256590.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 172,
|
|
"step_time": 72.47474573273212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1415.0,
|
|
"completions/max_terminated_length": 1415.0,
|
|
"completions/mean_length": 774.4375,
|
|
"completions/mean_terminated_length": 774.4375,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 1.0347714982926846,
|
|
"epoch": 0.09807256235827665,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 9.024943310657595e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 10316010.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5754139423370361,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 173,
|
|
"step_time": 54.19198610819876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1635.0,
|
|
"completions/max_terminated_length": 1635.0,
|
|
"completions/mean_length": 823.203125,
|
|
"completions/mean_terminated_length": 823.203125,
|
|
"completions/min_length": 434.0,
|
|
"completions/min_terminated_length": 434.0,
|
|
"entropy": 0.9763973355293274,
|
|
"epoch": 0.09863945578231292,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.255859375,
|
|
"learning_rate": 9.019274376417233e-07,
|
|
"loss": 0.0109,
|
|
"num_tokens": 10379279.0,
|
|
"reward": 0.015625,
|
|
"reward_std": 0.08768405020236969,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 174,
|
|
"step_time": 63.09931514598429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1525.0,
|
|
"completions/max_terminated_length": 1525.0,
|
|
"completions/mean_length": 809.859375,
|
|
"completions/mean_terminated_length": 809.859375,
|
|
"completions/min_length": 410.0,
|
|
"completions/min_terminated_length": 410.0,
|
|
"entropy": 0.8472088351845741,
|
|
"epoch": 0.0992063492063492,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 9.01360544217687e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 10444046.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 175,
|
|
"step_time": 59.40772425476462
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/max_terminated_length": 1202.0,
|
|
"completions/mean_length": 773.359375,
|
|
"completions/mean_terminated_length": 773.359375,
|
|
"completions/min_length": 403.0,
|
|
"completions/min_terminated_length": 403.0,
|
|
"entropy": 0.889940544962883,
|
|
"epoch": 0.09977324263038549,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 9.007936507936508e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 10503645.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 176,
|
|
"step_time": 45.98833618219942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1280.0,
|
|
"completions/max_terminated_length": 1280.0,
|
|
"completions/mean_length": 721.875,
|
|
"completions/mean_terminated_length": 721.875,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.5890161376446486,
|
|
"epoch": 0.10034013605442177,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 9.002267573696145e-07,
|
|
"loss": -0.0192,
|
|
"num_tokens": 10559589.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 177,
|
|
"step_time": 48.5907125454396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1195.0,
|
|
"completions/max_terminated_length": 1195.0,
|
|
"completions/mean_length": 707.140625,
|
|
"completions/mean_terminated_length": 707.140625,
|
|
"completions/min_length": 21.0,
|
|
"completions/min_terminated_length": 21.0,
|
|
"entropy": 1.0144722759723663,
|
|
"epoch": 0.10090702947845805,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 8.996598639455783e-07,
|
|
"loss": 0.0149,
|
|
"num_tokens": 10614134.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 178,
|
|
"step_time": 45.42839828785509
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1314.0,
|
|
"completions/max_terminated_length": 1314.0,
|
|
"completions/mean_length": 801.234375,
|
|
"completions/mean_terminated_length": 801.234375,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 0.7205090411007404,
|
|
"epoch": 0.10147392290249434,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"learning_rate": 8.990929705215419e-07,
|
|
"loss": -0.0033,
|
|
"num_tokens": 10677061.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 179,
|
|
"step_time": 50.42318244557828
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1579.0,
|
|
"completions/max_terminated_length": 1579.0,
|
|
"completions/mean_length": 756.328125,
|
|
"completions/mean_terminated_length": 756.328125,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.7697293981909752,
|
|
"epoch": 0.10204081632653061,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"learning_rate": 8.985260770975056e-07,
|
|
"loss": 0.0173,
|
|
"num_tokens": 10738306.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 180,
|
|
"step_time": 60.82005000580102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2502.0,
|
|
"completions/max_terminated_length": 2502.0,
|
|
"completions/mean_length": 806.46875,
|
|
"completions/mean_terminated_length": 806.46875,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.8854840584099293,
|
|
"epoch": 0.10260770975056689,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 8.979591836734693e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 10801224.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 181,
|
|
"step_time": 99.66593520808965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1445.0,
|
|
"completions/max_terminated_length": 1445.0,
|
|
"completions/mean_length": 681.46875,
|
|
"completions/mean_terminated_length": 681.46875,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.833324022591114,
|
|
"epoch": 0.10317460317460317,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 8.973922902494331e-07,
|
|
"loss": -0.0122,
|
|
"num_tokens": 10854622.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 182,
|
|
"step_time": 56.18346180021763
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 756.9375,
|
|
"completions/mean_terminated_length": 703.9365234375,
|
|
"completions/min_length": 127.0,
|
|
"completions/min_terminated_length": 127.0,
|
|
"entropy": 0.8603761028498411,
|
|
"epoch": 0.10374149659863946,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4765625,
|
|
"learning_rate": 8.968253968253968e-07,
|
|
"loss": -0.0064,
|
|
"num_tokens": 10912946.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 183,
|
|
"step_time": 176.22284319531173
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1420.0,
|
|
"completions/max_terminated_length": 1420.0,
|
|
"completions/mean_length": 771.984375,
|
|
"completions/mean_terminated_length": 771.984375,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.679333757609129,
|
|
"epoch": 0.10430839002267574,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 8.962585034013606e-07,
|
|
"loss": -0.0128,
|
|
"num_tokens": 10972257.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 184,
|
|
"step_time": 54.31486776098609
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2259.0,
|
|
"completions/max_terminated_length": 2259.0,
|
|
"completions/mean_length": 856.28125,
|
|
"completions/mean_terminated_length": 856.28125,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 0.8160925414413214,
|
|
"epoch": 0.10487528344671201,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59375,
|
|
"learning_rate": 8.956916099773242e-07,
|
|
"loss": 0.0073,
|
|
"num_tokens": 11036563.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.5434950590133667,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 185,
|
|
"step_time": 88.11578060686588
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 739.234375,
|
|
"completions/mean_terminated_length": 739.234375,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.7380147892981768,
|
|
"epoch": 0.1054421768707483,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.236328125,
|
|
"learning_rate": 8.95124716553288e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 11095386.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 186,
|
|
"step_time": 56.094907987862825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1792.0,
|
|
"completions/max_terminated_length": 1792.0,
|
|
"completions/mean_length": 750.96875,
|
|
"completions/mean_terminated_length": 750.96875,
|
|
"completions/min_length": 350.0,
|
|
"completions/min_terminated_length": 350.0,
|
|
"entropy": 0.642386669293046,
|
|
"epoch": 0.10600907029478458,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 8.945578231292517e-07,
|
|
"loss": 0.0299,
|
|
"num_tokens": 11153080.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 187,
|
|
"step_time": 67.72712143044919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1212.0,
|
|
"completions/max_terminated_length": 1212.0,
|
|
"completions/mean_length": 745.625,
|
|
"completions/mean_terminated_length": 745.625,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.6487344261258841,
|
|
"epoch": 0.10657596371882086,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 8.939909297052153e-07,
|
|
"loss": -0.0064,
|
|
"num_tokens": 11210912.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 188,
|
|
"step_time": 45.404655010439456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1832.0,
|
|
"completions/max_terminated_length": 1832.0,
|
|
"completions/mean_length": 784.28125,
|
|
"completions/mean_terminated_length": 784.28125,
|
|
"completions/min_length": 369.0,
|
|
"completions/min_terminated_length": 369.0,
|
|
"entropy": 0.9743715263903141,
|
|
"epoch": 0.10714285714285714,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 8.934240362811791e-07,
|
|
"loss": -0.0049,
|
|
"num_tokens": 11272802.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 189,
|
|
"step_time": 69.66925997380167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1579.0,
|
|
"completions/max_terminated_length": 1579.0,
|
|
"completions/mean_length": 730.0625,
|
|
"completions/mean_terminated_length": 730.0625,
|
|
"completions/min_length": 264.0,
|
|
"completions/min_terminated_length": 264.0,
|
|
"entropy": 0.8052809778600931,
|
|
"epoch": 0.10770975056689343,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 8.928571428571428e-07,
|
|
"loss": -0.0014,
|
|
"num_tokens": 11329454.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.489915668964386,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 190,
|
|
"step_time": 59.718092283234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1164.0,
|
|
"completions/max_terminated_length": 1164.0,
|
|
"completions/mean_length": 689.609375,
|
|
"completions/mean_terminated_length": 689.609375,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.8131397124379873,
|
|
"epoch": 0.1082766439909297,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 8.922902494331065e-07,
|
|
"loss": 0.0137,
|
|
"num_tokens": 11386005.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.745356023311615,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 191,
|
|
"step_time": 44.201239502988756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 770.609375,
|
|
"completions/mean_terminated_length": 770.609375,
|
|
"completions/min_length": 369.0,
|
|
"completions/min_terminated_length": 369.0,
|
|
"entropy": 0.8108047619462013,
|
|
"epoch": 0.10884353741496598,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.43359375,
|
|
"learning_rate": 8.917233560090702e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 11445404.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 192,
|
|
"step_time": 52.274463404901326
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1482.0,
|
|
"completions/max_terminated_length": 1482.0,
|
|
"completions/mean_length": 731.984375,
|
|
"completions/mean_terminated_length": 731.984375,
|
|
"completions/min_length": 320.0,
|
|
"completions/min_terminated_length": 320.0,
|
|
"entropy": 0.7214385736733675,
|
|
"epoch": 0.10941043083900227,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 8.91156462585034e-07,
|
|
"loss": 0.0048,
|
|
"num_tokens": 11501987.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 193,
|
|
"step_time": 56.17669538129121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1884.0,
|
|
"completions/max_terminated_length": 1884.0,
|
|
"completions/mean_length": 858.609375,
|
|
"completions/mean_terminated_length": 858.609375,
|
|
"completions/min_length": 422.0,
|
|
"completions/min_terminated_length": 422.0,
|
|
"entropy": 0.5905744098126888,
|
|
"epoch": 0.10997732426303855,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 8.905895691609977e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 11565898.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 194,
|
|
"step_time": 71.28613732848316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1306.0,
|
|
"completions/max_terminated_length": 1306.0,
|
|
"completions/mean_length": 710.1875,
|
|
"completions/mean_terminated_length": 710.1875,
|
|
"completions/min_length": 217.0,
|
|
"completions/min_terminated_length": 217.0,
|
|
"entropy": 0.8393162861466408,
|
|
"epoch": 0.11054421768707483,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 8.900226757369615e-07,
|
|
"loss": -0.012,
|
|
"num_tokens": 11620614.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5739033818244934,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 195,
|
|
"step_time": 48.9538035010919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 799.625,
|
|
"completions/mean_terminated_length": 799.625,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.7309198118746281,
|
|
"epoch": 0.1111111111111111,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 8.894557823129251e-07,
|
|
"loss": -0.0092,
|
|
"num_tokens": 11686094.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 196,
|
|
"step_time": 54.0608956348151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1248.0,
|
|
"completions/max_terminated_length": 1248.0,
|
|
"completions/mean_length": 708.078125,
|
|
"completions/mean_terminated_length": 708.078125,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.8973634522408247,
|
|
"epoch": 0.1116780045351474,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 8.888888888888888e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 11743739.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 197,
|
|
"step_time": 47.21101175248623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1249.0,
|
|
"completions/max_terminated_length": 1249.0,
|
|
"completions/mean_length": 664.328125,
|
|
"completions/mean_terminated_length": 664.328125,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.8888151589781046,
|
|
"epoch": 0.11224489795918367,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3359375,
|
|
"learning_rate": 8.883219954648525e-07,
|
|
"loss": -0.0159,
|
|
"num_tokens": 11797512.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 198,
|
|
"step_time": 47.433251860551536
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1767.0,
|
|
"completions/max_terminated_length": 1767.0,
|
|
"completions/mean_length": 736.75,
|
|
"completions/mean_terminated_length": 736.75,
|
|
"completions/min_length": 204.0,
|
|
"completions/min_terminated_length": 204.0,
|
|
"entropy": 0.617466252297163,
|
|
"epoch": 0.11281179138321995,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 8.877551020408163e-07,
|
|
"loss": 0.0137,
|
|
"num_tokens": 11853832.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 199,
|
|
"step_time": 67.18875078298151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1274.0,
|
|
"completions/max_terminated_length": 1274.0,
|
|
"completions/mean_length": 754.609375,
|
|
"completions/mean_terminated_length": 754.609375,
|
|
"completions/min_length": 461.0,
|
|
"completions/min_terminated_length": 461.0,
|
|
"entropy": 0.993266124278307,
|
|
"epoch": 0.11337868480725624,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 8.8718820861678e-07,
|
|
"loss": -0.0036,
|
|
"num_tokens": 11912055.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 200,
|
|
"step_time": 48.555252633057535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1785.0,
|
|
"completions/max_terminated_length": 1785.0,
|
|
"completions/mean_length": 734.9375,
|
|
"completions/mean_terminated_length": 734.9375,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.8032393399626017,
|
|
"epoch": 0.11394557823129252,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 8.866213151927438e-07,
|
|
"loss": 0.0151,
|
|
"num_tokens": 11969403.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6986381411552429,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 201,
|
|
"step_time": 68.53788980189711
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1584.0,
|
|
"completions/max_terminated_length": 1584.0,
|
|
"completions/mean_length": 687.6875,
|
|
"completions/mean_terminated_length": 687.6875,
|
|
"completions/min_length": 217.0,
|
|
"completions/min_terminated_length": 217.0,
|
|
"entropy": 0.8932653181254864,
|
|
"epoch": 0.1145124716553288,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 8.860544217687075e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 12025543.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 202,
|
|
"step_time": 61.52193641010672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1410.0,
|
|
"completions/max_terminated_length": 1410.0,
|
|
"completions/mean_length": 721.234375,
|
|
"completions/mean_terminated_length": 721.234375,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 1.0084965340793133,
|
|
"epoch": 0.11507936507936507,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3203125,
|
|
"learning_rate": 8.854875283446712e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 12081302.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 203,
|
|
"step_time": 53.666412461549044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 768.046875,
|
|
"completions/mean_terminated_length": 768.046875,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.9337150938808918,
|
|
"epoch": 0.11564625850340136,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.349609375,
|
|
"learning_rate": 8.849206349206348e-07,
|
|
"loss": 0.0497,
|
|
"num_tokens": 12140425.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 204,
|
|
"step_time": 52.04016512725502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1596.0,
|
|
"completions/max_terminated_length": 1596.0,
|
|
"completions/mean_length": 843.734375,
|
|
"completions/mean_terminated_length": 843.734375,
|
|
"completions/min_length": 194.0,
|
|
"completions/min_terminated_length": 194.0,
|
|
"entropy": 0.617034338414669,
|
|
"epoch": 0.11621315192743764,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1806640625,
|
|
"learning_rate": 8.843537414965985e-07,
|
|
"loss": 0.0015,
|
|
"num_tokens": 12203368.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.5379911661148071,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 205,
|
|
"step_time": 60.39713575411588
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2372.0,
|
|
"completions/max_terminated_length": 2372.0,
|
|
"completions/mean_length": 866.125,
|
|
"completions/mean_terminated_length": 866.125,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.6286310963332653,
|
|
"epoch": 0.11678004535147392,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 8.837868480725623e-07,
|
|
"loss": 0.021,
|
|
"num_tokens": 12270552.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6662480235099792,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 206,
|
|
"step_time": 92.44137762580067
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1399.0,
|
|
"completions/max_terminated_length": 1399.0,
|
|
"completions/mean_length": 693.765625,
|
|
"completions/mean_terminated_length": 693.765625,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.7869409117847681,
|
|
"epoch": 0.11734693877551021,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 8.83219954648526e-07,
|
|
"loss": 0.0016,
|
|
"num_tokens": 12326353.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 207,
|
|
"step_time": 53.71699774544686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1426.0,
|
|
"completions/max_terminated_length": 1426.0,
|
|
"completions/mean_length": 710.984375,
|
|
"completions/mean_terminated_length": 710.984375,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.8867442682385445,
|
|
"epoch": 0.11791383219954649,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 8.826530612244898e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 12381736.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 208,
|
|
"step_time": 54.03470201138407
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 801.8125,
|
|
"completions/mean_terminated_length": 801.8125,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.6637982688844204,
|
|
"epoch": 0.11848072562358276,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 8.820861678004535e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 12442460.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 209,
|
|
"step_time": 55.399893024004996
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1073.0,
|
|
"completions/max_terminated_length": 1073.0,
|
|
"completions/mean_length": 677.4375,
|
|
"completions/mean_terminated_length": 677.4375,
|
|
"completions/min_length": 335.0,
|
|
"completions/min_terminated_length": 335.0,
|
|
"entropy": 0.8804828450083733,
|
|
"epoch": 0.11904761904761904,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4609375,
|
|
"learning_rate": 8.815192743764172e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 12495944.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 210,
|
|
"step_time": 40.4301584828645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1643.0,
|
|
"completions/max_terminated_length": 1643.0,
|
|
"completions/mean_length": 783.734375,
|
|
"completions/mean_terminated_length": 783.734375,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.8268555216491222,
|
|
"epoch": 0.11961451247165533,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 8.809523809523809e-07,
|
|
"loss": 0.03,
|
|
"num_tokens": 12555895.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 211,
|
|
"step_time": 61.941131071187556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1534.0,
|
|
"completions/max_terminated_length": 1534.0,
|
|
"completions/mean_length": 808.421875,
|
|
"completions/mean_terminated_length": 808.421875,
|
|
"completions/min_length": 241.0,
|
|
"completions/min_terminated_length": 241.0,
|
|
"entropy": 0.6941155325621367,
|
|
"epoch": 0.12018140589569161,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 8.803854875283446e-07,
|
|
"loss": -0.0081,
|
|
"num_tokens": 12617538.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 212,
|
|
"step_time": 58.329736151732504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1460.0,
|
|
"completions/max_terminated_length": 1460.0,
|
|
"completions/mean_length": 784.46875,
|
|
"completions/mean_terminated_length": 784.46875,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.7919981423765421,
|
|
"epoch": 0.12074829931972789,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.19140625,
|
|
"learning_rate": 8.798185941043083e-07,
|
|
"loss": 0.0028,
|
|
"num_tokens": 12679336.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3398725986480713,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 213,
|
|
"step_time": 55.956489546224475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1449.0,
|
|
"completions/max_terminated_length": 1449.0,
|
|
"completions/mean_length": 754.890625,
|
|
"completions/mean_terminated_length": 754.890625,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7367960270494223,
|
|
"epoch": 0.12131519274376418,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 8.792517006802721e-07,
|
|
"loss": -0.017,
|
|
"num_tokens": 12738401.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 214,
|
|
"step_time": 55.02792660146952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1494.0,
|
|
"completions/max_terminated_length": 1494.0,
|
|
"completions/mean_length": 758.40625,
|
|
"completions/mean_terminated_length": 758.40625,
|
|
"completions/min_length": 418.0,
|
|
"completions/min_terminated_length": 418.0,
|
|
"entropy": 0.6321735177189112,
|
|
"epoch": 0.12188208616780045,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 8.786848072562358e-07,
|
|
"loss": 0.0405,
|
|
"num_tokens": 12797347.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 215,
|
|
"step_time": 58.47775779198855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1197.0,
|
|
"completions/max_terminated_length": 1197.0,
|
|
"completions/mean_length": 781.21875,
|
|
"completions/mean_terminated_length": 781.21875,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.970495417714119,
|
|
"epoch": 0.12244897959183673,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"learning_rate": 8.781179138321995e-07,
|
|
"loss": 0.0257,
|
|
"num_tokens": 12861937.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5422958731651306,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0078125,
|
|
"rewards/format_reward_func/std": 0.0625,
|
|
"step": 216,
|
|
"step_time": 48.29055262543261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1515.0,
|
|
"completions/max_terminated_length": 1515.0,
|
|
"completions/mean_length": 863.328125,
|
|
"completions/mean_terminated_length": 863.328125,
|
|
"completions/min_length": 445.0,
|
|
"completions/min_terminated_length": 445.0,
|
|
"entropy": 0.7418898530304432,
|
|
"epoch": 0.12301587301587301,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 8.775510204081632e-07,
|
|
"loss": -0.0217,
|
|
"num_tokens": 12927302.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 217,
|
|
"step_time": 57.822010475210845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1969.0,
|
|
"completions/max_terminated_length": 1969.0,
|
|
"completions/mean_length": 851.828125,
|
|
"completions/mean_terminated_length": 851.828125,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.876748263835907,
|
|
"epoch": 0.1235827664399093,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 8.76984126984127e-07,
|
|
"loss": 0.0152,
|
|
"num_tokens": 12992571.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5445777773857117,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 218,
|
|
"step_time": 76.29621310438961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2382.0,
|
|
"completions/max_terminated_length": 2382.0,
|
|
"completions/mean_length": 736.34375,
|
|
"completions/mean_terminated_length": 736.34375,
|
|
"completions/min_length": 170.0,
|
|
"completions/min_terminated_length": 170.0,
|
|
"entropy": 0.7815339379012585,
|
|
"epoch": 0.12414965986394558,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 8.764172335600907e-07,
|
|
"loss": -0.0102,
|
|
"num_tokens": 13049665.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 219,
|
|
"step_time": 90.68243482988328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1426.0,
|
|
"completions/max_terminated_length": 1426.0,
|
|
"completions/mean_length": 751.296875,
|
|
"completions/mean_terminated_length": 751.296875,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 0.8732363618910313,
|
|
"epoch": 0.12471655328798185,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2216796875,
|
|
"learning_rate": 8.758503401360544e-07,
|
|
"loss": -0.0087,
|
|
"num_tokens": 13108468.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 220,
|
|
"step_time": 54.02226519025862
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1302.0,
|
|
"completions/max_terminated_length": 1302.0,
|
|
"completions/mean_length": 800.765625,
|
|
"completions/mean_terminated_length": 800.765625,
|
|
"completions/min_length": 450.0,
|
|
"completions/min_terminated_length": 450.0,
|
|
"entropy": 0.7857203856110573,
|
|
"epoch": 0.12528344671201813,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.376953125,
|
|
"learning_rate": 8.752834467120181e-07,
|
|
"loss": 0.0214,
|
|
"num_tokens": 13170357.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 221,
|
|
"step_time": 49.31268990319222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1215.0,
|
|
"completions/max_terminated_length": 1215.0,
|
|
"completions/mean_length": 746.875,
|
|
"completions/mean_terminated_length": 746.875,
|
|
"completions/min_length": 366.0,
|
|
"completions/min_terminated_length": 366.0,
|
|
"entropy": 0.9006501343101263,
|
|
"epoch": 0.12585034013605442,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 8.747165532879818e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 13230621.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 222,
|
|
"step_time": 47.496775266714394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1563.0,
|
|
"completions/max_terminated_length": 1563.0,
|
|
"completions/mean_length": 678.0625,
|
|
"completions/mean_terminated_length": 678.0625,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.9045848734676838,
|
|
"epoch": 0.1264172335600907,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 8.741496598639455e-07,
|
|
"loss": -0.014,
|
|
"num_tokens": 13283545.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 223,
|
|
"step_time": 59.5722237136215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1134.0,
|
|
"completions/max_terminated_length": 1134.0,
|
|
"completions/mean_length": 696.53125,
|
|
"completions/mean_terminated_length": 696.53125,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.6746862567961216,
|
|
"epoch": 0.12698412698412698,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 8.735827664399093e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 13337867.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 224,
|
|
"step_time": 42.94442406669259
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1520.0,
|
|
"completions/max_terminated_length": 1520.0,
|
|
"completions/mean_length": 682.421875,
|
|
"completions/mean_terminated_length": 682.421875,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.9444940574467182,
|
|
"epoch": 0.12755102040816327,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"learning_rate": 8.73015873015873e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 13397838.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 225,
|
|
"step_time": 61.212527259252965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 4083.0,
|
|
"completions/mean_length": 863.296875,
|
|
"completions/mean_terminated_length": 811.9841918945312,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.5170101718977094,
|
|
"epoch": 0.12811791383219956,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.369140625,
|
|
"learning_rate": 8.724489795918368e-07,
|
|
"loss": 0.0063,
|
|
"num_tokens": 13462673.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.6784212589263916,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 226,
|
|
"step_time": 175.77780503034592
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1330.0,
|
|
"completions/max_terminated_length": 1330.0,
|
|
"completions/mean_length": 737.75,
|
|
"completions/mean_terminated_length": 737.75,
|
|
"completions/min_length": 28.0,
|
|
"completions/min_terminated_length": 28.0,
|
|
"entropy": 0.8340022526681423,
|
|
"epoch": 0.12868480725623582,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"learning_rate": 8.718820861678005e-07,
|
|
"loss": -0.0183,
|
|
"num_tokens": 13520017.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 227,
|
|
"step_time": 51.04294812493026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1305.0,
|
|
"completions/max_terminated_length": 1305.0,
|
|
"completions/mean_length": 688.578125,
|
|
"completions/mean_terminated_length": 688.578125,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.8905489556491375,
|
|
"epoch": 0.1292517006802721,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 8.71315192743764e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 13573430.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 228,
|
|
"step_time": 49.98010636493564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 823.40625,
|
|
"completions/mean_terminated_length": 823.40625,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.7021019477397203,
|
|
"epoch": 0.12981859410430838,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"learning_rate": 8.707482993197278e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 13636056.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6029338836669922,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 229,
|
|
"step_time": 56.998927718028426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 683.25,
|
|
"completions/mean_terminated_length": 683.25,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.8584277145564556,
|
|
"epoch": 0.13038548752834467,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 8.701814058956915e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 13690056.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 230,
|
|
"step_time": 54.34538010787219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1381.0,
|
|
"completions/max_terminated_length": 1381.0,
|
|
"completions/mean_length": 786.390625,
|
|
"completions/mean_terminated_length": 786.390625,
|
|
"completions/min_length": 239.0,
|
|
"completions/min_terminated_length": 239.0,
|
|
"entropy": 0.7083199769258499,
|
|
"epoch": 0.13095238095238096,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 8.696145124716553e-07,
|
|
"loss": 0.0133,
|
|
"num_tokens": 13749385.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 231,
|
|
"step_time": 52.82335822377354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 731.09375,
|
|
"completions/mean_terminated_length": 731.09375,
|
|
"completions/min_length": 427.0,
|
|
"completions/min_terminated_length": 427.0,
|
|
"entropy": 0.6988869290798903,
|
|
"epoch": 0.13151927437641722,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 8.69047619047619e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 13807071.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 232,
|
|
"step_time": 49.35344780609012
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1791.0,
|
|
"completions/max_terminated_length": 1791.0,
|
|
"completions/mean_length": 773.171875,
|
|
"completions/mean_terminated_length": 773.171875,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.7230974994599819,
|
|
"epoch": 0.1320861678004535,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 8.684807256235828e-07,
|
|
"loss": -0.0015,
|
|
"num_tokens": 13866754.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 233,
|
|
"step_time": 67.96496311109513
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1691.0,
|
|
"completions/max_terminated_length": 1691.0,
|
|
"completions/mean_length": 722.4375,
|
|
"completions/mean_terminated_length": 722.4375,
|
|
"completions/min_length": 198.0,
|
|
"completions/min_terminated_length": 198.0,
|
|
"entropy": 0.9679892286658287,
|
|
"epoch": 0.1326530612244898,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.423828125,
|
|
"learning_rate": 8.679138321995464e-07,
|
|
"loss": 0.0132,
|
|
"num_tokens": 13924182.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.36247265338897705,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 234,
|
|
"step_time": 64.32111782580614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1209.0,
|
|
"completions/max_terminated_length": 1209.0,
|
|
"completions/mean_length": 697.359375,
|
|
"completions/mean_terminated_length": 697.359375,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.8101966269314289,
|
|
"epoch": 0.13321995464852607,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 8.673469387755102e-07,
|
|
"loss": 0.036,
|
|
"num_tokens": 13983613.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 235,
|
|
"step_time": 46.66053644847125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1462.0,
|
|
"completions/max_terminated_length": 1462.0,
|
|
"completions/mean_length": 848.953125,
|
|
"completions/mean_terminated_length": 848.953125,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.745408920571208,
|
|
"epoch": 0.13378684807256236,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.197265625,
|
|
"learning_rate": 8.667800453514738e-07,
|
|
"loss": -0.0025,
|
|
"num_tokens": 14052034.0,
|
|
"reward": 0.0234375,
|
|
"reward_std": 0.10652101039886475,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 236,
|
|
"step_time": 55.998151410371065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 670.515625,
|
|
"completions/mean_terminated_length": 670.515625,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.8511352762579918,
|
|
"epoch": 0.13435374149659865,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 8.662131519274376e-07,
|
|
"loss": 0.0455,
|
|
"num_tokens": 14105475.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 237,
|
|
"step_time": 54.208372516557574
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2160.0,
|
|
"completions/max_terminated_length": 2160.0,
|
|
"completions/mean_length": 886.609375,
|
|
"completions/mean_terminated_length": 886.609375,
|
|
"completions/min_length": 306.0,
|
|
"completions/min_terminated_length": 306.0,
|
|
"entropy": 0.6885136272758245,
|
|
"epoch": 0.1349206349206349,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 8.656462585034013e-07,
|
|
"loss": -0.0186,
|
|
"num_tokens": 14172330.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 238,
|
|
"step_time": 82.4482323108241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1512.0,
|
|
"completions/max_terminated_length": 1512.0,
|
|
"completions/mean_length": 776.484375,
|
|
"completions/mean_terminated_length": 776.484375,
|
|
"completions/min_length": 336.0,
|
|
"completions/min_terminated_length": 336.0,
|
|
"entropy": 0.8836219422519207,
|
|
"epoch": 0.1354875283446712,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 8.650793650793651e-07,
|
|
"loss": -0.0082,
|
|
"num_tokens": 14231937.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 239,
|
|
"step_time": 56.863407094962895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1435.0,
|
|
"completions/max_terminated_length": 1435.0,
|
|
"completions/mean_length": 772.421875,
|
|
"completions/mean_terminated_length": 772.421875,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 0.8125546742230654,
|
|
"epoch": 0.1360544217687075,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 8.645124716553288e-07,
|
|
"loss": 0.0253,
|
|
"num_tokens": 14291156.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.426084041595459,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0,
|
|
"rewards/format_reward_func/std": 0.0,
|
|
"step": 240,
|
|
"step_time": 54.15421236958355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1214.0,
|
|
"completions/max_terminated_length": 1214.0,
|
|
"completions/mean_length": 717.5,
|
|
"completions/mean_terminated_length": 717.5,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.7670545168220997,
|
|
"epoch": 0.13662131519274376,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 8.639455782312925e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 14347132.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 241,
|
|
"step_time": 45.79305484518409
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1725.0,
|
|
"completions/max_terminated_length": 1725.0,
|
|
"completions/mean_length": 817.71875,
|
|
"completions/mean_terminated_length": 817.71875,
|
|
"completions/min_length": 408.0,
|
|
"completions/min_terminated_length": 408.0,
|
|
"entropy": 0.7369464822113514,
|
|
"epoch": 0.13718820861678005,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 8.633786848072562e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 14409370.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 242,
|
|
"step_time": 64.96719512436539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1240.0,
|
|
"completions/max_terminated_length": 1240.0,
|
|
"completions/mean_length": 643.890625,
|
|
"completions/mean_terminated_length": 643.890625,
|
|
"completions/min_length": 190.0,
|
|
"completions/min_terminated_length": 190.0,
|
|
"entropy": 0.760059030726552,
|
|
"epoch": 0.1377551020408163,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.431640625,
|
|
"learning_rate": 8.6281179138322e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 14461675.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 243,
|
|
"step_time": 47.31172960717231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1309.0,
|
|
"completions/max_terminated_length": 1309.0,
|
|
"completions/mean_length": 779.359375,
|
|
"completions/mean_terminated_length": 779.359375,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.711810328066349,
|
|
"epoch": 0.1383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"learning_rate": 8.622448979591836e-07,
|
|
"loss": -0.0117,
|
|
"num_tokens": 14521482.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 244,
|
|
"step_time": 49.666923959739506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1311.0,
|
|
"completions/max_terminated_length": 1311.0,
|
|
"completions/mean_length": 749.359375,
|
|
"completions/mean_terminated_length": 749.359375,
|
|
"completions/min_length": 363.0,
|
|
"completions/min_terminated_length": 363.0,
|
|
"entropy": 0.8499616701155901,
|
|
"epoch": 0.1388888888888889,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"learning_rate": 8.616780045351474e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 14582617.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 245,
|
|
"step_time": 52.20430337637663
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1238.0,
|
|
"completions/max_terminated_length": 1238.0,
|
|
"completions/mean_length": 779.84375,
|
|
"completions/mean_terminated_length": 779.84375,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.8445747178047895,
|
|
"epoch": 0.13945578231292516,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"learning_rate": 8.611111111111111e-07,
|
|
"loss": 0.0172,
|
|
"num_tokens": 14644599.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 246,
|
|
"step_time": 47.973739959299564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1796.0,
|
|
"completions/max_terminated_length": 1796.0,
|
|
"completions/mean_length": 858.078125,
|
|
"completions/mean_terminated_length": 858.078125,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.7805829998105764,
|
|
"epoch": 0.14002267573696145,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 8.605442176870747e-07,
|
|
"loss": -0.0442,
|
|
"num_tokens": 14714260.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.40703144669532776,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 247,
|
|
"step_time": 71.51869424339384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1315.0,
|
|
"completions/max_terminated_length": 1315.0,
|
|
"completions/mean_length": 689.59375,
|
|
"completions/mean_terminated_length": 689.59375,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.6935113929212093,
|
|
"epoch": 0.14058956916099774,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 8.599773242630385e-07,
|
|
"loss": -0.0056,
|
|
"num_tokens": 14767354.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 248,
|
|
"step_time": 50.06727380864322
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1367.0,
|
|
"completions/max_terminated_length": 1367.0,
|
|
"completions/mean_length": 817.921875,
|
|
"completions/mean_terminated_length": 817.921875,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.8837875053286552,
|
|
"epoch": 0.141156462585034,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 8.594104308390022e-07,
|
|
"loss": 0.028,
|
|
"num_tokens": 14831589.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 249,
|
|
"step_time": 52.77266918309033
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1598.0,
|
|
"completions/max_terminated_length": 1598.0,
|
|
"completions/mean_length": 743.78125,
|
|
"completions/mean_terminated_length": 743.78125,
|
|
"completions/min_length": 227.0,
|
|
"completions/min_terminated_length": 227.0,
|
|
"entropy": 0.8470449112355709,
|
|
"epoch": 0.1417233560090703,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 8.58843537414966e-07,
|
|
"loss": 0.0148,
|
|
"num_tokens": 14889359.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 250,
|
|
"step_time": 60.69317118264735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1382.0,
|
|
"completions/max_terminated_length": 1382.0,
|
|
"completions/mean_length": 639.578125,
|
|
"completions/mean_terminated_length": 639.578125,
|
|
"completions/min_length": 336.0,
|
|
"completions/min_terminated_length": 336.0,
|
|
"entropy": 1.1404040716588497,
|
|
"epoch": 0.14229024943310659,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 8.582766439909297e-07,
|
|
"loss": 0.0229,
|
|
"num_tokens": 14941284.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 251,
|
|
"step_time": 52.66949089244008
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1634.0,
|
|
"completions/max_terminated_length": 1634.0,
|
|
"completions/mean_length": 704.6875,
|
|
"completions/mean_terminated_length": 704.6875,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.7078406568616629,
|
|
"epoch": 0.14285714285714285,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 8.577097505668934e-07,
|
|
"loss": 0.0486,
|
|
"num_tokens": 14997264.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 252,
|
|
"step_time": 61.867045666091144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1101.0,
|
|
"completions/max_terminated_length": 1101.0,
|
|
"completions/mean_length": 730.359375,
|
|
"completions/mean_terminated_length": 730.359375,
|
|
"completions/min_length": 419.0,
|
|
"completions/min_terminated_length": 419.0,
|
|
"entropy": 0.9553987365216017,
|
|
"epoch": 0.14342403628117914,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.63671875,
|
|
"learning_rate": 8.57142857142857e-07,
|
|
"loss": 0.0092,
|
|
"num_tokens": 15053551.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 253,
|
|
"step_time": 42.036332870833576
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1391.0,
|
|
"completions/max_terminated_length": 1391.0,
|
|
"completions/mean_length": 762.953125,
|
|
"completions/mean_terminated_length": 762.953125,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.7707686368376017,
|
|
"epoch": 0.14399092970521543,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4296875,
|
|
"learning_rate": 8.565759637188208e-07,
|
|
"loss": 0.013,
|
|
"num_tokens": 15113372.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.46922925114631653,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 254,
|
|
"step_time": 52.543017198331654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2076.0,
|
|
"completions/max_terminated_length": 2076.0,
|
|
"completions/mean_length": 797.25,
|
|
"completions/mean_terminated_length": 797.25,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.7859501540660858,
|
|
"epoch": 0.1445578231292517,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 8.560090702947845e-07,
|
|
"loss": 0.0319,
|
|
"num_tokens": 15174468.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 255,
|
|
"step_time": 79.07321002520621
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1145.0,
|
|
"completions/max_terminated_length": 1145.0,
|
|
"completions/mean_length": 712.0625,
|
|
"completions/mean_terminated_length": 712.0625,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.9650670122355223,
|
|
"epoch": 0.14512471655328799,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 8.554421768707483e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 15232792.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 256,
|
|
"step_time": 44.803927617147565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1194.0,
|
|
"completions/max_terminated_length": 1194.0,
|
|
"completions/mean_length": 703.984375,
|
|
"completions/mean_terminated_length": 703.984375,
|
|
"completions/min_length": 324.0,
|
|
"completions/min_terminated_length": 324.0,
|
|
"entropy": 0.9428023956716061,
|
|
"epoch": 0.14569160997732428,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 8.54875283446712e-07,
|
|
"loss": 0.0024,
|
|
"num_tokens": 15290367.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 257,
|
|
"step_time": 46.41874376870692
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1018.0,
|
|
"completions/max_terminated_length": 1018.0,
|
|
"completions/mean_length": 664.9375,
|
|
"completions/mean_terminated_length": 664.9375,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8272994346916676,
|
|
"epoch": 0.14625850340136054,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 8.543083900226758e-07,
|
|
"loss": -0.0044,
|
|
"num_tokens": 15341947.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 258,
|
|
"step_time": 38.47307618521154
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1474.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 771.171875,
|
|
"completions/mean_terminated_length": 771.171875,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.9931114763021469,
|
|
"epoch": 0.14682539682539683,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"learning_rate": 8.537414965986394e-07,
|
|
"loss": -0.0203,
|
|
"num_tokens": 15401766.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 259,
|
|
"step_time": 56.232815210707486
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1481.0,
|
|
"completions/max_terminated_length": 1481.0,
|
|
"completions/mean_length": 723.03125,
|
|
"completions/mean_terminated_length": 723.03125,
|
|
"completions/min_length": 391.0,
|
|
"completions/min_terminated_length": 391.0,
|
|
"entropy": 0.8141548484563828,
|
|
"epoch": 0.1473922902494331,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 8.531746031746031e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 15457280.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 260,
|
|
"step_time": 56.52870470285416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1471.0,
|
|
"completions/max_terminated_length": 1471.0,
|
|
"completions/mean_length": 658.359375,
|
|
"completions/mean_terminated_length": 658.359375,
|
|
"completions/min_length": 230.0,
|
|
"completions/min_terminated_length": 230.0,
|
|
"entropy": 0.6927868071943521,
|
|
"epoch": 0.14795918367346939,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 8.526077097505668e-07,
|
|
"loss": 0.0249,
|
|
"num_tokens": 15510295.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 261,
|
|
"step_time": 56.96211295481771
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1513.0,
|
|
"completions/max_terminated_length": 1513.0,
|
|
"completions/mean_length": 752.953125,
|
|
"completions/mean_terminated_length": 752.953125,
|
|
"completions/min_length": 398.0,
|
|
"completions/min_terminated_length": 398.0,
|
|
"entropy": 0.9332785494625568,
|
|
"epoch": 0.14852607709750568,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"learning_rate": 8.520408163265306e-07,
|
|
"loss": 0.0098,
|
|
"num_tokens": 15569868.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 262,
|
|
"step_time": 59.106632246635854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1880.0,
|
|
"completions/max_terminated_length": 1880.0,
|
|
"completions/mean_length": 839.875,
|
|
"completions/mean_terminated_length": 839.875,
|
|
"completions/min_length": 340.0,
|
|
"completions/min_terminated_length": 340.0,
|
|
"entropy": 0.682667376473546,
|
|
"epoch": 0.14909297052154194,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.314453125,
|
|
"learning_rate": 8.514739229024943e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 15634316.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 263,
|
|
"step_time": 71.25024418812245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 722.6875,
|
|
"completions/mean_terminated_length": 722.6875,
|
|
"completions/min_length": 391.0,
|
|
"completions/min_terminated_length": 391.0,
|
|
"entropy": 0.9179340749979019,
|
|
"epoch": 0.14965986394557823,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 8.509070294784581e-07,
|
|
"loss": 0.018,
|
|
"num_tokens": 15690792.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 264,
|
|
"step_time": 47.271880991756916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1270.0,
|
|
"completions/max_terminated_length": 1270.0,
|
|
"completions/mean_length": 637.296875,
|
|
"completions/mean_terminated_length": 637.296875,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 1.086241863667965,
|
|
"epoch": 0.15022675736961452,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48828125,
|
|
"learning_rate": 8.503401360544217e-07,
|
|
"loss": 0.0132,
|
|
"num_tokens": 15742243.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.559903621673584,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 265,
|
|
"step_time": 47.730507844127715
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1571.0,
|
|
"completions/max_terminated_length": 1571.0,
|
|
"completions/mean_length": 706.125,
|
|
"completions/mean_terminated_length": 706.125,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.9066045880317688,
|
|
"epoch": 0.15079365079365079,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.333984375,
|
|
"learning_rate": 8.497732426303855e-07,
|
|
"loss": 0.0451,
|
|
"num_tokens": 15796715.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 266,
|
|
"step_time": 59.063225102610886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 697.734375,
|
|
"completions/mean_terminated_length": 697.734375,
|
|
"completions/min_length": 224.0,
|
|
"completions/min_terminated_length": 224.0,
|
|
"entropy": 0.8448955230414867,
|
|
"epoch": 0.15136054421768708,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"learning_rate": 8.492063492063492e-07,
|
|
"loss": 0.0121,
|
|
"num_tokens": 15852362.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.603396475315094,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 267,
|
|
"step_time": 52.62285229656845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1680.0,
|
|
"completions/max_terminated_length": 1680.0,
|
|
"completions/mean_length": 696.25,
|
|
"completions/mean_terminated_length": 696.25,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 0.7814523428678513,
|
|
"epoch": 0.15192743764172337,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 8.486394557823128e-07,
|
|
"loss": 0.0395,
|
|
"num_tokens": 15910290.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 268,
|
|
"step_time": 64.97410681564361
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1367.0,
|
|
"completions/max_terminated_length": 1367.0,
|
|
"completions/mean_length": 616.078125,
|
|
"completions/mean_terminated_length": 616.078125,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.8676711972802877,
|
|
"epoch": 0.15249433106575963,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.75390625,
|
|
"learning_rate": 8.480725623582766e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 15961415.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 269,
|
|
"step_time": 52.289122426882386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1256.0,
|
|
"completions/max_terminated_length": 1256.0,
|
|
"completions/mean_length": 735.8125,
|
|
"completions/mean_terminated_length": 735.8125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.8315059375017881,
|
|
"epoch": 0.15306122448979592,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.42578125,
|
|
"learning_rate": 8.475056689342403e-07,
|
|
"loss": -0.0034,
|
|
"num_tokens": 16019531.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 270,
|
|
"step_time": 48.43833992630243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2195.0,
|
|
"completions/max_terminated_length": 2195.0,
|
|
"completions/mean_length": 753.65625,
|
|
"completions/mean_terminated_length": 753.65625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 1.0189075618982315,
|
|
"epoch": 0.1536281179138322,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 8.469387755102041e-07,
|
|
"loss": 0.0254,
|
|
"num_tokens": 16078069.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 271,
|
|
"step_time": 84.29482378717512
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1597.0,
|
|
"completions/max_terminated_length": 1597.0,
|
|
"completions/mean_length": 761.625,
|
|
"completions/mean_terminated_length": 761.625,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.9707346335053444,
|
|
"epoch": 0.15419501133786848,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 8.463718820861677e-07,
|
|
"loss": 0.0053,
|
|
"num_tokens": 16137533.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6900206208229065,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 272,
|
|
"step_time": 60.21038067340851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2329.0,
|
|
"completions/max_terminated_length": 2329.0,
|
|
"completions/mean_length": 791.125,
|
|
"completions/mean_terminated_length": 791.125,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.7534265369176865,
|
|
"epoch": 0.15476190476190477,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 8.458049886621315e-07,
|
|
"loss": 0.0448,
|
|
"num_tokens": 16198797.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 273,
|
|
"step_time": 89.73313437681645
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1502.0,
|
|
"completions/max_terminated_length": 1502.0,
|
|
"completions/mean_length": 731.515625,
|
|
"completions/mean_terminated_length": 731.515625,
|
|
"completions/min_length": 213.0,
|
|
"completions/min_terminated_length": 213.0,
|
|
"entropy": 0.7331438101828098,
|
|
"epoch": 0.15532879818594103,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.427734375,
|
|
"learning_rate": 8.452380952380952e-07,
|
|
"loss": -0.0193,
|
|
"num_tokens": 16255470.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 274,
|
|
"step_time": 56.693840723484755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1401.0,
|
|
"completions/max_terminated_length": 1401.0,
|
|
"completions/mean_length": 697.0625,
|
|
"completions/mean_terminated_length": 697.0625,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 0.6416508182883263,
|
|
"epoch": 0.15589569160997732,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 8.44671201814059e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 16309378.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7210001945495605,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 275,
|
|
"step_time": 52.967795793898404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1338.0,
|
|
"completions/mean_length": 856.109375,
|
|
"completions/mean_terminated_length": 804.6825561523438,
|
|
"completions/min_length": 433.0,
|
|
"completions/min_terminated_length": 433.0,
|
|
"entropy": 0.6948063969612122,
|
|
"epoch": 0.1564625850340136,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37109375,
|
|
"learning_rate": 8.441043083900226e-07,
|
|
"loss": 0.0166,
|
|
"num_tokens": 16375169.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 276,
|
|
"step_time": 177.92747872509062
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1217.0,
|
|
"completions/max_terminated_length": 1217.0,
|
|
"completions/mean_length": 584.125,
|
|
"completions/mean_terminated_length": 584.125,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 0.6641750391572714,
|
|
"epoch": 0.15702947845804988,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6953125,
|
|
"learning_rate": 8.435374149659864e-07,
|
|
"loss": 0.0061,
|
|
"num_tokens": 16423729.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7724404335021973,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 277,
|
|
"step_time": 46.15311232954264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 868.453125,
|
|
"completions/mean_terminated_length": 868.453125,
|
|
"completions/min_length": 425.0,
|
|
"completions/min_terminated_length": 425.0,
|
|
"entropy": 0.7224074248224497,
|
|
"epoch": 0.15759637188208617,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.322265625,
|
|
"learning_rate": 8.4297052154195e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 16491150.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 278,
|
|
"step_time": 52.73080511763692
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1196.0,
|
|
"completions/max_terminated_length": 1196.0,
|
|
"completions/mean_length": 707.671875,
|
|
"completions/mean_terminated_length": 707.671875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.8151501063257456,
|
|
"epoch": 0.15816326530612246,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 8.424036281179138e-07,
|
|
"loss": -0.0161,
|
|
"num_tokens": 16546033.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 279,
|
|
"step_time": 44.939537251368165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2440.0,
|
|
"completions/max_terminated_length": 2440.0,
|
|
"completions/mean_length": 784.25,
|
|
"completions/mean_terminated_length": 784.25,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 1.0181800108402967,
|
|
"epoch": 0.15873015873015872,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 8.418367346938775e-07,
|
|
"loss": 0.0263,
|
|
"num_tokens": 16605617.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 280,
|
|
"step_time": 94.32256285753101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1421.0,
|
|
"completions/max_terminated_length": 1421.0,
|
|
"completions/mean_length": 723.46875,
|
|
"completions/mean_terminated_length": 723.46875,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.6927716005593538,
|
|
"epoch": 0.159297052154195,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 8.412698412698413e-07,
|
|
"loss": -0.0006,
|
|
"num_tokens": 16662055.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 281,
|
|
"step_time": 54.639340850524604
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1401.0,
|
|
"completions/max_terminated_length": 1401.0,
|
|
"completions/mean_length": 784.578125,
|
|
"completions/mean_terminated_length": 784.578125,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.7328203655779362,
|
|
"epoch": 0.1598639455782313,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.384765625,
|
|
"learning_rate": 8.40702947845805e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 16722388.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 282,
|
|
"step_time": 53.27179315127432
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1107.0,
|
|
"completions/mean_length": 722.609375,
|
|
"completions/mean_terminated_length": 669.0635375976562,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.7031647693365812,
|
|
"epoch": 0.16043083900226757,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 8.401360544217688e-07,
|
|
"loss": 0.0005,
|
|
"num_tokens": 16780995.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6777355074882507,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 283,
|
|
"step_time": 179.04967124108225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1520.0,
|
|
"completions/max_terminated_length": 1520.0,
|
|
"completions/mean_length": 677.578125,
|
|
"completions/mean_terminated_length": 677.578125,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.9004981126636267,
|
|
"epoch": 0.16099773242630386,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 8.395691609977323e-07,
|
|
"loss": 0.0095,
|
|
"num_tokens": 16834080.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 284,
|
|
"step_time": 57.85565869230777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1098.0,
|
|
"completions/max_terminated_length": 1098.0,
|
|
"completions/mean_length": 682.59375,
|
|
"completions/mean_terminated_length": 682.59375,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.8715041428804398,
|
|
"epoch": 0.16156462585034015,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 8.39002267573696e-07,
|
|
"loss": 0.0372,
|
|
"num_tokens": 16887894.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 285,
|
|
"step_time": 43.965926623903215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1345.0,
|
|
"completions/max_terminated_length": 1345.0,
|
|
"completions/mean_length": 683.46875,
|
|
"completions/mean_terminated_length": 683.46875,
|
|
"completions/min_length": 174.0,
|
|
"completions/min_terminated_length": 174.0,
|
|
"entropy": 1.0438571982085705,
|
|
"epoch": 0.1621315192743764,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 8.384353741496598e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 16942412.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 286,
|
|
"step_time": 50.78270637895912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2196.0,
|
|
"completions/max_terminated_length": 2196.0,
|
|
"completions/mean_length": 741.359375,
|
|
"completions/mean_terminated_length": 741.359375,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.883393669500947,
|
|
"epoch": 0.1626984126984127,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 8.378684807256235e-07,
|
|
"loss": 0.0164,
|
|
"num_tokens": 17000619.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 287,
|
|
"step_time": 79.346072871238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1470.0,
|
|
"completions/max_terminated_length": 1470.0,
|
|
"completions/mean_length": 860.609375,
|
|
"completions/mean_terminated_length": 860.609375,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7544728927314281,
|
|
"epoch": 0.16326530612244897,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 8.373015873015873e-07,
|
|
"loss": -0.0007,
|
|
"num_tokens": 17067378.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 288,
|
|
"step_time": 52.59334731567651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1235.0,
|
|
"completions/max_terminated_length": 1235.0,
|
|
"completions/mean_length": 705.0,
|
|
"completions/mean_terminated_length": 705.0,
|
|
"completions/min_length": 259.0,
|
|
"completions/min_terminated_length": 259.0,
|
|
"entropy": 0.8788765668869019,
|
|
"epoch": 0.16383219954648526,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.279296875,
|
|
"learning_rate": 8.36734693877551e-07,
|
|
"loss": -0.0093,
|
|
"num_tokens": 17123122.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 289,
|
|
"step_time": 43.87046900205314
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1443.0,
|
|
"completions/max_terminated_length": 1443.0,
|
|
"completions/mean_length": 705.03125,
|
|
"completions/mean_terminated_length": 705.03125,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.9619238749146461,
|
|
"epoch": 0.16439909297052155,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 8.361678004535147e-07,
|
|
"loss": -0.0029,
|
|
"num_tokens": 17178516.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 290,
|
|
"step_time": 50.73440036177635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1172.0,
|
|
"completions/max_terminated_length": 1172.0,
|
|
"completions/mean_length": 677.921875,
|
|
"completions/mean_terminated_length": 677.921875,
|
|
"completions/min_length": 366.0,
|
|
"completions/min_terminated_length": 366.0,
|
|
"entropy": 0.9480736441910267,
|
|
"epoch": 0.1649659863945578,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 8.356009070294783e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 17233575.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 291,
|
|
"step_time": 41.97542067337781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1791.0,
|
|
"completions/max_terminated_length": 1791.0,
|
|
"completions/mean_length": 695.1875,
|
|
"completions/mean_terminated_length": 695.1875,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 1.0936996825039387,
|
|
"epoch": 0.1655328798185941,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1845703125,
|
|
"learning_rate": 8.350340136054421e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 17292555.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3398725986480713,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 292,
|
|
"step_time": 64.84023329243064
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1295.0,
|
|
"completions/max_terminated_length": 1295.0,
|
|
"completions/mean_length": 668.34375,
|
|
"completions/mean_terminated_length": 668.34375,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.8729091212153435,
|
|
"epoch": 0.1660997732426304,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"learning_rate": 8.344671201814058e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 17346385.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 293,
|
|
"step_time": 46.00880840513855
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1544.0,
|
|
"completions/max_terminated_length": 1544.0,
|
|
"completions/mean_length": 830.59375,
|
|
"completions/mean_terminated_length": 830.59375,
|
|
"completions/min_length": 405.0,
|
|
"completions/min_terminated_length": 405.0,
|
|
"entropy": 0.8223886843770742,
|
|
"epoch": 0.16666666666666666,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.443359375,
|
|
"learning_rate": 8.339002267573696e-07,
|
|
"loss": -0.0125,
|
|
"num_tokens": 17412391.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6023679971694946,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 294,
|
|
"step_time": 55.6619848748669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 3081.0,
|
|
"completions/mean_length": 821.265625,
|
|
"completions/mean_terminated_length": 769.2857666015625,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.7634497378021479,
|
|
"epoch": 0.16723356009070295,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 8.333333333333333e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 17475888.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 295,
|
|
"step_time": 172.29678527265787
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1131.0,
|
|
"completions/max_terminated_length": 1131.0,
|
|
"completions/mean_length": 713.515625,
|
|
"completions/mean_terminated_length": 713.515625,
|
|
"completions/min_length": 263.0,
|
|
"completions/min_terminated_length": 263.0,
|
|
"entropy": 0.7846761327236891,
|
|
"epoch": 0.16780045351473924,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6640625,
|
|
"learning_rate": 8.327664399092971e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 17531785.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 296,
|
|
"step_time": 40.278807829134166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 781.171875,
|
|
"completions/mean_terminated_length": 781.171875,
|
|
"completions/min_length": 209.0,
|
|
"completions/min_terminated_length": 209.0,
|
|
"entropy": 0.6870723441243172,
|
|
"epoch": 0.1683673469387755,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 8.321995464852607e-07,
|
|
"loss": 0.0432,
|
|
"num_tokens": 17592324.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5896238088607788,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 297,
|
|
"step_time": 47.24058729223907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2147.0,
|
|
"completions/max_terminated_length": 2147.0,
|
|
"completions/mean_length": 755.96875,
|
|
"completions/mean_terminated_length": 755.96875,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 0.8424653597176075,
|
|
"epoch": 0.1689342403628118,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 8.316326530612245e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 17650874.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 298,
|
|
"step_time": 77.23045068792999
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1169.0,
|
|
"completions/max_terminated_length": 1169.0,
|
|
"completions/mean_length": 690.859375,
|
|
"completions/mean_terminated_length": 690.859375,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.9548549987375736,
|
|
"epoch": 0.16950113378684808,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 8.310657596371881e-07,
|
|
"loss": -0.0076,
|
|
"num_tokens": 17706529.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 299,
|
|
"step_time": 42.4382833102718
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1243.0,
|
|
"completions/max_terminated_length": 1243.0,
|
|
"completions/mean_length": 723.296875,
|
|
"completions/mean_terminated_length": 723.296875,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.8589233346283436,
|
|
"epoch": 0.17006802721088435,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3984375,
|
|
"learning_rate": 8.304988662131519e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 17764780.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 300,
|
|
"step_time": 44.784306516870856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1137.0,
|
|
"completions/max_terminated_length": 1137.0,
|
|
"completions/mean_length": 697.21875,
|
|
"completions/mean_terminated_length": 697.21875,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.9121045172214508,
|
|
"epoch": 0.17063492063492064,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 8.299319727891156e-07,
|
|
"loss": -0.0034,
|
|
"num_tokens": 17820530.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 301,
|
|
"step_time": 40.97146006766707
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1421.0,
|
|
"completions/max_terminated_length": 1421.0,
|
|
"completions/mean_length": 669.296875,
|
|
"completions/mean_terminated_length": 669.296875,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 0.7469332460314035,
|
|
"epoch": 0.1712018140589569,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 8.293650793650794e-07,
|
|
"loss": -0.0241,
|
|
"num_tokens": 17874285.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5033986568450928,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.1015625,
|
|
"rewards/format_reward_func/std": 0.20275264978408813,
|
|
"step": 302,
|
|
"step_time": 50.62312275916338
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 736.5,
|
|
"completions/mean_terminated_length": 736.5,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.8989239111542702,
|
|
"epoch": 0.1717687074829932,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 8.28798185941043e-07,
|
|
"loss": 0.0223,
|
|
"num_tokens": 17931741.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 303,
|
|
"step_time": 51.81090355012566
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1204.0,
|
|
"completions/max_terminated_length": 1204.0,
|
|
"completions/mean_length": 676.96875,
|
|
"completions/mean_terminated_length": 676.96875,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.9559312872588634,
|
|
"epoch": 0.17233560090702948,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 8.282312925170068e-07,
|
|
"loss": -0.0022,
|
|
"num_tokens": 17983251.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4917473793029785,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 304,
|
|
"step_time": 42.032407058402896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1426.0,
|
|
"completions/max_terminated_length": 1426.0,
|
|
"completions/mean_length": 815.46875,
|
|
"completions/mean_terminated_length": 815.46875,
|
|
"completions/min_length": 204.0,
|
|
"completions/min_terminated_length": 204.0,
|
|
"entropy": 0.8452925551682711,
|
|
"epoch": 0.17290249433106575,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 8.276643990929705e-07,
|
|
"loss": 0.0129,
|
|
"num_tokens": 18046369.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 305,
|
|
"step_time": 51.47114187292755
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1221.0,
|
|
"completions/max_terminated_length": 1221.0,
|
|
"completions/mean_length": 743.65625,
|
|
"completions/mean_terminated_length": 743.65625,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.7419566698372364,
|
|
"epoch": 0.17346938775510204,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.416015625,
|
|
"learning_rate": 8.270975056689343e-07,
|
|
"loss": 0.0102,
|
|
"num_tokens": 18106523.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 306,
|
|
"step_time": 44.13747341185808
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1436.0,
|
|
"completions/max_terminated_length": 1436.0,
|
|
"completions/mean_length": 771.5625,
|
|
"completions/mean_terminated_length": 771.5625,
|
|
"completions/min_length": 435.0,
|
|
"completions/min_terminated_length": 435.0,
|
|
"entropy": 0.9274702072143555,
|
|
"epoch": 0.17403628117913833,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 8.26530612244898e-07,
|
|
"loss": -0.0092,
|
|
"num_tokens": 18166095.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 307,
|
|
"step_time": 51.33585764002055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1881.0,
|
|
"completions/max_terminated_length": 1881.0,
|
|
"completions/mean_length": 726.3125,
|
|
"completions/mean_terminated_length": 726.3125,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.7567149810492992,
|
|
"epoch": 0.1746031746031746,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 8.259637188208616e-07,
|
|
"loss": 0.0355,
|
|
"num_tokens": 18229043.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6978833675384521,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 308,
|
|
"step_time": 69.37814402300864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1125.0,
|
|
"completions/max_terminated_length": 1125.0,
|
|
"completions/mean_length": 691.328125,
|
|
"completions/mean_terminated_length": 691.328125,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.820183027535677,
|
|
"epoch": 0.17517006802721088,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 8.253968253968253e-07,
|
|
"loss": 0.0132,
|
|
"num_tokens": 18284560.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6416434645652771,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 309,
|
|
"step_time": 39.77160437498242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1591.0,
|
|
"completions/max_terminated_length": 1591.0,
|
|
"completions/mean_length": 733.546875,
|
|
"completions/mean_terminated_length": 733.546875,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.7642414346337318,
|
|
"epoch": 0.17573696145124718,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 8.24829931972789e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 18341827.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 310,
|
|
"step_time": 56.358331645838916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1484.0,
|
|
"completions/max_terminated_length": 1484.0,
|
|
"completions/mean_length": 762.4375,
|
|
"completions/mean_terminated_length": 762.4375,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.8820738904178143,
|
|
"epoch": 0.17630385487528344,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 8.242630385487528e-07,
|
|
"loss": 0.0081,
|
|
"num_tokens": 18400719.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 311,
|
|
"step_time": 51.908669171854854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1463.0,
|
|
"completions/max_terminated_length": 1463.0,
|
|
"completions/mean_length": 725.21875,
|
|
"completions/mean_terminated_length": 725.21875,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.7850626893341541,
|
|
"epoch": 0.17687074829931973,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6953125,
|
|
"learning_rate": 8.236961451247165e-07,
|
|
"loss": 0.0387,
|
|
"num_tokens": 18457485.0,
|
|
"reward": 0.515625,
|
|
"reward_std": 0.8681706190109253,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 312,
|
|
"step_time": 51.591500581242144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1707.0,
|
|
"completions/max_terminated_length": 1707.0,
|
|
"completions/mean_length": 686.421875,
|
|
"completions/mean_terminated_length": 686.421875,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.8607305027544498,
|
|
"epoch": 0.17743764172335602,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 8.231292517006803e-07,
|
|
"loss": 0.0428,
|
|
"num_tokens": 18511328.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 313,
|
|
"step_time": 59.51167070213705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1308.0,
|
|
"completions/max_terminated_length": 1308.0,
|
|
"completions/mean_length": 766.125,
|
|
"completions/mean_terminated_length": 766.125,
|
|
"completions/min_length": 422.0,
|
|
"completions/min_terminated_length": 422.0,
|
|
"entropy": 0.9440110493451357,
|
|
"epoch": 0.17800453514739228,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 8.22562358276644e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 18570056.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 314,
|
|
"step_time": 46.22242483403534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1031.0,
|
|
"completions/max_terminated_length": 1031.0,
|
|
"completions/mean_length": 611.09375,
|
|
"completions/mean_terminated_length": 611.09375,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.8531279899179935,
|
|
"epoch": 0.17857142857142858,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"learning_rate": 8.219954648526076e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 18620142.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 315,
|
|
"step_time": 37.25250487215817
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1888.0,
|
|
"completions/max_terminated_length": 1888.0,
|
|
"completions/mean_length": 812.453125,
|
|
"completions/mean_terminated_length": 812.453125,
|
|
"completions/min_length": 345.0,
|
|
"completions/min_terminated_length": 345.0,
|
|
"entropy": 0.7896588500589132,
|
|
"epoch": 0.17913832199546487,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 8.214285714285713e-07,
|
|
"loss": 0.0044,
|
|
"num_tokens": 18683939.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.5274578928947449,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 316,
|
|
"step_time": 68.10056507680565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1211.0,
|
|
"completions/max_terminated_length": 1211.0,
|
|
"completions/mean_length": 701.328125,
|
|
"completions/mean_terminated_length": 701.328125,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 0.915490660816431,
|
|
"epoch": 0.17970521541950113,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 8.208616780045351e-07,
|
|
"loss": -0.016,
|
|
"num_tokens": 18739544.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 317,
|
|
"step_time": 42.40765083394945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1132.0,
|
|
"completions/max_terminated_length": 1132.0,
|
|
"completions/mean_length": 684.890625,
|
|
"completions/mean_terminated_length": 684.890625,
|
|
"completions/min_length": 363.0,
|
|
"completions/min_terminated_length": 363.0,
|
|
"entropy": 0.6934215240180492,
|
|
"epoch": 0.18027210884353742,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.609375,
|
|
"learning_rate": 8.202947845804988e-07,
|
|
"loss": -0.0324,
|
|
"num_tokens": 18794417.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 318,
|
|
"step_time": 40.335626715794206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1182.0,
|
|
"completions/max_terminated_length": 1182.0,
|
|
"completions/mean_length": 653.875,
|
|
"completions/mean_terminated_length": 653.875,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 1.1461282074451447,
|
|
"epoch": 0.18083900226757368,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 8.197278911564626e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 18845817.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 319,
|
|
"step_time": 41.016207785345614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1723.0,
|
|
"completions/max_terminated_length": 1723.0,
|
|
"completions/mean_length": 781.03125,
|
|
"completions/mean_terminated_length": 781.03125,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.7973394598811865,
|
|
"epoch": 0.18140589569160998,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 8.191609977324263e-07,
|
|
"loss": -0.0217,
|
|
"num_tokens": 18905923.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.4292735755443573,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 320,
|
|
"step_time": 60.48013670463115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1551.0,
|
|
"completions/max_terminated_length": 1551.0,
|
|
"completions/mean_length": 760.359375,
|
|
"completions/mean_terminated_length": 760.359375,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.6594368945807219,
|
|
"epoch": 0.18197278911564627,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 8.1859410430839e-07,
|
|
"loss": -0.0598,
|
|
"num_tokens": 18965026.0,
|
|
"reward": 0.4296875,
|
|
"reward_std": 0.7759642601013184,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 321,
|
|
"step_time": 54.980602267198265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 709.921875,
|
|
"completions/mean_terminated_length": 709.921875,
|
|
"completions/min_length": 207.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"entropy": 0.7112272661179304,
|
|
"epoch": 0.18253968253968253,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 8.180272108843537e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 19021141.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 322,
|
|
"step_time": 51.63816947955638
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1665.0,
|
|
"completions/max_terminated_length": 1665.0,
|
|
"completions/mean_length": 861.578125,
|
|
"completions/mean_terminated_length": 861.578125,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.6500704195350409,
|
|
"epoch": 0.18310657596371882,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.384765625,
|
|
"learning_rate": 8.174603174603174e-07,
|
|
"loss": -0.0154,
|
|
"num_tokens": 19085874.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 323,
|
|
"step_time": 58.53681822679937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1450.0,
|
|
"completions/max_terminated_length": 1450.0,
|
|
"completions/mean_length": 767.546875,
|
|
"completions/mean_terminated_length": 767.546875,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.8385312054306269,
|
|
"epoch": 0.1836734693877551,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 8.168934240362811e-07,
|
|
"loss": -0.0013,
|
|
"num_tokens": 19144493.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7210001945495605,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 324,
|
|
"step_time": 51.178459596820176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1679.0,
|
|
"completions/max_terminated_length": 1679.0,
|
|
"completions/mean_length": 688.390625,
|
|
"completions/mean_terminated_length": 688.390625,
|
|
"completions/min_length": 153.0,
|
|
"completions/min_terminated_length": 153.0,
|
|
"entropy": 0.8860599715262651,
|
|
"epoch": 0.18424036281179137,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 8.163265306122449e-07,
|
|
"loss": -0.0275,
|
|
"num_tokens": 19199726.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 325,
|
|
"step_time": 59.97611542791128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1238.0,
|
|
"completions/max_terminated_length": 1238.0,
|
|
"completions/mean_length": 694.484375,
|
|
"completions/mean_terminated_length": 694.484375,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.8037873115390539,
|
|
"epoch": 0.18480725623582767,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 8.157596371882086e-07,
|
|
"loss": 0.0105,
|
|
"num_tokens": 19255077.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 326,
|
|
"step_time": 43.73556720651686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2087.0,
|
|
"completions/max_terminated_length": 2087.0,
|
|
"completions/mean_length": 821.625,
|
|
"completions/mean_terminated_length": 821.625,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"entropy": 0.6347903534770012,
|
|
"epoch": 0.18537414965986396,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 8.151927437641724e-07,
|
|
"loss": -0.0118,
|
|
"num_tokens": 19319261.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.654606282711029,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 327,
|
|
"step_time": 75.29317637253553
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1285.0,
|
|
"completions/max_terminated_length": 1285.0,
|
|
"completions/mean_length": 807.21875,
|
|
"completions/mean_terminated_length": 807.21875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.7769613545387983,
|
|
"epoch": 0.18594104308390022,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 8.14625850340136e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 19380979.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 328,
|
|
"step_time": 44.861584201455116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1734.0,
|
|
"completions/max_terminated_length": 1734.0,
|
|
"completions/mean_length": 747.203125,
|
|
"completions/mean_terminated_length": 747.203125,
|
|
"completions/min_length": 392.0,
|
|
"completions/min_terminated_length": 392.0,
|
|
"entropy": 0.8658007867634296,
|
|
"epoch": 0.1865079365079365,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.26171875,
|
|
"learning_rate": 8.140589569160997e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 19440032.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 329,
|
|
"step_time": 61.10620769113302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1219.0,
|
|
"completions/max_terminated_length": 1219.0,
|
|
"completions/mean_length": 709.296875,
|
|
"completions/mean_terminated_length": 709.296875,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.8043992314487696,
|
|
"epoch": 0.1870748299319728,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.75,
|
|
"learning_rate": 8.134920634920635e-07,
|
|
"loss": -0.0316,
|
|
"num_tokens": 19495355.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 330,
|
|
"step_time": 42.660171108320355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1563.0,
|
|
"completions/max_terminated_length": 1563.0,
|
|
"completions/mean_length": 638.28125,
|
|
"completions/mean_terminated_length": 638.28125,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 1.0374234952032566,
|
|
"epoch": 0.18764172335600907,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 8.129251700680271e-07,
|
|
"loss": -0.0039,
|
|
"num_tokens": 19545749.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 331,
|
|
"step_time": 54.728871271014214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2653.0,
|
|
"completions/max_terminated_length": 2653.0,
|
|
"completions/mean_length": 864.421875,
|
|
"completions/mean_terminated_length": 864.421875,
|
|
"completions/min_length": 291.0,
|
|
"completions/min_terminated_length": 291.0,
|
|
"entropy": 0.7442141883075237,
|
|
"epoch": 0.18820861678004536,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.197265625,
|
|
"learning_rate": 8.123582766439909e-07,
|
|
"loss": -0.0118,
|
|
"num_tokens": 19614520.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 332,
|
|
"step_time": 98.94229808729142
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 883.140625,
|
|
"completions/mean_terminated_length": 883.140625,
|
|
"completions/min_length": 434.0,
|
|
"completions/min_terminated_length": 434.0,
|
|
"entropy": 0.6293924115598202,
|
|
"epoch": 0.18877551020408162,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 8.117913832199546e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 19681265.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 333,
|
|
"step_time": 52.70649899542332
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1195.0,
|
|
"completions/max_terminated_length": 1195.0,
|
|
"completions/mean_length": 748.640625,
|
|
"completions/mean_terminated_length": 748.640625,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.8612051568925381,
|
|
"epoch": 0.1893424036281179,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 8.112244897959183e-07,
|
|
"loss": 0.0353,
|
|
"num_tokens": 19738930.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 334,
|
|
"step_time": 41.320436858572066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1750.0,
|
|
"completions/max_terminated_length": 1750.0,
|
|
"completions/mean_length": 853.046875,
|
|
"completions/mean_terminated_length": 853.046875,
|
|
"completions/min_length": 333.0,
|
|
"completions/min_terminated_length": 333.0,
|
|
"entropy": 0.5430580489337444,
|
|
"epoch": 0.1899092970521542,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 8.10657596371882e-07,
|
|
"loss": 0.0235,
|
|
"num_tokens": 19804261.0,
|
|
"reward": 0.515625,
|
|
"reward_std": 0.9299630522727966,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 335,
|
|
"step_time": 60.53818303346634
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1238.0,
|
|
"completions/max_terminated_length": 1238.0,
|
|
"completions/mean_length": 609.25,
|
|
"completions/mean_terminated_length": 609.25,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.9052265919744968,
|
|
"epoch": 0.19047619047619047,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 8.100907029478458e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 19852877.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 336,
|
|
"step_time": 42.84950289875269
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1399.0,
|
|
"completions/max_terminated_length": 1399.0,
|
|
"completions/mean_length": 850.046875,
|
|
"completions/mean_terminated_length": 850.046875,
|
|
"completions/min_length": 405.0,
|
|
"completions/min_terminated_length": 405.0,
|
|
"entropy": 0.6594413593411446,
|
|
"epoch": 0.19104308390022676,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.443359375,
|
|
"learning_rate": 8.095238095238095e-07,
|
|
"loss": 0.0106,
|
|
"num_tokens": 19917728.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 337,
|
|
"step_time": 48.65508953295648
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1175.0,
|
|
"completions/max_terminated_length": 1175.0,
|
|
"completions/mean_length": 688.390625,
|
|
"completions/mean_terminated_length": 688.390625,
|
|
"completions/min_length": 198.0,
|
|
"completions/min_terminated_length": 198.0,
|
|
"entropy": 0.9500348977744579,
|
|
"epoch": 0.19160997732426305,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.423828125,
|
|
"learning_rate": 8.089569160997733e-07,
|
|
"loss": 0.002,
|
|
"num_tokens": 19974329.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 338,
|
|
"step_time": 41.63581903837621
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1230.0,
|
|
"completions/max_terminated_length": 1230.0,
|
|
"completions/mean_length": 739.140625,
|
|
"completions/mean_terminated_length": 739.140625,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.8920802474021912,
|
|
"epoch": 0.1921768707482993,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 8.083900226757369e-07,
|
|
"loss": -0.0019,
|
|
"num_tokens": 20033394.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.46922925114631653,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 339,
|
|
"step_time": 43.43856304511428
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1392.0,
|
|
"completions/max_terminated_length": 1392.0,
|
|
"completions/mean_length": 806.859375,
|
|
"completions/mean_terminated_length": 806.859375,
|
|
"completions/min_length": 352.0,
|
|
"completions/min_terminated_length": 352.0,
|
|
"entropy": 0.8819306213408709,
|
|
"epoch": 0.1927437641723356,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 8.078231292517006e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 20095577.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.5039526224136353,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 340,
|
|
"step_time": 48.59819230623543
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1377.0,
|
|
"completions/max_terminated_length": 1377.0,
|
|
"completions/mean_length": 826.03125,
|
|
"completions/mean_terminated_length": 826.03125,
|
|
"completions/min_length": 440.0,
|
|
"completions/min_terminated_length": 440.0,
|
|
"entropy": 1.1564702279865742,
|
|
"epoch": 0.1933106575963719,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.38671875,
|
|
"learning_rate": 8.072562358276643e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 20161203.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 341,
|
|
"step_time": 48.10737384017557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 849.359375,
|
|
"completions/mean_terminated_length": 849.359375,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.6847367249429226,
|
|
"epoch": 0.19387755102040816,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 8.066893424036281e-07,
|
|
"loss": -0.02,
|
|
"num_tokens": 20225370.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 342,
|
|
"step_time": 52.841353938914835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1541.0,
|
|
"completions/max_terminated_length": 1541.0,
|
|
"completions/mean_length": 911.234375,
|
|
"completions/mean_terminated_length": 911.234375,
|
|
"completions/min_length": 434.0,
|
|
"completions/min_terminated_length": 434.0,
|
|
"entropy": 0.6147547662258148,
|
|
"epoch": 0.19444444444444445,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.416015625,
|
|
"learning_rate": 8.061224489795918e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 20294697.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 343,
|
|
"step_time": 53.379401203244925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1495.0,
|
|
"completions/max_terminated_length": 1495.0,
|
|
"completions/mean_length": 765.8125,
|
|
"completions/mean_terminated_length": 765.8125,
|
|
"completions/min_length": 383.0,
|
|
"completions/min_terminated_length": 383.0,
|
|
"entropy": 0.7179710138589144,
|
|
"epoch": 0.19501133786848074,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.61328125,
|
|
"learning_rate": 8.055555555555556e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 20353541.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 344,
|
|
"step_time": 52.01110478769988
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 773.6875,
|
|
"completions/mean_terminated_length": 773.6875,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 0.9038583058863878,
|
|
"epoch": 0.195578231292517,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25390625,
|
|
"learning_rate": 8.049886621315193e-07,
|
|
"loss": -0.0172,
|
|
"num_tokens": 20415897.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 345,
|
|
"step_time": 48.716181214898825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1084.0,
|
|
"completions/max_terminated_length": 1084.0,
|
|
"completions/mean_length": 632.28125,
|
|
"completions/mean_terminated_length": 632.28125,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.9033645987510681,
|
|
"epoch": 0.1961451247165533,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7578125,
|
|
"learning_rate": 8.04421768707483e-07,
|
|
"loss": 0.0293,
|
|
"num_tokens": 20467507.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.8158888816833496,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 346,
|
|
"step_time": 37.71913564391434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1516.0,
|
|
"completions/max_terminated_length": 1516.0,
|
|
"completions/mean_length": 728.0,
|
|
"completions/mean_terminated_length": 728.0,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.512436656281352,
|
|
"epoch": 0.19671201814058956,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 8.038548752834466e-07,
|
|
"loss": 0.0255,
|
|
"num_tokens": 20522883.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.654606282711029,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 347,
|
|
"step_time": 52.17997798230499
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1552.0,
|
|
"completions/max_terminated_length": 1552.0,
|
|
"completions/mean_length": 782.9375,
|
|
"completions/mean_terminated_length": 782.9375,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.8477420397102833,
|
|
"epoch": 0.19727891156462585,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 8.032879818594103e-07,
|
|
"loss": -0.0042,
|
|
"num_tokens": 20582663.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 348,
|
|
"step_time": 52.993523797020316
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1714.0,
|
|
"completions/max_terminated_length": 1714.0,
|
|
"completions/mean_length": 686.515625,
|
|
"completions/mean_terminated_length": 686.515625,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 1.0396155789494514,
|
|
"epoch": 0.19784580498866214,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 8.027210884353741e-07,
|
|
"loss": 0.012,
|
|
"num_tokens": 20637984.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7426892518997192,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 349,
|
|
"step_time": 59.609021263197064
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2002.0,
|
|
"completions/max_terminated_length": 2002.0,
|
|
"completions/mean_length": 756.28125,
|
|
"completions/mean_terminated_length": 756.28125,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.7620594538748264,
|
|
"epoch": 0.1984126984126984,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 8.021541950113378e-07,
|
|
"loss": -0.0012,
|
|
"num_tokens": 20701946.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.47709178924560547,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 350,
|
|
"step_time": 74.73007242754102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1187.0,
|
|
"completions/max_terminated_length": 1187.0,
|
|
"completions/mean_length": 698.890625,
|
|
"completions/mean_terminated_length": 698.890625,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.77883305773139,
|
|
"epoch": 0.1989795918367347,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 8.015873015873016e-07,
|
|
"loss": -0.0028,
|
|
"num_tokens": 20758195.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 351,
|
|
"step_time": 41.66911055613309
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1134.0,
|
|
"completions/max_terminated_length": 1134.0,
|
|
"completions/mean_length": 720.765625,
|
|
"completions/mean_terminated_length": 720.765625,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"entropy": 0.8328768480569124,
|
|
"epoch": 0.19954648526077098,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41796875,
|
|
"learning_rate": 8.010204081632652e-07,
|
|
"loss": -0.0443,
|
|
"num_tokens": 20814124.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 352,
|
|
"step_time": 38.688570947386324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1271.0,
|
|
"completions/max_terminated_length": 1271.0,
|
|
"completions/mean_length": 740.203125,
|
|
"completions/mean_terminated_length": 740.203125,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 1.0927176661789417,
|
|
"epoch": 0.20011337868480725,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 8.00453514739229e-07,
|
|
"loss": -0.0254,
|
|
"num_tokens": 20871633.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 353,
|
|
"step_time": 43.63492278382182
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1773.0,
|
|
"completions/max_terminated_length": 1773.0,
|
|
"completions/mean_length": 745.609375,
|
|
"completions/mean_terminated_length": 745.609375,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.582351628690958,
|
|
"epoch": 0.20068027210884354,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6640625,
|
|
"learning_rate": 7.998866213151927e-07,
|
|
"loss": 0.0291,
|
|
"num_tokens": 20929368.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.8723031282424927,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 354,
|
|
"step_time": 60.86191135738045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1450.0,
|
|
"completions/max_terminated_length": 1450.0,
|
|
"completions/mean_length": 767.15625,
|
|
"completions/mean_terminated_length": 767.15625,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.8429246470332146,
|
|
"epoch": 0.20124716553287983,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 7.993197278911564e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 20987434.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 355,
|
|
"step_time": 49.143246746622026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1267.0,
|
|
"completions/max_terminated_length": 1267.0,
|
|
"completions/mean_length": 740.703125,
|
|
"completions/mean_terminated_length": 740.703125,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"entropy": 0.6812925096601248,
|
|
"epoch": 0.2018140589569161,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 7.987528344671201e-07,
|
|
"loss": 0.0336,
|
|
"num_tokens": 21049383.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 356,
|
|
"step_time": 44.72687572520226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1656.0,
|
|
"completions/max_terminated_length": 1656.0,
|
|
"completions/mean_length": 764.78125,
|
|
"completions/mean_terminated_length": 764.78125,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 0.7700738701969385,
|
|
"epoch": 0.20238095238095238,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 7.981859410430839e-07,
|
|
"loss": -0.0,
|
|
"num_tokens": 21107657.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7545743584632874,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 357,
|
|
"step_time": 56.46399795729667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2142.0,
|
|
"completions/max_terminated_length": 2142.0,
|
|
"completions/mean_length": 775.359375,
|
|
"completions/mean_terminated_length": 775.359375,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 0.6098148673772812,
|
|
"epoch": 0.20294784580498867,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4765625,
|
|
"learning_rate": 7.976190476190476e-07,
|
|
"loss": 0.0411,
|
|
"num_tokens": 21168424.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 358,
|
|
"step_time": 75.8587139165029
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1602.0,
|
|
"completions/max_terminated_length": 1602.0,
|
|
"completions/mean_length": 665.546875,
|
|
"completions/mean_terminated_length": 665.546875,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.8294234424829483,
|
|
"epoch": 0.20351473922902494,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 7.970521541950113e-07,
|
|
"loss": -0.0151,
|
|
"num_tokens": 21221739.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 359,
|
|
"step_time": 55.06628938857466
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2727.0,
|
|
"completions/max_terminated_length": 2727.0,
|
|
"completions/mean_length": 776.59375,
|
|
"completions/mean_terminated_length": 776.59375,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.6692650187760592,
|
|
"epoch": 0.20408163265306123,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 7.96485260770975e-07,
|
|
"loss": -0.0408,
|
|
"num_tokens": 21280881.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7559289336204529,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 360,
|
|
"step_time": 99.08741320390254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1522.0,
|
|
"completions/max_terminated_length": 1522.0,
|
|
"completions/mean_length": 735.9375,
|
|
"completions/mean_terminated_length": 735.9375,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.9531947933137417,
|
|
"epoch": 0.2046485260770975,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 7.959183673469388e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 21339093.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 361,
|
|
"step_time": 52.08562697377056
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1211.0,
|
|
"completions/max_terminated_length": 1211.0,
|
|
"completions/mean_length": 733.140625,
|
|
"completions/mean_terminated_length": 733.140625,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.937473900616169,
|
|
"epoch": 0.20521541950113378,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 7.953514739229025e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 21395806.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 362,
|
|
"step_time": 41.219106522388756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/max_terminated_length": 1142.0,
|
|
"completions/mean_length": 679.3125,
|
|
"completions/mean_terminated_length": 679.3125,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 0.9143323600292206,
|
|
"epoch": 0.20578231292517007,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40625,
|
|
"learning_rate": 7.947845804988662e-07,
|
|
"loss": 0.0205,
|
|
"num_tokens": 21449338.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 363,
|
|
"step_time": 38.76798083074391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1269.0,
|
|
"completions/max_terminated_length": 1269.0,
|
|
"completions/mean_length": 633.0,
|
|
"completions/mean_terminated_length": 633.0,
|
|
"completions/min_length": 24.0,
|
|
"completions/min_terminated_length": 24.0,
|
|
"entropy": 0.8506261613219976,
|
|
"epoch": 0.20634920634920634,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 7.942176870748299e-07,
|
|
"loss": -0.0036,
|
|
"num_tokens": 21502546.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 364,
|
|
"step_time": 43.8399096513167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1347.0,
|
|
"completions/max_terminated_length": 1347.0,
|
|
"completions/mean_length": 703.875,
|
|
"completions/mean_terminated_length": 703.875,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.8162322528660297,
|
|
"epoch": 0.20691609977324263,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.671875,
|
|
"learning_rate": 7.936507936507936e-07,
|
|
"loss": 0.0111,
|
|
"num_tokens": 21556562.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 365,
|
|
"step_time": 46.54994601197541
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1237.0,
|
|
"completions/max_terminated_length": 1237.0,
|
|
"completions/mean_length": 737.890625,
|
|
"completions/mean_terminated_length": 737.890625,
|
|
"completions/min_length": 392.0,
|
|
"completions/min_terminated_length": 392.0,
|
|
"entropy": 0.6952655706554651,
|
|
"epoch": 0.20748299319727892,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 7.930839002267573e-07,
|
|
"loss": 0.0004,
|
|
"num_tokens": 21616107.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 366,
|
|
"step_time": 42.6978315198794
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2991.0,
|
|
"completions/max_terminated_length": 2991.0,
|
|
"completions/mean_length": 821.578125,
|
|
"completions/mean_terminated_length": 821.578125,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.8010297678411007,
|
|
"epoch": 0.20804988662131518,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 7.92517006802721e-07,
|
|
"loss": -0.0124,
|
|
"num_tokens": 21679152.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 367,
|
|
"step_time": 110.2668658932671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1148.0,
|
|
"completions/max_terminated_length": 1148.0,
|
|
"completions/mean_length": 692.90625,
|
|
"completions/mean_terminated_length": 692.90625,
|
|
"completions/min_length": 24.0,
|
|
"completions/min_terminated_length": 24.0,
|
|
"entropy": 0.9686912037432194,
|
|
"epoch": 0.20861678004535147,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 7.919501133786848e-07,
|
|
"loss": -0.0076,
|
|
"num_tokens": 21734586.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 368,
|
|
"step_time": 41.391304216347635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1621.0,
|
|
"completions/max_terminated_length": 1621.0,
|
|
"completions/mean_length": 831.9375,
|
|
"completions/mean_terminated_length": 831.9375,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.8207549266517162,
|
|
"epoch": 0.20918367346938777,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 7.913832199546485e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 21798350.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5900968313217163,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 369,
|
|
"step_time": 57.505966384895146
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2418.0,
|
|
"completions/max_terminated_length": 2418.0,
|
|
"completions/mean_length": 753.28125,
|
|
"completions/mean_terminated_length": 753.28125,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 0.897388132289052,
|
|
"epoch": 0.20975056689342403,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 7.908163265306123e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 21857816.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.41539984941482544,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 370,
|
|
"step_time": 86.28427155595273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1559.0,
|
|
"completions/max_terminated_length": 1559.0,
|
|
"completions/mean_length": 707.015625,
|
|
"completions/mean_terminated_length": 707.015625,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.9148596245795488,
|
|
"epoch": 0.21031746031746032,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 7.902494331065758e-07,
|
|
"loss": 0.0293,
|
|
"num_tokens": 21913553.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 371,
|
|
"step_time": 53.49216768797487
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1359.0,
|
|
"completions/max_terminated_length": 1359.0,
|
|
"completions/mean_length": 789.859375,
|
|
"completions/mean_terminated_length": 789.859375,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.7241196781396866,
|
|
"epoch": 0.2108843537414966,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 7.896825396825396e-07,
|
|
"loss": -0.0076,
|
|
"num_tokens": 21973280.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 372,
|
|
"step_time": 46.46333046257496
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1379.0,
|
|
"completions/max_terminated_length": 1379.0,
|
|
"completions/mean_length": 793.625,
|
|
"completions/mean_terminated_length": 793.625,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.9126806035637856,
|
|
"epoch": 0.21145124716553287,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.333984375,
|
|
"learning_rate": 7.891156462585033e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 22033992.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 373,
|
|
"step_time": 47.039062136784196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/max_terminated_length": 1225.0,
|
|
"completions/mean_length": 658.15625,
|
|
"completions/mean_terminated_length": 658.15625,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.9130817167460918,
|
|
"epoch": 0.21201814058956917,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 7.885487528344671e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 22086234.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.559903621673584,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 374,
|
|
"step_time": 41.82396076992154
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1455.0,
|
|
"completions/max_terminated_length": 1455.0,
|
|
"completions/mean_length": 768.046875,
|
|
"completions/mean_terminated_length": 768.046875,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.6956988330930471,
|
|
"epoch": 0.21258503401360543,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 7.879818594104308e-07,
|
|
"loss": 0.0286,
|
|
"num_tokens": 22145509.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 375,
|
|
"step_time": 49.9479739703238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1196.0,
|
|
"completions/max_terminated_length": 1196.0,
|
|
"completions/mean_length": 712.703125,
|
|
"completions/mean_terminated_length": 712.703125,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.7200741954147816,
|
|
"epoch": 0.21315192743764172,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 7.874149659863946e-07,
|
|
"loss": 0.0405,
|
|
"num_tokens": 22201378.0,
|
|
"reward": 0.484375,
|
|
"reward_std": 0.8589803576469421,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 376,
|
|
"step_time": 40.888242116197944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1387.0,
|
|
"completions/max_terminated_length": 1387.0,
|
|
"completions/mean_length": 698.328125,
|
|
"completions/mean_terminated_length": 698.328125,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.6802572142332792,
|
|
"epoch": 0.213718820861678,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 7.868480725623582e-07,
|
|
"loss": -0.0053,
|
|
"num_tokens": 22254615.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.33174845576286316,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 377,
|
|
"step_time": 46.84081286471337
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1334.0,
|
|
"completions/max_terminated_length": 1334.0,
|
|
"completions/mean_length": 697.703125,
|
|
"completions/mean_terminated_length": 697.703125,
|
|
"completions/min_length": 379.0,
|
|
"completions/min_terminated_length": 379.0,
|
|
"entropy": 0.7381638363003731,
|
|
"epoch": 0.21428571428571427,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.640625,
|
|
"learning_rate": 7.86281179138322e-07,
|
|
"loss": 0.0413,
|
|
"num_tokens": 22309500.0,
|
|
"reward": 0.4765625,
|
|
"reward_std": 0.8234156966209412,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 378,
|
|
"step_time": 45.019169168546796
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1311.0,
|
|
"completions/max_terminated_length": 1311.0,
|
|
"completions/mean_length": 734.625,
|
|
"completions/mean_terminated_length": 734.625,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 1.001370683312416,
|
|
"epoch": 0.21485260770975056,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 7.857142857142856e-07,
|
|
"loss": 0.0238,
|
|
"num_tokens": 22368140.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 379,
|
|
"step_time": 44.70761434081942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1632.0,
|
|
"completions/max_terminated_length": 1632.0,
|
|
"completions/mean_length": 736.71875,
|
|
"completions/mean_terminated_length": 736.71875,
|
|
"completions/min_length": 172.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"entropy": 0.8948388043791056,
|
|
"epoch": 0.21541950113378686,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 7.851473922902494e-07,
|
|
"loss": -0.0129,
|
|
"num_tokens": 22430162.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 380,
|
|
"step_time": 56.97926906310022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1786.0,
|
|
"completions/max_terminated_length": 1786.0,
|
|
"completions/mean_length": 808.390625,
|
|
"completions/mean_terminated_length": 808.390625,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.6405720394104719,
|
|
"epoch": 0.21598639455782312,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 7.845804988662131e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 22492107.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 381,
|
|
"step_time": 62.248050440102816
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2058.0,
|
|
"completions/max_terminated_length": 2058.0,
|
|
"completions/mean_length": 756.203125,
|
|
"completions/mean_terminated_length": 756.203125,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.7315251268446445,
|
|
"epoch": 0.2165532879818594,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.369140625,
|
|
"learning_rate": 7.840136054421769e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 22550776.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 382,
|
|
"step_time": 71.68382942955941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1127.0,
|
|
"completions/max_terminated_length": 1127.0,
|
|
"completions/mean_length": 559.9375,
|
|
"completions/mean_terminated_length": 559.9375,
|
|
"completions/min_length": 210.0,
|
|
"completions/min_terminated_length": 210.0,
|
|
"entropy": 0.8677006028592587,
|
|
"epoch": 0.2171201814058957,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7578125,
|
|
"learning_rate": 7.834467120181405e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 22597796.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 383,
|
|
"step_time": 38.73852313775569
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1409.0,
|
|
"completions/max_terminated_length": 1409.0,
|
|
"completions/mean_length": 873.34375,
|
|
"completions/mean_terminated_length": 873.34375,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 1.0035483818501234,
|
|
"epoch": 0.21768707482993196,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.185546875,
|
|
"learning_rate": 7.828798185941043e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 22666370.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.33592739701271057,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 384,
|
|
"step_time": 48.65996228437871
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 731.875,
|
|
"completions/mean_terminated_length": 678.4762573242188,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 1.0197479017078876,
|
|
"epoch": 0.21825396825396826,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 7.82312925170068e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 22723634.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 385,
|
|
"step_time": 170.17113096639514
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1078.0,
|
|
"completions/max_terminated_length": 1078.0,
|
|
"completions/mean_length": 688.109375,
|
|
"completions/mean_terminated_length": 688.109375,
|
|
"completions/min_length": 276.0,
|
|
"completions/min_terminated_length": 276.0,
|
|
"entropy": 0.9092415310442448,
|
|
"epoch": 0.21882086167800455,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6015625,
|
|
"learning_rate": 7.817460317460318e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 22782593.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 386,
|
|
"step_time": 39.27197832521051
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1406.0,
|
|
"completions/max_terminated_length": 1406.0,
|
|
"completions/mean_length": 674.8125,
|
|
"completions/mean_terminated_length": 674.8125,
|
|
"completions/min_length": 194.0,
|
|
"completions/min_terminated_length": 194.0,
|
|
"entropy": 0.8119625337421894,
|
|
"epoch": 0.2193877551020408,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 7.811791383219954e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 22837981.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 387,
|
|
"step_time": 49.66723500657827
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1304.0,
|
|
"completions/max_terminated_length": 1304.0,
|
|
"completions/mean_length": 739.5625,
|
|
"completions/mean_terminated_length": 739.5625,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.7483879532665014,
|
|
"epoch": 0.2199546485260771,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 7.806122448979591e-07,
|
|
"loss": -0.0148,
|
|
"num_tokens": 22895241.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 388,
|
|
"step_time": 46.12076739780605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1423.0,
|
|
"completions/max_terminated_length": 1423.0,
|
|
"completions/mean_length": 783.90625,
|
|
"completions/mean_terminated_length": 783.90625,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.9556456692516804,
|
|
"epoch": 0.2205215419501134,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.24609375,
|
|
"learning_rate": 7.800453514739229e-07,
|
|
"loss": -0.008,
|
|
"num_tokens": 22957635.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.41277948021888733,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 389,
|
|
"step_time": 49.99157304875553
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1285.0,
|
|
"completions/max_terminated_length": 1285.0,
|
|
"completions/mean_length": 772.96875,
|
|
"completions/mean_terminated_length": 772.96875,
|
|
"completions/min_length": 433.0,
|
|
"completions/min_terminated_length": 433.0,
|
|
"entropy": 0.7684030756354332,
|
|
"epoch": 0.22108843537414966,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 7.794784580498865e-07,
|
|
"loss": 0.0129,
|
|
"num_tokens": 23017057.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 390,
|
|
"step_time": 43.90013815369457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1277.0,
|
|
"completions/max_terminated_length": 1277.0,
|
|
"completions/mean_length": 611.40625,
|
|
"completions/mean_terminated_length": 611.40625,
|
|
"completions/min_length": 303.0,
|
|
"completions/min_terminated_length": 303.0,
|
|
"entropy": 0.855836596339941,
|
|
"epoch": 0.22165532879818595,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 7.789115646258503e-07,
|
|
"loss": 0.0222,
|
|
"num_tokens": 23066291.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 391,
|
|
"step_time": 42.97751526068896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1177.0,
|
|
"completions/max_terminated_length": 1177.0,
|
|
"completions/mean_length": 690.765625,
|
|
"completions/mean_terminated_length": 690.765625,
|
|
"completions/min_length": 336.0,
|
|
"completions/min_terminated_length": 336.0,
|
|
"entropy": 0.7666698209941387,
|
|
"epoch": 0.2222222222222222,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 7.78344671201814e-07,
|
|
"loss": 0.0101,
|
|
"num_tokens": 23120540.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6900655627250671,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 392,
|
|
"step_time": 40.13614541944116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1224.0,
|
|
"completions/max_terminated_length": 1224.0,
|
|
"completions/mean_length": 711.40625,
|
|
"completions/mean_terminated_length": 711.40625,
|
|
"completions/min_length": 350.0,
|
|
"completions/min_terminated_length": 350.0,
|
|
"entropy": 0.9489793553948402,
|
|
"epoch": 0.2227891156462585,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.494140625,
|
|
"learning_rate": 7.777777777777778e-07,
|
|
"loss": -0.0162,
|
|
"num_tokens": 23175838.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 393,
|
|
"step_time": 42.049905179068446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1489.0,
|
|
"completions/max_terminated_length": 1489.0,
|
|
"completions/mean_length": 808.875,
|
|
"completions/mean_terminated_length": 808.875,
|
|
"completions/min_length": 405.0,
|
|
"completions/min_terminated_length": 405.0,
|
|
"entropy": 0.784135427325964,
|
|
"epoch": 0.2233560090702948,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 7.772108843537415e-07,
|
|
"loss": 0.0124,
|
|
"num_tokens": 23239006.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 394,
|
|
"step_time": 50.6201617596671
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1621.0,
|
|
"completions/max_terminated_length": 1621.0,
|
|
"completions/mean_length": 824.5,
|
|
"completions/mean_terminated_length": 824.5,
|
|
"completions/min_length": 213.0,
|
|
"completions/min_terminated_length": 213.0,
|
|
"entropy": 0.6317049227654934,
|
|
"epoch": 0.22392290249433106,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 7.766439909297052e-07,
|
|
"loss": 0.01,
|
|
"num_tokens": 23302790.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 395,
|
|
"step_time": 56.24066119547933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1463.0,
|
|
"completions/max_terminated_length": 1463.0,
|
|
"completions/mean_length": 704.796875,
|
|
"completions/mean_terminated_length": 704.796875,
|
|
"completions/min_length": 168.0,
|
|
"completions/min_terminated_length": 168.0,
|
|
"entropy": 0.9040312692523003,
|
|
"epoch": 0.22448979591836735,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 7.760770975056688e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 23358881.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.6950790882110596,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 396,
|
|
"step_time": 50.88810185715556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1553.0,
|
|
"completions/max_terminated_length": 1553.0,
|
|
"completions/mean_length": 616.921875,
|
|
"completions/mean_terminated_length": 616.921875,
|
|
"completions/min_length": 24.0,
|
|
"completions/min_terminated_length": 24.0,
|
|
"entropy": 0.7749507799744606,
|
|
"epoch": 0.22505668934240364,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 7.755102040816326e-07,
|
|
"loss": -0.0083,
|
|
"num_tokens": 23407412.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5589615106582642,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 397,
|
|
"step_time": 52.4311613375321
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1303.0,
|
|
"completions/max_terminated_length": 1303.0,
|
|
"completions/mean_length": 745.265625,
|
|
"completions/mean_terminated_length": 745.265625,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 1.037154708057642,
|
|
"epoch": 0.2256235827664399,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 7.749433106575963e-07,
|
|
"loss": -0.005,
|
|
"num_tokens": 23467309.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 398,
|
|
"step_time": 44.10899902880192
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1357.0,
|
|
"completions/max_terminated_length": 1357.0,
|
|
"completions/mean_length": 698.6875,
|
|
"completions/mean_terminated_length": 698.6875,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.9808155223727226,
|
|
"epoch": 0.2261904761904762,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 7.743764172335601e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 23522145.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7676090598106384,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 399,
|
|
"step_time": 46.3060958776623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1181.0,
|
|
"completions/max_terminated_length": 1181.0,
|
|
"completions/mean_length": 683.59375,
|
|
"completions/mean_terminated_length": 683.59375,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"entropy": 0.6724383383989334,
|
|
"epoch": 0.22675736961451248,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 7.738095238095238e-07,
|
|
"loss": 0.0142,
|
|
"num_tokens": 23574207.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6986381411552429,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.09375,
|
|
"rewards/format_reward_func/std": 0.19669894874095917,
|
|
"step": 400,
|
|
"step_time": 40.15125029999763
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1227.0,
|
|
"completions/max_terminated_length": 1227.0,
|
|
"completions/mean_length": 771.9375,
|
|
"completions/mean_terminated_length": 771.9375,
|
|
"completions/min_length": 370.0,
|
|
"completions/min_terminated_length": 370.0,
|
|
"entropy": 0.8765030633658171,
|
|
"epoch": 0.22732426303854875,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"learning_rate": 7.732426303854876e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 23635267.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 401,
|
|
"step_time": 42.207340964116156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2591.0,
|
|
"completions/max_terminated_length": 2591.0,
|
|
"completions/mean_length": 801.46875,
|
|
"completions/mean_terminated_length": 801.46875,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.8337623011320829,
|
|
"epoch": 0.22789115646258504,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 7.726757369614512e-07,
|
|
"loss": 0.0211,
|
|
"num_tokens": 23699601.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 402,
|
|
"step_time": 93.77474882733077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1147.0,
|
|
"completions/max_terminated_length": 1147.0,
|
|
"completions/mean_length": 650.578125,
|
|
"completions/mean_terminated_length": 650.578125,
|
|
"completions/min_length": 189.0,
|
|
"completions/min_terminated_length": 189.0,
|
|
"entropy": 0.7326683849096298,
|
|
"epoch": 0.22845804988662133,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 7.721088435374149e-07,
|
|
"loss": -0.0558,
|
|
"num_tokens": 23750414.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7185450792312622,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 403,
|
|
"step_time": 38.47740055248141
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1746.0,
|
|
"completions/max_terminated_length": 1746.0,
|
|
"completions/mean_length": 761.71875,
|
|
"completions/mean_terminated_length": 761.71875,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 0.6371756438165903,
|
|
"epoch": 0.2290249433106576,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 7.715419501133786e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 23810716.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 404,
|
|
"step_time": 61.017441344447434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1338.0,
|
|
"completions/max_terminated_length": 1338.0,
|
|
"completions/mean_length": 607.6875,
|
|
"completions/mean_terminated_length": 607.6875,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 1.2175236158072948,
|
|
"epoch": 0.22959183673469388,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 7.709750566893424e-07,
|
|
"loss": -0.0044,
|
|
"num_tokens": 23861824.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 405,
|
|
"step_time": 45.624282780103385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1400.0,
|
|
"completions/max_terminated_length": 1400.0,
|
|
"completions/mean_length": 785.640625,
|
|
"completions/mean_terminated_length": 785.640625,
|
|
"completions/min_length": 238.0,
|
|
"completions/min_terminated_length": 238.0,
|
|
"entropy": 0.659934913739562,
|
|
"epoch": 0.23015873015873015,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 7.704081632653061e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 23920985.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 406,
|
|
"step_time": 48.06194389145821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1513.0,
|
|
"completions/max_terminated_length": 1513.0,
|
|
"completions/mean_length": 768.203125,
|
|
"completions/mean_terminated_length": 768.203125,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.7393356766551733,
|
|
"epoch": 0.23072562358276644,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"learning_rate": 7.698412698412699e-07,
|
|
"loss": 0.0011,
|
|
"num_tokens": 23981334.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 407,
|
|
"step_time": 51.93799236509949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1655.0,
|
|
"completions/max_terminated_length": 1655.0,
|
|
"completions/mean_length": 791.578125,
|
|
"completions/mean_terminated_length": 791.578125,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.6468864362686872,
|
|
"epoch": 0.23129251700680273,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 7.692743764172335e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 24041171.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 408,
|
|
"step_time": 56.41847629472613
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 863.390625,
|
|
"completions/mean_terminated_length": 863.390625,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.6300061997026205,
|
|
"epoch": 0.231859410430839,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.333984375,
|
|
"learning_rate": 7.687074829931972e-07,
|
|
"loss": 0.0041,
|
|
"num_tokens": 24107764.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 409,
|
|
"step_time": 49.124212984927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1315.0,
|
|
"completions/max_terminated_length": 1315.0,
|
|
"completions/mean_length": 682.546875,
|
|
"completions/mean_terminated_length": 682.546875,
|
|
"completions/min_length": 207.0,
|
|
"completions/min_terminated_length": 207.0,
|
|
"entropy": 0.8103759475052357,
|
|
"epoch": 0.23242630385487528,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.76953125,
|
|
"learning_rate": 7.68140589569161e-07,
|
|
"loss": 0.0294,
|
|
"num_tokens": 24164863.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 410,
|
|
"step_time": 47.03771211300045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1504.0,
|
|
"completions/max_terminated_length": 1504.0,
|
|
"completions/mean_length": 867.53125,
|
|
"completions/mean_terminated_length": 867.53125,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.6705813817679882,
|
|
"epoch": 0.23299319727891157,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 7.675736961451246e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 24231129.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 411,
|
|
"step_time": 51.962798967026174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1661.0,
|
|
"completions/max_terminated_length": 1661.0,
|
|
"completions/mean_length": 765.484375,
|
|
"completions/mean_terminated_length": 765.484375,
|
|
"completions/min_length": 289.0,
|
|
"completions/min_terminated_length": 289.0,
|
|
"entropy": 0.7321729604154825,
|
|
"epoch": 0.23356009070294784,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 7.670068027210884e-07,
|
|
"loss": 0.027,
|
|
"num_tokens": 24289264.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7949103116989136,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 412,
|
|
"step_time": 56.802216346375644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1495.0,
|
|
"completions/max_terminated_length": 1495.0,
|
|
"completions/mean_length": 775.515625,
|
|
"completions/mean_terminated_length": 775.515625,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.8015652969479561,
|
|
"epoch": 0.23412698412698413,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 7.664399092970521e-07,
|
|
"loss": 0.0053,
|
|
"num_tokens": 24347777.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.6259417533874512,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.1015625,
|
|
"rewards/format_reward_func/std": 0.20275264978408813,
|
|
"step": 413,
|
|
"step_time": 50.851755070500076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 812.984375,
|
|
"completions/mean_terminated_length": 812.984375,
|
|
"completions/min_length": 350.0,
|
|
"completions/min_terminated_length": 350.0,
|
|
"entropy": 0.5214714016765356,
|
|
"epoch": 0.23469387755102042,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.455078125,
|
|
"learning_rate": 7.658730158730159e-07,
|
|
"loss": 0.0112,
|
|
"num_tokens": 24409896.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7531349658966064,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 414,
|
|
"step_time": 44.565095531754196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1319.0,
|
|
"completions/max_terminated_length": 1319.0,
|
|
"completions/mean_length": 730.359375,
|
|
"completions/mean_terminated_length": 730.359375,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.8997184168547392,
|
|
"epoch": 0.23526077097505668,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 7.653061224489795e-07,
|
|
"loss": 0.012,
|
|
"num_tokens": 24467431.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 415,
|
|
"step_time": 45.2768179923296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/max_terminated_length": 1396.0,
|
|
"completions/mean_length": 712.03125,
|
|
"completions/mean_terminated_length": 712.03125,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"entropy": 0.8183233700692654,
|
|
"epoch": 0.23582766439909297,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.380859375,
|
|
"learning_rate": 7.647392290249433e-07,
|
|
"loss": -0.0093,
|
|
"num_tokens": 24525633.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6529466509819031,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 416,
|
|
"step_time": 48.1918881079182
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1402.0,
|
|
"completions/max_terminated_length": 1402.0,
|
|
"completions/mean_length": 726.4375,
|
|
"completions/mean_terminated_length": 726.4375,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 0.8129748981446028,
|
|
"epoch": 0.23639455782312926,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.296875,
|
|
"learning_rate": 7.64172335600907e-07,
|
|
"loss": -0.0106,
|
|
"num_tokens": 24584173.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 417,
|
|
"step_time": 48.32587588019669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1301.0,
|
|
"completions/max_terminated_length": 1301.0,
|
|
"completions/mean_length": 623.875,
|
|
"completions/mean_terminated_length": 623.875,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 0.9808018021285534,
|
|
"epoch": 0.23696145124716553,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.453125,
|
|
"learning_rate": 7.636054421768708e-07,
|
|
"loss": -0.0203,
|
|
"num_tokens": 24635549.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 418,
|
|
"step_time": 44.927674242295325
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1632.0,
|
|
"completions/max_terminated_length": 1632.0,
|
|
"completions/mean_length": 732.84375,
|
|
"completions/mean_terminated_length": 732.84375,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.6691880654543638,
|
|
"epoch": 0.23752834467120182,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 7.630385487528344e-07,
|
|
"loss": -0.0094,
|
|
"num_tokens": 24692691.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.47709178924560547,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 419,
|
|
"step_time": 55.825158819556236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1570.0,
|
|
"completions/max_terminated_length": 1570.0,
|
|
"completions/mean_length": 767.515625,
|
|
"completions/mean_terminated_length": 767.515625,
|
|
"completions/min_length": 414.0,
|
|
"completions/min_terminated_length": 414.0,
|
|
"entropy": 0.8011331092566252,
|
|
"epoch": 0.23809523809523808,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"learning_rate": 7.624716553287982e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 24752844.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 420,
|
|
"step_time": 53.255850152112544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1497.0,
|
|
"completions/max_terminated_length": 1497.0,
|
|
"completions/mean_length": 771.734375,
|
|
"completions/mean_terminated_length": 771.734375,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.8095507547259331,
|
|
"epoch": 0.23866213151927437,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.25,
|
|
"learning_rate": 7.619047619047618e-07,
|
|
"loss": -0.0171,
|
|
"num_tokens": 24812779.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 421,
|
|
"step_time": 50.773611281998456
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1176.0,
|
|
"completions/max_terminated_length": 1176.0,
|
|
"completions/mean_length": 681.765625,
|
|
"completions/mean_terminated_length": 681.765625,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.7349203620105982,
|
|
"epoch": 0.23922902494331066,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 7.613378684807256e-07,
|
|
"loss": -0.0037,
|
|
"num_tokens": 24865948.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7007648944854736,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 422,
|
|
"step_time": 39.631040232256055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1510.0,
|
|
"completions/max_terminated_length": 1510.0,
|
|
"completions/mean_length": 722.5,
|
|
"completions/mean_terminated_length": 722.5,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.7520661912858486,
|
|
"epoch": 0.23979591836734693,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 7.607709750566893e-07,
|
|
"loss": -0.0196,
|
|
"num_tokens": 24922852.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 423,
|
|
"step_time": 51.616779503412545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1843.0,
|
|
"completions/max_terminated_length": 1843.0,
|
|
"completions/mean_length": 719.765625,
|
|
"completions/mean_terminated_length": 719.765625,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.7724943663924932,
|
|
"epoch": 0.24036281179138322,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 7.602040816326531e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 24979133.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 424,
|
|
"step_time": 62.50873214751482
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1671.0,
|
|
"completions/max_terminated_length": 1671.0,
|
|
"completions/mean_length": 736.859375,
|
|
"completions/mean_terminated_length": 736.859375,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 0.8050760552287102,
|
|
"epoch": 0.2409297052154195,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 7.596371882086168e-07,
|
|
"loss": -0.027,
|
|
"num_tokens": 25037660.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 425,
|
|
"step_time": 57.295719363726676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1315.0,
|
|
"completions/max_terminated_length": 1315.0,
|
|
"completions/mean_length": 641.71875,
|
|
"completions/mean_terminated_length": 641.71875,
|
|
"completions/min_length": 271.0,
|
|
"completions/min_terminated_length": 271.0,
|
|
"entropy": 1.2077321372926235,
|
|
"epoch": 0.24149659863945577,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 7.590702947845806e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 25092130.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 426,
|
|
"step_time": 46.93154688831419
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1814.0,
|
|
"completions/max_terminated_length": 1814.0,
|
|
"completions/mean_length": 727.703125,
|
|
"completions/mean_terminated_length": 727.703125,
|
|
"completions/min_length": 254.0,
|
|
"completions/min_terminated_length": 254.0,
|
|
"entropy": 0.9418045226484537,
|
|
"epoch": 0.24206349206349206,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34375,
|
|
"learning_rate": 7.585034013605441e-07,
|
|
"loss": 0.0003,
|
|
"num_tokens": 25150535.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4787135720252991,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 427,
|
|
"step_time": 63.36056375131011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1510.0,
|
|
"completions/max_terminated_length": 1510.0,
|
|
"completions/mean_length": 753.890625,
|
|
"completions/mean_terminated_length": 753.890625,
|
|
"completions/min_length": 206.0,
|
|
"completions/min_terminated_length": 206.0,
|
|
"entropy": 0.6319617219269276,
|
|
"epoch": 0.24263038548752835,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34375,
|
|
"learning_rate": 7.579365079365078e-07,
|
|
"loss": -0.0097,
|
|
"num_tokens": 25209680.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 428,
|
|
"step_time": 52.11038209684193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1242.0,
|
|
"completions/max_terminated_length": 1242.0,
|
|
"completions/mean_length": 728.234375,
|
|
"completions/mean_terminated_length": 728.234375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.7340629417449236,
|
|
"epoch": 0.24319727891156462,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.275390625,
|
|
"learning_rate": 7.573696145124716e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 25265807.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 429,
|
|
"step_time": 41.60768271610141
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1277.0,
|
|
"completions/max_terminated_length": 1277.0,
|
|
"completions/mean_length": 741.234375,
|
|
"completions/mean_terminated_length": 741.234375,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.8030328117311001,
|
|
"epoch": 0.2437641723356009,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 7.568027210884353e-07,
|
|
"loss": -0.0157,
|
|
"num_tokens": 25325862.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 430,
|
|
"step_time": 43.42780246119946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 676.265625,
|
|
"completions/mean_terminated_length": 676.265625,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.8896284252405167,
|
|
"epoch": 0.2443310657596372,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 7.562358276643991e-07,
|
|
"loss": -0.0126,
|
|
"num_tokens": 25378679.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8042545318603516,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 431,
|
|
"step_time": 46.09310553781688
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 705.390625,
|
|
"completions/mean_terminated_length": 705.390625,
|
|
"completions/min_length": 258.0,
|
|
"completions/min_terminated_length": 258.0,
|
|
"entropy": 0.7383911740034819,
|
|
"epoch": 0.24489795918367346,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 7.556689342403628e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 25433072.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 432,
|
|
"step_time": 49.770350963808596
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1634.0,
|
|
"completions/max_terminated_length": 1634.0,
|
|
"completions/mean_length": 823.46875,
|
|
"completions/mean_terminated_length": 823.46875,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.6049559805542231,
|
|
"epoch": 0.24546485260770975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 7.551020408163265e-07,
|
|
"loss": 0.0026,
|
|
"num_tokens": 25497070.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 433,
|
|
"step_time": 57.097911885008216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1264.0,
|
|
"completions/max_terminated_length": 1264.0,
|
|
"completions/mean_length": 715.09375,
|
|
"completions/mean_terminated_length": 715.09375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 1.0062689557671547,
|
|
"epoch": 0.24603174603174602,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 7.545351473922901e-07,
|
|
"loss": -0.0195,
|
|
"num_tokens": 25562236.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.6784212589263916,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 434,
|
|
"step_time": 46.80837841890752
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1566.0,
|
|
"completions/max_terminated_length": 1566.0,
|
|
"completions/mean_length": 745.890625,
|
|
"completions/mean_terminated_length": 745.890625,
|
|
"completions/min_length": 332.0,
|
|
"completions/min_terminated_length": 332.0,
|
|
"entropy": 0.8993903696537018,
|
|
"epoch": 0.2465986394557823,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6484375,
|
|
"learning_rate": 7.539682539682539e-07,
|
|
"loss": 0.005,
|
|
"num_tokens": 25619405.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 435,
|
|
"step_time": 53.25824845582247
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1661.0,
|
|
"completions/max_terminated_length": 1661.0,
|
|
"completions/mean_length": 792.34375,
|
|
"completions/mean_terminated_length": 792.34375,
|
|
"completions/min_length": 216.0,
|
|
"completions/min_terminated_length": 216.0,
|
|
"entropy": 0.8956113327294588,
|
|
"epoch": 0.2471655328798186,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.32421875,
|
|
"learning_rate": 7.534013605442176e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 25679243.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 436,
|
|
"step_time": 56.02351733762771
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1229.0,
|
|
"completions/max_terminated_length": 1229.0,
|
|
"completions/mean_length": 709.609375,
|
|
"completions/mean_terminated_length": 709.609375,
|
|
"completions/min_length": 325.0,
|
|
"completions/min_terminated_length": 325.0,
|
|
"entropy": 0.8032096195966005,
|
|
"epoch": 0.24773242630385486,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 7.528344671201814e-07,
|
|
"loss": 0.0304,
|
|
"num_tokens": 25734386.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 437,
|
|
"step_time": 42.16178113408387
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1268.0,
|
|
"completions/max_terminated_length": 1268.0,
|
|
"completions/mean_length": 850.6875,
|
|
"completions/mean_terminated_length": 850.6875,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.6397589351981878,
|
|
"epoch": 0.24829931972789115,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.43359375,
|
|
"learning_rate": 7.522675736961451e-07,
|
|
"loss": -0.0119,
|
|
"num_tokens": 25798718.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7263124585151672,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 438,
|
|
"step_time": 42.7511393353343
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1236.0,
|
|
"completions/max_terminated_length": 1236.0,
|
|
"completions/mean_length": 618.953125,
|
|
"completions/mean_terminated_length": 618.953125,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.6891420613974333,
|
|
"epoch": 0.24886621315192745,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 7.517006802721088e-07,
|
|
"loss": 0.0294,
|
|
"num_tokens": 25849811.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 439,
|
|
"step_time": 42.715612224303186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1131.0,
|
|
"completions/max_terminated_length": 1131.0,
|
|
"completions/mean_length": 618.09375,
|
|
"completions/mean_terminated_length": 618.09375,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.9189383015036583,
|
|
"epoch": 0.2494331065759637,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"learning_rate": 7.511337868480725e-07,
|
|
"loss": 0.0251,
|
|
"num_tokens": 25899305.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 440,
|
|
"step_time": 39.003290640190244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1666.0,
|
|
"completions/max_terminated_length": 1666.0,
|
|
"completions/mean_length": 823.5,
|
|
"completions/mean_terminated_length": 823.5,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.7929299920797348,
|
|
"epoch": 0.25,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.255859375,
|
|
"learning_rate": 7.505668934240363e-07,
|
|
"loss": 0.0022,
|
|
"num_tokens": 25963049.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 441,
|
|
"step_time": 58.207637903280556
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2147.0,
|
|
"completions/max_terminated_length": 2147.0,
|
|
"completions/mean_length": 761.40625,
|
|
"completions/mean_terminated_length": 761.40625,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.7989441230893135,
|
|
"epoch": 0.25056689342403626,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 7.5e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 26020443.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6544642448425293,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 442,
|
|
"step_time": 74.96575433388352
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1505.0,
|
|
"completions/max_terminated_length": 1505.0,
|
|
"completions/mean_length": 830.8125,
|
|
"completions/mean_terminated_length": 830.8125,
|
|
"completions/min_length": 457.0,
|
|
"completions/min_terminated_length": 457.0,
|
|
"entropy": 0.8002884350717068,
|
|
"epoch": 0.2511337868480726,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 7.494331065759637e-07,
|
|
"loss": 0.0268,
|
|
"num_tokens": 26087447.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5261042714118958,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 443,
|
|
"step_time": 52.967568409629166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 780.5625,
|
|
"completions/mean_terminated_length": 780.5625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8361520990729332,
|
|
"epoch": 0.25170068027210885,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.474609375,
|
|
"learning_rate": 7.488662131519274e-07,
|
|
"loss": -0.0088,
|
|
"num_tokens": 26147027.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 444,
|
|
"step_time": 50.80671252217144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1494.0,
|
|
"completions/max_terminated_length": 1494.0,
|
|
"completions/mean_length": 736.25,
|
|
"completions/mean_terminated_length": 736.25,
|
|
"completions/min_length": 212.0,
|
|
"completions/min_terminated_length": 212.0,
|
|
"entropy": 0.7818868961185217,
|
|
"epoch": 0.2522675736961451,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.43359375,
|
|
"learning_rate": 7.482993197278912e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 26206291.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 445,
|
|
"step_time": 53.251479097642004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1633.0,
|
|
"completions/max_terminated_length": 1633.0,
|
|
"completions/mean_length": 726.9375,
|
|
"completions/mean_terminated_length": 726.9375,
|
|
"completions/min_length": 369.0,
|
|
"completions/min_terminated_length": 369.0,
|
|
"entropy": 0.8610232416540384,
|
|
"epoch": 0.2528344671201814,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7421875,
|
|
"learning_rate": 7.477324263038548e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 26262095.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 446,
|
|
"step_time": 55.98789916560054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1180.0,
|
|
"completions/max_terminated_length": 1180.0,
|
|
"completions/mean_length": 606.203125,
|
|
"completions/mean_terminated_length": 606.203125,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.7205122336745262,
|
|
"epoch": 0.2534013605442177,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 7.471655328798186e-07,
|
|
"loss": -0.036,
|
|
"num_tokens": 26314428.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.7943251132965088,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 447,
|
|
"step_time": 41.76088255736977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 799.953125,
|
|
"completions/mean_terminated_length": 799.953125,
|
|
"completions/min_length": 454.0,
|
|
"completions/min_terminated_length": 454.0,
|
|
"entropy": 0.833087008446455,
|
|
"epoch": 0.25396825396825395,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 7.465986394557823e-07,
|
|
"loss": 0.0343,
|
|
"num_tokens": 26375929.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 448,
|
|
"step_time": 53.082594824023545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1251.0,
|
|
"completions/max_terminated_length": 1251.0,
|
|
"completions/mean_length": 723.78125,
|
|
"completions/mean_terminated_length": 723.78125,
|
|
"completions/min_length": 257.0,
|
|
"completions/min_terminated_length": 257.0,
|
|
"entropy": 0.8315397277474403,
|
|
"epoch": 0.2545351473922903,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 7.46031746031746e-07,
|
|
"loss": -0.027,
|
|
"num_tokens": 26433067.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 449,
|
|
"step_time": 42.740751163102686
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1752.0,
|
|
"completions/max_terminated_length": 1752.0,
|
|
"completions/mean_length": 851.140625,
|
|
"completions/mean_terminated_length": 851.140625,
|
|
"completions/min_length": 2.0,
|
|
"completions/min_terminated_length": 2.0,
|
|
"entropy": 0.6886486634612083,
|
|
"epoch": 0.25510204081632654,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2578125,
|
|
"learning_rate": 7.454648526077098e-07,
|
|
"loss": 0.0061,
|
|
"num_tokens": 26498692.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4542567729949951,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 450,
|
|
"step_time": 60.04871977586299
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1121.0,
|
|
"completions/max_terminated_length": 1121.0,
|
|
"completions/mean_length": 693.03125,
|
|
"completions/mean_terminated_length": 693.03125,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 0.6884783115237951,
|
|
"epoch": 0.2556689342403628,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 7.448979591836734e-07,
|
|
"loss": 0.0503,
|
|
"num_tokens": 26552750.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 451,
|
|
"step_time": 38.398378430865705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2723.0,
|
|
"completions/max_terminated_length": 2723.0,
|
|
"completions/mean_length": 824.140625,
|
|
"completions/mean_terminated_length": 824.140625,
|
|
"completions/min_length": 187.0,
|
|
"completions/min_terminated_length": 187.0,
|
|
"entropy": 0.5746772699058056,
|
|
"epoch": 0.2562358276643991,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 7.443310657596371e-07,
|
|
"loss": -0.016,
|
|
"num_tokens": 26617207.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 452,
|
|
"step_time": 100.30535710975528
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 735.796875,
|
|
"completions/mean_terminated_length": 735.796875,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.822858165949583,
|
|
"epoch": 0.2568027210884354,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 7.437641723356008e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 26674298.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 453,
|
|
"step_time": 51.56850883085281
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1242.0,
|
|
"completions/max_terminated_length": 1242.0,
|
|
"completions/mean_length": 722.234375,
|
|
"completions/mean_terminated_length": 722.234375,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.8533442914485931,
|
|
"epoch": 0.25736961451247165,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 7.431972789115646e-07,
|
|
"loss": -0.0024,
|
|
"num_tokens": 26730257.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.49344661831855774,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 454,
|
|
"step_time": 41.675769814290106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1861.0,
|
|
"completions/max_terminated_length": 1861.0,
|
|
"completions/mean_length": 818.953125,
|
|
"completions/mean_terminated_length": 818.953125,
|
|
"completions/min_length": 369.0,
|
|
"completions/min_terminated_length": 369.0,
|
|
"entropy": 0.8123147170990705,
|
|
"epoch": 0.25793650793650796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 7.426303854875283e-07,
|
|
"loss": 0.0174,
|
|
"num_tokens": 26794046.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 455,
|
|
"step_time": 64.55787387117743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1733.0,
|
|
"completions/max_terminated_length": 1733.0,
|
|
"completions/mean_length": 788.671875,
|
|
"completions/mean_terminated_length": 788.671875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.8716447334736586,
|
|
"epoch": 0.2585034013605442,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"learning_rate": 7.420634920634921e-07,
|
|
"loss": -0.0143,
|
|
"num_tokens": 26853513.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 456,
|
|
"step_time": 59.41806235536933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 806.65625,
|
|
"completions/mean_terminated_length": 806.65625,
|
|
"completions/min_length": 366.0,
|
|
"completions/min_terminated_length": 366.0,
|
|
"entropy": 0.8578362837433815,
|
|
"epoch": 0.2590702947845805,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.287109375,
|
|
"learning_rate": 7.414965986394558e-07,
|
|
"loss": -0.0013,
|
|
"num_tokens": 26916163.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 457,
|
|
"step_time": 47.249078824184835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1290.0,
|
|
"completions/max_terminated_length": 1290.0,
|
|
"completions/mean_length": 809.015625,
|
|
"completions/mean_terminated_length": 809.015625,
|
|
"completions/min_length": 433.0,
|
|
"completions/min_terminated_length": 433.0,
|
|
"entropy": 0.8555043283849955,
|
|
"epoch": 0.25963718820861675,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 7.409297052154194e-07,
|
|
"loss": -0.0103,
|
|
"num_tokens": 26980372.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 458,
|
|
"step_time": 45.47941083367914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1132.0,
|
|
"completions/max_terminated_length": 1132.0,
|
|
"completions/mean_length": 731.34375,
|
|
"completions/mean_terminated_length": 731.34375,
|
|
"completions/min_length": 476.0,
|
|
"completions/min_terminated_length": 476.0,
|
|
"entropy": 0.8994519170373678,
|
|
"epoch": 0.2602040816326531,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 7.403628117913831e-07,
|
|
"loss": 0.0145,
|
|
"num_tokens": 27037674.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5739033818244934,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 459,
|
|
"step_time": 39.012475188821554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2292.0,
|
|
"completions/max_terminated_length": 2292.0,
|
|
"completions/mean_length": 607.640625,
|
|
"completions/mean_terminated_length": 607.640625,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.6151527259498835,
|
|
"epoch": 0.26077097505668934,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.609375,
|
|
"learning_rate": 7.397959183673469e-07,
|
|
"loss": -0.0369,
|
|
"num_tokens": 27089427.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 460,
|
|
"step_time": 82.16680842358619
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/max_terminated_length": 1202.0,
|
|
"completions/mean_length": 692.390625,
|
|
"completions/mean_terminated_length": 692.390625,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.6963042225688696,
|
|
"epoch": 0.2613378684807256,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 7.392290249433106e-07,
|
|
"loss": -0.0166,
|
|
"num_tokens": 27142900.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 461,
|
|
"step_time": 40.6895647700876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1794.0,
|
|
"completions/max_terminated_length": 1794.0,
|
|
"completions/mean_length": 783.640625,
|
|
"completions/mean_terminated_length": 783.640625,
|
|
"completions/min_length": 263.0,
|
|
"completions/min_terminated_length": 263.0,
|
|
"entropy": 0.7272162567824125,
|
|
"epoch": 0.2619047619047619,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.38671875,
|
|
"learning_rate": 7.386621315192744e-07,
|
|
"loss": -0.0249,
|
|
"num_tokens": 27203557.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4734388589859009,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 462,
|
|
"step_time": 60.710756020620465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1483.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 762.0625,
|
|
"completions/mean_terminated_length": 762.0625,
|
|
"completions/min_length": 312.0,
|
|
"completions/min_terminated_length": 312.0,
|
|
"entropy": 0.8299924489110708,
|
|
"epoch": 0.2624716553287982,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 7.380952380952381e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 27262929.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 463,
|
|
"step_time": 50.03462992142886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1250.0,
|
|
"completions/max_terminated_length": 1250.0,
|
|
"completions/mean_length": 708.78125,
|
|
"completions/mean_terminated_length": 708.78125,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.8047873191535473,
|
|
"epoch": 0.26303854875283444,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"learning_rate": 7.375283446712018e-07,
|
|
"loss": 0.0096,
|
|
"num_tokens": 27317939.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 464,
|
|
"step_time": 42.26102858968079
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 829.0,
|
|
"completions/mean_terminated_length": 829.0,
|
|
"completions/min_length": 457.0,
|
|
"completions/min_terminated_length": 457.0,
|
|
"entropy": 0.7290856074541807,
|
|
"epoch": 0.26360544217687076,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.333984375,
|
|
"learning_rate": 7.369614512471655e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 27382139.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 465,
|
|
"step_time": 48.702748977579176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1627.0,
|
|
"completions/mean_length": 806.71875,
|
|
"completions/mean_terminated_length": 754.5079956054688,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.7747468892484903,
|
|
"epoch": 0.264172335600907,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"learning_rate": 7.363945578231292e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 27444345.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 466,
|
|
"step_time": 167.59687008708715
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1456.0,
|
|
"completions/max_terminated_length": 1456.0,
|
|
"completions/mean_length": 695.59375,
|
|
"completions/mean_terminated_length": 695.59375,
|
|
"completions/min_length": 179.0,
|
|
"completions/min_terminated_length": 179.0,
|
|
"entropy": 0.8323264606297016,
|
|
"epoch": 0.2647392290249433,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.66796875,
|
|
"learning_rate": 7.358276643990929e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 27501527.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.71200031042099,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 467,
|
|
"step_time": 49.73436775896698
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1436.0,
|
|
"completions/mean_length": 799.671875,
|
|
"completions/mean_terminated_length": 747.3492431640625,
|
|
"completions/min_length": 418.0,
|
|
"completions/min_terminated_length": 418.0,
|
|
"entropy": 0.864380856975913,
|
|
"epoch": 0.2653061224489796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 7.352607709750567e-07,
|
|
"loss": -0.0071,
|
|
"num_tokens": 27565346.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 468,
|
|
"step_time": 168.72100699786097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1620.0,
|
|
"completions/max_terminated_length": 1620.0,
|
|
"completions/mean_length": 656.609375,
|
|
"completions/mean_terminated_length": 656.609375,
|
|
"completions/min_length": 209.0,
|
|
"completions/min_terminated_length": 209.0,
|
|
"entropy": 1.0048782378435135,
|
|
"epoch": 0.26587301587301587,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.734375,
|
|
"learning_rate": 7.346938775510204e-07,
|
|
"loss": 0.0027,
|
|
"num_tokens": 27617289.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.7777600884437561,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 469,
|
|
"step_time": 54.89007071964443
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1607.0,
|
|
"completions/max_terminated_length": 1607.0,
|
|
"completions/mean_length": 850.734375,
|
|
"completions/mean_terminated_length": 850.734375,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.6241070311516523,
|
|
"epoch": 0.26643990929705214,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.255859375,
|
|
"learning_rate": 7.34126984126984e-07,
|
|
"loss": -0.0052,
|
|
"num_tokens": 27683528.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 470,
|
|
"step_time": 54.522400417365134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1613.0,
|
|
"completions/max_terminated_length": 1613.0,
|
|
"completions/mean_length": 817.828125,
|
|
"completions/mean_terminated_length": 817.828125,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.6263278722763062,
|
|
"epoch": 0.26700680272108845,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 7.335600907029478e-07,
|
|
"loss": -0.0038,
|
|
"num_tokens": 27745725.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 471,
|
|
"step_time": 54.59293780755252
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1464.0,
|
|
"completions/max_terminated_length": 1464.0,
|
|
"completions/mean_length": 676.078125,
|
|
"completions/mean_terminated_length": 676.078125,
|
|
"completions/min_length": 174.0,
|
|
"completions/min_terminated_length": 174.0,
|
|
"entropy": 0.9488864932209253,
|
|
"epoch": 0.2675736961451247,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.453125,
|
|
"learning_rate": 7.329931972789115e-07,
|
|
"loss": 0.0229,
|
|
"num_tokens": 27800218.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 472,
|
|
"step_time": 50.35415969602764
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1091.0,
|
|
"completions/max_terminated_length": 1091.0,
|
|
"completions/mean_length": 610.6875,
|
|
"completions/mean_terminated_length": 610.6875,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 0.9655479695647955,
|
|
"epoch": 0.268140589569161,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 7.324263038548753e-07,
|
|
"loss": 0.0249,
|
|
"num_tokens": 27849822.0,
|
|
"reward": 0.4140625,
|
|
"reward_std": 0.8191879987716675,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 473,
|
|
"step_time": 36.97484754025936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1558.0,
|
|
"completions/mean_length": 865.90625,
|
|
"completions/mean_terminated_length": 814.6349487304688,
|
|
"completions/min_length": 402.0,
|
|
"completions/min_terminated_length": 402.0,
|
|
"entropy": 0.7376750633120537,
|
|
"epoch": 0.2687074829931973,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.326171875,
|
|
"learning_rate": 7.318594104308389e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 27916312.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 474,
|
|
"step_time": 168.47027471847832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1392.0,
|
|
"completions/max_terminated_length": 1392.0,
|
|
"completions/mean_length": 762.484375,
|
|
"completions/mean_terminated_length": 762.484375,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.8245761692523956,
|
|
"epoch": 0.26927437641723356,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 7.312925170068027e-07,
|
|
"loss": 0.0077,
|
|
"num_tokens": 27976319.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 475,
|
|
"step_time": 47.97840517479926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1287.0,
|
|
"completions/max_terminated_length": 1287.0,
|
|
"completions/mean_length": 744.515625,
|
|
"completions/mean_terminated_length": 744.515625,
|
|
"completions/min_length": 355.0,
|
|
"completions/min_terminated_length": 355.0,
|
|
"entropy": 0.8160289656370878,
|
|
"epoch": 0.2698412698412698,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.376953125,
|
|
"learning_rate": 7.307256235827664e-07,
|
|
"loss": -0.0182,
|
|
"num_tokens": 28037056.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 476,
|
|
"step_time": 45.09215791430324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1155.0,
|
|
"completions/max_terminated_length": 1155.0,
|
|
"completions/mean_length": 685.59375,
|
|
"completions/mean_terminated_length": 685.59375,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.9145476706326008,
|
|
"epoch": 0.27040816326530615,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.59375,
|
|
"learning_rate": 7.301587301587301e-07,
|
|
"loss": -0.0174,
|
|
"num_tokens": 28091230.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 477,
|
|
"step_time": 42.135603019967675
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 745.109375,
|
|
"completions/mean_terminated_length": 745.109375,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"entropy": 0.7281346414238214,
|
|
"epoch": 0.2709750566893424,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.416015625,
|
|
"learning_rate": 7.295918367346938e-07,
|
|
"loss": -0.0439,
|
|
"num_tokens": 28151277.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 478,
|
|
"step_time": 49.57602191157639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 777.015625,
|
|
"completions/mean_terminated_length": 777.015625,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.8162975553423166,
|
|
"epoch": 0.27154195011337867,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 7.290249433106576e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 28213142.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6986381411552429,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 479,
|
|
"step_time": 51.75872884038836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1412.0,
|
|
"completions/max_terminated_length": 1412.0,
|
|
"completions/mean_length": 683.921875,
|
|
"completions/mean_terminated_length": 683.921875,
|
|
"completions/min_length": 279.0,
|
|
"completions/min_terminated_length": 279.0,
|
|
"entropy": 0.7339695952832699,
|
|
"epoch": 0.272108843537415,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 7.284580498866213e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 28269049.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7319250702857971,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 480,
|
|
"step_time": 48.186254777014256
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2134.0,
|
|
"completions/max_terminated_length": 2134.0,
|
|
"completions/mean_length": 858.9375,
|
|
"completions/mean_terminated_length": 858.9375,
|
|
"completions/min_length": 373.0,
|
|
"completions/min_terminated_length": 373.0,
|
|
"entropy": 0.8460619766265154,
|
|
"epoch": 0.27267573696145125,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 7.278911564625851e-07,
|
|
"loss": 0.0113,
|
|
"num_tokens": 28334053.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 481,
|
|
"step_time": 74.01236078422517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/max_terminated_length": 1600.0,
|
|
"completions/mean_length": 711.5,
|
|
"completions/mean_terminated_length": 711.5,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 1.0337553285062313,
|
|
"epoch": 0.2732426303854875,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 7.273242630385487e-07,
|
|
"loss": 0.0149,
|
|
"num_tokens": 28390757.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 482,
|
|
"step_time": 55.5663038874045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1871.0,
|
|
"completions/max_terminated_length": 1871.0,
|
|
"completions/mean_length": 743.484375,
|
|
"completions/mean_terminated_length": 743.484375,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.6883541475981474,
|
|
"epoch": 0.27380952380952384,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 7.267573696145124e-07,
|
|
"loss": 0.0269,
|
|
"num_tokens": 28447796.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 483,
|
|
"step_time": 63.777127023786306
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1145.0,
|
|
"completions/max_terminated_length": 1145.0,
|
|
"completions/mean_length": 683.71875,
|
|
"completions/mean_terminated_length": 683.71875,
|
|
"completions/min_length": 244.0,
|
|
"completions/min_terminated_length": 244.0,
|
|
"entropy": 0.9639867581427097,
|
|
"epoch": 0.2743764172335601,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 7.261904761904761e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 28501322.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 484,
|
|
"step_time": 39.343465231359005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1419.0,
|
|
"completions/max_terminated_length": 1419.0,
|
|
"completions/mean_length": 747.109375,
|
|
"completions/mean_terminated_length": 747.109375,
|
|
"completions/min_length": 311.0,
|
|
"completions/min_terminated_length": 311.0,
|
|
"entropy": 0.7536885496228933,
|
|
"epoch": 0.27494331065759636,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 7.256235827664399e-07,
|
|
"loss": -0.0134,
|
|
"num_tokens": 28560233.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 485,
|
|
"step_time": 48.31962897256017
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1981.0,
|
|
"completions/max_terminated_length": 1981.0,
|
|
"completions/mean_length": 643.703125,
|
|
"completions/mean_terminated_length": 643.703125,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"entropy": 1.0371637865900993,
|
|
"epoch": 0.2755102040816326,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 7.250566893424036e-07,
|
|
"loss": 0.0181,
|
|
"num_tokens": 28610966.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 486,
|
|
"step_time": 68.0433338470757
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1365.0,
|
|
"completions/max_terminated_length": 1365.0,
|
|
"completions/mean_length": 752.03125,
|
|
"completions/mean_terminated_length": 752.03125,
|
|
"completions/min_length": 397.0,
|
|
"completions/min_terminated_length": 397.0,
|
|
"entropy": 0.9433378670364618,
|
|
"epoch": 0.27607709750566894,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.37890625,
|
|
"learning_rate": 7.244897959183674e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 28668464.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 487,
|
|
"step_time": 45.741264135576785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1221.0,
|
|
"completions/mean_length": 745.34375,
|
|
"completions/mean_terminated_length": 692.1587524414062,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.9549574814736843,
|
|
"epoch": 0.2766439909297052,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 7.239229024943311e-07,
|
|
"loss": 0.0035,
|
|
"num_tokens": 28726022.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 488,
|
|
"step_time": 167.2989202560857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2674.0,
|
|
"completions/max_terminated_length": 2674.0,
|
|
"completions/mean_length": 736.71875,
|
|
"completions/mean_terminated_length": 736.71875,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.7453363202512264,
|
|
"epoch": 0.27721088435374147,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 7.233560090702947e-07,
|
|
"loss": 0.0189,
|
|
"num_tokens": 28782500.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8188852071762085,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 489,
|
|
"step_time": 95.1553288353607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2410.0,
|
|
"completions/max_terminated_length": 2410.0,
|
|
"completions/mean_length": 697.984375,
|
|
"completions/mean_terminated_length": 697.984375,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.8135911952704191,
|
|
"epoch": 0.2777777777777778,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 7.227891156462584e-07,
|
|
"loss": 0.0218,
|
|
"num_tokens": 28836923.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 490,
|
|
"step_time": 85.06263784877956
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 707.46875,
|
|
"completions/mean_terminated_length": 707.46875,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.7418342307209969,
|
|
"epoch": 0.27834467120181405,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 7.222222222222221e-07,
|
|
"loss": 0.0243,
|
|
"num_tokens": 28891929.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7867957949638367,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 491,
|
|
"step_time": 46.50618886575103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1376.0,
|
|
"completions/max_terminated_length": 1376.0,
|
|
"completions/mean_length": 776.40625,
|
|
"completions/mean_terminated_length": 776.40625,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.7166824005544186,
|
|
"epoch": 0.2789115646258503,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 7.216553287981859e-07,
|
|
"loss": 0.0466,
|
|
"num_tokens": 28952427.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 492,
|
|
"step_time": 46.96396184246987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1663.0,
|
|
"completions/max_terminated_length": 1663.0,
|
|
"completions/mean_length": 819.046875,
|
|
"completions/mean_terminated_length": 819.046875,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.6781535688787699,
|
|
"epoch": 0.27947845804988664,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 7.210884353741496e-07,
|
|
"loss": 0.0083,
|
|
"num_tokens": 29014574.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 493,
|
|
"step_time": 56.07055533491075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 765.6875,
|
|
"completions/mean_terminated_length": 765.6875,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.8276981525123119,
|
|
"epoch": 0.2800453514739229,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 7.205215419501134e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 29074026.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 494,
|
|
"step_time": 49.20934342965484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1183.0,
|
|
"completions/max_terminated_length": 1183.0,
|
|
"completions/mean_length": 709.609375,
|
|
"completions/mean_terminated_length": 709.609375,
|
|
"completions/min_length": 316.0,
|
|
"completions/min_terminated_length": 316.0,
|
|
"entropy": 1.0416027531027794,
|
|
"epoch": 0.28061224489795916,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 7.19954648526077e-07,
|
|
"loss": -0.0205,
|
|
"num_tokens": 29131681.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7489659190177917,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 495,
|
|
"step_time": 40.29257717169821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3069.0,
|
|
"completions/max_terminated_length": 3069.0,
|
|
"completions/mean_length": 841.96875,
|
|
"completions/mean_terminated_length": 841.96875,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 1.0242768581956625,
|
|
"epoch": 0.2811791383219955,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.275390625,
|
|
"learning_rate": 7.193877551020408e-07,
|
|
"loss": 0.0039,
|
|
"num_tokens": 29196975.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 496,
|
|
"step_time": 115.6932109799236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1292.0,
|
|
"completions/max_terminated_length": 1292.0,
|
|
"completions/mean_length": 748.203125,
|
|
"completions/mean_terminated_length": 748.203125,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.7642087694257498,
|
|
"epoch": 0.28174603174603174,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 7.188208616780045e-07,
|
|
"loss": 0.0442,
|
|
"num_tokens": 29257428.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7263124585151672,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 497,
|
|
"step_time": 45.00180886685848
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1546.0,
|
|
"completions/max_terminated_length": 1546.0,
|
|
"completions/mean_length": 781.859375,
|
|
"completions/mean_terminated_length": 781.859375,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.9330013245344162,
|
|
"epoch": 0.282312925170068,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 7.182539682539682e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 29319491.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.3587760329246521,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 498,
|
|
"step_time": 52.68529424164444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1461.0,
|
|
"completions/max_terminated_length": 1461.0,
|
|
"completions/mean_length": 793.0625,
|
|
"completions/mean_terminated_length": 793.0625,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 0.6890045292675495,
|
|
"epoch": 0.2828798185941043,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 7.176870748299319e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 29380775.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 499,
|
|
"step_time": 49.33592140302062
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1679.0,
|
|
"completions/max_terminated_length": 1679.0,
|
|
"completions/mean_length": 749.6875,
|
|
"completions/mean_terminated_length": 749.6875,
|
|
"completions/min_length": 19.0,
|
|
"completions/min_terminated_length": 19.0,
|
|
"entropy": 0.9031962808221579,
|
|
"epoch": 0.2834467120181406,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 7.171201814058957e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 29440547.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 500,
|
|
"step_time": 57.61708065588027
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1302.0,
|
|
"completions/max_terminated_length": 1302.0,
|
|
"completions/mean_length": 669.671875,
|
|
"completions/mean_terminated_length": 669.671875,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.8602342400699854,
|
|
"epoch": 0.28401360544217685,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 7.165532879818594e-07,
|
|
"loss": 0.0134,
|
|
"num_tokens": 29493126.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.6950790882110596,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 501,
|
|
"step_time": 47.38943115156144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/max_terminated_length": 1202.0,
|
|
"completions/mean_length": 768.4375,
|
|
"completions/mean_terminated_length": 768.4375,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.7709177304059267,
|
|
"epoch": 0.28458049886621317,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 7.159863945578231e-07,
|
|
"loss": 0.0025,
|
|
"num_tokens": 29553514.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7210001945495605,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 502,
|
|
"step_time": 41.020229395478964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1671.0,
|
|
"completions/max_terminated_length": 1671.0,
|
|
"completions/mean_length": 762.71875,
|
|
"completions/mean_terminated_length": 762.71875,
|
|
"completions/min_length": 231.0,
|
|
"completions/min_terminated_length": 231.0,
|
|
"entropy": 0.8904999010264874,
|
|
"epoch": 0.28514739229024944,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 7.154195011337868e-07,
|
|
"loss": 0.0084,
|
|
"num_tokens": 29613840.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 503,
|
|
"step_time": 56.85954328905791
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1644.0,
|
|
"completions/max_terminated_length": 1644.0,
|
|
"completions/mean_length": 879.484375,
|
|
"completions/mean_terminated_length": 879.484375,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.6220353841781616,
|
|
"epoch": 0.2857142857142857,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.28125,
|
|
"learning_rate": 7.148526077097506e-07,
|
|
"loss": -0.0277,
|
|
"num_tokens": 29679599.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 504,
|
|
"step_time": 55.55082425381988
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1626.0,
|
|
"completions/max_terminated_length": 1626.0,
|
|
"completions/mean_length": 709.75,
|
|
"completions/mean_terminated_length": 709.75,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.698782479390502,
|
|
"epoch": 0.286281179138322,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 7.142857142857143e-07,
|
|
"loss": 0.0252,
|
|
"num_tokens": 29732879.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.7539166808128357,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 505,
|
|
"step_time": 54.8224318632856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2076.0,
|
|
"completions/max_terminated_length": 2076.0,
|
|
"completions/mean_length": 793.15625,
|
|
"completions/mean_terminated_length": 793.15625,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.883378142490983,
|
|
"epoch": 0.2868480725623583,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.369140625,
|
|
"learning_rate": 7.13718820861678e-07,
|
|
"loss": -0.0302,
|
|
"num_tokens": 29793113.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 506,
|
|
"step_time": 71.06204910762608
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1252.0,
|
|
"completions/max_terminated_length": 1252.0,
|
|
"completions/mean_length": 669.5625,
|
|
"completions/mean_terminated_length": 669.5625,
|
|
"completions/min_length": 249.0,
|
|
"completions/min_terminated_length": 249.0,
|
|
"entropy": 0.8932874873280525,
|
|
"epoch": 0.28741496598639454,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 7.131519274376417e-07,
|
|
"loss": -0.0184,
|
|
"num_tokens": 29848925.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 507,
|
|
"step_time": 44.22843154706061
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1084.0,
|
|
"completions/max_terminated_length": 1084.0,
|
|
"completions/mean_length": 629.3125,
|
|
"completions/mean_terminated_length": 629.3125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.733006564900279,
|
|
"epoch": 0.28798185941043086,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 7.125850340136053e-07,
|
|
"loss": 0.0016,
|
|
"num_tokens": 29898377.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 508,
|
|
"step_time": 36.84416067041457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1565.0,
|
|
"completions/max_terminated_length": 1565.0,
|
|
"completions/mean_length": 763.890625,
|
|
"completions/mean_terminated_length": 763.890625,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.8635252937674522,
|
|
"epoch": 0.2885487528344671,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 7.120181405895691e-07,
|
|
"loss": 0.0158,
|
|
"num_tokens": 29959178.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.7419792413711548,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 509,
|
|
"step_time": 53.838072184473276
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1169.0,
|
|
"completions/max_terminated_length": 1169.0,
|
|
"completions/mean_length": 704.0625,
|
|
"completions/mean_terminated_length": 704.0625,
|
|
"completions/min_length": 13.0,
|
|
"completions/min_terminated_length": 13.0,
|
|
"entropy": 0.8978853225708008,
|
|
"epoch": 0.2891156462585034,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 7.114512471655328e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 30012262.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 510,
|
|
"step_time": 39.582871126011014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1647.0,
|
|
"completions/max_terminated_length": 1647.0,
|
|
"completions/mean_length": 783.859375,
|
|
"completions/mean_terminated_length": 783.859375,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.799968246370554,
|
|
"epoch": 0.2896825396825397,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 7.108843537414966e-07,
|
|
"loss": 0.0272,
|
|
"num_tokens": 30074093.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5754139423370361,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 511,
|
|
"step_time": 57.25170711334795
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1238.0,
|
|
"completions/max_terminated_length": 1238.0,
|
|
"completions/mean_length": 717.03125,
|
|
"completions/mean_terminated_length": 717.03125,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.8698889967054129,
|
|
"epoch": 0.29024943310657597,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 7.103174603174603e-07,
|
|
"loss": 0.0059,
|
|
"num_tokens": 30131095.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 512,
|
|
"step_time": 42.829598689451814
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1260.0,
|
|
"completions/max_terminated_length": 1260.0,
|
|
"completions/mean_length": 771.09375,
|
|
"completions/mean_terminated_length": 771.09375,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.7532858662307262,
|
|
"epoch": 0.29081632653061223,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 7.097505668934241e-07,
|
|
"loss": -0.003,
|
|
"num_tokens": 30190981.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 513,
|
|
"step_time": 42.77709187101573
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 754.40625,
|
|
"completions/mean_terminated_length": 754.40625,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.8620533980429173,
|
|
"epoch": 0.29138321995464855,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 7.091836734693876e-07,
|
|
"loss": 0.023,
|
|
"num_tokens": 30250655.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 514,
|
|
"step_time": 45.86416158825159
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1073.0,
|
|
"completions/max_terminated_length": 1073.0,
|
|
"completions/mean_length": 645.046875,
|
|
"completions/mean_terminated_length": 645.046875,
|
|
"completions/min_length": 274.0,
|
|
"completions/min_terminated_length": 274.0,
|
|
"entropy": 0.8946611173450947,
|
|
"epoch": 0.2919501133786848,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 7.086167800453514e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 30305274.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 515,
|
|
"step_time": 38.31303133163601
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1503.0,
|
|
"completions/max_terminated_length": 1503.0,
|
|
"completions/mean_length": 831.09375,
|
|
"completions/mean_terminated_length": 831.09375,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 0.7219805084168911,
|
|
"epoch": 0.2925170068027211,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2421875,
|
|
"learning_rate": 7.080498866213151e-07,
|
|
"loss": -0.0237,
|
|
"num_tokens": 30370616.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.26899558305740356,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 516,
|
|
"step_time": 51.80148950777948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 776.765625,
|
|
"completions/mean_terminated_length": 776.765625,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.7232088558375835,
|
|
"epoch": 0.29308390022675734,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.27734375,
|
|
"learning_rate": 7.074829931972789e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 30430017.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 517,
|
|
"step_time": 45.39673104416579
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1546.0,
|
|
"completions/max_terminated_length": 1546.0,
|
|
"completions/mean_length": 677.609375,
|
|
"completions/mean_terminated_length": 677.609375,
|
|
"completions/min_length": 267.0,
|
|
"completions/min_terminated_length": 267.0,
|
|
"entropy": 0.6846966706216335,
|
|
"epoch": 0.29365079365079366,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 7.069160997732426e-07,
|
|
"loss": 0.016,
|
|
"num_tokens": 30483504.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 518,
|
|
"step_time": 52.81310435663909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1577.0,
|
|
"completions/max_terminated_length": 1577.0,
|
|
"completions/mean_length": 750.40625,
|
|
"completions/mean_terminated_length": 750.40625,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.8029123768210411,
|
|
"epoch": 0.2942176870748299,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.42578125,
|
|
"learning_rate": 7.063492063492064e-07,
|
|
"loss": 0.0076,
|
|
"num_tokens": 30540378.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 519,
|
|
"step_time": 53.39374658744782
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1460.0,
|
|
"completions/max_terminated_length": 1460.0,
|
|
"completions/mean_length": 910.828125,
|
|
"completions/mean_terminated_length": 910.828125,
|
|
"completions/min_length": 452.0,
|
|
"completions/min_terminated_length": 452.0,
|
|
"entropy": 0.6312018167227507,
|
|
"epoch": 0.2947845804988662,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 7.0578231292517e-07,
|
|
"loss": 0.0107,
|
|
"num_tokens": 30611967.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 520,
|
|
"step_time": 51.096797372214496
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1223.0,
|
|
"completions/max_terminated_length": 1223.0,
|
|
"completions/mean_length": 810.53125,
|
|
"completions/mean_terminated_length": 810.53125,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.811397809535265,
|
|
"epoch": 0.2953514739229025,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 7.052154195011338e-07,
|
|
"loss": 0.0079,
|
|
"num_tokens": 30676281.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 521,
|
|
"step_time": 42.19156005140394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2165.0,
|
|
"completions/max_terminated_length": 2165.0,
|
|
"completions/mean_length": 684.265625,
|
|
"completions/mean_terminated_length": 684.265625,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.7709549944847822,
|
|
"epoch": 0.29591836734693877,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 7.046485260770974e-07,
|
|
"loss": 0.0225,
|
|
"num_tokens": 30730258.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 522,
|
|
"step_time": 74.87661859579384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1502.0,
|
|
"completions/max_terminated_length": 1502.0,
|
|
"completions/mean_length": 756.203125,
|
|
"completions/mean_terminated_length": 756.203125,
|
|
"completions/min_length": 208.0,
|
|
"completions/min_terminated_length": 208.0,
|
|
"entropy": 0.7819712497293949,
|
|
"epoch": 0.29648526077097503,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.23828125,
|
|
"learning_rate": 7.040816326530612e-07,
|
|
"loss": 0.0001,
|
|
"num_tokens": 30789095.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 523,
|
|
"step_time": 51.73434377089143
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1584.0,
|
|
"completions/max_terminated_length": 1584.0,
|
|
"completions/mean_length": 696.265625,
|
|
"completions/mean_terminated_length": 696.265625,
|
|
"completions/min_length": 284.0,
|
|
"completions/min_terminated_length": 284.0,
|
|
"entropy": 0.6800563894212246,
|
|
"epoch": 0.29705215419501135,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3828125,
|
|
"learning_rate": 7.035147392290249e-07,
|
|
"loss": 0.0017,
|
|
"num_tokens": 30845672.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.5092765688896179,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 524,
|
|
"step_time": 55.245374178513885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1404.0,
|
|
"completions/max_terminated_length": 1404.0,
|
|
"completions/mean_length": 750.734375,
|
|
"completions/mean_terminated_length": 750.734375,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.7135310284793377,
|
|
"epoch": 0.2976190476190476,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 7.029478458049887e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 30904439.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 525,
|
|
"step_time": 47.8006643243134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1607.0,
|
|
"completions/max_terminated_length": 1607.0,
|
|
"completions/mean_length": 836.84375,
|
|
"completions/mean_terminated_length": 836.84375,
|
|
"completions/min_length": 430.0,
|
|
"completions/min_terminated_length": 430.0,
|
|
"entropy": 0.7955428976565599,
|
|
"epoch": 0.2981859410430839,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 7.023809523809523e-07,
|
|
"loss": -0.0231,
|
|
"num_tokens": 30968933.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 526,
|
|
"step_time": 55.37982105277479
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 773.140625,
|
|
"completions/mean_terminated_length": 773.140625,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 0.7125669363886118,
|
|
"epoch": 0.2987528344671202,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 7.018140589569161e-07,
|
|
"loss": 0.0143,
|
|
"num_tokens": 31029470.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 527,
|
|
"step_time": 45.26804610528052
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1274.0,
|
|
"completions/max_terminated_length": 1274.0,
|
|
"completions/mean_length": 675.0,
|
|
"completions/mean_terminated_length": 675.0,
|
|
"completions/min_length": 230.0,
|
|
"completions/min_terminated_length": 230.0,
|
|
"entropy": 0.8840137869119644,
|
|
"epoch": 0.29931972789115646,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 7.012471655328798e-07,
|
|
"loss": -0.0026,
|
|
"num_tokens": 31086014.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 528,
|
|
"step_time": 43.771443992853165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1389.0,
|
|
"completions/max_terminated_length": 1389.0,
|
|
"completions/mean_length": 843.234375,
|
|
"completions/mean_terminated_length": 843.234375,
|
|
"completions/min_length": 487.0,
|
|
"completions/min_terminated_length": 487.0,
|
|
"entropy": 0.8427235707640648,
|
|
"epoch": 0.2998866213151927,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 7.006802721088436e-07,
|
|
"loss": -0.0055,
|
|
"num_tokens": 31150757.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 529,
|
|
"step_time": 47.427061734721065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1196.0,
|
|
"completions/max_terminated_length": 1196.0,
|
|
"completions/mean_length": 727.515625,
|
|
"completions/mean_terminated_length": 727.515625,
|
|
"completions/min_length": 374.0,
|
|
"completions/min_terminated_length": 374.0,
|
|
"entropy": 0.7390955612063408,
|
|
"epoch": 0.30045351473922904,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 7.001133786848072e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 31209062.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 530,
|
|
"step_time": 41.13703847210854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1342.0,
|
|
"completions/max_terminated_length": 1342.0,
|
|
"completions/mean_length": 743.859375,
|
|
"completions/mean_terminated_length": 743.859375,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.8050366118550301,
|
|
"epoch": 0.3010204081632653,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 6.995464852607709e-07,
|
|
"loss": -0.0348,
|
|
"num_tokens": 31267789.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 531,
|
|
"step_time": 46.66273658722639
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1820.0,
|
|
"completions/max_terminated_length": 1820.0,
|
|
"completions/mean_length": 713.109375,
|
|
"completions/mean_terminated_length": 713.109375,
|
|
"completions/min_length": 335.0,
|
|
"completions/min_terminated_length": 335.0,
|
|
"entropy": 0.8603307083249092,
|
|
"epoch": 0.30158730158730157,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 6.989795918367347e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 31324644.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5739033818244934,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 532,
|
|
"step_time": 62.559358855709434
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1071.0,
|
|
"completions/max_terminated_length": 1071.0,
|
|
"completions/mean_length": 673.40625,
|
|
"completions/mean_terminated_length": 673.40625,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.8667759243398905,
|
|
"epoch": 0.3021541950113379,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 6.984126984126983e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 31377774.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 533,
|
|
"step_time": 36.42737630009651
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1292.0,
|
|
"completions/max_terminated_length": 1292.0,
|
|
"completions/mean_length": 813.75,
|
|
"completions/mean_terminated_length": 813.75,
|
|
"completions/min_length": 383.0,
|
|
"completions/min_terminated_length": 383.0,
|
|
"entropy": 0.57998451218009,
|
|
"epoch": 0.30272108843537415,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 6.978458049886621e-07,
|
|
"loss": -0.0264,
|
|
"num_tokens": 31440150.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 534,
|
|
"step_time": 44.09220365062356
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 789.1875,
|
|
"completions/mean_terminated_length": 789.1875,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.6913237422704697,
|
|
"epoch": 0.3032879818594104,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 6.972789115646258e-07,
|
|
"loss": 0.0284,
|
|
"num_tokens": 31500722.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.654606282711029,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 535,
|
|
"step_time": 46.49318843707442
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1232.0,
|
|
"completions/max_terminated_length": 1232.0,
|
|
"completions/mean_length": 754.171875,
|
|
"completions/mean_terminated_length": 754.171875,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.7832220420241356,
|
|
"epoch": 0.30385487528344673,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 6.967120181405896e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 31558805.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 536,
|
|
"step_time": 42.142516100779176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 747.328125,
|
|
"completions/mean_terminated_length": 747.328125,
|
|
"completions/min_length": 249.0,
|
|
"completions/min_terminated_length": 249.0,
|
|
"entropy": 0.744468754157424,
|
|
"epoch": 0.304421768707483,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 6.961451247165533e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 31621434.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7676090598106384,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 537,
|
|
"step_time": 50.692077185958624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1630.0,
|
|
"completions/max_terminated_length": 1630.0,
|
|
"completions/mean_length": 696.609375,
|
|
"completions/mean_terminated_length": 696.609375,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 0.7965532336384058,
|
|
"epoch": 0.30498866213151926,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 6.95578231292517e-07,
|
|
"loss": -0.0114,
|
|
"num_tokens": 31678865.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 538,
|
|
"step_time": 56.480417009443045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1730.0,
|
|
"completions/max_terminated_length": 1730.0,
|
|
"completions/mean_length": 849.265625,
|
|
"completions/mean_terminated_length": 849.265625,
|
|
"completions/min_length": 437.0,
|
|
"completions/min_terminated_length": 437.0,
|
|
"entropy": 0.8315318040549755,
|
|
"epoch": 0.3055555555555556,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 6.950113378684806e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 31745010.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 539,
|
|
"step_time": 59.35373664740473
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1554.0,
|
|
"completions/max_terminated_length": 1554.0,
|
|
"completions/mean_length": 796.84375,
|
|
"completions/mean_terminated_length": 796.84375,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.7862520944327116,
|
|
"epoch": 0.30612244897959184,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 6.944444444444444e-07,
|
|
"loss": 0.001,
|
|
"num_tokens": 31807048.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 540,
|
|
"step_time": 53.72292930446565
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1538.0,
|
|
"completions/max_terminated_length": 1538.0,
|
|
"completions/mean_length": 725.5625,
|
|
"completions/mean_terminated_length": 725.5625,
|
|
"completions/min_length": 373.0,
|
|
"completions/min_terminated_length": 373.0,
|
|
"entropy": 0.9693691655993462,
|
|
"epoch": 0.3066893424036281,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 6.938775510204081e-07,
|
|
"loss": -0.0146,
|
|
"num_tokens": 31864004.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 541,
|
|
"step_time": 51.74457472469658
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1515.0,
|
|
"completions/max_terminated_length": 1515.0,
|
|
"completions/mean_length": 712.453125,
|
|
"completions/mean_terminated_length": 712.453125,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.9037486426532269,
|
|
"epoch": 0.3072562358276644,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 6.933106575963719e-07,
|
|
"loss": -0.0017,
|
|
"num_tokens": 31919913.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 542,
|
|
"step_time": 51.879045808687806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1188.0,
|
|
"completions/max_terminated_length": 1188.0,
|
|
"completions/mean_length": 651.984375,
|
|
"completions/mean_terminated_length": 651.984375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 1.141062118113041,
|
|
"epoch": 0.3078231292517007,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62109375,
|
|
"learning_rate": 6.927437641723356e-07,
|
|
"loss": -0.0188,
|
|
"num_tokens": 31972304.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 543,
|
|
"step_time": 40.88096096273512
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1417.0,
|
|
"completions/max_terminated_length": 1417.0,
|
|
"completions/mean_length": 826.140625,
|
|
"completions/mean_terminated_length": 826.140625,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.7591061405837536,
|
|
"epoch": 0.30839002267573695,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 6.921768707482994e-07,
|
|
"loss": 0.0363,
|
|
"num_tokens": 32036385.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 544,
|
|
"step_time": 48.60988793708384
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1401.0,
|
|
"completions/max_terminated_length": 1401.0,
|
|
"completions/mean_length": 744.8125,
|
|
"completions/mean_terminated_length": 744.8125,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.7927939239889383,
|
|
"epoch": 0.3089569160997732,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 6.91609977324263e-07,
|
|
"loss": 0.0235,
|
|
"num_tokens": 32097493.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 545,
|
|
"step_time": 48.86637980118394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1425.0,
|
|
"completions/max_terminated_length": 1425.0,
|
|
"completions/mean_length": 732.296875,
|
|
"completions/mean_terminated_length": 732.296875,
|
|
"completions/min_length": 186.0,
|
|
"completions/min_terminated_length": 186.0,
|
|
"entropy": 0.7569485493004322,
|
|
"epoch": 0.30952380952380953,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 6.910430839002267e-07,
|
|
"loss": 0.0135,
|
|
"num_tokens": 32156384.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 546,
|
|
"step_time": 49.739972153678536
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1347.0,
|
|
"completions/max_terminated_length": 1347.0,
|
|
"completions/mean_length": 680.53125,
|
|
"completions/mean_terminated_length": 680.53125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.6933835446834564,
|
|
"epoch": 0.3100907029478458,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 6.904761904761904e-07,
|
|
"loss": 0.0049,
|
|
"num_tokens": 32209106.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 547,
|
|
"step_time": 45.50701228249818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1271.0,
|
|
"completions/max_terminated_length": 1271.0,
|
|
"completions/mean_length": 768.578125,
|
|
"completions/mean_terminated_length": 768.578125,
|
|
"completions/min_length": 331.0,
|
|
"completions/min_terminated_length": 331.0,
|
|
"entropy": 0.902186393737793,
|
|
"epoch": 0.31065759637188206,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 6.899092970521542e-07,
|
|
"loss": -0.0052,
|
|
"num_tokens": 32269119.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 548,
|
|
"step_time": 43.688417446799576
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1454.0,
|
|
"completions/max_terminated_length": 1454.0,
|
|
"completions/mean_length": 817.125,
|
|
"completions/mean_terminated_length": 817.125,
|
|
"completions/min_length": 359.0,
|
|
"completions/min_terminated_length": 359.0,
|
|
"entropy": 0.8277923595160246,
|
|
"epoch": 0.3112244897959184,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40625,
|
|
"learning_rate": 6.893424036281179e-07,
|
|
"loss": 0.0245,
|
|
"num_tokens": 32331335.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 549,
|
|
"step_time": 50.39040738157928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 746.328125,
|
|
"completions/mean_terminated_length": 746.328125,
|
|
"completions/min_length": 176.0,
|
|
"completions/min_terminated_length": 176.0,
|
|
"entropy": 0.6842802502214909,
|
|
"epoch": 0.31179138321995464,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 6.887755102040817e-07,
|
|
"loss": 0.014,
|
|
"num_tokens": 32390716.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 550,
|
|
"step_time": 44.9338699048385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 683.09375,
|
|
"completions/mean_terminated_length": 683.09375,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 0.8729349300265312,
|
|
"epoch": 0.3123582766439909,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5234375,
|
|
"learning_rate": 6.882086167800453e-07,
|
|
"loss": 0.0186,
|
|
"num_tokens": 32445186.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 551,
|
|
"step_time": 44.33868500776589
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1002.0,
|
|
"completions/max_terminated_length": 1002.0,
|
|
"completions/mean_length": 542.546875,
|
|
"completions/mean_terminated_length": 542.546875,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"entropy": 0.8894463591277599,
|
|
"epoch": 0.3129251700680272,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6640625,
|
|
"learning_rate": 6.87641723356009e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 32490925.0,
|
|
"reward": 0.59375,
|
|
"reward_std": 0.9380620121955872,
|
|
"rewards/accuracy_reward_func/mean": 0.53125,
|
|
"rewards/accuracy_reward_func/std": 0.8903138637542725,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 552,
|
|
"step_time": 34.315698532387614
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1125.0,
|
|
"completions/max_terminated_length": 1125.0,
|
|
"completions/mean_length": 669.484375,
|
|
"completions/mean_terminated_length": 669.484375,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"entropy": 0.9448998272418976,
|
|
"epoch": 0.3134920634920635,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.65625,
|
|
"learning_rate": 6.870748299319728e-07,
|
|
"loss": 0.0165,
|
|
"num_tokens": 32543484.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 553,
|
|
"step_time": 38.50360931828618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1497.0,
|
|
"completions/max_terminated_length": 1497.0,
|
|
"completions/mean_length": 745.609375,
|
|
"completions/mean_terminated_length": 745.609375,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.9086252637207508,
|
|
"epoch": 0.31405895691609975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 6.865079365079364e-07,
|
|
"loss": 0.0064,
|
|
"num_tokens": 32603171.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.548943817615509,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 554,
|
|
"step_time": 51.281183240935206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1619.0,
|
|
"completions/max_terminated_length": 1619.0,
|
|
"completions/mean_length": 915.921875,
|
|
"completions/mean_terminated_length": 915.921875,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.671868585050106,
|
|
"epoch": 0.31462585034013607,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"learning_rate": 6.859410430839002e-07,
|
|
"loss": 0.0095,
|
|
"num_tokens": 32673350.0,
|
|
"reward": 0.1015625,
|
|
"reward_std": 0.3692515790462494,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 555,
|
|
"step_time": 55.37346922699362
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1241.0,
|
|
"completions/max_terminated_length": 1241.0,
|
|
"completions/mean_length": 665.46875,
|
|
"completions/mean_terminated_length": 665.46875,
|
|
"completions/min_length": 248.0,
|
|
"completions/min_terminated_length": 248.0,
|
|
"entropy": 0.8857528772205114,
|
|
"epoch": 0.31519274376417233,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 6.853741496598639e-07,
|
|
"loss": 0.0301,
|
|
"num_tokens": 32726140.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 556,
|
|
"step_time": 42.94716017693281
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1541.0,
|
|
"completions/max_terminated_length": 1541.0,
|
|
"completions/mean_length": 826.265625,
|
|
"completions/mean_terminated_length": 826.265625,
|
|
"completions/min_length": 372.0,
|
|
"completions/min_terminated_length": 372.0,
|
|
"entropy": 0.7831900082528591,
|
|
"epoch": 0.3157596371882086,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 6.848072562358276e-07,
|
|
"loss": -0.0003,
|
|
"num_tokens": 32789085.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.666620135307312,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 557,
|
|
"step_time": 52.977177309803665
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1338.0,
|
|
"completions/max_terminated_length": 1338.0,
|
|
"completions/mean_length": 778.609375,
|
|
"completions/mean_terminated_length": 778.609375,
|
|
"completions/min_length": 310.0,
|
|
"completions/min_terminated_length": 310.0,
|
|
"entropy": 0.7681636288762093,
|
|
"epoch": 0.3163265306122449,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 6.842403628117913e-07,
|
|
"loss": 0.0189,
|
|
"num_tokens": 32848660.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.7111289501190186,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 558,
|
|
"step_time": 46.01967536751181
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1359.0,
|
|
"completions/max_terminated_length": 1359.0,
|
|
"completions/mean_length": 695.90625,
|
|
"completions/mean_terminated_length": 695.90625,
|
|
"completions/min_length": 205.0,
|
|
"completions/min_terminated_length": 205.0,
|
|
"entropy": 0.8793341629207134,
|
|
"epoch": 0.3168934240362812,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 6.836734693877551e-07,
|
|
"loss": -0.0091,
|
|
"num_tokens": 32903478.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 559,
|
|
"step_time": 46.370093904435635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1436.0,
|
|
"completions/max_terminated_length": 1436.0,
|
|
"completions/mean_length": 724.828125,
|
|
"completions/mean_terminated_length": 724.828125,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 0.6435755137354136,
|
|
"epoch": 0.31746031746031744,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 6.831065759637188e-07,
|
|
"loss": -0.0127,
|
|
"num_tokens": 32960851.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 560,
|
|
"step_time": 48.38396042305976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1861.0,
|
|
"completions/max_terminated_length": 1861.0,
|
|
"completions/mean_length": 750.765625,
|
|
"completions/mean_terminated_length": 750.765625,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.7577685285359621,
|
|
"epoch": 0.31802721088435376,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 6.825396825396826e-07,
|
|
"loss": 0.0172,
|
|
"num_tokens": 33020532.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5589615106582642,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 561,
|
|
"step_time": 64.55832711420953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1674.0,
|
|
"completions/max_terminated_length": 1674.0,
|
|
"completions/mean_length": 758.71875,
|
|
"completions/mean_terminated_length": 758.71875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.6104346979409456,
|
|
"epoch": 0.31859410430839,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 6.819727891156462e-07,
|
|
"loss": -0.0132,
|
|
"num_tokens": 33078402.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6782841682434082,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 562,
|
|
"step_time": 56.82965278439224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1167.0,
|
|
"completions/max_terminated_length": 1167.0,
|
|
"completions/mean_length": 708.859375,
|
|
"completions/mean_terminated_length": 708.859375,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 0.9760242644697428,
|
|
"epoch": 0.3191609977324263,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.392578125,
|
|
"learning_rate": 6.8140589569161e-07,
|
|
"loss": -0.0077,
|
|
"num_tokens": 33134577.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 563,
|
|
"step_time": 39.54180992022157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2076.0,
|
|
"completions/max_terminated_length": 2076.0,
|
|
"completions/mean_length": 790.15625,
|
|
"completions/mean_terminated_length": 790.15625,
|
|
"completions/min_length": 162.0,
|
|
"completions/min_terminated_length": 162.0,
|
|
"entropy": 0.7078715600073338,
|
|
"epoch": 0.3197278911564626,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 6.808390022675736e-07,
|
|
"loss": -0.023,
|
|
"num_tokens": 33196979.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7708735466003418,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 564,
|
|
"step_time": 74.30706621333957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1752.0,
|
|
"completions/max_terminated_length": 1752.0,
|
|
"completions/mean_length": 705.828125,
|
|
"completions/mean_terminated_length": 705.828125,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 1.0469638369977474,
|
|
"epoch": 0.32029478458049887,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 6.802721088435374e-07,
|
|
"loss": 0.0184,
|
|
"num_tokens": 33253344.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 565,
|
|
"step_time": 59.81270517874509
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 780.859375,
|
|
"completions/mean_terminated_length": 780.859375,
|
|
"completions/min_length": 416.0,
|
|
"completions/min_terminated_length": 416.0,
|
|
"entropy": 0.9410541523247957,
|
|
"epoch": 0.32086167800453513,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 6.797052154195011e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 33314319.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 566,
|
|
"step_time": 45.85976626165211
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1599.0,
|
|
"completions/max_terminated_length": 1599.0,
|
|
"completions/mean_length": 898.859375,
|
|
"completions/mean_terminated_length": 898.859375,
|
|
"completions/min_length": 492.0,
|
|
"completions/min_terminated_length": 492.0,
|
|
"entropy": 0.7099578492343426,
|
|
"epoch": 0.32142857142857145,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.380859375,
|
|
"learning_rate": 6.791383219954649e-07,
|
|
"loss": -0.0052,
|
|
"num_tokens": 33382998.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 567,
|
|
"step_time": 54.45179197285324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1702.0,
|
|
"completions/max_terminated_length": 1702.0,
|
|
"completions/mean_length": 778.390625,
|
|
"completions/mean_terminated_length": 778.390625,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.7448876928538084,
|
|
"epoch": 0.3219954648526077,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 6.785714285714286e-07,
|
|
"loss": -0.0014,
|
|
"num_tokens": 33444663.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 568,
|
|
"step_time": 58.32504893839359
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1237.0,
|
|
"completions/max_terminated_length": 1237.0,
|
|
"completions/mean_length": 729.28125,
|
|
"completions/mean_terminated_length": 729.28125,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.7822460401803255,
|
|
"epoch": 0.322562358276644,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41796875,
|
|
"learning_rate": 6.780045351473924e-07,
|
|
"loss": -0.0207,
|
|
"num_tokens": 33502609.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 569,
|
|
"step_time": 42.56434412673116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1705.0,
|
|
"completions/max_terminated_length": 1705.0,
|
|
"completions/mean_length": 750.046875,
|
|
"completions/mean_terminated_length": 750.046875,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"entropy": 0.7973836399614811,
|
|
"epoch": 0.3231292517006803,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 6.774376417233559e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 33561988.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6170124411582947,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 570,
|
|
"step_time": 59.071029322221875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1406.0,
|
|
"completions/max_terminated_length": 1406.0,
|
|
"completions/mean_length": 794.96875,
|
|
"completions/mean_terminated_length": 794.96875,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.7202172577381134,
|
|
"epoch": 0.32369614512471656,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 6.768707482993196e-07,
|
|
"loss": 0.0092,
|
|
"num_tokens": 33623810.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 571,
|
|
"step_time": 48.03664357494563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1233.0,
|
|
"completions/max_terminated_length": 1233.0,
|
|
"completions/mean_length": 762.6875,
|
|
"completions/mean_terminated_length": 762.6875,
|
|
"completions/min_length": 418.0,
|
|
"completions/min_terminated_length": 418.0,
|
|
"entropy": 0.831648550927639,
|
|
"epoch": 0.3242630385487528,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 6.763038548752834e-07,
|
|
"loss": 0.0042,
|
|
"num_tokens": 33682734.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 572,
|
|
"step_time": 42.203618557192385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1349.0,
|
|
"completions/mean_length": 783.515625,
|
|
"completions/mean_terminated_length": 730.9365234375,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.9939813017845154,
|
|
"epoch": 0.32482993197278914,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"learning_rate": 6.757369614512471e-07,
|
|
"loss": 0.0058,
|
|
"num_tokens": 33744327.0,
|
|
"reward": 0.3515625,
|
|
"reward_std": 0.7274640202522278,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 573,
|
|
"step_time": 170.5334463007748
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1742.0,
|
|
"completions/max_terminated_length": 1742.0,
|
|
"completions/mean_length": 832.96875,
|
|
"completions/mean_terminated_length": 832.96875,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.7291863579303026,
|
|
"epoch": 0.3253968253968254,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"learning_rate": 6.751700680272109e-07,
|
|
"loss": -0.0244,
|
|
"num_tokens": 33807605.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6419816017150879,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 574,
|
|
"step_time": 59.69654292333871
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3735.0,
|
|
"completions/max_terminated_length": 3735.0,
|
|
"completions/mean_length": 877.171875,
|
|
"completions/mean_terminated_length": 877.171875,
|
|
"completions/min_length": 348.0,
|
|
"completions/min_terminated_length": 348.0,
|
|
"entropy": 1.0489157401025295,
|
|
"epoch": 0.32596371882086167,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.384765625,
|
|
"learning_rate": 6.746031746031746e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 33877088.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 575,
|
|
"step_time": 151.11537385731936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1673.0,
|
|
"completions/max_terminated_length": 1673.0,
|
|
"completions/mean_length": 745.53125,
|
|
"completions/mean_terminated_length": 745.53125,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.8816084526479244,
|
|
"epoch": 0.32653061224489793,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 6.740362811791383e-07,
|
|
"loss": 0.0166,
|
|
"num_tokens": 33934002.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 576,
|
|
"step_time": 56.5695292269811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1446.0,
|
|
"completions/max_terminated_length": 1446.0,
|
|
"completions/mean_length": 740.953125,
|
|
"completions/mean_terminated_length": 740.953125,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.7327089421451092,
|
|
"epoch": 0.32709750566893425,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 6.734693877551019e-07,
|
|
"loss": -0.009,
|
|
"num_tokens": 33992335.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 577,
|
|
"step_time": 49.20490532647818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1336.0,
|
|
"completions/max_terminated_length": 1336.0,
|
|
"completions/mean_length": 738.65625,
|
|
"completions/mean_terminated_length": 738.65625,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.7540786378085613,
|
|
"epoch": 0.3276643990929705,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 6.729024943310657e-07,
|
|
"loss": -0.0208,
|
|
"num_tokens": 34049585.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 578,
|
|
"step_time": 45.131568097509444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1294.0,
|
|
"completions/max_terminated_length": 1294.0,
|
|
"completions/mean_length": 670.796875,
|
|
"completions/mean_terminated_length": 670.796875,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.8483943492174149,
|
|
"epoch": 0.3282312925170068,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 6.723356009070294e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 34102884.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 579,
|
|
"step_time": 43.97271020244807
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2467.0,
|
|
"completions/max_terminated_length": 2467.0,
|
|
"completions/mean_length": 831.953125,
|
|
"completions/mean_terminated_length": 831.953125,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.6686179060488939,
|
|
"epoch": 0.3287981859410431,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 6.717687074829932e-07,
|
|
"loss": 0.013,
|
|
"num_tokens": 34167137.0,
|
|
"reward": 0.421875,
|
|
"reward_std": 0.7777600884437561,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 580,
|
|
"step_time": 87.3412484433502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1763.0,
|
|
"completions/max_terminated_length": 1763.0,
|
|
"completions/mean_length": 801.453125,
|
|
"completions/mean_terminated_length": 801.453125,
|
|
"completions/min_length": 3.0,
|
|
"completions/min_terminated_length": 3.0,
|
|
"entropy": 0.8247601091861725,
|
|
"epoch": 0.32936507936507936,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.326171875,
|
|
"learning_rate": 6.712018140589569e-07,
|
|
"loss": -0.0017,
|
|
"num_tokens": 34230310.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.2745848298072815,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 581,
|
|
"step_time": 60.85651118867099
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1618.0,
|
|
"completions/max_terminated_length": 1618.0,
|
|
"completions/mean_length": 792.46875,
|
|
"completions/mean_terminated_length": 792.46875,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.5867024697363377,
|
|
"epoch": 0.3299319727891156,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 6.706349206349206e-07,
|
|
"loss": 0.0166,
|
|
"num_tokens": 34290940.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 582,
|
|
"step_time": 54.818203169852495
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1397.0,
|
|
"completions/max_terminated_length": 1397.0,
|
|
"completions/mean_length": 779.015625,
|
|
"completions/mean_terminated_length": 779.015625,
|
|
"completions/min_length": 209.0,
|
|
"completions/min_terminated_length": 209.0,
|
|
"entropy": 0.6622234340757132,
|
|
"epoch": 0.33049886621315194,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 6.700680272108843e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 34351981.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 583,
|
|
"step_time": 47.57754575368017
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1248.0,
|
|
"completions/max_terminated_length": 1248.0,
|
|
"completions/mean_length": 693.1875,
|
|
"completions/mean_terminated_length": 693.1875,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.8937420938163996,
|
|
"epoch": 0.3310657596371882,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"learning_rate": 6.695011337868481e-07,
|
|
"loss": 0.0229,
|
|
"num_tokens": 34407801.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 584,
|
|
"step_time": 42.56495556328446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1394.0,
|
|
"completions/max_terminated_length": 1394.0,
|
|
"completions/mean_length": 931.484375,
|
|
"completions/mean_terminated_length": 931.484375,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.8077186606824398,
|
|
"epoch": 0.33163265306122447,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2099609375,
|
|
"learning_rate": 6.689342403628118e-07,
|
|
"loss": 0.0117,
|
|
"num_tokens": 34478968.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 585,
|
|
"step_time": 47.231089257635176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1672.0,
|
|
"completions/max_terminated_length": 1672.0,
|
|
"completions/mean_length": 716.265625,
|
|
"completions/mean_terminated_length": 716.265625,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.8019145056605339,
|
|
"epoch": 0.3321995464852608,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 6.683673469387755e-07,
|
|
"loss": 0.0029,
|
|
"num_tokens": 34546209.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 586,
|
|
"step_time": 67.00292479712516
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1335.0,
|
|
"completions/max_terminated_length": 1335.0,
|
|
"completions/mean_length": 736.53125,
|
|
"completions/mean_terminated_length": 736.53125,
|
|
"completions/min_length": 202.0,
|
|
"completions/min_terminated_length": 202.0,
|
|
"entropy": 0.7051402106881142,
|
|
"epoch": 0.33276643990929705,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.678004535147392e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 34604299.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.1666666716337204,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 587,
|
|
"step_time": 46.34575128182769
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1202.0,
|
|
"completions/max_terminated_length": 1202.0,
|
|
"completions/mean_length": 708.546875,
|
|
"completions/mean_terminated_length": 708.546875,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.8992711044847965,
|
|
"epoch": 0.3333333333333333,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 6.672335600907029e-07,
|
|
"loss": -0.0137,
|
|
"num_tokens": 34664494.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4753340482711792,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 588,
|
|
"step_time": 42.4019844615832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2654.0,
|
|
"completions/max_terminated_length": 2654.0,
|
|
"completions/mean_length": 766.90625,
|
|
"completions/mean_terminated_length": 766.90625,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.5499698556959629,
|
|
"epoch": 0.33390022675736963,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"learning_rate": 6.666666666666666e-07,
|
|
"loss": 0.029,
|
|
"num_tokens": 34722872.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6993034482002258,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 589,
|
|
"step_time": 95.65712734311819
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1403.0,
|
|
"completions/max_terminated_length": 1403.0,
|
|
"completions/mean_length": 744.34375,
|
|
"completions/mean_terminated_length": 744.34375,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.5921298786997795,
|
|
"epoch": 0.3344671201814059,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 6.660997732426304e-07,
|
|
"loss": -0.0128,
|
|
"num_tokens": 34780398.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7175520658493042,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 590,
|
|
"step_time": 47.90016901958734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1064.0,
|
|
"completions/max_terminated_length": 1064.0,
|
|
"completions/mean_length": 686.203125,
|
|
"completions/mean_terminated_length": 686.203125,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 1.0365900918841362,
|
|
"epoch": 0.33503401360544216,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.43359375,
|
|
"learning_rate": 6.655328798185941e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 34835507.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 591,
|
|
"step_time": 36.734624343924224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1436.0,
|
|
"completions/max_terminated_length": 1436.0,
|
|
"completions/mean_length": 772.140625,
|
|
"completions/mean_terminated_length": 772.140625,
|
|
"completions/min_length": 404.0,
|
|
"completions/min_terminated_length": 404.0,
|
|
"entropy": 0.806842315942049,
|
|
"epoch": 0.3356009070294785,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.462890625,
|
|
"learning_rate": 6.649659863945578e-07,
|
|
"loss": 0.0047,
|
|
"num_tokens": 34895524.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 592,
|
|
"step_time": 50.1835395488888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 918.0,
|
|
"completions/max_terminated_length": 918.0,
|
|
"completions/mean_length": 660.421875,
|
|
"completions/mean_terminated_length": 660.421875,
|
|
"completions/min_length": 318.0,
|
|
"completions/min_terminated_length": 318.0,
|
|
"entropy": 0.8969474509358406,
|
|
"epoch": 0.33616780045351474,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 6.643990929705216e-07,
|
|
"loss": -0.0263,
|
|
"num_tokens": 34947367.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 593,
|
|
"step_time": 31.30379052646458
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1442.0,
|
|
"completions/max_terminated_length": 1442.0,
|
|
"completions/mean_length": 755.59375,
|
|
"completions/mean_terminated_length": 755.59375,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.690056225284934,
|
|
"epoch": 0.336734693877551,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.41015625,
|
|
"learning_rate": 6.638321995464852e-07,
|
|
"loss": -0.0176,
|
|
"num_tokens": 35004309.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 594,
|
|
"step_time": 49.0545499343425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1356.0,
|
|
"completions/max_terminated_length": 1356.0,
|
|
"completions/mean_length": 795.78125,
|
|
"completions/mean_terminated_length": 795.78125,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.6606776397675276,
|
|
"epoch": 0.3373015873015873,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.357421875,
|
|
"learning_rate": 6.632653061224489e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 35065127.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 595,
|
|
"step_time": 46.109949179925025
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1506.0,
|
|
"completions/max_terminated_length": 1506.0,
|
|
"completions/mean_length": 769.59375,
|
|
"completions/mean_terminated_length": 769.59375,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.647127928212285,
|
|
"epoch": 0.3378684807256236,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.427734375,
|
|
"learning_rate": 6.626984126984126e-07,
|
|
"loss": 0.0099,
|
|
"num_tokens": 35124485.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 596,
|
|
"step_time": 51.34399641864002
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1460.0,
|
|
"completions/max_terminated_length": 1460.0,
|
|
"completions/mean_length": 785.75,
|
|
"completions/mean_terminated_length": 785.75,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 0.8237004764378071,
|
|
"epoch": 0.33843537414965985,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 6.621315192743764e-07,
|
|
"loss": -0.0073,
|
|
"num_tokens": 35183677.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 597,
|
|
"step_time": 49.71723074000329
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1332.0,
|
|
"completions/max_terminated_length": 1332.0,
|
|
"completions/mean_length": 763.015625,
|
|
"completions/mean_terminated_length": 763.015625,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.7137241717427969,
|
|
"epoch": 0.33900226757369617,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 6.615646258503401e-07,
|
|
"loss": 0.0363,
|
|
"num_tokens": 35242974.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 598,
|
|
"step_time": 45.134176816791296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1623.0,
|
|
"completions/max_terminated_length": 1623.0,
|
|
"completions/mean_length": 823.53125,
|
|
"completions/mean_terminated_length": 823.53125,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.6909013129770756,
|
|
"epoch": 0.33956916099773243,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 6.609977324263039e-07,
|
|
"loss": 0.0313,
|
|
"num_tokens": 35308888.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.728528618812561,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 599,
|
|
"step_time": 58.116622598841786
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 772.953125,
|
|
"completions/mean_terminated_length": 772.953125,
|
|
"completions/min_length": 409.0,
|
|
"completions/min_terminated_length": 409.0,
|
|
"entropy": 0.8247405681759119,
|
|
"epoch": 0.3401360544217687,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 6.604308390022676e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 35370845.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 600,
|
|
"step_time": 52.88959987089038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1297.0,
|
|
"completions/max_terminated_length": 1297.0,
|
|
"completions/mean_length": 705.8125,
|
|
"completions/mean_terminated_length": 705.8125,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.6509308498352766,
|
|
"epoch": 0.340702947845805,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 6.598639455782312e-07,
|
|
"loss": 0.0099,
|
|
"num_tokens": 35425377.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 601,
|
|
"step_time": 44.81054830830544
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1102.0,
|
|
"completions/max_terminated_length": 1102.0,
|
|
"completions/mean_length": 680.453125,
|
|
"completions/mean_terminated_length": 680.453125,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.7608289979398251,
|
|
"epoch": 0.3412698412698413,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 6.592970521541949e-07,
|
|
"loss": 0.0142,
|
|
"num_tokens": 35478118.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 602,
|
|
"step_time": 37.77187288366258
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1385.0,
|
|
"completions/max_terminated_length": 1385.0,
|
|
"completions/mean_length": 672.109375,
|
|
"completions/mean_terminated_length": 672.109375,
|
|
"completions/min_length": 237.0,
|
|
"completions/min_terminated_length": 237.0,
|
|
"entropy": 0.6571997348219156,
|
|
"epoch": 0.34183673469387754,
|
|
"frac_reward_zero_std": 0.125,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 6.587301587301587e-07,
|
|
"loss": 0.0388,
|
|
"num_tokens": 35530517.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.9236745238304138,
|
|
"rewards/accuracy_reward_func/mean": 0.625,
|
|
"rewards/accuracy_reward_func/std": 0.934353232383728,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 603,
|
|
"step_time": 47.63913592789322
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1856.0,
|
|
"completions/max_terminated_length": 1856.0,
|
|
"completions/mean_length": 659.203125,
|
|
"completions/mean_terminated_length": 659.203125,
|
|
"completions/min_length": 245.0,
|
|
"completions/min_terminated_length": 245.0,
|
|
"entropy": 0.9442145936191082,
|
|
"epoch": 0.3424036281179138,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 6.581632653061224e-07,
|
|
"loss": 0.0245,
|
|
"num_tokens": 35583354.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 604,
|
|
"step_time": 63.76840851083398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1444.0,
|
|
"completions/max_terminated_length": 1444.0,
|
|
"completions/mean_length": 697.515625,
|
|
"completions/mean_terminated_length": 697.515625,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"entropy": 0.8977196142077446,
|
|
"epoch": 0.3429705215419501,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.458984375,
|
|
"learning_rate": 6.575963718820862e-07,
|
|
"loss": 0.0122,
|
|
"num_tokens": 35638179.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 605,
|
|
"step_time": 49.88650964573026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1533.0,
|
|
"completions/max_terminated_length": 1533.0,
|
|
"completions/mean_length": 657.5,
|
|
"completions/mean_terminated_length": 657.5,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.8332506120204926,
|
|
"epoch": 0.3435374149659864,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 6.570294784580499e-07,
|
|
"loss": 0.0057,
|
|
"num_tokens": 35691707.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 606,
|
|
"step_time": 53.01526586804539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1416.0,
|
|
"completions/max_terminated_length": 1416.0,
|
|
"completions/mean_length": 648.828125,
|
|
"completions/mean_terminated_length": 648.828125,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 1.0154879465699196,
|
|
"epoch": 0.34410430839002265,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.73046875,
|
|
"learning_rate": 6.564625850340136e-07,
|
|
"loss": 0.0016,
|
|
"num_tokens": 35742760.0,
|
|
"reward": 0.4921875,
|
|
"reward_std": 0.8380447030067444,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 607,
|
|
"step_time": 48.210856564342976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 725.875,
|
|
"completions/mean_terminated_length": 725.875,
|
|
"completions/min_length": 293.0,
|
|
"completions/min_terminated_length": 293.0,
|
|
"entropy": 0.944462962448597,
|
|
"epoch": 0.34467120181405897,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 6.558956916099773e-07,
|
|
"loss": 0.0032,
|
|
"num_tokens": 35799640.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 608,
|
|
"step_time": 48.60132034402341
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1429.0,
|
|
"completions/max_terminated_length": 1429.0,
|
|
"completions/mean_length": 846.515625,
|
|
"completions/mean_terminated_length": 846.515625,
|
|
"completions/min_length": 515.0,
|
|
"completions/min_terminated_length": 515.0,
|
|
"entropy": 0.8410714901983738,
|
|
"epoch": 0.34523809523809523,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 6.55328798185941e-07,
|
|
"loss": 0.0339,
|
|
"num_tokens": 35864201.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6299408078193665,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 609,
|
|
"step_time": 48.66408775374293
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1459.0,
|
|
"completions/max_terminated_length": 1459.0,
|
|
"completions/mean_length": 736.625,
|
|
"completions/mean_terminated_length": 736.625,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.7623574510216713,
|
|
"epoch": 0.3458049886621315,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 6.547619047619047e-07,
|
|
"loss": 0.0185,
|
|
"num_tokens": 35922593.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 610,
|
|
"step_time": 50.23085585050285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1625.0,
|
|
"completions/max_terminated_length": 1625.0,
|
|
"completions/mean_length": 805.015625,
|
|
"completions/mean_terminated_length": 805.015625,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.8921775817871094,
|
|
"epoch": 0.3463718820861678,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 6.541950113378684e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 35985210.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 611,
|
|
"step_time": 55.43881920911372
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 772.203125,
|
|
"completions/mean_terminated_length": 772.203125,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.9786438681185246,
|
|
"epoch": 0.3469387755102041,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 6.536281179138322e-07,
|
|
"loss": 0.0244,
|
|
"num_tokens": 36045959.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 612,
|
|
"step_time": 48.973512183874846
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.03125,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1547.0,
|
|
"completions/mean_length": 944.84375,
|
|
"completions/mean_terminated_length": 843.1935424804688,
|
|
"completions/min_length": 332.0,
|
|
"completions/min_terminated_length": 332.0,
|
|
"entropy": 0.7050858326256275,
|
|
"epoch": 0.34750566893424034,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 6.530612244897958e-07,
|
|
"loss": 0.0271,
|
|
"num_tokens": 36116885.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.6664806604385376,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 613,
|
|
"step_time": 167.45049969945103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1424.0,
|
|
"completions/max_terminated_length": 1424.0,
|
|
"completions/mean_length": 714.84375,
|
|
"completions/mean_terminated_length": 714.84375,
|
|
"completions/min_length": 324.0,
|
|
"completions/min_terminated_length": 324.0,
|
|
"entropy": 0.99871014803648,
|
|
"epoch": 0.34807256235827666,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.3125,
|
|
"learning_rate": 6.524943310657596e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 36172699.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 614,
|
|
"step_time": 47.915361465886235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2035.0,
|
|
"completions/max_terminated_length": 2035.0,
|
|
"completions/mean_length": 719.6875,
|
|
"completions/mean_terminated_length": 719.6875,
|
|
"completions/min_length": 175.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"entropy": 0.8770802449434996,
|
|
"epoch": 0.3486394557823129,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 6.519274376417233e-07,
|
|
"loss": -0.0263,
|
|
"num_tokens": 36228455.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 615,
|
|
"step_time": 71.19476261176169
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1373.0,
|
|
"completions/max_terminated_length": 1373.0,
|
|
"completions/mean_length": 780.3125,
|
|
"completions/mean_terminated_length": 780.3125,
|
|
"completions/min_length": 341.0,
|
|
"completions/min_terminated_length": 341.0,
|
|
"entropy": 0.7977379746735096,
|
|
"epoch": 0.3492063492063492,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"learning_rate": 6.513605442176871e-07,
|
|
"loss": 0.03,
|
|
"num_tokens": 36289075.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 616,
|
|
"step_time": 46.82819330506027
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2057.0,
|
|
"completions/max_terminated_length": 2057.0,
|
|
"completions/mean_length": 898.3125,
|
|
"completions/mean_terminated_length": 898.3125,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.5450865644961596,
|
|
"epoch": 0.3497732426303855,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 6.507936507936507e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 36357823.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 617,
|
|
"step_time": 73.42446285765618
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1304.0,
|
|
"completions/max_terminated_length": 1304.0,
|
|
"completions/mean_length": 753.21875,
|
|
"completions/mean_terminated_length": 753.21875,
|
|
"completions/min_length": 288.0,
|
|
"completions/min_terminated_length": 288.0,
|
|
"entropy": 0.9117572195827961,
|
|
"epoch": 0.35034013605442177,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 6.502267573696145e-07,
|
|
"loss": -0.0248,
|
|
"num_tokens": 36416093.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 618,
|
|
"step_time": 44.56446181051433
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1483.0,
|
|
"completions/mean_length": 864.9375,
|
|
"completions/mean_terminated_length": 813.6508178710938,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.7935023289173841,
|
|
"epoch": 0.35090702947845803,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.46484375,
|
|
"learning_rate": 6.496598639455782e-07,
|
|
"loss": 0.0169,
|
|
"num_tokens": 36483521.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 619,
|
|
"step_time": 170.8946770299226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1218.0,
|
|
"completions/max_terminated_length": 1218.0,
|
|
"completions/mean_length": 710.234375,
|
|
"completions/mean_terminated_length": 710.234375,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.8544355668127537,
|
|
"epoch": 0.35147392290249435,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 6.490929705215419e-07,
|
|
"loss": 0.0179,
|
|
"num_tokens": 36539160.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 620,
|
|
"step_time": 42.336375730112195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1233.0,
|
|
"completions/max_terminated_length": 1233.0,
|
|
"completions/mean_length": 711.09375,
|
|
"completions/mean_terminated_length": 711.09375,
|
|
"completions/min_length": 356.0,
|
|
"completions/min_terminated_length": 356.0,
|
|
"entropy": 0.9783514887094498,
|
|
"epoch": 0.3520408163265306,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 6.485260770975056e-07,
|
|
"loss": 0.0062,
|
|
"num_tokens": 36596734.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.81524258852005,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 621,
|
|
"step_time": 42.53386629931629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1196.0,
|
|
"completions/max_terminated_length": 1196.0,
|
|
"completions/mean_length": 740.265625,
|
|
"completions/mean_terminated_length": 740.265625,
|
|
"completions/min_length": 18.0,
|
|
"completions/min_terminated_length": 18.0,
|
|
"entropy": 0.9278873167932034,
|
|
"epoch": 0.3526077097505669,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 6.479591836734694e-07,
|
|
"loss": 0.0076,
|
|
"num_tokens": 36654535.0,
|
|
"reward": 0.0390625,
|
|
"reward_std": 0.13524484634399414,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 622,
|
|
"step_time": 40.763666816055775
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1187.0,
|
|
"completions/max_terminated_length": 1187.0,
|
|
"completions/mean_length": 697.25,
|
|
"completions/mean_terminated_length": 697.25,
|
|
"completions/min_length": 318.0,
|
|
"completions/min_terminated_length": 318.0,
|
|
"entropy": 0.8700597248971462,
|
|
"epoch": 0.3531746031746032,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 6.473922902494331e-07,
|
|
"loss": -0.0035,
|
|
"num_tokens": 36713311.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 623,
|
|
"step_time": 40.7892361106351
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 767.1875,
|
|
"completions/mean_terminated_length": 767.1875,
|
|
"completions/min_length": 225.0,
|
|
"completions/min_terminated_length": 225.0,
|
|
"entropy": 0.7525980323553085,
|
|
"epoch": 0.35374149659863946,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 6.468253968253969e-07,
|
|
"loss": -0.006,
|
|
"num_tokens": 36772115.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 624,
|
|
"step_time": 49.61737298686057
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1688.0,
|
|
"completions/max_terminated_length": 1688.0,
|
|
"completions/mean_length": 818.109375,
|
|
"completions/mean_terminated_length": 818.109375,
|
|
"completions/min_length": 304.0,
|
|
"completions/min_terminated_length": 304.0,
|
|
"entropy": 0.7521986719220877,
|
|
"epoch": 0.3543083900226757,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 6.462585034013605e-07,
|
|
"loss": -0.0161,
|
|
"num_tokens": 36835690.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.6167611479759216,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 625,
|
|
"step_time": 58.91802252270281
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1658.0,
|
|
"completions/max_terminated_length": 1658.0,
|
|
"completions/mean_length": 840.390625,
|
|
"completions/mean_terminated_length": 840.390625,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.717966303229332,
|
|
"epoch": 0.35487528344671204,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 6.456916099773242e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 36901123.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.43408530950546265,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 626,
|
|
"step_time": 57.560720714740455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1388.0,
|
|
"completions/max_terminated_length": 1388.0,
|
|
"completions/mean_length": 777.40625,
|
|
"completions/mean_terminated_length": 777.40625,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.8006776142865419,
|
|
"epoch": 0.3554421768707483,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"learning_rate": 6.451247165532879e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 36962941.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 627,
|
|
"step_time": 47.950291788205504
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 757.0,
|
|
"completions/mean_terminated_length": 757.0,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"entropy": 0.6608790792524815,
|
|
"epoch": 0.35600907029478457,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 6.445578231292517e-07,
|
|
"loss": 0.0192,
|
|
"num_tokens": 37020725.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.8158888816833496,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 628,
|
|
"step_time": 46.2614265633747
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1352.0,
|
|
"completions/max_terminated_length": 1352.0,
|
|
"completions/mean_length": 818.3125,
|
|
"completions/mean_terminated_length": 818.3125,
|
|
"completions/min_length": 218.0,
|
|
"completions/min_terminated_length": 218.0,
|
|
"entropy": 0.6299032997339964,
|
|
"epoch": 0.3565759637188209,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 6.439909297052154e-07,
|
|
"loss": 0.0221,
|
|
"num_tokens": 37082113.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 629,
|
|
"step_time": 46.16052764002234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1500.0,
|
|
"completions/max_terminated_length": 1500.0,
|
|
"completions/mean_length": 783.0,
|
|
"completions/mean_terminated_length": 783.0,
|
|
"completions/min_length": 411.0,
|
|
"completions/min_terminated_length": 411.0,
|
|
"entropy": 0.6570486966520548,
|
|
"epoch": 0.35714285714285715,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 6.434240362811792e-07,
|
|
"loss": -0.0167,
|
|
"num_tokens": 37143633.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 630,
|
|
"step_time": 51.87512902356684
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1419.0,
|
|
"completions/max_terminated_length": 1419.0,
|
|
"completions/mean_length": 766.234375,
|
|
"completions/mean_terminated_length": 766.234375,
|
|
"completions/min_length": 394.0,
|
|
"completions/min_terminated_length": 394.0,
|
|
"entropy": 0.9654701687395573,
|
|
"epoch": 0.3577097505668934,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2578125,
|
|
"learning_rate": 6.428571428571429e-07,
|
|
"loss": -0.0025,
|
|
"num_tokens": 37202672.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 631,
|
|
"step_time": 48.53060559555888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1952.0,
|
|
"completions/max_terminated_length": 1952.0,
|
|
"completions/mean_length": 642.921875,
|
|
"completions/mean_terminated_length": 642.921875,
|
|
"completions/min_length": 220.0,
|
|
"completions/min_terminated_length": 220.0,
|
|
"entropy": 0.8917891718447208,
|
|
"epoch": 0.35827664399092973,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 6.422902494331065e-07,
|
|
"loss": -0.0179,
|
|
"num_tokens": 37254427.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 632,
|
|
"step_time": 68.72827850095928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1191.0,
|
|
"completions/max_terminated_length": 1191.0,
|
|
"completions/mean_length": 758.796875,
|
|
"completions/mean_terminated_length": 758.796875,
|
|
"completions/min_length": 447.0,
|
|
"completions/min_terminated_length": 447.0,
|
|
"entropy": 0.8271527048200369,
|
|
"epoch": 0.358843537414966,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 6.417233560090702e-07,
|
|
"loss": -0.0232,
|
|
"num_tokens": 37312790.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 633,
|
|
"step_time": 40.42411103658378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1263.0,
|
|
"completions/max_terminated_length": 1263.0,
|
|
"completions/mean_length": 733.765625,
|
|
"completions/mean_terminated_length": 733.765625,
|
|
"completions/min_length": 269.0,
|
|
"completions/min_terminated_length": 269.0,
|
|
"entropy": 0.7152926605194807,
|
|
"epoch": 0.35941043083900226,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 6.411564625850339e-07,
|
|
"loss": 0.0052,
|
|
"num_tokens": 37370775.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 634,
|
|
"step_time": 43.42288984358311
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1476.0,
|
|
"completions/max_terminated_length": 1476.0,
|
|
"completions/mean_length": 807.0,
|
|
"completions/mean_terminated_length": 807.0,
|
|
"completions/min_length": 346.0,
|
|
"completions/min_terminated_length": 346.0,
|
|
"entropy": 0.524667102843523,
|
|
"epoch": 0.3599773242630385,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.42578125,
|
|
"learning_rate": 6.405895691609977e-07,
|
|
"loss": 0.0139,
|
|
"num_tokens": 37432231.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5754139423370361,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 635,
|
|
"step_time": 50.549458659254014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1161.0,
|
|
"completions/max_terminated_length": 1161.0,
|
|
"completions/mean_length": 664.625,
|
|
"completions/mean_terminated_length": 664.625,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.8426170144230127,
|
|
"epoch": 0.36054421768707484,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 6.400226757369614e-07,
|
|
"loss": 0.0249,
|
|
"num_tokens": 37484247.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 636,
|
|
"step_time": 38.97255809511989
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1452.0,
|
|
"completions/max_terminated_length": 1452.0,
|
|
"completions/mean_length": 751.203125,
|
|
"completions/mean_terminated_length": 751.203125,
|
|
"completions/min_length": 419.0,
|
|
"completions/min_terminated_length": 419.0,
|
|
"entropy": 0.8785478807985783,
|
|
"epoch": 0.3611111111111111,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 6.394557823129252e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 37543156.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 637,
|
|
"step_time": 49.94559401087463
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1330.0,
|
|
"completions/max_terminated_length": 1330.0,
|
|
"completions/mean_length": 747.765625,
|
|
"completions/mean_terminated_length": 747.765625,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 0.7077295072376728,
|
|
"epoch": 0.36167800453514737,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"learning_rate": 6.388888888888888e-07,
|
|
"loss": -0.0281,
|
|
"num_tokens": 37600861.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6023679971694946,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 638,
|
|
"step_time": 45.29154534637928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1479.0,
|
|
"completions/max_terminated_length": 1479.0,
|
|
"completions/mean_length": 856.625,
|
|
"completions/mean_terminated_length": 856.625,
|
|
"completions/min_length": 399.0,
|
|
"completions/min_terminated_length": 399.0,
|
|
"entropy": 0.7673860471695662,
|
|
"epoch": 0.3622448979591837,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.361328125,
|
|
"learning_rate": 6.383219954648526e-07,
|
|
"loss": -0.009,
|
|
"num_tokens": 37666237.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 639,
|
|
"step_time": 50.89554338529706
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1670.0,
|
|
"completions/max_terminated_length": 1670.0,
|
|
"completions/mean_length": 699.71875,
|
|
"completions/mean_terminated_length": 699.71875,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.7384172566235065,
|
|
"epoch": 0.36281179138321995,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 6.377551020408163e-07,
|
|
"loss": -0.0041,
|
|
"num_tokens": 37721307.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 640,
|
|
"step_time": 56.23733188863844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1224.0,
|
|
"completions/max_terminated_length": 1224.0,
|
|
"completions/mean_length": 644.109375,
|
|
"completions/mean_terminated_length": 644.109375,
|
|
"completions/min_length": 22.0,
|
|
"completions/min_terminated_length": 22.0,
|
|
"entropy": 0.7242608647793531,
|
|
"epoch": 0.3633786848072562,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 6.3718820861678e-07,
|
|
"loss": -0.0272,
|
|
"num_tokens": 37774218.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 641,
|
|
"step_time": 42.41266290098429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2104.0,
|
|
"completions/max_terminated_length": 2104.0,
|
|
"completions/mean_length": 825.53125,
|
|
"completions/mean_terminated_length": 825.53125,
|
|
"completions/min_length": 236.0,
|
|
"completions/min_terminated_length": 236.0,
|
|
"entropy": 0.810003375634551,
|
|
"epoch": 0.36394557823129253,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 6.366213151927437e-07,
|
|
"loss": -0.0085,
|
|
"num_tokens": 37838356.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.725073516368866,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 642,
|
|
"step_time": 73.07388027384877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1489.0,
|
|
"completions/max_terminated_length": 1489.0,
|
|
"completions/mean_length": 803.25,
|
|
"completions/mean_terminated_length": 803.25,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.7468740493059158,
|
|
"epoch": 0.3645124716553288,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 6.360544217687075e-07,
|
|
"loss": 0.0256,
|
|
"num_tokens": 37900724.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 643,
|
|
"step_time": 51.518991655670106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1696.0,
|
|
"completions/max_terminated_length": 1696.0,
|
|
"completions/mean_length": 850.265625,
|
|
"completions/mean_terminated_length": 850.265625,
|
|
"completions/min_length": 371.0,
|
|
"completions/min_terminated_length": 371.0,
|
|
"entropy": 0.673087963834405,
|
|
"epoch": 0.36507936507936506,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.328125,
|
|
"learning_rate": 6.354875283446711e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 37964213.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 644,
|
|
"step_time": 58.71310798358172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1975.0,
|
|
"completions/max_terminated_length": 1975.0,
|
|
"completions/mean_length": 778.71875,
|
|
"completions/mean_terminated_length": 778.71875,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 0.9261895325034857,
|
|
"epoch": 0.3656462585034014,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 6.349206349206349e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 38025019.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 645,
|
|
"step_time": 68.84365547727793
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1166.0,
|
|
"completions/max_terminated_length": 1166.0,
|
|
"completions/mean_length": 729.046875,
|
|
"completions/mean_terminated_length": 729.046875,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.8771541491150856,
|
|
"epoch": 0.36621315192743764,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 6.343537414965986e-07,
|
|
"loss": -0.01,
|
|
"num_tokens": 38083278.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 646,
|
|
"step_time": 40.352095168083906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1622.0,
|
|
"completions/max_terminated_length": 1622.0,
|
|
"completions/mean_length": 723.34375,
|
|
"completions/mean_terminated_length": 723.34375,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 1.0137921087443829,
|
|
"epoch": 0.3667800453514739,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 6.337868480725624e-07,
|
|
"loss": 0.0016,
|
|
"num_tokens": 38142580.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 647,
|
|
"step_time": 57.25881090760231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1098.0,
|
|
"completions/max_terminated_length": 1098.0,
|
|
"completions/mean_length": 685.1875,
|
|
"completions/mean_terminated_length": 685.1875,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.7464550426229835,
|
|
"epoch": 0.3673469387755102,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 6.332199546485261e-07,
|
|
"loss": 0.0115,
|
|
"num_tokens": 38200488.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 648,
|
|
"step_time": 38.424966879189014
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1172.0,
|
|
"completions/max_terminated_length": 1172.0,
|
|
"completions/mean_length": 693.640625,
|
|
"completions/mean_terminated_length": 693.640625,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.8691299445927143,
|
|
"epoch": 0.3679138321995465,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 6.326530612244898e-07,
|
|
"loss": -0.0586,
|
|
"num_tokens": 38256145.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 649,
|
|
"step_time": 41.102212926372886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1296.0,
|
|
"completions/max_terminated_length": 1296.0,
|
|
"completions/mean_length": 707.546875,
|
|
"completions/mean_terminated_length": 707.546875,
|
|
"completions/min_length": 208.0,
|
|
"completions/min_terminated_length": 208.0,
|
|
"entropy": 0.919795224443078,
|
|
"epoch": 0.36848072562358275,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.640625,
|
|
"learning_rate": 6.320861678004535e-07,
|
|
"loss": 0.0195,
|
|
"num_tokens": 38313532.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 650,
|
|
"step_time": 45.144144896417856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1353.0,
|
|
"completions/max_terminated_length": 1353.0,
|
|
"completions/mean_length": 709.59375,
|
|
"completions/mean_terminated_length": 709.59375,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.7981938906013966,
|
|
"epoch": 0.36904761904761907,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 6.315192743764171e-07,
|
|
"loss": -0.0176,
|
|
"num_tokens": 38370258.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 651,
|
|
"step_time": 47.05556992441416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 764.5,
|
|
"completions/mean_terminated_length": 764.5,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.7629802487790585,
|
|
"epoch": 0.36961451247165533,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 6.309523809523809e-07,
|
|
"loss": -0.0039,
|
|
"num_tokens": 38428738.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 652,
|
|
"step_time": 48.13539344444871
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1088.0,
|
|
"completions/max_terminated_length": 1088.0,
|
|
"completions/mean_length": 655.28125,
|
|
"completions/mean_terminated_length": 655.28125,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 0.8418986964970827,
|
|
"epoch": 0.3701814058956916,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.671875,
|
|
"learning_rate": 6.303854875283446e-07,
|
|
"loss": 0.0087,
|
|
"num_tokens": 38482532.0,
|
|
"reward": 0.5390625,
|
|
"reward_std": 0.89666348695755,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 653,
|
|
"step_time": 38.0635462095961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1431.0,
|
|
"completions/max_terminated_length": 1431.0,
|
|
"completions/mean_length": 750.359375,
|
|
"completions/mean_terminated_length": 750.359375,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.7542352508753538,
|
|
"epoch": 0.3707482993197279,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 6.298185941043084e-07,
|
|
"loss": -0.0113,
|
|
"num_tokens": 38541355.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8042545318603516,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 654,
|
|
"step_time": 49.74752385355532
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1817.0,
|
|
"completions/max_terminated_length": 1817.0,
|
|
"completions/mean_length": 693.984375,
|
|
"completions/mean_terminated_length": 693.984375,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.7433206718415022,
|
|
"epoch": 0.3713151927437642,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.61328125,
|
|
"learning_rate": 6.292517006802721e-07,
|
|
"loss": 0.0248,
|
|
"num_tokens": 38596466.0,
|
|
"reward": 0.4140625,
|
|
"reward_std": 0.7794721722602844,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 655,
|
|
"step_time": 62.506738415919244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1489.0,
|
|
"completions/max_terminated_length": 1489.0,
|
|
"completions/mean_length": 774.453125,
|
|
"completions/mean_terminated_length": 774.453125,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.8884105794131756,
|
|
"epoch": 0.37188208616780044,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 6.286848072562359e-07,
|
|
"loss": -0.0053,
|
|
"num_tokens": 38656367.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 656,
|
|
"step_time": 50.6804611440748
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1346.0,
|
|
"completions/max_terminated_length": 1346.0,
|
|
"completions/mean_length": 798.75,
|
|
"completions/mean_terminated_length": 798.75,
|
|
"completions/min_length": 531.0,
|
|
"completions/min_terminated_length": 531.0,
|
|
"entropy": 0.9627105258405209,
|
|
"epoch": 0.37244897959183676,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.265625,
|
|
"learning_rate": 6.281179138321994e-07,
|
|
"loss": 0.0014,
|
|
"num_tokens": 38718191.0,
|
|
"reward": 0.03125,
|
|
"reward_std": 0.12198750674724579,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 657,
|
|
"step_time": 46.4745169011876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 671.015625,
|
|
"completions/mean_terminated_length": 671.015625,
|
|
"completions/min_length": 244.0,
|
|
"completions/min_terminated_length": 244.0,
|
|
"entropy": 0.6412702202796936,
|
|
"epoch": 0.373015873015873,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 6.275510204081632e-07,
|
|
"loss": 0.0115,
|
|
"num_tokens": 38770584.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.75,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 658,
|
|
"step_time": 43.05835712514818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1335.0,
|
|
"completions/max_terminated_length": 1335.0,
|
|
"completions/mean_length": 660.75,
|
|
"completions/mean_terminated_length": 660.75,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 0.9306098241358995,
|
|
"epoch": 0.3735827664399093,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 6.269841269841269e-07,
|
|
"loss": 0.0206,
|
|
"num_tokens": 38824848.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6542273163795471,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 659,
|
|
"step_time": 47.23293770570308
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1761.0,
|
|
"completions/max_terminated_length": 1761.0,
|
|
"completions/mean_length": 663.109375,
|
|
"completions/mean_terminated_length": 663.109375,
|
|
"completions/min_length": 268.0,
|
|
"completions/min_terminated_length": 268.0,
|
|
"entropy": 0.7533489614725113,
|
|
"epoch": 0.3741496598639456,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 6.264172335600907e-07,
|
|
"loss": 0.0452,
|
|
"num_tokens": 38877199.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 660,
|
|
"step_time": 60.841483088210225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2161.0,
|
|
"completions/max_terminated_length": 2161.0,
|
|
"completions/mean_length": 950.015625,
|
|
"completions/mean_terminated_length": 950.015625,
|
|
"completions/min_length": 97.0,
|
|
"completions/min_terminated_length": 97.0,
|
|
"entropy": 0.6504107564687729,
|
|
"epoch": 0.37471655328798187,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.431640625,
|
|
"learning_rate": 6.258503401360544e-07,
|
|
"loss": 0.0196,
|
|
"num_tokens": 38949080.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 661,
|
|
"step_time": 75.70265740901232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1536.0,
|
|
"completions/mean_length": 739.921875,
|
|
"completions/mean_terminated_length": 739.921875,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.7864415291696787,
|
|
"epoch": 0.37528344671201813,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 6.252834467120182e-07,
|
|
"loss": -0.0188,
|
|
"num_tokens": 39006931.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 662,
|
|
"step_time": 52.27386832423508
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1872.0,
|
|
"completions/max_terminated_length": 1872.0,
|
|
"completions/mean_length": 828.875,
|
|
"completions/mean_terminated_length": 828.875,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.8523445185273886,
|
|
"epoch": 0.3758503401360544,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 6.247165532879818e-07,
|
|
"loss": -0.0363,
|
|
"num_tokens": 39070923.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 663,
|
|
"step_time": 63.8998099565506
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 973.0,
|
|
"completions/max_terminated_length": 973.0,
|
|
"completions/mean_length": 619.65625,
|
|
"completions/mean_terminated_length": 619.65625,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.8749851696193218,
|
|
"epoch": 0.3764172335600907,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 6.241496598639456e-07,
|
|
"loss": 0.0023,
|
|
"num_tokens": 39120301.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6538955569267273,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 664,
|
|
"step_time": 33.00213658250868
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1175.0,
|
|
"completions/max_terminated_length": 1175.0,
|
|
"completions/mean_length": 712.78125,
|
|
"completions/mean_terminated_length": 712.78125,
|
|
"completions/min_length": 297.0,
|
|
"completions/min_terminated_length": 297.0,
|
|
"entropy": 0.79150197096169,
|
|
"epoch": 0.376984126984127,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 6.235827664399092e-07,
|
|
"loss": 0.0167,
|
|
"num_tokens": 39175031.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7724404335021973,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 665,
|
|
"step_time": 39.8501486890018
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1400.0,
|
|
"completions/max_terminated_length": 1400.0,
|
|
"completions/mean_length": 698.96875,
|
|
"completions/mean_terminated_length": 698.96875,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 1.027171328663826,
|
|
"epoch": 0.37755102040816324,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7421875,
|
|
"learning_rate": 6.23015873015873e-07,
|
|
"loss": -0.0039,
|
|
"num_tokens": 39232597.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 666,
|
|
"step_time": 48.249097941443324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1352.0,
|
|
"completions/max_terminated_length": 1352.0,
|
|
"completions/mean_length": 667.390625,
|
|
"completions/mean_terminated_length": 667.390625,
|
|
"completions/min_length": 246.0,
|
|
"completions/min_terminated_length": 246.0,
|
|
"entropy": 0.7203786689788103,
|
|
"epoch": 0.37811791383219956,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 6.224489795918367e-07,
|
|
"loss": 0.0172,
|
|
"num_tokens": 39285686.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 667,
|
|
"step_time": 45.993100920692086
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2115.0,
|
|
"completions/max_terminated_length": 2115.0,
|
|
"completions/mean_length": 832.96875,
|
|
"completions/mean_terminated_length": 832.96875,
|
|
"completions/min_length": 309.0,
|
|
"completions/min_terminated_length": 309.0,
|
|
"entropy": 0.7268731407821178,
|
|
"epoch": 0.3786848072562358,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.302734375,
|
|
"learning_rate": 6.218820861678005e-07,
|
|
"loss": -0.0119,
|
|
"num_tokens": 39352492.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 668,
|
|
"step_time": 76.51206331700087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1736.0,
|
|
"completions/max_terminated_length": 1736.0,
|
|
"completions/mean_length": 795.03125,
|
|
"completions/mean_terminated_length": 795.03125,
|
|
"completions/min_length": 393.0,
|
|
"completions/min_terminated_length": 393.0,
|
|
"entropy": 1.0189481489360332,
|
|
"epoch": 0.3792517006802721,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.205078125,
|
|
"learning_rate": 6.213151927437641e-07,
|
|
"loss": 0.0107,
|
|
"num_tokens": 39415294.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 669,
|
|
"step_time": 61.95392454788089
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1400.0,
|
|
"completions/max_terminated_length": 1400.0,
|
|
"completions/mean_length": 678.34375,
|
|
"completions/mean_terminated_length": 678.34375,
|
|
"completions/min_length": 275.0,
|
|
"completions/min_terminated_length": 275.0,
|
|
"entropy": 0.7759978659451008,
|
|
"epoch": 0.3798185941043084,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 6.207482993197279e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 39467188.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 670,
|
|
"step_time": 47.91178299859166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1682.0,
|
|
"completions/max_terminated_length": 1682.0,
|
|
"completions/mean_length": 692.203125,
|
|
"completions/mean_terminated_length": 692.203125,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.7530367765575647,
|
|
"epoch": 0.38038548752834467,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"learning_rate": 6.201814058956916e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 39524657.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4364357888698578,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 671,
|
|
"step_time": 60.2888721851632
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1403.0,
|
|
"completions/max_terminated_length": 1403.0,
|
|
"completions/mean_length": 649.578125,
|
|
"completions/mean_terminated_length": 649.578125,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.9415013194084167,
|
|
"epoch": 0.38095238095238093,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 6.196145124716554e-07,
|
|
"loss": 0.0153,
|
|
"num_tokens": 39576438.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.603396475315094,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 672,
|
|
"step_time": 47.99685996212065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1588.0,
|
|
"completions/max_terminated_length": 1588.0,
|
|
"completions/mean_length": 696.28125,
|
|
"completions/mean_terminated_length": 696.28125,
|
|
"completions/min_length": 301.0,
|
|
"completions/min_terminated_length": 301.0,
|
|
"entropy": 0.9544547498226166,
|
|
"epoch": 0.38151927437641725,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4375,
|
|
"learning_rate": 6.19047619047619e-07,
|
|
"loss": -0.0038,
|
|
"num_tokens": 39630928.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 673,
|
|
"step_time": 54.064920268021524
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1364.0,
|
|
"completions/max_terminated_length": 1364.0,
|
|
"completions/mean_length": 698.296875,
|
|
"completions/mean_terminated_length": 698.296875,
|
|
"completions/min_length": 388.0,
|
|
"completions/min_terminated_length": 388.0,
|
|
"entropy": 0.8187709338963032,
|
|
"epoch": 0.3820861678004535,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 6.184807256235827e-07,
|
|
"loss": 0.0406,
|
|
"num_tokens": 39686555.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 674,
|
|
"step_time": 46.89131363015622
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1380.0,
|
|
"completions/max_terminated_length": 1380.0,
|
|
"completions/mean_length": 753.46875,
|
|
"completions/mean_terminated_length": 753.46875,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.7073788288980722,
|
|
"epoch": 0.3826530612244898,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 6.179138321995464e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 39745129.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 675,
|
|
"step_time": 47.17899635992944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 805.90625,
|
|
"completions/mean_terminated_length": 805.90625,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.7654253169894218,
|
|
"epoch": 0.3832199546485261,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 6.173469387755101e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 39806963.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 676,
|
|
"step_time": 50.05393888242543
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1368.0,
|
|
"completions/max_terminated_length": 1368.0,
|
|
"completions/mean_length": 702.984375,
|
|
"completions/mean_terminated_length": 702.984375,
|
|
"completions/min_length": 280.0,
|
|
"completions/min_terminated_length": 280.0,
|
|
"entropy": 0.8189000021666288,
|
|
"epoch": 0.38378684807256236,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 6.167800453514739e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 39863642.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 677,
|
|
"step_time": 47.03352596703917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 996.0,
|
|
"completions/max_terminated_length": 996.0,
|
|
"completions/mean_length": 544.640625,
|
|
"completions/mean_terminated_length": 544.640625,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 1.0272091627120972,
|
|
"epoch": 0.3843537414965986,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 6.162131519274376e-07,
|
|
"loss": 0.0088,
|
|
"num_tokens": 39908995.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 678,
|
|
"step_time": 34.143078247085214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2551.0,
|
|
"completions/max_terminated_length": 2551.0,
|
|
"completions/mean_length": 708.828125,
|
|
"completions/mean_terminated_length": 708.828125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.8408221248537302,
|
|
"epoch": 0.38492063492063494,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 6.156462585034014e-07,
|
|
"loss": 0.0159,
|
|
"num_tokens": 39970608.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.6878606677055359,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 679,
|
|
"step_time": 101.4084204416722
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1166.0,
|
|
"completions/max_terminated_length": 1166.0,
|
|
"completions/mean_length": 666.5,
|
|
"completions/mean_terminated_length": 666.5,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.720644436776638,
|
|
"epoch": 0.3854875283446712,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 6.150793650793651e-07,
|
|
"loss": 0.009,
|
|
"num_tokens": 40024016.0,
|
|
"reward": 0.484375,
|
|
"reward_std": 0.8402982950210571,
|
|
"rewards/accuracy_reward_func/mean": 0.4375,
|
|
"rewards/accuracy_reward_func/std": 0.8333333730697632,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 680,
|
|
"step_time": 40.70588407944888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1545.0,
|
|
"completions/max_terminated_length": 1545.0,
|
|
"completions/mean_length": 814.765625,
|
|
"completions/mean_terminated_length": 814.765625,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 0.5790644586086273,
|
|
"epoch": 0.38605442176870747,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 6.145124716553288e-07,
|
|
"loss": -0.0126,
|
|
"num_tokens": 40085937.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.8506752252578735,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 681,
|
|
"step_time": 53.31244630552828
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1194.0,
|
|
"completions/max_terminated_length": 1194.0,
|
|
"completions/mean_length": 700.921875,
|
|
"completions/mean_terminated_length": 700.921875,
|
|
"completions/min_length": 250.0,
|
|
"completions/min_terminated_length": 250.0,
|
|
"entropy": 0.8838515989482403,
|
|
"epoch": 0.3866213151927438,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.287109375,
|
|
"learning_rate": 6.139455782312924e-07,
|
|
"loss": 0.0115,
|
|
"num_tokens": 40140412.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 682,
|
|
"step_time": 40.56008638162166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1335.0,
|
|
"completions/max_terminated_length": 1335.0,
|
|
"completions/mean_length": 706.5625,
|
|
"completions/mean_terminated_length": 706.5625,
|
|
"completions/min_length": 239.0,
|
|
"completions/min_terminated_length": 239.0,
|
|
"entropy": 0.7888701613992453,
|
|
"epoch": 0.38718820861678005,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 6.133786848072562e-07,
|
|
"loss": 0.0037,
|
|
"num_tokens": 40196552.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 683,
|
|
"step_time": 45.72881556674838
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 667.0,
|
|
"completions/mean_terminated_length": 667.0,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.7939537465572357,
|
|
"epoch": 0.3877551020408163,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.609375,
|
|
"learning_rate": 6.128117913832199e-07,
|
|
"loss": -0.0012,
|
|
"num_tokens": 40250568.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5037065148353577,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.109375,
|
|
"rewards/format_reward_func/std": 0.2083333432674408,
|
|
"step": 684,
|
|
"step_time": 42.73227133601904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1704.0,
|
|
"completions/max_terminated_length": 1704.0,
|
|
"completions/mean_length": 827.609375,
|
|
"completions/mean_terminated_length": 827.609375,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.6719684153795242,
|
|
"epoch": 0.38832199546485263,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 6.122448979591837e-07,
|
|
"loss": -0.0568,
|
|
"num_tokens": 40313031.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 685,
|
|
"step_time": 57.538355994969606
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1648.0,
|
|
"completions/max_terminated_length": 1648.0,
|
|
"completions/mean_length": 861.515625,
|
|
"completions/mean_terminated_length": 861.515625,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.5714195594191551,
|
|
"epoch": 0.3888888888888889,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.240234375,
|
|
"learning_rate": 6.116780045351474e-07,
|
|
"loss": -0.0039,
|
|
"num_tokens": 40380608.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 686,
|
|
"step_time": 57.4830530770123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1139.0,
|
|
"completions/max_terminated_length": 1139.0,
|
|
"completions/mean_length": 732.8125,
|
|
"completions/mean_terminated_length": 732.8125,
|
|
"completions/min_length": 382.0,
|
|
"completions/min_terminated_length": 382.0,
|
|
"entropy": 0.8867952115833759,
|
|
"epoch": 0.38945578231292516,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 6.111111111111112e-07,
|
|
"loss": 0.0081,
|
|
"num_tokens": 40437204.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.6950790882110596,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 687,
|
|
"step_time": 39.095103594474494
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 987.0,
|
|
"completions/max_terminated_length": 987.0,
|
|
"completions/mean_length": 572.3125,
|
|
"completions/mean_terminated_length": 572.3125,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.8474721796810627,
|
|
"epoch": 0.3900226757369615,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 6.105442176870748e-07,
|
|
"loss": -0.001,
|
|
"num_tokens": 40484848.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 688,
|
|
"step_time": 34.06386910099536
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1576.0,
|
|
"completions/max_terminated_length": 1576.0,
|
|
"completions/mean_length": 756.40625,
|
|
"completions/mean_terminated_length": 756.40625,
|
|
"completions/min_length": 466.0,
|
|
"completions/min_terminated_length": 466.0,
|
|
"entropy": 0.8588584586977959,
|
|
"epoch": 0.39058956916099774,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 6.099773242630385e-07,
|
|
"loss": -0.0057,
|
|
"num_tokens": 40543778.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7724404335021973,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 689,
|
|
"step_time": 53.495136791840196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1401.0,
|
|
"completions/max_terminated_length": 1401.0,
|
|
"completions/mean_length": 732.53125,
|
|
"completions/mean_terminated_length": 732.53125,
|
|
"completions/min_length": 294.0,
|
|
"completions/min_terminated_length": 294.0,
|
|
"entropy": 0.7606502324342728,
|
|
"epoch": 0.391156462585034,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"learning_rate": 6.094104308390022e-07,
|
|
"loss": 0.0096,
|
|
"num_tokens": 40600692.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 690,
|
|
"step_time": 47.67858843225986
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1372.0,
|
|
"completions/max_terminated_length": 1372.0,
|
|
"completions/mean_length": 734.78125,
|
|
"completions/mean_terminated_length": 734.78125,
|
|
"completions/min_length": 351.0,
|
|
"completions/min_terminated_length": 351.0,
|
|
"entropy": 0.7689732853323221,
|
|
"epoch": 0.39172335600907027,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.388671875,
|
|
"learning_rate": 6.08843537414966e-07,
|
|
"loss": 0.0183,
|
|
"num_tokens": 40657950.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4753340482711792,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 691,
|
|
"step_time": 46.17201771680266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1816.0,
|
|
"completions/max_terminated_length": 1816.0,
|
|
"completions/mean_length": 755.828125,
|
|
"completions/mean_terminated_length": 755.828125,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 0.786823371425271,
|
|
"epoch": 0.3922902494331066,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.2734375,
|
|
"learning_rate": 6.082766439909297e-07,
|
|
"loss": 0.0068,
|
|
"num_tokens": 40716715.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.41539984941482544,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 692,
|
|
"step_time": 62.187869605608284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1218.0,
|
|
"completions/max_terminated_length": 1218.0,
|
|
"completions/mean_length": 647.65625,
|
|
"completions/mean_terminated_length": 647.65625,
|
|
"completions/min_length": 202.0,
|
|
"completions/min_terminated_length": 202.0,
|
|
"entropy": 0.9059557132422924,
|
|
"epoch": 0.39285714285714285,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 6.077097505668934e-07,
|
|
"loss": -0.0175,
|
|
"num_tokens": 40767525.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6900206208229065,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 693,
|
|
"step_time": 41.2340766787529
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1199.0,
|
|
"completions/max_terminated_length": 1199.0,
|
|
"completions/mean_length": 719.21875,
|
|
"completions/mean_terminated_length": 719.21875,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.9956004079431295,
|
|
"epoch": 0.3934240362811791,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 6.071428571428571e-07,
|
|
"loss": 0.0411,
|
|
"num_tokens": 40823899.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.7835976481437683,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 694,
|
|
"step_time": 40.48257102072239
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1414.0,
|
|
"completions/max_terminated_length": 1414.0,
|
|
"completions/mean_length": 756.078125,
|
|
"completions/mean_terminated_length": 756.078125,
|
|
"completions/min_length": 233.0,
|
|
"completions/min_terminated_length": 233.0,
|
|
"entropy": 0.830492103472352,
|
|
"epoch": 0.39399092970521543,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2041015625,
|
|
"learning_rate": 6.065759637188208e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 40884488.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 695,
|
|
"step_time": 49.3948492500931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1581.0,
|
|
"completions/max_terminated_length": 1581.0,
|
|
"completions/mean_length": 739.84375,
|
|
"completions/mean_terminated_length": 739.84375,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.6636850759387016,
|
|
"epoch": 0.3945578231292517,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 6.060090702947846e-07,
|
|
"loss": -0.0156,
|
|
"num_tokens": 40943910.0,
|
|
"reward": 0.40625,
|
|
"reward_std": 0.7708735466003418,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 696,
|
|
"step_time": 55.97520697116852
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1650.0,
|
|
"completions/max_terminated_length": 1650.0,
|
|
"completions/mean_length": 697.921875,
|
|
"completions/mean_terminated_length": 697.921875,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.73609808832407,
|
|
"epoch": 0.39512471655328796,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 6.054421768707482e-07,
|
|
"loss": 0.0007,
|
|
"num_tokens": 40999065.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 697,
|
|
"step_time": 55.76855263393372
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1517.0,
|
|
"completions/max_terminated_length": 1517.0,
|
|
"completions/mean_length": 726.25,
|
|
"completions/mean_terminated_length": 726.25,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.784137524664402,
|
|
"epoch": 0.3956916099773243,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 6.04875283446712e-07,
|
|
"loss": 0.0198,
|
|
"num_tokens": 41055809.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 698,
|
|
"step_time": 51.082175908610225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1463.0,
|
|
"completions/max_terminated_length": 1463.0,
|
|
"completions/mean_length": 730.53125,
|
|
"completions/mean_terminated_length": 730.53125,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.7605953272432089,
|
|
"epoch": 0.39625850340136054,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.48046875,
|
|
"learning_rate": 6.043083900226757e-07,
|
|
"loss": 0.0312,
|
|
"num_tokens": 41114627.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 699,
|
|
"step_time": 50.327455514110625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1494.0,
|
|
"completions/max_terminated_length": 1494.0,
|
|
"completions/mean_length": 744.046875,
|
|
"completions/mean_terminated_length": 744.046875,
|
|
"completions/min_length": 319.0,
|
|
"completions/min_terminated_length": 319.0,
|
|
"entropy": 0.7873789332807064,
|
|
"epoch": 0.3968253968253968,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3671875,
|
|
"learning_rate": 6.037414965986394e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 41172598.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.37524792551994324,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 700,
|
|
"step_time": 50.620650958269835
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1719.0,
|
|
"completions/max_terminated_length": 1719.0,
|
|
"completions/mean_length": 822.015625,
|
|
"completions/mean_terminated_length": 822.015625,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.6169170867651701,
|
|
"epoch": 0.3973922902494331,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 6.031746031746031e-07,
|
|
"loss": 0.0103,
|
|
"num_tokens": 41236375.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 701,
|
|
"step_time": 58.50305821374059
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1276.0,
|
|
"completions/max_terminated_length": 1276.0,
|
|
"completions/mean_length": 702.59375,
|
|
"completions/mean_terminated_length": 702.59375,
|
|
"completions/min_length": 244.0,
|
|
"completions/min_terminated_length": 244.0,
|
|
"entropy": 0.8059696778655052,
|
|
"epoch": 0.3979591836734694,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 6.026077097505669e-07,
|
|
"loss": 0.0069,
|
|
"num_tokens": 41293101.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5455447435379028,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 702,
|
|
"step_time": 43.96138078626245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1250.0,
|
|
"completions/max_terminated_length": 1250.0,
|
|
"completions/mean_length": 648.046875,
|
|
"completions/mean_terminated_length": 648.046875,
|
|
"completions/min_length": 280.0,
|
|
"completions/min_terminated_length": 280.0,
|
|
"entropy": 0.6702596955001354,
|
|
"epoch": 0.39852607709750565,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 6.020408163265306e-07,
|
|
"loss": -0.0121,
|
|
"num_tokens": 41346528.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.745356023311615,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 703,
|
|
"step_time": 42.75202510878444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1140.0,
|
|
"completions/max_terminated_length": 1140.0,
|
|
"completions/mean_length": 687.78125,
|
|
"completions/mean_terminated_length": 687.78125,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.7277284692972898,
|
|
"epoch": 0.39909297052154197,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.63671875,
|
|
"learning_rate": 6.014739229024944e-07,
|
|
"loss": 0.0301,
|
|
"num_tokens": 41401434.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 704,
|
|
"step_time": 39.50285739637911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1286.0,
|
|
"completions/max_terminated_length": 1286.0,
|
|
"completions/mean_length": 665.953125,
|
|
"completions/mean_terminated_length": 665.953125,
|
|
"completions/min_length": 174.0,
|
|
"completions/min_terminated_length": 174.0,
|
|
"entropy": 0.800261564552784,
|
|
"epoch": 0.39965986394557823,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 6.00907029478458e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 41454063.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 705,
|
|
"step_time": 43.9979402134195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1239.0,
|
|
"completions/max_terminated_length": 1239.0,
|
|
"completions/mean_length": 719.078125,
|
|
"completions/mean_terminated_length": 719.078125,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.877237593755126,
|
|
"epoch": 0.4002267573696145,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 6.003401360544217e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 41512276.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.701074481010437,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 706,
|
|
"step_time": 43.27953645866364
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1338.0,
|
|
"completions/max_terminated_length": 1338.0,
|
|
"completions/mean_length": 786.453125,
|
|
"completions/mean_terminated_length": 786.453125,
|
|
"completions/min_length": 437.0,
|
|
"completions/min_terminated_length": 437.0,
|
|
"entropy": 0.6615323107689619,
|
|
"epoch": 0.4007936507936508,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.474609375,
|
|
"learning_rate": 5.997732426303854e-07,
|
|
"loss": 0.0054,
|
|
"num_tokens": 41573089.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 707,
|
|
"step_time": 46.12602823320776
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1576.0,
|
|
"completions/max_terminated_length": 1576.0,
|
|
"completions/mean_length": 776.859375,
|
|
"completions/mean_terminated_length": 776.859375,
|
|
"completions/min_length": 302.0,
|
|
"completions/min_terminated_length": 302.0,
|
|
"entropy": 0.7004927899688482,
|
|
"epoch": 0.4013605442176871,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 5.992063492063492e-07,
|
|
"loss": 0.023,
|
|
"num_tokens": 41632328.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.774084210395813,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 708,
|
|
"step_time": 54.11020789574832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1239.0,
|
|
"completions/max_terminated_length": 1239.0,
|
|
"completions/mean_length": 697.984375,
|
|
"completions/mean_terminated_length": 697.984375,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 0.8287873566150665,
|
|
"epoch": 0.40192743764172334,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 5.986394557823129e-07,
|
|
"loss": 0.0374,
|
|
"num_tokens": 41688127.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 709,
|
|
"step_time": 42.59092733357102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1351.0,
|
|
"completions/max_terminated_length": 1351.0,
|
|
"completions/mean_length": 758.484375,
|
|
"completions/mean_terminated_length": 758.484375,
|
|
"completions/min_length": 392.0,
|
|
"completions/min_terminated_length": 392.0,
|
|
"entropy": 0.9395110942423344,
|
|
"epoch": 0.40249433106575966,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 5.980725623582767e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 41746894.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 710,
|
|
"step_time": 46.619201021268964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1842.0,
|
|
"completions/max_terminated_length": 1842.0,
|
|
"completions/mean_length": 898.625,
|
|
"completions/mean_terminated_length": 898.625,
|
|
"completions/min_length": 363.0,
|
|
"completions/min_terminated_length": 363.0,
|
|
"entropy": 0.6419065445661545,
|
|
"epoch": 0.4030612244897959,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 5.975056689342404e-07,
|
|
"loss": 0.0082,
|
|
"num_tokens": 41813894.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.8796644806861877,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 711,
|
|
"step_time": 63.49873128719628
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1299.0,
|
|
"completions/max_terminated_length": 1299.0,
|
|
"completions/mean_length": 734.046875,
|
|
"completions/mean_terminated_length": 734.046875,
|
|
"completions/min_length": 266.0,
|
|
"completions/min_terminated_length": 266.0,
|
|
"entropy": 1.0022796988487244,
|
|
"epoch": 0.4036281179138322,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.423828125,
|
|
"learning_rate": 5.969387755102042e-07,
|
|
"loss": 0.003,
|
|
"num_tokens": 41872089.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.2630521357059479,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.015625,
|
|
"rewards/format_reward_func/std": 0.08768405020236969,
|
|
"step": 712,
|
|
"step_time": 44.153327917680144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2780.0,
|
|
"completions/max_terminated_length": 2780.0,
|
|
"completions/mean_length": 812.296875,
|
|
"completions/mean_terminated_length": 812.296875,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.6370504721999168,
|
|
"epoch": 0.4041950113378685,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 5.963718820861677e-07,
|
|
"loss": -0.0073,
|
|
"num_tokens": 41934788.0,
|
|
"reward": 0.4765625,
|
|
"reward_std": 0.763397216796875,
|
|
"rewards/accuracy_reward_func/mean": 0.375,
|
|
"rewards/accuracy_reward_func/std": 0.7867957949638367,
|
|
"rewards/format_reward_func/mean": 0.1015625,
|
|
"rewards/format_reward_func/std": 0.20275264978408813,
|
|
"step": 713,
|
|
"step_time": 101.39591556601226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1720.0,
|
|
"completions/max_terminated_length": 1720.0,
|
|
"completions/mean_length": 701.71875,
|
|
"completions/mean_terminated_length": 701.71875,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.7805192265659571,
|
|
"epoch": 0.40476190476190477,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 5.958049886621314e-07,
|
|
"loss": 0.0228,
|
|
"num_tokens": 41992266.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 714,
|
|
"step_time": 60.487199569121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1879.0,
|
|
"completions/max_terminated_length": 1879.0,
|
|
"completions/mean_length": 710.25,
|
|
"completions/mean_terminated_length": 710.25,
|
|
"completions/min_length": 226.0,
|
|
"completions/min_terminated_length": 226.0,
|
|
"entropy": 0.6370216235518456,
|
|
"epoch": 0.40532879818594103,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 5.952380952380952e-07,
|
|
"loss": 0.0107,
|
|
"num_tokens": 42047082.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7867957949638367,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 715,
|
|
"step_time": 63.83074046112597
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1340.0,
|
|
"completions/max_terminated_length": 1340.0,
|
|
"completions/mean_length": 781.703125,
|
|
"completions/mean_terminated_length": 781.703125,
|
|
"completions/min_length": 344.0,
|
|
"completions/min_terminated_length": 344.0,
|
|
"entropy": 0.6871910858899355,
|
|
"epoch": 0.40589569160997735,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46484375,
|
|
"learning_rate": 5.946712018140589e-07,
|
|
"loss": -0.0113,
|
|
"num_tokens": 42106431.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 716,
|
|
"step_time": 45.26214159745723
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1261.0,
|
|
"completions/max_terminated_length": 1261.0,
|
|
"completions/mean_length": 712.53125,
|
|
"completions/mean_terminated_length": 712.53125,
|
|
"completions/min_length": 281.0,
|
|
"completions/min_terminated_length": 281.0,
|
|
"entropy": 0.8684989213943481,
|
|
"epoch": 0.4064625850340136,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 5.941043083900227e-07,
|
|
"loss": 0.0168,
|
|
"num_tokens": 42163337.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 717,
|
|
"step_time": 43.39038709178567
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1401.0,
|
|
"completions/max_terminated_length": 1401.0,
|
|
"completions/mean_length": 746.5,
|
|
"completions/mean_terminated_length": 746.5,
|
|
"completions/min_length": 267.0,
|
|
"completions/min_terminated_length": 267.0,
|
|
"entropy": 0.577209796756506,
|
|
"epoch": 0.4070294784580499,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.494140625,
|
|
"learning_rate": 5.935374149659864e-07,
|
|
"loss": 0.0105,
|
|
"num_tokens": 42220441.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 718,
|
|
"step_time": 48.270371528342366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1201.0,
|
|
"completions/max_terminated_length": 1201.0,
|
|
"completions/mean_length": 654.03125,
|
|
"completions/mean_terminated_length": 654.03125,
|
|
"completions/min_length": 251.0,
|
|
"completions/min_terminated_length": 251.0,
|
|
"entropy": 0.8232200630009174,
|
|
"epoch": 0.4075963718820862,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 5.929705215419501e-07,
|
|
"loss": 0.02,
|
|
"num_tokens": 42273035.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6538955569267273,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 719,
|
|
"step_time": 41.617827150039375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1354.0,
|
|
"completions/max_terminated_length": 1354.0,
|
|
"completions/mean_length": 725.734375,
|
|
"completions/mean_terminated_length": 725.734375,
|
|
"completions/min_length": 419.0,
|
|
"completions/min_terminated_length": 419.0,
|
|
"entropy": 0.7096963319927454,
|
|
"epoch": 0.40816326530612246,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 5.924036281179138e-07,
|
|
"loss": 0.0038,
|
|
"num_tokens": 42334130.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6172134280204773,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 720,
|
|
"step_time": 47.07885314244777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1600.0,
|
|
"completions/max_terminated_length": 1600.0,
|
|
"completions/mean_length": 808.625,
|
|
"completions/mean_terminated_length": 808.625,
|
|
"completions/min_length": 290.0,
|
|
"completions/min_terminated_length": 290.0,
|
|
"entropy": 0.62769565731287,
|
|
"epoch": 0.4087301587301587,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.28125,
|
|
"learning_rate": 5.918367346938775e-07,
|
|
"loss": -0.004,
|
|
"num_tokens": 42395810.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 721,
|
|
"step_time": 54.02559690643102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 740.375,
|
|
"completions/mean_terminated_length": 740.375,
|
|
"completions/min_length": 366.0,
|
|
"completions/min_terminated_length": 366.0,
|
|
"entropy": 0.8444711156189442,
|
|
"epoch": 0.409297052154195,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"learning_rate": 5.912698412698412e-07,
|
|
"loss": -0.0021,
|
|
"num_tokens": 42454954.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 722,
|
|
"step_time": 43.73231762833893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1242.0,
|
|
"completions/max_terminated_length": 1242.0,
|
|
"completions/mean_length": 728.609375,
|
|
"completions/mean_terminated_length": 728.609375,
|
|
"completions/min_length": 298.0,
|
|
"completions/min_terminated_length": 298.0,
|
|
"entropy": 0.8367441222071648,
|
|
"epoch": 0.4098639455782313,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.310546875,
|
|
"learning_rate": 5.90702947845805e-07,
|
|
"loss": 0.013,
|
|
"num_tokens": 42519009.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.47709178924560547,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 723,
|
|
"step_time": 45.36817385815084
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1386.0,
|
|
"completions/max_terminated_length": 1386.0,
|
|
"completions/mean_length": 816.359375,
|
|
"completions/mean_terminated_length": 816.359375,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.7131411526352167,
|
|
"epoch": 0.41043083900226757,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 5.901360544217687e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 42582088.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6777355074882507,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 724,
|
|
"step_time": 46.9804265871644
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1515.0,
|
|
"completions/max_terminated_length": 1515.0,
|
|
"completions/mean_length": 737.046875,
|
|
"completions/mean_terminated_length": 737.046875,
|
|
"completions/min_length": 375.0,
|
|
"completions/min_terminated_length": 375.0,
|
|
"entropy": 0.7451744675636292,
|
|
"epoch": 0.41099773242630383,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.6484375,
|
|
"learning_rate": 5.895691609977324e-07,
|
|
"loss": 0.0135,
|
|
"num_tokens": 42641243.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 725,
|
|
"step_time": 52.16097322292626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1119.0,
|
|
"completions/max_terminated_length": 1119.0,
|
|
"completions/mean_length": 705.15625,
|
|
"completions/mean_terminated_length": 705.15625,
|
|
"completions/min_length": 339.0,
|
|
"completions/min_terminated_length": 339.0,
|
|
"entropy": 0.6900949291884899,
|
|
"epoch": 0.41156462585034015,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.62109375,
|
|
"learning_rate": 5.890022675736961e-07,
|
|
"loss": 0.0551,
|
|
"num_tokens": 42696653.0,
|
|
"reward": 0.3671875,
|
|
"reward_std": 0.7880163192749023,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 726,
|
|
"step_time": 38.13649579510093
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 735.984375,
|
|
"completions/mean_terminated_length": 735.984375,
|
|
"completions/min_length": 235.0,
|
|
"completions/min_terminated_length": 235.0,
|
|
"entropy": 0.8310915026813745,
|
|
"epoch": 0.4121315192743764,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 5.884353741496599e-07,
|
|
"loss": 0.0281,
|
|
"num_tokens": 42753748.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 727,
|
|
"step_time": 49.61503871437162
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1372.0,
|
|
"completions/max_terminated_length": 1372.0,
|
|
"completions/mean_length": 731.25,
|
|
"completions/mean_terminated_length": 731.25,
|
|
"completions/min_length": 383.0,
|
|
"completions/min_terminated_length": 383.0,
|
|
"entropy": 0.9786618016660213,
|
|
"epoch": 0.4126984126984127,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 5.878684807256236e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 42813036.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 728,
|
|
"step_time": 46.887156607583165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1585.0,
|
|
"completions/max_terminated_length": 1585.0,
|
|
"completions/mean_length": 759.34375,
|
|
"completions/mean_terminated_length": 759.34375,
|
|
"completions/min_length": 265.0,
|
|
"completions/min_terminated_length": 265.0,
|
|
"entropy": 0.919614166021347,
|
|
"epoch": 0.413265306122449,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 5.873015873015873e-07,
|
|
"loss": 0.0107,
|
|
"num_tokens": 42875178.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 729,
|
|
"step_time": 55.13844683393836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1396.0,
|
|
"completions/max_terminated_length": 1396.0,
|
|
"completions/mean_length": 728.9375,
|
|
"completions/mean_terminated_length": 728.9375,
|
|
"completions/min_length": 295.0,
|
|
"completions/min_terminated_length": 295.0,
|
|
"entropy": 0.8809193521738052,
|
|
"epoch": 0.41383219954648526,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 5.86734693877551e-07,
|
|
"loss": 0.0294,
|
|
"num_tokens": 42931734.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 730,
|
|
"step_time": 48.121805551461875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1328.0,
|
|
"completions/max_terminated_length": 1328.0,
|
|
"completions/mean_length": 667.140625,
|
|
"completions/mean_terminated_length": 667.140625,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 0.7360779270529747,
|
|
"epoch": 0.4143990929705215,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.62109375,
|
|
"learning_rate": 5.861678004535146e-07,
|
|
"loss": -0.001,
|
|
"num_tokens": 42985119.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 731,
|
|
"step_time": 45.864870502613485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1301.0,
|
|
"completions/max_terminated_length": 1301.0,
|
|
"completions/mean_length": 739.34375,
|
|
"completions/mean_terminated_length": 739.34375,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.6361839286983013,
|
|
"epoch": 0.41496598639455784,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.7109375,
|
|
"learning_rate": 5.856009070294784e-07,
|
|
"loss": 0.0104,
|
|
"num_tokens": 43043989.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 732,
|
|
"step_time": 44.30827684234828
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1193.0,
|
|
"completions/max_terminated_length": 1193.0,
|
|
"completions/mean_length": 679.5625,
|
|
"completions/mean_terminated_length": 679.5625,
|
|
"completions/min_length": 326.0,
|
|
"completions/min_terminated_length": 326.0,
|
|
"entropy": 1.0744093619287014,
|
|
"epoch": 0.4155328798185941,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 5.850340136054421e-07,
|
|
"loss": 0.0066,
|
|
"num_tokens": 43098857.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6029338836669922,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 733,
|
|
"step_time": 40.71691221278161
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1097.0,
|
|
"completions/max_terminated_length": 1097.0,
|
|
"completions/mean_length": 688.671875,
|
|
"completions/mean_terminated_length": 688.671875,
|
|
"completions/min_length": 370.0,
|
|
"completions/min_terminated_length": 370.0,
|
|
"entropy": 0.9368883110582829,
|
|
"epoch": 0.41609977324263037,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 5.844671201814059e-07,
|
|
"loss": 0.0075,
|
|
"num_tokens": 43153788.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 734,
|
|
"step_time": 37.3967783851549
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1316.0,
|
|
"completions/max_terminated_length": 1316.0,
|
|
"completions/mean_length": 795.484375,
|
|
"completions/mean_terminated_length": 795.484375,
|
|
"completions/min_length": 407.0,
|
|
"completions/min_terminated_length": 407.0,
|
|
"entropy": 0.8395189568400383,
|
|
"epoch": 0.4166666666666667,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 5.839002267573696e-07,
|
|
"loss": -0.007,
|
|
"num_tokens": 43214323.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 735,
|
|
"step_time": 45.11113391537219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1152.0,
|
|
"completions/max_terminated_length": 1152.0,
|
|
"completions/mean_length": 792.1875,
|
|
"completions/mean_terminated_length": 792.1875,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.8356929533183575,
|
|
"epoch": 0.41723356009070295,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 5.833333333333334e-07,
|
|
"loss": 0.0282,
|
|
"num_tokens": 43276127.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 736,
|
|
"step_time": 40.03395030833781
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1551.0,
|
|
"completions/max_terminated_length": 1551.0,
|
|
"completions/mean_length": 724.71875,
|
|
"completions/mean_terminated_length": 724.71875,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.735107097774744,
|
|
"epoch": 0.4178004535147392,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 5.82766439909297e-07,
|
|
"loss": -0.0089,
|
|
"num_tokens": 43336573.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 737,
|
|
"step_time": 55.217949592508376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1433.0,
|
|
"completions/max_terminated_length": 1433.0,
|
|
"completions/mean_length": 751.375,
|
|
"completions/mean_terminated_length": 751.375,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.8345095999538898,
|
|
"epoch": 0.41836734693877553,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 5.821995464852607e-07,
|
|
"loss": 0.0365,
|
|
"num_tokens": 43394461.0,
|
|
"reward": 0.5546875,
|
|
"reward_std": 0.8734041452407837,
|
|
"rewards/accuracy_reward_func/mean": 0.5,
|
|
"rewards/accuracy_reward_func/std": 0.8728715777397156,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 738,
|
|
"step_time": 48.906748646870255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1473.0,
|
|
"completions/max_terminated_length": 1473.0,
|
|
"completions/mean_length": 708.40625,
|
|
"completions/mean_terminated_length": 708.40625,
|
|
"completions/min_length": 368.0,
|
|
"completions/min_terminated_length": 368.0,
|
|
"entropy": 0.8683152124285698,
|
|
"epoch": 0.4189342403628118,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 5.816326530612244e-07,
|
|
"loss": -0.0016,
|
|
"num_tokens": 43453271.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 739,
|
|
"step_time": 51.43727010767907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1472.0,
|
|
"completions/max_terminated_length": 1472.0,
|
|
"completions/mean_length": 810.96875,
|
|
"completions/mean_terminated_length": 810.96875,
|
|
"completions/min_length": 353.0,
|
|
"completions/min_terminated_length": 353.0,
|
|
"entropy": 0.6964520383626223,
|
|
"epoch": 0.41950113378684806,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3125,
|
|
"learning_rate": 5.810657596371882e-07,
|
|
"loss": -0.0012,
|
|
"num_tokens": 43514821.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.5107961297035217,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 740,
|
|
"step_time": 49.67749553266913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1840.0,
|
|
"completions/mean_length": 845.1875,
|
|
"completions/mean_terminated_length": 793.5873413085938,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.5888213403522968,
|
|
"epoch": 0.4200680272108844,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 5.804988662131519e-07,
|
|
"loss": 0.0107,
|
|
"num_tokens": 43579161.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 741,
|
|
"step_time": 168.525633610785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1242.0,
|
|
"completions/max_terminated_length": 1242.0,
|
|
"completions/mean_length": 743.546875,
|
|
"completions/mean_terminated_length": 743.546875,
|
|
"completions/min_length": 54.0,
|
|
"completions/min_terminated_length": 54.0,
|
|
"entropy": 0.5299877347424626,
|
|
"epoch": 0.42063492063492064,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 5.799319727891157e-07,
|
|
"loss": 0.0112,
|
|
"num_tokens": 43635908.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.665922224521637,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 742,
|
|
"step_time": 42.83841699268669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1542.0,
|
|
"completions/max_terminated_length": 1542.0,
|
|
"completions/mean_length": 720.21875,
|
|
"completions/mean_terminated_length": 720.21875,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 1.0659764669835567,
|
|
"epoch": 0.4212018140589569,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 5.793650793650794e-07,
|
|
"loss": 0.0152,
|
|
"num_tokens": 43693034.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 743,
|
|
"step_time": 53.25322303455323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1190.0,
|
|
"completions/max_terminated_length": 1190.0,
|
|
"completions/mean_length": 750.3125,
|
|
"completions/mean_terminated_length": 750.3125,
|
|
"completions/min_length": 372.0,
|
|
"completions/min_terminated_length": 372.0,
|
|
"entropy": 0.6504237297922373,
|
|
"epoch": 0.4217687074829932,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 5.78798185941043e-07,
|
|
"loss": -0.0099,
|
|
"num_tokens": 43751678.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.7007648944854736,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 744,
|
|
"step_time": 41.41377501748502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2257.0,
|
|
"completions/max_terminated_length": 2257.0,
|
|
"completions/mean_length": 768.09375,
|
|
"completions/mean_terminated_length": 768.09375,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.694596691057086,
|
|
"epoch": 0.4223356009070295,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 5.782312925170067e-07,
|
|
"loss": 0.06,
|
|
"num_tokens": 43811748.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 745,
|
|
"step_time": 79.29288134723902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1441.0,
|
|
"completions/max_terminated_length": 1441.0,
|
|
"completions/mean_length": 838.765625,
|
|
"completions/mean_terminated_length": 838.765625,
|
|
"completions/min_length": 358.0,
|
|
"completions/min_terminated_length": 358.0,
|
|
"entropy": 0.5027836207300425,
|
|
"epoch": 0.42290249433106575,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 5.776643990929705e-07,
|
|
"loss": 0.0122,
|
|
"num_tokens": 43876501.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 746,
|
|
"step_time": 49.38363088015467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1466.0,
|
|
"completions/max_terminated_length": 1466.0,
|
|
"completions/mean_length": 795.109375,
|
|
"completions/mean_terminated_length": 795.109375,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.9084397703409195,
|
|
"epoch": 0.42346938775510207,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 5.770975056689342e-07,
|
|
"loss": -0.0214,
|
|
"num_tokens": 43937892.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 747,
|
|
"step_time": 50.44422111287713
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1628.0,
|
|
"completions/max_terminated_length": 1628.0,
|
|
"completions/mean_length": 747.75,
|
|
"completions/mean_terminated_length": 747.75,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.6967819910496473,
|
|
"epoch": 0.42403628117913833,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.443359375,
|
|
"learning_rate": 5.76530612244898e-07,
|
|
"loss": 0.0251,
|
|
"num_tokens": 43999332.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7185450792312622,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 748,
|
|
"step_time": 57.57845904864371
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1853.0,
|
|
"completions/max_terminated_length": 1853.0,
|
|
"completions/mean_length": 806.46875,
|
|
"completions/mean_terminated_length": 806.46875,
|
|
"completions/min_length": 425.0,
|
|
"completions/min_terminated_length": 425.0,
|
|
"entropy": 0.8299741912633181,
|
|
"epoch": 0.4246031746031746,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.376953125,
|
|
"learning_rate": 5.759637188208617e-07,
|
|
"loss": 0.0229,
|
|
"num_tokens": 44061122.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.46922925114631653,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 749,
|
|
"step_time": 63.8022293895483
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2944.0,
|
|
"completions/max_terminated_length": 2944.0,
|
|
"completions/mean_length": 783.640625,
|
|
"completions/mean_terminated_length": 783.640625,
|
|
"completions/min_length": 272.0,
|
|
"completions/min_terminated_length": 272.0,
|
|
"entropy": 0.960544366389513,
|
|
"epoch": 0.42517006802721086,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.48046875,
|
|
"learning_rate": 5.753968253968254e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 44127067.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 750,
|
|
"step_time": 118.26526685990393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1354.0,
|
|
"completions/max_terminated_length": 1354.0,
|
|
"completions/mean_length": 724.109375,
|
|
"completions/mean_terminated_length": 724.109375,
|
|
"completions/min_length": 222.0,
|
|
"completions/min_terminated_length": 222.0,
|
|
"entropy": 0.9830201044678688,
|
|
"epoch": 0.4257369614512472,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 5.748299319727891e-07,
|
|
"loss": 0.0244,
|
|
"num_tokens": 44184498.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 751,
|
|
"step_time": 45.99378525465727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1508.0,
|
|
"completions/max_terminated_length": 1508.0,
|
|
"completions/mean_length": 784.0625,
|
|
"completions/mean_terminated_length": 784.0625,
|
|
"completions/min_length": 397.0,
|
|
"completions/min_terminated_length": 397.0,
|
|
"entropy": 0.8110213577747345,
|
|
"epoch": 0.42630385487528344,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 5.742630385487527e-07,
|
|
"loss": 0.0261,
|
|
"num_tokens": 44246126.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 752,
|
|
"step_time": 52.3525614021346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 665.4375,
|
|
"completions/mean_terminated_length": 665.4375,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.8430373854935169,
|
|
"epoch": 0.4268707482993197,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.400390625,
|
|
"learning_rate": 5.736961451247165e-07,
|
|
"loss": 0.032,
|
|
"num_tokens": 44299570.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5764904618263245,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 753,
|
|
"step_time": 42.80218448769301
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1264.0,
|
|
"completions/max_terminated_length": 1264.0,
|
|
"completions/mean_length": 743.78125,
|
|
"completions/mean_terminated_length": 743.78125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.8105163779109716,
|
|
"epoch": 0.427437641723356,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 5.731292517006802e-07,
|
|
"loss": -0.0031,
|
|
"num_tokens": 44357388.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 754,
|
|
"step_time": 42.49784342665225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1602.0,
|
|
"completions/max_terminated_length": 1602.0,
|
|
"completions/mean_length": 701.390625,
|
|
"completions/mean_terminated_length": 701.390625,
|
|
"completions/min_length": 194.0,
|
|
"completions/min_terminated_length": 194.0,
|
|
"entropy": 0.668464969843626,
|
|
"epoch": 0.4280045351473923,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 5.72562358276644e-07,
|
|
"loss": 0.0018,
|
|
"num_tokens": 44411381.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 755,
|
|
"step_time": 54.76838449109346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1142.0,
|
|
"completions/max_terminated_length": 1142.0,
|
|
"completions/mean_length": 645.34375,
|
|
"completions/mean_terminated_length": 645.34375,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 0.9178903736174107,
|
|
"epoch": 0.42857142857142855,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.8125,
|
|
"learning_rate": 5.719954648526076e-07,
|
|
"loss": -0.0107,
|
|
"num_tokens": 44461635.0,
|
|
"reward": 0.3984375,
|
|
"reward_std": 0.7516103982925415,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 756,
|
|
"step_time": 38.94769489020109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1241.0,
|
|
"completions/max_terminated_length": 1241.0,
|
|
"completions/mean_length": 675.90625,
|
|
"completions/mean_terminated_length": 675.90625,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 1.0502110309898853,
|
|
"epoch": 0.42913832199546487,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.298828125,
|
|
"learning_rate": 5.714285714285714e-07,
|
|
"loss": -0.0109,
|
|
"num_tokens": 44515069.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 757,
|
|
"step_time": 42.03664774727076
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1222.0,
|
|
"completions/max_terminated_length": 1222.0,
|
|
"completions/mean_length": 719.71875,
|
|
"completions/mean_terminated_length": 719.71875,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"entropy": 0.7671445775777102,
|
|
"epoch": 0.42970521541950113,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 5.708616780045351e-07,
|
|
"loss": -0.0067,
|
|
"num_tokens": 44570131.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 758,
|
|
"step_time": 40.8751755123958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1571.0,
|
|
"completions/max_terminated_length": 1571.0,
|
|
"completions/mean_length": 829.53125,
|
|
"completions/mean_terminated_length": 829.53125,
|
|
"completions/min_length": 366.0,
|
|
"completions/min_terminated_length": 366.0,
|
|
"entropy": 0.7642947994172573,
|
|
"epoch": 0.4302721088435374,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 5.702947845804989e-07,
|
|
"loss": 0.0352,
|
|
"num_tokens": 44634093.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6649904847145081,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 759,
|
|
"step_time": 54.131978866644204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1333.0,
|
|
"completions/max_terminated_length": 1333.0,
|
|
"completions/mean_length": 744.859375,
|
|
"completions/mean_terminated_length": 744.859375,
|
|
"completions/min_length": 349.0,
|
|
"completions/min_terminated_length": 349.0,
|
|
"entropy": 0.95208290964365,
|
|
"epoch": 0.4308390022675737,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.462890625,
|
|
"learning_rate": 5.697278911564625e-07,
|
|
"loss": -0.0062,
|
|
"num_tokens": 44692580.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.49501484632492065,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 760,
|
|
"step_time": 45.60193131119013
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1383.0,
|
|
"completions/max_terminated_length": 1383.0,
|
|
"completions/mean_length": 820.046875,
|
|
"completions/mean_terminated_length": 820.046875,
|
|
"completions/min_length": 370.0,
|
|
"completions/min_terminated_length": 370.0,
|
|
"entropy": 0.7668710723519325,
|
|
"epoch": 0.43140589569161,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3203125,
|
|
"learning_rate": 5.691609977324263e-07,
|
|
"loss": -0.0045,
|
|
"num_tokens": 44759479.0,
|
|
"reward": 0.078125,
|
|
"reward_std": 0.2847827076911926,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 761,
|
|
"step_time": 49.02586789615452
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1593.0,
|
|
"completions/max_terminated_length": 1593.0,
|
|
"completions/mean_length": 860.4375,
|
|
"completions/mean_terminated_length": 860.4375,
|
|
"completions/min_length": 480.0,
|
|
"completions/min_terminated_length": 480.0,
|
|
"entropy": 0.725027572363615,
|
|
"epoch": 0.43197278911564624,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 5.685941043083899e-07,
|
|
"loss": 0.0034,
|
|
"num_tokens": 44824139.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7185450792312622,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 762,
|
|
"step_time": 54.259508294053376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1319.0,
|
|
"completions/max_terminated_length": 1319.0,
|
|
"completions/mean_length": 729.15625,
|
|
"completions/mean_terminated_length": 729.15625,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"entropy": 0.789761122316122,
|
|
"epoch": 0.43253968253968256,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4609375,
|
|
"learning_rate": 5.680272108843537e-07,
|
|
"loss": -0.0118,
|
|
"num_tokens": 44882285.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 763,
|
|
"step_time": 45.12244441639632
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1804.0,
|
|
"completions/max_terminated_length": 1804.0,
|
|
"completions/mean_length": 765.96875,
|
|
"completions/mean_terminated_length": 765.96875,
|
|
"completions/min_length": 186.0,
|
|
"completions/min_terminated_length": 186.0,
|
|
"entropy": 0.8189289644360542,
|
|
"epoch": 0.4331065759637188,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.37109375,
|
|
"learning_rate": 5.674603174603174e-07,
|
|
"loss": 0.0002,
|
|
"num_tokens": 44941499.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5298036932945251,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 764,
|
|
"step_time": 61.34843339771032
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1769.0,
|
|
"completions/max_terminated_length": 1769.0,
|
|
"completions/mean_length": 747.53125,
|
|
"completions/mean_terminated_length": 747.53125,
|
|
"completions/min_length": 430.0,
|
|
"completions/min_terminated_length": 430.0,
|
|
"entropy": 0.8964242096990347,
|
|
"epoch": 0.4336734693877551,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 5.668934240362812e-07,
|
|
"loss": 0.0091,
|
|
"num_tokens": 45001397.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 765,
|
|
"step_time": 60.81541761662811
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 995.0,
|
|
"completions/max_terminated_length": 995.0,
|
|
"completions/mean_length": 586.203125,
|
|
"completions/mean_terminated_length": 586.203125,
|
|
"completions/min_length": 162.0,
|
|
"completions/min_terminated_length": 162.0,
|
|
"entropy": 1.0916917622089386,
|
|
"epoch": 0.4342403628117914,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.6328125,
|
|
"learning_rate": 5.663265306122449e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 45050914.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 766,
|
|
"step_time": 34.052269238978624
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1876.0,
|
|
"completions/max_terminated_length": 1876.0,
|
|
"completions/mean_length": 671.640625,
|
|
"completions/mean_terminated_length": 671.640625,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.8228061571717262,
|
|
"epoch": 0.43480725623582767,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 5.657596371882087e-07,
|
|
"loss": 0.0232,
|
|
"num_tokens": 45103571.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 767,
|
|
"step_time": 63.753929750062525
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2744.0,
|
|
"completions/max_terminated_length": 2744.0,
|
|
"completions/mean_length": 776.796875,
|
|
"completions/mean_terminated_length": 776.796875,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.6937824469059706,
|
|
"epoch": 0.43537414965986393,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4921875,
|
|
"learning_rate": 5.651927437641723e-07,
|
|
"loss": 0.0839,
|
|
"num_tokens": 45163486.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 768,
|
|
"step_time": 99.16287459339947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1625.0,
|
|
"completions/max_terminated_length": 1625.0,
|
|
"completions/mean_length": 919.171875,
|
|
"completions/mean_terminated_length": 919.171875,
|
|
"completions/min_length": 533.0,
|
|
"completions/min_terminated_length": 533.0,
|
|
"entropy": 0.5660981275141239,
|
|
"epoch": 0.43594104308390025,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.416015625,
|
|
"learning_rate": 5.64625850340136e-07,
|
|
"loss": 0.0097,
|
|
"num_tokens": 45231185.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 769,
|
|
"step_time": 55.40216701198369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1838.0,
|
|
"completions/max_terminated_length": 1838.0,
|
|
"completions/mean_length": 738.65625,
|
|
"completions/mean_terminated_length": 738.65625,
|
|
"completions/min_length": 197.0,
|
|
"completions/min_terminated_length": 197.0,
|
|
"entropy": 0.6767476070672274,
|
|
"epoch": 0.4365079365079365,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 5.640589569160997e-07,
|
|
"loss": 0.015,
|
|
"num_tokens": 45289083.0,
|
|
"reward": 0.359375,
|
|
"reward_std": 0.7478474378585815,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 770,
|
|
"step_time": 64.3451043413952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1220.0,
|
|
"completions/max_terminated_length": 1220.0,
|
|
"completions/mean_length": 698.328125,
|
|
"completions/mean_terminated_length": 698.328125,
|
|
"completions/min_length": 282.0,
|
|
"completions/min_terminated_length": 282.0,
|
|
"entropy": 1.005146386101842,
|
|
"epoch": 0.4370748299319728,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 5.634920634920635e-07,
|
|
"loss": -0.0048,
|
|
"num_tokens": 45342728.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 771,
|
|
"step_time": 41.5515520805493
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1205.0,
|
|
"completions/max_terminated_length": 1205.0,
|
|
"completions/mean_length": 756.921875,
|
|
"completions/mean_terminated_length": 756.921875,
|
|
"completions/min_length": 354.0,
|
|
"completions/min_terminated_length": 354.0,
|
|
"entropy": 0.7958099376410246,
|
|
"epoch": 0.4376417233560091,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 5.629251700680272e-07,
|
|
"loss": -0.0216,
|
|
"num_tokens": 45400107.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 772,
|
|
"step_time": 41.31011986825615
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1242.0,
|
|
"completions/max_terminated_length": 1242.0,
|
|
"completions/mean_length": 751.96875,
|
|
"completions/mean_terminated_length": 751.96875,
|
|
"completions/min_length": 392.0,
|
|
"completions/min_terminated_length": 392.0,
|
|
"entropy": 0.9773392900824547,
|
|
"epoch": 0.43820861678004536,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 5.62358276643991e-07,
|
|
"loss": -0.0111,
|
|
"num_tokens": 45458601.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 773,
|
|
"step_time": 42.69651668332517
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1407.0,
|
|
"completions/max_terminated_length": 1407.0,
|
|
"completions/mean_length": 806.125,
|
|
"completions/mean_terminated_length": 806.125,
|
|
"completions/min_length": 338.0,
|
|
"completions/min_terminated_length": 338.0,
|
|
"entropy": 0.9133709631860256,
|
|
"epoch": 0.4387755102040816,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 5.617913832199547e-07,
|
|
"loss": -0.0011,
|
|
"num_tokens": 45522753.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 774,
|
|
"step_time": 49.31818789802492
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/max_terminated_length": 1344.0,
|
|
"completions/mean_length": 812.515625,
|
|
"completions/mean_terminated_length": 812.515625,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.5701410993933678,
|
|
"epoch": 0.43934240362811794,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 5.612244897959183e-07,
|
|
"loss": 0.0478,
|
|
"num_tokens": 45586578.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.8351171612739563,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 775,
|
|
"step_time": 47.40858102682978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1495.0,
|
|
"completions/max_terminated_length": 1495.0,
|
|
"completions/mean_length": 826.625,
|
|
"completions/mean_terminated_length": 826.625,
|
|
"completions/min_length": 364.0,
|
|
"completions/min_terminated_length": 364.0,
|
|
"entropy": 0.7575418595224619,
|
|
"epoch": 0.4399092970521542,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 5.60657596371882e-07,
|
|
"loss": 0.0051,
|
|
"num_tokens": 45652490.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6780555844306946,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 776,
|
|
"step_time": 52.221214167773724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1353.0,
|
|
"completions/max_terminated_length": 1353.0,
|
|
"completions/mean_length": 707.953125,
|
|
"completions/mean_terminated_length": 707.953125,
|
|
"completions/min_length": 15.0,
|
|
"completions/min_terminated_length": 15.0,
|
|
"entropy": 0.887224867939949,
|
|
"epoch": 0.44047619047619047,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 5.600907029478457e-07,
|
|
"loss": 0.0102,
|
|
"num_tokens": 45708751.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 777,
|
|
"step_time": 46.73268238455057
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1164.0,
|
|
"completions/max_terminated_length": 1164.0,
|
|
"completions/mean_length": 673.015625,
|
|
"completions/mean_terminated_length": 673.015625,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"entropy": 0.8263971842825413,
|
|
"epoch": 0.4410430839002268,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 5.595238095238095e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 45761592.0,
|
|
"reward": 0.3828125,
|
|
"reward_std": 0.7545743584632874,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 778,
|
|
"step_time": 39.5863815927878
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1420.0,
|
|
"completions/max_terminated_length": 1420.0,
|
|
"completions/mean_length": 768.234375,
|
|
"completions/mean_terminated_length": 768.234375,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.6901191417127848,
|
|
"epoch": 0.44160997732426305,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.59375,
|
|
"learning_rate": 5.589569160997732e-07,
|
|
"loss": -0.0535,
|
|
"num_tokens": 45819247.0,
|
|
"reward": 0.2890625,
|
|
"reward_std": 0.6655030846595764,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 779,
|
|
"step_time": 48.00127135589719
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1542.0,
|
|
"completions/max_terminated_length": 1542.0,
|
|
"completions/mean_length": 713.59375,
|
|
"completions/mean_terminated_length": 713.59375,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 0.7308294903486967,
|
|
"epoch": 0.4421768707482993,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 5.58390022675737e-07,
|
|
"loss": -0.0285,
|
|
"num_tokens": 45875405.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 780,
|
|
"step_time": 52.49343588668853
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1265.0,
|
|
"completions/max_terminated_length": 1265.0,
|
|
"completions/mean_length": 807.234375,
|
|
"completions/mean_terminated_length": 807.234375,
|
|
"completions/min_length": 412.0,
|
|
"completions/min_terminated_length": 412.0,
|
|
"entropy": 0.7114958334714174,
|
|
"epoch": 0.4427437641723356,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 5.578231292517006e-07,
|
|
"loss": 0.0114,
|
|
"num_tokens": 45936076.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 781,
|
|
"step_time": 42.76526521053165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1711.0,
|
|
"completions/max_terminated_length": 1711.0,
|
|
"completions/mean_length": 731.65625,
|
|
"completions/mean_terminated_length": 731.65625,
|
|
"completions/min_length": 239.0,
|
|
"completions/min_terminated_length": 239.0,
|
|
"entropy": 0.8945471830666065,
|
|
"epoch": 0.4433106575963719,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 5.572562358276644e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 45993326.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.528690755367279,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 782,
|
|
"step_time": 58.49189604818821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1119.0,
|
|
"completions/max_terminated_length": 1119.0,
|
|
"completions/mean_length": 668.390625,
|
|
"completions/mean_terminated_length": 668.390625,
|
|
"completions/min_length": 285.0,
|
|
"completions/min_terminated_length": 285.0,
|
|
"entropy": 0.9769992753863335,
|
|
"epoch": 0.44387755102040816,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 5.566893424036281e-07,
|
|
"loss": 0.0043,
|
|
"num_tokens": 46045031.0,
|
|
"reward": 0.0546875,
|
|
"reward_std": 0.15728822350502014,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 783,
|
|
"step_time": 38.11941893771291
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1154.0,
|
|
"completions/max_terminated_length": 1154.0,
|
|
"completions/mean_length": 618.921875,
|
|
"completions/mean_terminated_length": 618.921875,
|
|
"completions/min_length": 249.0,
|
|
"completions/min_terminated_length": 249.0,
|
|
"entropy": 0.8294611759483814,
|
|
"epoch": 0.4444444444444444,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6484375,
|
|
"learning_rate": 5.561224489795918e-07,
|
|
"loss": 0.0218,
|
|
"num_tokens": 46095178.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.6970388293266296,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 784,
|
|
"step_time": 39.68534651771188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1344.0,
|
|
"completions/max_terminated_length": 1344.0,
|
|
"completions/mean_length": 697.859375,
|
|
"completions/mean_terminated_length": 697.859375,
|
|
"completions/min_length": 247.0,
|
|
"completions/min_terminated_length": 247.0,
|
|
"entropy": 0.813194153830409,
|
|
"epoch": 0.44501133786848074,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 5.555555555555555e-07,
|
|
"loss": -0.0149,
|
|
"num_tokens": 46149177.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.6636837720870972,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 785,
|
|
"step_time": 45.37823559436947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1234.0,
|
|
"completions/max_terminated_length": 1234.0,
|
|
"completions/mean_length": 724.953125,
|
|
"completions/mean_terminated_length": 724.953125,
|
|
"completions/min_length": 287.0,
|
|
"completions/min_terminated_length": 287.0,
|
|
"entropy": 0.9080928973853588,
|
|
"epoch": 0.445578231292517,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 5.549886621315193e-07,
|
|
"loss": 0.01,
|
|
"num_tokens": 46209878.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 786,
|
|
"step_time": 44.43291334249079
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1106.0,
|
|
"completions/max_terminated_length": 1106.0,
|
|
"completions/mean_length": 631.15625,
|
|
"completions/mean_terminated_length": 631.15625,
|
|
"completions/min_length": 11.0,
|
|
"completions/min_terminated_length": 11.0,
|
|
"entropy": 0.857480525970459,
|
|
"epoch": 0.44614512471655327,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6015625,
|
|
"learning_rate": 5.544217687074829e-07,
|
|
"loss": 0.0463,
|
|
"num_tokens": 46260248.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 787,
|
|
"step_time": 37.46732331532985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2864.0,
|
|
"completions/max_terminated_length": 2864.0,
|
|
"completions/mean_length": 785.546875,
|
|
"completions/mean_terminated_length": 785.546875,
|
|
"completions/min_length": 262.0,
|
|
"completions/min_terminated_length": 262.0,
|
|
"entropy": 0.6720971409231424,
|
|
"epoch": 0.4467120181405896,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 5.538548752834467e-07,
|
|
"loss": -0.0103,
|
|
"num_tokens": 46320851.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 788,
|
|
"step_time": 106.06966610439122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1913.0,
|
|
"completions/max_terminated_length": 1913.0,
|
|
"completions/mean_length": 817.0,
|
|
"completions/mean_terminated_length": 817.0,
|
|
"completions/min_length": 313.0,
|
|
"completions/min_terminated_length": 313.0,
|
|
"entropy": 0.6204305812716484,
|
|
"epoch": 0.44727891156462585,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 5.532879818594104e-07,
|
|
"loss": -0.0054,
|
|
"num_tokens": 46382315.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7185450792312622,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 789,
|
|
"step_time": 65.26288800500333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1375.0,
|
|
"completions/max_terminated_length": 1375.0,
|
|
"completions/mean_length": 758.734375,
|
|
"completions/mean_terminated_length": 758.734375,
|
|
"completions/min_length": 277.0,
|
|
"completions/min_terminated_length": 277.0,
|
|
"entropy": 0.7401899695396423,
|
|
"epoch": 0.4478458049886621,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 5.527210884353742e-07,
|
|
"loss": -0.0334,
|
|
"num_tokens": 46443322.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 790,
|
|
"step_time": 48.78369199298322
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1325.0,
|
|
"completions/max_terminated_length": 1325.0,
|
|
"completions/mean_length": 714.921875,
|
|
"completions/mean_terminated_length": 714.921875,
|
|
"completions/min_length": 190.0,
|
|
"completions/min_terminated_length": 190.0,
|
|
"entropy": 0.7558332365006208,
|
|
"epoch": 0.44841269841269843,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.326171875,
|
|
"learning_rate": 5.521541950113379e-07,
|
|
"loss": 0.004,
|
|
"num_tokens": 46499925.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.6900655627250671,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 791,
|
|
"step_time": 47.22886026557535
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1854.0,
|
|
"completions/max_terminated_length": 1854.0,
|
|
"completions/mean_length": 864.578125,
|
|
"completions/mean_terminated_length": 864.578125,
|
|
"completions/min_length": 390.0,
|
|
"completions/min_terminated_length": 390.0,
|
|
"entropy": 0.6858900133520365,
|
|
"epoch": 0.4489795918367347,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 5.515873015873016e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 46565514.0,
|
|
"reward": 0.5234375,
|
|
"reward_std": 0.8927827477455139,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 792,
|
|
"step_time": 64.46240985859185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1498.0,
|
|
"completions/max_terminated_length": 1498.0,
|
|
"completions/mean_length": 851.265625,
|
|
"completions/mean_terminated_length": 851.265625,
|
|
"completions/min_length": 535.0,
|
|
"completions/min_terminated_length": 535.0,
|
|
"entropy": 0.7471986319869757,
|
|
"epoch": 0.44954648526077096,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 5.510204081632652e-07,
|
|
"loss": -0.0267,
|
|
"num_tokens": 46630859.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.576006293296814,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 793,
|
|
"step_time": 51.35069953557104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1723.0,
|
|
"completions/max_terminated_length": 1723.0,
|
|
"completions/mean_length": 807.609375,
|
|
"completions/mean_terminated_length": 807.609375,
|
|
"completions/min_length": 27.0,
|
|
"completions/min_terminated_length": 27.0,
|
|
"entropy": 0.7217428702861071,
|
|
"epoch": 0.4501133786848073,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.1494140625,
|
|
"learning_rate": 5.504535147392289e-07,
|
|
"loss": -0.0005,
|
|
"num_tokens": 46697194.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.4542567729949951,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 794,
|
|
"step_time": 60.17362304404378
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1281.0,
|
|
"completions/max_terminated_length": 1281.0,
|
|
"completions/mean_length": 727.15625,
|
|
"completions/mean_terminated_length": 727.15625,
|
|
"completions/min_length": 433.0,
|
|
"completions/min_terminated_length": 433.0,
|
|
"entropy": 0.9020949602127075,
|
|
"epoch": 0.45068027210884354,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 5.498866213151927e-07,
|
|
"loss": -0.0134,
|
|
"num_tokens": 46754660.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7175520658493042,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 795,
|
|
"step_time": 43.84019714873284
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1260.0,
|
|
"completions/max_terminated_length": 1260.0,
|
|
"completions/mean_length": 795.78125,
|
|
"completions/mean_terminated_length": 795.78125,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.8808070737868547,
|
|
"epoch": 0.4512471655328798,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 5.493197278911564e-07,
|
|
"loss": -0.0046,
|
|
"num_tokens": 46823054.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.5589615106582642,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 796,
|
|
"step_time": 46.92815616633743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1434.0,
|
|
"completions/max_terminated_length": 1434.0,
|
|
"completions/mean_length": 835.515625,
|
|
"completions/mean_terminated_length": 835.515625,
|
|
"completions/min_length": 476.0,
|
|
"completions/min_terminated_length": 476.0,
|
|
"entropy": 0.8159170430153608,
|
|
"epoch": 0.4518140589569161,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40625,
|
|
"learning_rate": 5.487528344671202e-07,
|
|
"loss": -0.0259,
|
|
"num_tokens": 46886367.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7319250702857971,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 797,
|
|
"step_time": 49.25248962920159
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1486.0,
|
|
"completions/max_terminated_length": 1486.0,
|
|
"completions/mean_length": 704.609375,
|
|
"completions/mean_terminated_length": 704.609375,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.7818028852343559,
|
|
"epoch": 0.4523809523809524,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 5.481859410430839e-07,
|
|
"loss": 0.026,
|
|
"num_tokens": 46941838.0,
|
|
"reward": 0.1796875,
|
|
"reward_std": 0.49894481897354126,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 798,
|
|
"step_time": 50.434957961551845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1493.0,
|
|
"completions/max_terminated_length": 1493.0,
|
|
"completions/mean_length": 755.640625,
|
|
"completions/mean_terminated_length": 755.640625,
|
|
"completions/min_length": 292.0,
|
|
"completions/min_terminated_length": 292.0,
|
|
"entropy": 0.6671258062124252,
|
|
"epoch": 0.45294784580498865,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 5.476190476190477e-07,
|
|
"loss": 0.0306,
|
|
"num_tokens": 47000215.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.7403060793876648,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 799,
|
|
"step_time": 51.44981790706515
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1775.0,
|
|
"completions/max_terminated_length": 1775.0,
|
|
"completions/mean_length": 724.5,
|
|
"completions/mean_terminated_length": 724.5,
|
|
"completions/min_length": 215.0,
|
|
"completions/min_terminated_length": 215.0,
|
|
"entropy": 0.8277410194277763,
|
|
"epoch": 0.45351473922902497,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.361328125,
|
|
"learning_rate": 5.470521541950112e-07,
|
|
"loss": -0.0069,
|
|
"num_tokens": 47056583.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 800,
|
|
"step_time": 61.59256974142045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1541.0,
|
|
"completions/max_terminated_length": 1541.0,
|
|
"completions/mean_length": 760.0,
|
|
"completions/mean_terminated_length": 760.0,
|
|
"completions/min_length": 378.0,
|
|
"completions/min_terminated_length": 378.0,
|
|
"entropy": 0.8334639333188534,
|
|
"epoch": 0.45408163265306123,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 5.46485260770975e-07,
|
|
"loss": 0.0134,
|
|
"num_tokens": 47117263.0,
|
|
"reward": 0.296875,
|
|
"reward_std": 0.6884463429450989,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 801,
|
|
"step_time": 53.386829040013254
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 769.96875,
|
|
"completions/mean_terminated_length": 769.96875,
|
|
"completions/min_length": 395.0,
|
|
"completions/min_terminated_length": 395.0,
|
|
"entropy": 0.7062572371214628,
|
|
"epoch": 0.4546485260770975,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 5.459183673469387e-07,
|
|
"loss": -0.0063,
|
|
"num_tokens": 47176485.0,
|
|
"reward": 0.3203125,
|
|
"reward_std": 0.7202688455581665,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 802,
|
|
"step_time": 46.564074200578034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1534.0,
|
|
"completions/max_terminated_length": 1534.0,
|
|
"completions/mean_length": 746.078125,
|
|
"completions/mean_terminated_length": 746.078125,
|
|
"completions/min_length": 300.0,
|
|
"completions/min_terminated_length": 300.0,
|
|
"entropy": 0.7608544230461121,
|
|
"epoch": 0.4552154195011338,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 5.453514739229025e-07,
|
|
"loss": 0.0283,
|
|
"num_tokens": 47236530.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 803,
|
|
"step_time": 53.19833767414093
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1015.0,
|
|
"completions/max_terminated_length": 1015.0,
|
|
"completions/mean_length": 679.671875,
|
|
"completions/mean_terminated_length": 679.671875,
|
|
"completions/min_length": 417.0,
|
|
"completions/min_terminated_length": 417.0,
|
|
"entropy": 0.8538750410079956,
|
|
"epoch": 0.4557823129251701,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 5.447845804988662e-07,
|
|
"loss": 0.0345,
|
|
"num_tokens": 47288997.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.664384126663208,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.1171875,
|
|
"rewards/format_reward_func/std": 0.21347814798355103,
|
|
"step": 804,
|
|
"step_time": 34.811470283195376
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1071.0,
|
|
"completions/max_terminated_length": 1071.0,
|
|
"completions/mean_length": 733.859375,
|
|
"completions/mean_terminated_length": 733.859375,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 0.8062970414757729,
|
|
"epoch": 0.45634920634920634,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 5.4421768707483e-07,
|
|
"loss": 0.0118,
|
|
"num_tokens": 47346980.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.6029338836669922,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 805,
|
|
"step_time": 37.23000880237669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1310.0,
|
|
"completions/max_terminated_length": 1310.0,
|
|
"completions/mean_length": 742.328125,
|
|
"completions/mean_terminated_length": 742.328125,
|
|
"completions/min_length": 4.0,
|
|
"completions/min_terminated_length": 4.0,
|
|
"entropy": 0.7718549035489559,
|
|
"epoch": 0.45691609977324266,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 5.436507936507936e-07,
|
|
"loss": -0.0047,
|
|
"num_tokens": 47406801.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6423677802085876,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 806,
|
|
"step_time": 45.8796949069947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1474.0,
|
|
"completions/mean_length": 770.40625,
|
|
"completions/mean_terminated_length": 717.6190795898438,
|
|
"completions/min_length": 221.0,
|
|
"completions/min_terminated_length": 221.0,
|
|
"entropy": 0.9605542048811913,
|
|
"epoch": 0.4574829931972789,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"learning_rate": 5.430839002267574e-07,
|
|
"loss": 0.0068,
|
|
"num_tokens": 47466827.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 807,
|
|
"step_time": 167.76192534156144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1547.0,
|
|
"completions/max_terminated_length": 1547.0,
|
|
"completions/mean_length": 810.625,
|
|
"completions/mean_terminated_length": 810.625,
|
|
"completions/min_length": 321.0,
|
|
"completions/min_terminated_length": 321.0,
|
|
"entropy": 0.6971529796719551,
|
|
"epoch": 0.4580498866213152,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 5.42517006802721e-07,
|
|
"loss": 0.0369,
|
|
"num_tokens": 47529947.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 808,
|
|
"step_time": 52.849554646760225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1357.0,
|
|
"completions/max_terminated_length": 1357.0,
|
|
"completions/mean_length": 643.953125,
|
|
"completions/mean_terminated_length": 643.953125,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.7967690993100405,
|
|
"epoch": 0.45861678004535145,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 5.419501133786848e-07,
|
|
"loss": 0.0332,
|
|
"num_tokens": 47583520.0,
|
|
"reward": 0.3359375,
|
|
"reward_std": 0.6961041688919067,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 809,
|
|
"step_time": 46.43441923055798
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1437.0,
|
|
"completions/max_terminated_length": 1437.0,
|
|
"completions/mean_length": 768.640625,
|
|
"completions/mean_terminated_length": 768.640625,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.7078898008912802,
|
|
"epoch": 0.45918367346938777,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 5.413832199546485e-07,
|
|
"loss": -0.0138,
|
|
"num_tokens": 47642713.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 810,
|
|
"step_time": 49.29645659215748
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1661.0,
|
|
"completions/max_terminated_length": 1661.0,
|
|
"completions/mean_length": 821.890625,
|
|
"completions/mean_terminated_length": 821.890625,
|
|
"completions/min_length": 260.0,
|
|
"completions/min_terminated_length": 260.0,
|
|
"entropy": 0.5739525891840458,
|
|
"epoch": 0.45975056689342403,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 5.408163265306123e-07,
|
|
"loss": 0.0403,
|
|
"num_tokens": 47705402.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 811,
|
|
"step_time": 56.428977597504854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1159.0,
|
|
"completions/max_terminated_length": 1159.0,
|
|
"completions/mean_length": 737.75,
|
|
"completions/mean_terminated_length": 737.75,
|
|
"completions/min_length": 337.0,
|
|
"completions/min_terminated_length": 337.0,
|
|
"entropy": 0.8854955472052097,
|
|
"epoch": 0.4603174603174603,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.3203125,
|
|
"learning_rate": 5.402494331065759e-07,
|
|
"loss": 0.0161,
|
|
"num_tokens": 47764842.0,
|
|
"reward": 0.1171875,
|
|
"reward_std": 0.41539984941482544,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 812,
|
|
"step_time": 40.85682630725205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1157.0,
|
|
"completions/max_terminated_length": 1157.0,
|
|
"completions/mean_length": 693.078125,
|
|
"completions/mean_terminated_length": 693.078125,
|
|
"completions/min_length": 51.0,
|
|
"completions/min_terminated_length": 51.0,
|
|
"entropy": 0.7058844473212957,
|
|
"epoch": 0.4608843537414966,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 5.396825396825396e-07,
|
|
"loss": -0.0421,
|
|
"num_tokens": 47819767.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 813,
|
|
"step_time": 39.65966281667352
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1656.0,
|
|
"completions/max_terminated_length": 1656.0,
|
|
"completions/mean_length": 839.90625,
|
|
"completions/mean_terminated_length": 839.90625,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.8400525860488415,
|
|
"epoch": 0.4614512471655329,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.259765625,
|
|
"learning_rate": 5.391156462585034e-07,
|
|
"loss": 0.0086,
|
|
"num_tokens": 47884801.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.29378482699394226,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 814,
|
|
"step_time": 57.51062997803092
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1467.0,
|
|
"completions/max_terminated_length": 1467.0,
|
|
"completions/mean_length": 711.03125,
|
|
"completions/mean_terminated_length": 711.03125,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.6923054903745651,
|
|
"epoch": 0.46201814058956914,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 5.385487528344671e-07,
|
|
"loss": -0.0013,
|
|
"num_tokens": 47939843.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.8261595964431763,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 815,
|
|
"step_time": 50.49040416907519
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1228.0,
|
|
"completions/max_terminated_length": 1228.0,
|
|
"completions/mean_length": 705.359375,
|
|
"completions/mean_terminated_length": 705.359375,
|
|
"completions/min_length": 79.0,
|
|
"completions/min_terminated_length": 79.0,
|
|
"entropy": 0.8476248569786549,
|
|
"epoch": 0.46258503401360546,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 5.379818594104308e-07,
|
|
"loss": -0.0071,
|
|
"num_tokens": 47998610.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.5471335649490356,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 816,
|
|
"step_time": 44.53605140373111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1465.0,
|
|
"completions/max_terminated_length": 1465.0,
|
|
"completions/mean_length": 845.4375,
|
|
"completions/mean_terminated_length": 845.4375,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.6319690588861704,
|
|
"epoch": 0.4631519274376417,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.396484375,
|
|
"learning_rate": 5.374149659863945e-07,
|
|
"loss": 0.0163,
|
|
"num_tokens": 48063238.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 817,
|
|
"step_time": 50.90306873712689
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1241.0,
|
|
"completions/max_terminated_length": 1241.0,
|
|
"completions/mean_length": 721.46875,
|
|
"completions/mean_terminated_length": 721.46875,
|
|
"completions/min_length": 12.0,
|
|
"completions/min_terminated_length": 12.0,
|
|
"entropy": 0.7134434916079044,
|
|
"epoch": 0.463718820861678,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 5.368480725623582e-07,
|
|
"loss": -0.0008,
|
|
"num_tokens": 48120948.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.6171631813049316,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 818,
|
|
"step_time": 43.22208777628839
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1312.0,
|
|
"completions/max_terminated_length": 1312.0,
|
|
"completions/mean_length": 751.03125,
|
|
"completions/mean_terminated_length": 751.03125,
|
|
"completions/min_length": 308.0,
|
|
"completions/min_terminated_length": 308.0,
|
|
"entropy": 0.7312752678990364,
|
|
"epoch": 0.4642857142857143,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.271484375,
|
|
"learning_rate": 5.362811791383219e-07,
|
|
"loss": -0.003,
|
|
"num_tokens": 48179742.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.653468668460846,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 819,
|
|
"step_time": 45.742073049768806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1257.0,
|
|
"completions/max_terminated_length": 1257.0,
|
|
"completions/mean_length": 728.859375,
|
|
"completions/mean_terminated_length": 728.859375,
|
|
"completions/min_length": 323.0,
|
|
"completions/min_terminated_length": 323.0,
|
|
"entropy": 0.8979786559939384,
|
|
"epoch": 0.46485260770975056,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 5.357142857142857e-07,
|
|
"loss": 0.0156,
|
|
"num_tokens": 48236013.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.7233612537384033,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 820,
|
|
"step_time": 43.38824546709657
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1334.0,
|
|
"completions/max_terminated_length": 1334.0,
|
|
"completions/mean_length": 799.515625,
|
|
"completions/mean_terminated_length": 799.515625,
|
|
"completions/min_length": 307.0,
|
|
"completions/min_terminated_length": 307.0,
|
|
"entropy": 0.7565864995121956,
|
|
"epoch": 0.46541950113378683,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.435546875,
|
|
"learning_rate": 5.351473922902494e-07,
|
|
"loss": 0.018,
|
|
"num_tokens": 48297742.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 821,
|
|
"step_time": 46.17744138929993
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2680.0,
|
|
"completions/max_terminated_length": 2680.0,
|
|
"completions/mean_length": 796.25,
|
|
"completions/mean_terminated_length": 796.25,
|
|
"completions/min_length": 362.0,
|
|
"completions/min_terminated_length": 362.0,
|
|
"entropy": 0.7098961528390646,
|
|
"epoch": 0.46598639455782315,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 5.345804988662132e-07,
|
|
"loss": -0.0113,
|
|
"num_tokens": 48357630.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 822,
|
|
"step_time": 95.88210507016629
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1413.0,
|
|
"completions/max_terminated_length": 1413.0,
|
|
"completions/mean_length": 812.828125,
|
|
"completions/mean_terminated_length": 812.828125,
|
|
"completions/min_length": 228.0,
|
|
"completions/min_terminated_length": 228.0,
|
|
"entropy": 0.6991224847733974,
|
|
"epoch": 0.4665532879818594,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.322265625,
|
|
"learning_rate": 5.340136054421769e-07,
|
|
"loss": 0.0055,
|
|
"num_tokens": 48418683.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.417855441570282,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 823,
|
|
"step_time": 47.68941381480545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1683.0,
|
|
"completions/max_terminated_length": 1683.0,
|
|
"completions/mean_length": 721.296875,
|
|
"completions/mean_terminated_length": 721.296875,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.8589720036834478,
|
|
"epoch": 0.4671201814058957,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 5.334467120181405e-07,
|
|
"loss": 0.0059,
|
|
"num_tokens": 48474638.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.6950790882110596,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 824,
|
|
"step_time": 57.212311525829136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1902.0,
|
|
"completions/max_terminated_length": 1902.0,
|
|
"completions/mean_length": 761.53125,
|
|
"completions/mean_terminated_length": 761.53125,
|
|
"completions/min_length": 240.0,
|
|
"completions/min_terminated_length": 240.0,
|
|
"entropy": 0.9465934336185455,
|
|
"epoch": 0.467687074829932,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5234375,
|
|
"learning_rate": 5.328798185941042e-07,
|
|
"loss": 0.0177,
|
|
"num_tokens": 48534288.0,
|
|
"reward": 0.46875,
|
|
"reward_std": 0.8351171612739563,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 825,
|
|
"step_time": 65.78962198086083
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 3657.0,
|
|
"completions/max_terminated_length": 3657.0,
|
|
"completions/mean_length": 713.09375,
|
|
"completions/mean_terminated_length": 713.09375,
|
|
"completions/min_length": 201.0,
|
|
"completions/min_terminated_length": 201.0,
|
|
"entropy": 0.7143605556339025,
|
|
"epoch": 0.46825396825396826,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 5.32312925170068e-07,
|
|
"loss": 0.0157,
|
|
"num_tokens": 48589630.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7237468957901001,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 826,
|
|
"step_time": 143.11971341632307
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1320.0,
|
|
"completions/max_terminated_length": 1320.0,
|
|
"completions/mean_length": 657.390625,
|
|
"completions/mean_terminated_length": 657.390625,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.601087711751461,
|
|
"epoch": 0.4688208616780045,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 5.317460317460317e-07,
|
|
"loss": 0.0268,
|
|
"num_tokens": 48642895.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7393422722816467,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 827,
|
|
"step_time": 44.84990187827498
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1361.0,
|
|
"completions/max_terminated_length": 1361.0,
|
|
"completions/mean_length": 743.296875,
|
|
"completions/mean_terminated_length": 743.296875,
|
|
"completions/min_length": 283.0,
|
|
"completions/min_terminated_length": 283.0,
|
|
"entropy": 0.7138791903853416,
|
|
"epoch": 0.46938775510204084,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 5.311791383219955e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 48700538.0,
|
|
"reward": 0.265625,
|
|
"reward_std": 0.6297439336776733,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 828,
|
|
"step_time": 46.322299542836845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1342.0,
|
|
"completions/max_terminated_length": 1342.0,
|
|
"completions/mean_length": 810.390625,
|
|
"completions/mean_terminated_length": 810.390625,
|
|
"completions/min_length": 328.0,
|
|
"completions/min_terminated_length": 328.0,
|
|
"entropy": 0.7545554023236036,
|
|
"epoch": 0.4699546485260771,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2119140625,
|
|
"learning_rate": 5.306122448979592e-07,
|
|
"loss": 0.0128,
|
|
"num_tokens": 48765875.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.37796446681022644,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 829,
|
|
"step_time": 46.48511579725891
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 960.0,
|
|
"completions/max_terminated_length": 960.0,
|
|
"completions/mean_length": 623.640625,
|
|
"completions/mean_terminated_length": 623.640625,
|
|
"completions/min_length": 334.0,
|
|
"completions/min_terminated_length": 334.0,
|
|
"entropy": 0.9303254708647728,
|
|
"epoch": 0.47052154195011336,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 5.30045351473923e-07,
|
|
"loss": -0.0114,
|
|
"num_tokens": 48819396.0,
|
|
"reward": 0.328125,
|
|
"reward_std": 0.7411849498748779,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 830,
|
|
"step_time": 35.616590317338705
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1225.0,
|
|
"completions/max_terminated_length": 1225.0,
|
|
"completions/mean_length": 755.953125,
|
|
"completions/mean_terminated_length": 755.953125,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"entropy": 0.7408284954726696,
|
|
"epoch": 0.4710884353741497,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.384765625,
|
|
"learning_rate": 5.294784580498866e-07,
|
|
"loss": -0.018,
|
|
"num_tokens": 48877153.0,
|
|
"reward": 0.34375,
|
|
"reward_std": 0.7175520658493042,
|
|
"rewards/accuracy_reward_func/mean": 0.28125,
|
|
"rewards/accuracy_reward_func/std": 0.7007648944854736,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 831,
|
|
"step_time": 41.72119013313204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2270.0,
|
|
"completions/max_terminated_length": 2270.0,
|
|
"completions/mean_length": 674.15625,
|
|
"completions/mean_terminated_length": 674.15625,
|
|
"completions/min_length": 223.0,
|
|
"completions/min_terminated_length": 223.0,
|
|
"entropy": 0.8973985128104687,
|
|
"epoch": 0.47165532879818595,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.57421875,
|
|
"learning_rate": 5.289115646258502e-07,
|
|
"loss": 0.0078,
|
|
"num_tokens": 48929299.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 832,
|
|
"step_time": 78.77375955600291
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2071.0,
|
|
"completions/max_terminated_length": 2071.0,
|
|
"completions/mean_length": 877.78125,
|
|
"completions/mean_terminated_length": 877.78125,
|
|
"completions/min_length": 229.0,
|
|
"completions/min_terminated_length": 229.0,
|
|
"entropy": 0.6091449167579412,
|
|
"epoch": 0.4722222222222222,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.29296875,
|
|
"learning_rate": 5.28344671201814e-07,
|
|
"loss": -0.0131,
|
|
"num_tokens": 48994805.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 833,
|
|
"step_time": 71.98725274205208
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1381.0,
|
|
"completions/max_terminated_length": 1381.0,
|
|
"completions/mean_length": 705.6875,
|
|
"completions/mean_terminated_length": 705.6875,
|
|
"completions/min_length": 270.0,
|
|
"completions/min_terminated_length": 270.0,
|
|
"entropy": 0.8348685037344694,
|
|
"epoch": 0.47278911564625853,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6015625,
|
|
"learning_rate": 5.277777777777777e-07,
|
|
"loss": -0.0051,
|
|
"num_tokens": 49050097.0,
|
|
"reward": 0.2734375,
|
|
"reward_std": 0.6294977068901062,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 834,
|
|
"step_time": 48.15299692656845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1339.0,
|
|
"completions/max_terminated_length": 1339.0,
|
|
"completions/mean_length": 761.5,
|
|
"completions/mean_terminated_length": 761.5,
|
|
"completions/min_length": 214.0,
|
|
"completions/min_terminated_length": 214.0,
|
|
"entropy": 0.8017330877482891,
|
|
"epoch": 0.4733560090702948,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"learning_rate": 5.272108843537415e-07,
|
|
"loss": -0.0032,
|
|
"num_tokens": 49109337.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 835,
|
|
"step_time": 45.73853251710534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1237.0,
|
|
"completions/max_terminated_length": 1237.0,
|
|
"completions/mean_length": 691.53125,
|
|
"completions/mean_terminated_length": 691.53125,
|
|
"completions/min_length": 363.0,
|
|
"completions/min_terminated_length": 363.0,
|
|
"entropy": 0.7132383771240711,
|
|
"epoch": 0.47392290249433106,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 5.266439909297052e-07,
|
|
"loss": 0.0144,
|
|
"num_tokens": 49163435.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 836,
|
|
"step_time": 42.433180400170386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1627.0,
|
|
"completions/max_terminated_length": 1627.0,
|
|
"completions/mean_length": 780.0625,
|
|
"completions/mean_terminated_length": 780.0625,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.8584453668445349,
|
|
"epoch": 0.4744897959183674,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 5.260770975056689e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 49223639.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 837,
|
|
"step_time": 56.13675133977085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1243.0,
|
|
"completions/max_terminated_length": 1243.0,
|
|
"completions/mean_length": 686.5625,
|
|
"completions/mean_terminated_length": 686.5625,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.7876420617103577,
|
|
"epoch": 0.47505668934240364,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.6328125,
|
|
"learning_rate": 5.255102040816326e-07,
|
|
"loss": 0.0108,
|
|
"num_tokens": 49276699.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.8351171612739563,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 838,
|
|
"step_time": 42.725214873440564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1267.0,
|
|
"completions/max_terminated_length": 1267.0,
|
|
"completions/mean_length": 661.171875,
|
|
"completions/mean_terminated_length": 661.171875,
|
|
"completions/min_length": 255.0,
|
|
"completions/min_terminated_length": 255.0,
|
|
"entropy": 0.8045499790459871,
|
|
"epoch": 0.4756235827664399,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 5.249433106575964e-07,
|
|
"loss": 0.0008,
|
|
"num_tokens": 49330302.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 839,
|
|
"step_time": 44.34828465804458
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1132.0,
|
|
"completions/max_terminated_length": 1132.0,
|
|
"completions/mean_length": 679.234375,
|
|
"completions/mean_terminated_length": 679.234375,
|
|
"completions/min_length": 357.0,
|
|
"completions/min_terminated_length": 357.0,
|
|
"entropy": 0.954418022185564,
|
|
"epoch": 0.47619047619047616,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 5.2437641723356e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 49386573.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 840,
|
|
"step_time": 42.85355908796191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1559.0,
|
|
"completions/max_terminated_length": 1559.0,
|
|
"completions/mean_length": 733.1875,
|
|
"completions/mean_terminated_length": 733.1875,
|
|
"completions/min_length": 278.0,
|
|
"completions/min_terminated_length": 278.0,
|
|
"entropy": 0.8145475946366787,
|
|
"epoch": 0.4767573696145125,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.494140625,
|
|
"learning_rate": 5.238095238095238e-07,
|
|
"loss": 0.0337,
|
|
"num_tokens": 49444049.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
|
"rewards/format_reward_func/std": 0.10652101784944534,
|
|
"step": 841,
|
|
"step_time": 53.10136365517974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2173.0,
|
|
"completions/max_terminated_length": 2173.0,
|
|
"completions/mean_length": 740.796875,
|
|
"completions/mean_terminated_length": 740.796875,
|
|
"completions/min_length": 123.0,
|
|
"completions/min_terminated_length": 123.0,
|
|
"entropy": 0.8114655651152134,
|
|
"epoch": 0.47732426303854875,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.34765625,
|
|
"learning_rate": 5.232426303854875e-07,
|
|
"loss": 0.0271,
|
|
"num_tokens": 49502836.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 842,
|
|
"step_time": 77.42696913983673
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1642.0,
|
|
"completions/max_terminated_length": 1642.0,
|
|
"completions/mean_length": 754.03125,
|
|
"completions/mean_terminated_length": 754.03125,
|
|
"completions/min_length": 199.0,
|
|
"completions/min_terminated_length": 199.0,
|
|
"entropy": 0.7055961042642593,
|
|
"epoch": 0.477891156462585,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.42578125,
|
|
"learning_rate": 5.226757369614512e-07,
|
|
"loss": 0.017,
|
|
"num_tokens": 49560110.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7739239931106567,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 843,
|
|
"step_time": 55.73219994083047
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1322.0,
|
|
"completions/max_terminated_length": 1322.0,
|
|
"completions/mean_length": 715.125,
|
|
"completions/mean_terminated_length": 715.125,
|
|
"completions/min_length": 252.0,
|
|
"completions/min_terminated_length": 252.0,
|
|
"entropy": 0.8245087638497353,
|
|
"epoch": 0.47845804988662133,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 5.221088435374149e-07,
|
|
"loss": 0.0008,
|
|
"num_tokens": 49615438.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.6538955569267273,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 844,
|
|
"step_time": 44.68207171745598
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1597.0,
|
|
"completions/max_terminated_length": 1597.0,
|
|
"completions/mean_length": 798.625,
|
|
"completions/mean_terminated_length": 798.625,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.7979188617318869,
|
|
"epoch": 0.4790249433106576,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.333984375,
|
|
"learning_rate": 5.215419501133787e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 49678918.0,
|
|
"reward": 0.1640625,
|
|
"reward_std": 0.496453195810318,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 845,
|
|
"step_time": 55.64507510140538
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1729.0,
|
|
"completions/max_terminated_length": 1729.0,
|
|
"completions/mean_length": 701.15625,
|
|
"completions/mean_terminated_length": 701.15625,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.8943630382418633,
|
|
"epoch": 0.47959183673469385,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 5.209750566893424e-07,
|
|
"loss": 0.0344,
|
|
"num_tokens": 49733448.0,
|
|
"reward": 0.140625,
|
|
"reward_std": 0.4406772553920746,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 846,
|
|
"step_time": 59.579053840599954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2160.0,
|
|
"completions/max_terminated_length": 2160.0,
|
|
"completions/mean_length": 817.359375,
|
|
"completions/mean_terminated_length": 817.359375,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.5982284694910049,
|
|
"epoch": 0.4801587301587302,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4296875,
|
|
"learning_rate": 5.204081632653062e-07,
|
|
"loss": -0.0004,
|
|
"num_tokens": 49795631.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 847,
|
|
"step_time": 75.91410423349589
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1697.0,
|
|
"completions/max_terminated_length": 1697.0,
|
|
"completions/mean_length": 869.609375,
|
|
"completions/mean_terminated_length": 869.609375,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.6736322026699781,
|
|
"epoch": 0.48072562358276644,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.2236328125,
|
|
"learning_rate": 5.198412698412698e-07,
|
|
"loss": 0.0009,
|
|
"num_tokens": 49860606.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.3398725986480713,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 848,
|
|
"step_time": 58.301382817327976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1979.0,
|
|
"completions/max_terminated_length": 1979.0,
|
|
"completions/mean_length": 634.765625,
|
|
"completions/mean_terminated_length": 634.765625,
|
|
"completions/min_length": 192.0,
|
|
"completions/min_terminated_length": 192.0,
|
|
"entropy": 0.8133383151143789,
|
|
"epoch": 0.4812925170068027,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 5.192743764172335e-07,
|
|
"loss": 0.007,
|
|
"num_tokens": 49912231.0,
|
|
"reward": 0.109375,
|
|
"reward_std": 0.37234508991241455,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 849,
|
|
"step_time": 69.65134237147868
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1630.0,
|
|
"completions/max_terminated_length": 1630.0,
|
|
"completions/mean_length": 652.640625,
|
|
"completions/mean_terminated_length": 652.640625,
|
|
"completions/min_length": 171.0,
|
|
"completions/min_terminated_length": 171.0,
|
|
"entropy": 0.7148718424141407,
|
|
"epoch": 0.481859410430839,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 5.187074829931972e-07,
|
|
"loss": 0.0006,
|
|
"num_tokens": 49964464.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 850,
|
|
"step_time": 57.249084785580635
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1642.0,
|
|
"completions/max_terminated_length": 1642.0,
|
|
"completions/mean_length": 751.140625,
|
|
"completions/mean_terminated_length": 751.140625,
|
|
"completions/min_length": 211.0,
|
|
"completions/min_terminated_length": 211.0,
|
|
"entropy": 0.7809609100222588,
|
|
"epoch": 0.4824263038548753,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.30859375,
|
|
"learning_rate": 5.18140589569161e-07,
|
|
"loss": 0.0036,
|
|
"num_tokens": 50023073.0,
|
|
"reward": 0.0703125,
|
|
"reward_std": 0.27984103560447693,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 851,
|
|
"step_time": 56.52398924808949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1234.0,
|
|
"completions/max_terminated_length": 1234.0,
|
|
"completions/mean_length": 677.6875,
|
|
"completions/mean_terminated_length": 677.6875,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.9891784265637398,
|
|
"epoch": 0.48299319727891155,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.47265625,
|
|
"learning_rate": 5.175736961451247e-07,
|
|
"loss": 0.0033,
|
|
"num_tokens": 50076749.0,
|
|
"reward": 0.0859375,
|
|
"reward_std": 0.28942590951919556,
|
|
"rewards/accuracy_reward_func/mean": 0.03125,
|
|
"rewards/accuracy_reward_func/std": 0.25,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 852,
|
|
"step_time": 42.11584579385817
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.015625,
|
|
"completions/max_length": 4096.0,
|
|
"completions/max_terminated_length": 1387.0,
|
|
"completions/mean_length": 783.21875,
|
|
"completions/mean_terminated_length": 730.6349487304688,
|
|
"completions/min_length": 329.0,
|
|
"completions/min_terminated_length": 329.0,
|
|
"entropy": 0.8308954164385796,
|
|
"epoch": 0.48356009070294786,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 5.170068027210885e-07,
|
|
"loss": -0.0198,
|
|
"num_tokens": 50137371.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.7099072337150574,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 853,
|
|
"step_time": 169.93684468977153
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1159.0,
|
|
"completions/max_terminated_length": 1159.0,
|
|
"completions/mean_length": 713.6875,
|
|
"completions/mean_terminated_length": 713.6875,
|
|
"completions/min_length": 336.0,
|
|
"completions/min_terminated_length": 336.0,
|
|
"entropy": 0.9584300182759762,
|
|
"epoch": 0.48412698412698413,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 5.164399092970522e-07,
|
|
"loss": 0.0098,
|
|
"num_tokens": 50194703.0,
|
|
"reward": 0.046875,
|
|
"reward_std": 0.14689241349697113,
|
|
"rewards/accuracy_reward_func/mean": 0.0,
|
|
"rewards/accuracy_reward_func/std": 0.0,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 854,
|
|
"step_time": 39.95798596646637
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 855.171875,
|
|
"completions/mean_terminated_length": 855.171875,
|
|
"completions/min_length": 421.0,
|
|
"completions/min_terminated_length": 421.0,
|
|
"entropy": 0.6795213408768177,
|
|
"epoch": 0.4846938775510204,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 5.15873015873016e-07,
|
|
"loss": 0.0094,
|
|
"num_tokens": 50259994.0,
|
|
"reward": 0.1953125,
|
|
"reward_std": 0.5463964939117432,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 855,
|
|
"step_time": 52.30651016253978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1287.0,
|
|
"completions/max_terminated_length": 1287.0,
|
|
"completions/mean_length": 647.09375,
|
|
"completions/mean_terminated_length": 647.09375,
|
|
"completions/min_length": 317.0,
|
|
"completions/min_terminated_length": 317.0,
|
|
"entropy": 0.8219214268028736,
|
|
"epoch": 0.4852607709750567,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.87109375,
|
|
"learning_rate": 5.153061224489795e-07,
|
|
"loss": -0.0243,
|
|
"num_tokens": 50311496.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.8351171612739563,
|
|
"rewards/accuracy_reward_func/mean": 0.46875,
|
|
"rewards/accuracy_reward_func/std": 0.8539125919342041,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 856,
|
|
"step_time": 43.74736983794719
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1514.0,
|
|
"completions/max_terminated_length": 1514.0,
|
|
"completions/mean_length": 630.875,
|
|
"completions/mean_terminated_length": 630.875,
|
|
"completions/min_length": 314.0,
|
|
"completions/min_terminated_length": 314.0,
|
|
"entropy": 0.8346275314688683,
|
|
"epoch": 0.485827664399093,
|
|
"frac_reward_zero_std": 0.375,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 5.147392290249432e-07,
|
|
"loss": 0.0067,
|
|
"num_tokens": 50363360.0,
|
|
"reward": 0.4609375,
|
|
"reward_std": 0.7983350157737732,
|
|
"rewards/accuracy_reward_func/mean": 0.40625,
|
|
"rewards/accuracy_reward_func/std": 0.8110105991363525,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 857,
|
|
"step_time": 52.10660233069211
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1283.0,
|
|
"completions/max_terminated_length": 1283.0,
|
|
"completions/mean_length": 737.171875,
|
|
"completions/mean_terminated_length": 737.171875,
|
|
"completions/min_length": 377.0,
|
|
"completions/min_terminated_length": 377.0,
|
|
"entropy": 0.8310398850589991,
|
|
"epoch": 0.48639455782312924,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 5.14172335600907e-07,
|
|
"loss": -0.0303,
|
|
"num_tokens": 50421827.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 858,
|
|
"step_time": 44.48652577307075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1220.0,
|
|
"completions/max_terminated_length": 1220.0,
|
|
"completions/mean_length": 673.171875,
|
|
"completions/mean_terminated_length": 673.171875,
|
|
"completions/min_length": 330.0,
|
|
"completions/min_terminated_length": 330.0,
|
|
"entropy": 0.9571273773908615,
|
|
"epoch": 0.48696145124716556,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 5.136054421768707e-07,
|
|
"loss": 0.0135,
|
|
"num_tokens": 50474806.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 859,
|
|
"step_time": 41.45254014246166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1179.0,
|
|
"completions/max_terminated_length": 1179.0,
|
|
"completions/mean_length": 666.890625,
|
|
"completions/mean_terminated_length": 666.890625,
|
|
"completions/min_length": 384.0,
|
|
"completions/min_terminated_length": 384.0,
|
|
"entropy": 0.92544767819345,
|
|
"epoch": 0.4875283446712018,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 5.130385487528345e-07,
|
|
"loss": 0.0019,
|
|
"num_tokens": 50526975.0,
|
|
"reward": 0.3046875,
|
|
"reward_std": 0.710561990737915,
|
|
"rewards/accuracy_reward_func/mean": 0.25,
|
|
"rewards/accuracy_reward_func/std": 0.6666666865348816,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 860,
|
|
"step_time": 40.417448453605175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1511.0,
|
|
"completions/max_terminated_length": 1511.0,
|
|
"completions/mean_length": 820.453125,
|
|
"completions/mean_terminated_length": 820.453125,
|
|
"completions/min_length": 381.0,
|
|
"completions/min_terminated_length": 381.0,
|
|
"entropy": 0.7471801303327084,
|
|
"epoch": 0.4880952380952381,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.392578125,
|
|
"learning_rate": 5.124716553287982e-07,
|
|
"loss": -0.0161,
|
|
"num_tokens": 50590172.0,
|
|
"reward": 0.09375,
|
|
"reward_std": 0.36596253514289856,
|
|
"rewards/accuracy_reward_func/mean": 0.0625,
|
|
"rewards/accuracy_reward_func/std": 0.35073620080947876,
|
|
"rewards/format_reward_func/mean": 0.03125,
|
|
"rewards/format_reward_func/std": 0.12198751419782639,
|
|
"step": 861,
|
|
"step_time": 52.75329654570669
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1692.0,
|
|
"completions/max_terminated_length": 1692.0,
|
|
"completions/mean_length": 791.78125,
|
|
"completions/mean_terminated_length": 791.78125,
|
|
"completions/min_length": 327.0,
|
|
"completions/min_terminated_length": 327.0,
|
|
"entropy": 0.6955524003133178,
|
|
"epoch": 0.4886621315192744,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 5.119047619047619e-07,
|
|
"loss": 0.0071,
|
|
"num_tokens": 50650750.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 862,
|
|
"step_time": 58.22629368491471
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1316.0,
|
|
"completions/max_terminated_length": 1316.0,
|
|
"completions/mean_length": 611.46875,
|
|
"completions/mean_terminated_length": 611.46875,
|
|
"completions/min_length": 173.0,
|
|
"completions/min_terminated_length": 173.0,
|
|
"entropy": 0.7979342136532068,
|
|
"epoch": 0.48922902494331066,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 5.113378684807256e-07,
|
|
"loss": -0.0108,
|
|
"num_tokens": 50699988.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 863,
|
|
"step_time": 45.322847423143685
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 999.0,
|
|
"completions/max_terminated_length": 999.0,
|
|
"completions/mean_length": 644.953125,
|
|
"completions/mean_terminated_length": 644.953125,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 1.2054049521684647,
|
|
"epoch": 0.4897959183673469,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 5.107709750566893e-07,
|
|
"loss": -0.0018,
|
|
"num_tokens": 50752945.0,
|
|
"reward": 0.2265625,
|
|
"reward_std": 0.5904644727706909,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 864,
|
|
"step_time": 34.44790306407958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 2064.0,
|
|
"completions/max_terminated_length": 2064.0,
|
|
"completions/mean_length": 805.4375,
|
|
"completions/mean_terminated_length": 805.4375,
|
|
"completions/min_length": 343.0,
|
|
"completions/min_terminated_length": 343.0,
|
|
"entropy": 0.729393633082509,
|
|
"epoch": 0.49036281179138325,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4375,
|
|
"learning_rate": 5.10204081632653e-07,
|
|
"loss": 0.0146,
|
|
"num_tokens": 50815317.0,
|
|
"reward": 0.1328125,
|
|
"reward_std": 0.4386323094367981,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 865,
|
|
"step_time": 71.76423513516784
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1230.0,
|
|
"completions/max_terminated_length": 1230.0,
|
|
"completions/mean_length": 704.65625,
|
|
"completions/mean_terminated_length": 704.65625,
|
|
"completions/min_length": 315.0,
|
|
"completions/min_terminated_length": 315.0,
|
|
"entropy": 1.010851725935936,
|
|
"epoch": 0.4909297052154195,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.419921875,
|
|
"learning_rate": 5.096371882086168e-07,
|
|
"loss": -0.0135,
|
|
"num_tokens": 50871743.0,
|
|
"reward": 0.203125,
|
|
"reward_std": 0.6281666159629822,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 866,
|
|
"step_time": 41.98374019283801
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1325.0,
|
|
"completions/max_terminated_length": 1325.0,
|
|
"completions/mean_length": 740.65625,
|
|
"completions/mean_terminated_length": 740.65625,
|
|
"completions/min_length": 365.0,
|
|
"completions/min_terminated_length": 365.0,
|
|
"entropy": 0.8532387372106314,
|
|
"epoch": 0.4914965986394558,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 5.090702947845805e-07,
|
|
"loss": -0.0021,
|
|
"num_tokens": 50935129.0,
|
|
"reward": 0.2421875,
|
|
"reward_std": 0.5908843874931335,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 867,
|
|
"step_time": 46.843373676761985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1258.0,
|
|
"completions/max_terminated_length": 1258.0,
|
|
"completions/mean_length": 708.3125,
|
|
"completions/mean_terminated_length": 708.3125,
|
|
"completions/min_length": 273.0,
|
|
"completions/min_terminated_length": 273.0,
|
|
"entropy": 0.6399582121521235,
|
|
"epoch": 0.49206349206349204,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.486328125,
|
|
"learning_rate": 5.085034013605442e-07,
|
|
"loss": -0.0179,
|
|
"num_tokens": 50990165.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.5909368395805359,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 868,
|
|
"step_time": 43.424545377492905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1910.0,
|
|
"completions/max_terminated_length": 1910.0,
|
|
"completions/mean_length": 780.140625,
|
|
"completions/mean_terminated_length": 780.140625,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.810400640591979,
|
|
"epoch": 0.49263038548752835,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.455078125,
|
|
"learning_rate": 5.079365079365079e-07,
|
|
"loss": 0.0313,
|
|
"num_tokens": 51049406.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 869,
|
|
"step_time": 65.45391212496907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1219.0,
|
|
"completions/max_terminated_length": 1219.0,
|
|
"completions/mean_length": 731.515625,
|
|
"completions/mean_terminated_length": 731.515625,
|
|
"completions/min_length": 367.0,
|
|
"completions/min_terminated_length": 367.0,
|
|
"entropy": 0.8992546834051609,
|
|
"epoch": 0.4931972789115646,
|
|
"frac_reward_zero_std": 0.875,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 5.073696145124717e-07,
|
|
"loss": 0.0113,
|
|
"num_tokens": 51108247.0,
|
|
"reward": 0.15625,
|
|
"reward_std": 0.4443204402923584,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 870,
|
|
"step_time": 41.870989022776484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1449.0,
|
|
"completions/max_terminated_length": 1449.0,
|
|
"completions/mean_length": 744.078125,
|
|
"completions/mean_terminated_length": 744.078125,
|
|
"completions/min_length": 380.0,
|
|
"completions/min_terminated_length": 380.0,
|
|
"entropy": 0.8174138925969601,
|
|
"epoch": 0.4937641723356009,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.4375,
|
|
"learning_rate": 5.068027210884354e-07,
|
|
"loss": -0.0256,
|
|
"num_tokens": 51167020.0,
|
|
"reward": 0.1484375,
|
|
"reward_std": 0.44257262349128723,
|
|
"rewards/accuracy_reward_func/mean": 0.09375,
|
|
"rewards/accuracy_reward_func/std": 0.42608407139778137,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 871,
|
|
"step_time": 49.688143651001155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1795.0,
|
|
"completions/max_terminated_length": 1795.0,
|
|
"completions/mean_length": 705.75,
|
|
"completions/mean_terminated_length": 705.75,
|
|
"completions/min_length": 232.0,
|
|
"completions/min_terminated_length": 232.0,
|
|
"entropy": 0.6099121104925871,
|
|
"epoch": 0.4943310657596372,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.462890625,
|
|
"learning_rate": 5.062358276643991e-07,
|
|
"loss": -0.0043,
|
|
"num_tokens": 51221348.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 872,
|
|
"step_time": 61.60922578629106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1014.0,
|
|
"completions/max_terminated_length": 1014.0,
|
|
"completions/mean_length": 680.453125,
|
|
"completions/mean_terminated_length": 680.453125,
|
|
"completions/min_length": 376.0,
|
|
"completions/min_terminated_length": 376.0,
|
|
"entropy": 1.0266446508467197,
|
|
"epoch": 0.49489795918367346,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 5.056689342403628e-07,
|
|
"loss": -0.0002,
|
|
"num_tokens": 51275617.0,
|
|
"reward": 0.28125,
|
|
"reward_std": 0.6291528940200806,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 873,
|
|
"step_time": 34.654701344668865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1562.0,
|
|
"completions/max_terminated_length": 1562.0,
|
|
"completions/mean_length": 823.9375,
|
|
"completions/mean_terminated_length": 823.9375,
|
|
"completions/min_length": 439.0,
|
|
"completions/min_terminated_length": 439.0,
|
|
"entropy": 0.8492737971246243,
|
|
"epoch": 0.4954648526077097,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.431640625,
|
|
"learning_rate": 5.051020408163264e-07,
|
|
"loss": 0.0176,
|
|
"num_tokens": 51341557.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.5,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 874,
|
|
"step_time": 53.710381127893925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1091.0,
|
|
"completions/max_terminated_length": 1091.0,
|
|
"completions/mean_length": 645.59375,
|
|
"completions/mean_terminated_length": 645.59375,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 1.10162472166121,
|
|
"epoch": 0.49603174603174605,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.76171875,
|
|
"learning_rate": 5.045351473922902e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 51401931.0,
|
|
"reward": 0.2578125,
|
|
"reward_std": 0.629891574382782,
|
|
"rewards/accuracy_reward_func/mean": 0.21875,
|
|
"rewards/accuracy_reward_func/std": 0.6291528940200806,
|
|
"rewards/format_reward_func/mean": 0.0390625,
|
|
"rewards/format_reward_func/std": 0.13524486124515533,
|
|
"step": 875,
|
|
"step_time": 42.514109442941844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1422.0,
|
|
"completions/max_terminated_length": 1422.0,
|
|
"completions/mean_length": 661.0625,
|
|
"completions/mean_terminated_length": 661.0625,
|
|
"completions/min_length": 318.0,
|
|
"completions/min_terminated_length": 318.0,
|
|
"entropy": 0.8126551173627377,
|
|
"epoch": 0.4965986394557823,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 5.039682539682539e-07,
|
|
"loss": 0.0154,
|
|
"num_tokens": 51454799.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.7867957949638367,
|
|
"rewards/accuracy_reward_func/mean": 0.3125,
|
|
"rewards/accuracy_reward_func/std": 0.7319250702857971,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 876,
|
|
"step_time": 49.50599535834044
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1308.0,
|
|
"completions/max_terminated_length": 1308.0,
|
|
"completions/mean_length": 736.921875,
|
|
"completions/mean_terminated_length": 736.921875,
|
|
"completions/min_length": 261.0,
|
|
"completions/min_terminated_length": 261.0,
|
|
"entropy": 0.8450348358601332,
|
|
"epoch": 0.4971655328798186,
|
|
"frac_reward_zero_std": 0.75,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 5.034013605442177e-07,
|
|
"loss": 0.0187,
|
|
"num_tokens": 51514826.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 877,
|
|
"step_time": 45.9421418486163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1363.0,
|
|
"completions/max_terminated_length": 1363.0,
|
|
"completions/mean_length": 681.6875,
|
|
"completions/mean_terminated_length": 681.6875,
|
|
"completions/min_length": 242.0,
|
|
"completions/min_terminated_length": 242.0,
|
|
"entropy": 0.9226914681494236,
|
|
"epoch": 0.4977324263038549,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 5.028344671201814e-07,
|
|
"loss": 0.0093,
|
|
"num_tokens": 51568558.0,
|
|
"reward": 0.2109375,
|
|
"reward_std": 0.5477564930915833,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0546875,
|
|
"rewards/format_reward_func/std": 0.15728822350502014,
|
|
"step": 878,
|
|
"step_time": 47.00045523699373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1278.0,
|
|
"completions/max_terminated_length": 1278.0,
|
|
"completions/mean_length": 725.0625,
|
|
"completions/mean_terminated_length": 725.0625,
|
|
"completions/min_length": 299.0,
|
|
"completions/min_terminated_length": 299.0,
|
|
"entropy": 0.8534124288707972,
|
|
"epoch": 0.49829931972789115,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 5.022675736961452e-07,
|
|
"loss": -0.0423,
|
|
"num_tokens": 51626098.0,
|
|
"reward": 0.21875,
|
|
"reward_std": 0.5482656359672546,
|
|
"rewards/accuracy_reward_func/mean": 0.15625,
|
|
"rewards/accuracy_reward_func/std": 0.5409794449806213,
|
|
"rewards/format_reward_func/mean": 0.0625,
|
|
"rewards/format_reward_func/std": 0.1666666716337204,
|
|
"step": 879,
|
|
"step_time": 44.45098415762186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1383.0,
|
|
"completions/max_terminated_length": 1383.0,
|
|
"completions/mean_length": 698.984375,
|
|
"completions/mean_terminated_length": 698.984375,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 0.6996864043176174,
|
|
"epoch": 0.4988662131519274,
|
|
"frac_reward_zero_std": 0.25,
|
|
"grad_norm": 0.81640625,
|
|
"learning_rate": 5.017006802721087e-07,
|
|
"loss": -0.0009,
|
|
"num_tokens": 51680737.0,
|
|
"reward": 0.390625,
|
|
"reward_std": 0.7531349658966064,
|
|
"rewards/accuracy_reward_func/mean": 0.34375,
|
|
"rewards/accuracy_reward_func/std": 0.7605084180831909,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 880,
|
|
"step_time": 47.530423319898546
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1462.0,
|
|
"completions/max_terminated_length": 1462.0,
|
|
"completions/mean_length": 766.0,
|
|
"completions/mean_terminated_length": 766.0,
|
|
"completions/min_length": 296.0,
|
|
"completions/min_terminated_length": 296.0,
|
|
"entropy": 0.8448827806860209,
|
|
"epoch": 0.49943310657596374,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 5.011337868480725e-07,
|
|
"loss": 0.0227,
|
|
"num_tokens": 51739321.0,
|
|
"reward": 0.171875,
|
|
"reward_std": 0.49776285886764526,
|
|
"rewards/accuracy_reward_func/mean": 0.125,
|
|
"rewards/accuracy_reward_func/std": 0.48795005679130554,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 881,
|
|
"step_time": 50.53285096306354
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 1428.0,
|
|
"completions/max_terminated_length": 1428.0,
|
|
"completions/mean_length": 702.734375,
|
|
"completions/mean_terminated_length": 702.734375,
|
|
"completions/min_length": 226.0,
|
|
"completions/min_terminated_length": 226.0,
|
|
"entropy": 0.733205184340477,
|
|
"epoch": 0.5,
|
|
"frac_reward_zero_std": 0.625,
|
|
"grad_norm": 0.484375,
|
|
"learning_rate": 5.005668934240362e-07,
|
|
"loss": -0.0118,
|
|
"num_tokens": 51794136.0,
|
|
"reward": 0.234375,
|
|
"reward_std": 0.5907269716262817,
|
|
"rewards/accuracy_reward_func/mean": 0.1875,
|
|
"rewards/accuracy_reward_func/std": 0.5875696539878845,
|
|
"rewards/format_reward_func/mean": 0.046875,
|
|
"rewards/format_reward_func/std": 0.14689241349697113,
|
|
"step": 882,
|
|
"step_time": 49.36098977178335
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 1764,
|
|
"num_input_tokens_seen": 51794136,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|